diff --git a/lib/matplotlib/artist.py b/lib/matplotlib/artist.py
index cb6e650a94cc..e3e6b3ca2d7c 100644
--- a/lib/matplotlib/artist.py
+++ b/lib/matplotlib/artist.py
@@ -255,10 +255,12 @@ def remove(self):
             # clear stale callback
             self.stale_callback = None
             _ax_flag = False
-            if hasattr(self, 'axes') and self.axes:
+            ax = getattr(self, 'axes', None)
+            mouseover_set = getattr(ax, '_mouseover_set', None)
+            if mouseover_set is not None:
                 # remove from the mouse hit list
-                self.axes._mouseover_set.discard(self)
-                self.axes.stale = True
+                mouseover_set.discard(self)
+                ax.stale = True
                 self.axes = None  # decouple the artist from the Axes
                 _ax_flag = True
 
diff --git a/lib/matplotlib/tests/test_figure.py b/lib/matplotlib/tests/test_figure.py
index 93aff4b1a632..744f12846bb9 100644
--- a/lib/matplotlib/tests/test_figure.py
+++ b/lib/matplotlib/tests/test_figure.py
@@ -1575,6 +1575,7 @@ def test_subfigures_wspace_hspace():
 def test_subfigure_remove():
     fig = plt.figure()
     sfs = fig.subfigures(2, 2)
+    sfs[1, 1].subplots()
     sfs[1, 1].remove()
     assert len(fig.subfigs) == 3