diff --git a/Orange/widgets/visualize/owheatmap.py b/Orange/widgets/visualize/owheatmap.py
index 53d1845331d..4ba24e8db84 100644
--- a/Orange/widgets/visualize/owheatmap.py
+++ b/Orange/widgets/visualize/owheatmap.py
@@ -146,9 +146,9 @@ class Outputs:
         selected_data = Output("Selected Data", Table, default=True)
         annotated_data = Output(ANNOTATED_DATA_SIGNAL_NAME, Table)
 
-    settings_version = 3
+    settings_version = 4
 
-    settingsHandler = settings.DomainContextHandler()
+    settingsHandler = settings.DomainContextHandlerPosition()
 
     # Disable clustering for inputs bigger than this
     MaxClustering = 25000
@@ -170,7 +170,7 @@ class Outputs:
     #: text row annotation (row names)
     annotation_var = settings.ContextSetting(None)
     #: color row annotation
-    annotation_color_var = settings.ContextSetting(None)
+    annotation_color_var = settings.ContextSetting(None, exclude_attributes=True)
     column_annotation_color_key: Optional[Tuple[str, str]] = settings.ContextSetting(None)
 
     # Discrete variable used to split that data/heatmaps (vertically)
diff --git a/Orange/widgets/visualize/tests/test_owheatmap.py b/Orange/widgets/visualize/tests/test_owheatmap.py
index d6f42f29154..7649a3299ef 100644
--- a/Orange/widgets/visualize/tests/test_owheatmap.py
+++ b/Orange/widgets/visualize/tests/test_owheatmap.py
@@ -365,6 +365,20 @@ def test_row_color_annotations_with_na(self):
         widget.set_annotation_color_var(None)
         self.assertFalse(widget.scene.widget.right_side_colors[0].isVisible())
 
+    def test_row_color_annotations_invalid_context(self):
+        widget = self.widget
+        data = self.brown_selected[::5]
+        self.send_signal(widget.Inputs.data, data, widget=widget)
+        widget.set_annotation_color_var(data.domain["function"])
+        self.assertTrue(widget.scene.widget.right_side_colors[0].isVisible())
+        # attributes are ignored in for annotation_color_var, so the following
+        # data should not match any context
+        data_attributes = self.brown_selected.transform(
+            Domain(data.domain.attributes + data.domain.class_vars))
+        self.send_signal(widget.Inputs.data, data_attributes, widget=widget)
+        self.assertEqual(widget.row_side_color_cb.currentText(), "(None)")
+        self.assertFalse(widget.scene.widget.right_side_colors[0].isVisible())
+
     def test_col_color_annotations(self):
         widget = self.widget
         data = self._brown_selected_10()