Datasets: Move files to root; make translatable

Orange/widgets/data/owdatasets.py

@@ -35,6 +35,12 @@
 
 log = logging.getLogger(__name__)
 
+# These two constants are used in settings (and in the proxy filter model).
+# The corresponding options in the combo box are translatable, therefore
+# the settings must be stored in language-independent form.
+GENERAL_DOMAIN = None
+ALL_DOMAINS = ""  # The setting is Optional[str], so don't use other types here
+
 
 def ensure_local(index_url, file_path, local_cache_path,
                  force=False, progress_advance=None):
@@ -124,7 +130,7 @@ def __init__(self, **kwargs):
         self.seealso = []
         self.tags = []
         self.language = "English"
-        self.domain = "core"
+        self.domain = None
         self.publication_status = self.PUBLISHED
 
         super(Namespace, self).__init__(**kwargs)
@@ -174,7 +180,7 @@ def filterAcceptsRow(self, row, parent):
         data = source.index(row, 0).data(Qt.UserRole)
         return (super().filterAcceptsRow(row, parent)
             and (self.__language is None or data.language == self.__language)
-            and (self.__domain is None or data.domain == self.__domain)
+            and (self.__domain == ALL_DOMAINS or data.domain == self.__domain)
             and (data.publication_status == Namespace.PUBLISHED or (
                  self.__filter is not None
                  and len(self.__filter) >= 5
@@ -200,7 +206,10 @@ class OWDataSets(OWWidget):
     DATASET_DIR = "datasets"
     DEFAULT_LANG = "English"
     ALL_LANGUAGES = "All Languages"
-    ALL_DOMAINS = "(general)"
+
+    # These two combo options are translatable; others (domain names) are not
+    GENERAL_DOMAIN_LABEL = "(General)"
+    ALL_DOMAINS_LABEL = "(Show all)"
 
     # override HEADER_SCHEMA to define new columns
     # if schema is changed override methods: self.assign_delegates and
@@ -230,6 +239,8 @@ class Outputs:
     #: Selected dataset id
     selected_id = Setting(None)   # type: Optional[str]
     language = Setting(DEFAULT_LANG)
+    domain = Setting(GENERAL_DOMAIN)
+    settings_version = 2
 
     #: main area splitter state
     splitter_state = Setting(b'')  # type: bytes
@@ -272,9 +283,9 @@ def __init__(self):
         layout.addWidget(lang_combo)
 
         layout.addSpacing(20)
-        layout.addWidget(QLabel("Domain: "))
+        layout.addWidget(QLabel("Domain:"))
         domain_combo = self.domain_combo = QComboBox()
-        domain_combo.addItem(self.ALL_DOMAINS)
+        domain_combo.addItem(self.GENERAL_DOMAIN_LABEL)
         domain_combo.activated.connect(self._on_domain_changed)
         layout.addWidget(domain_combo)
 
@@ -410,15 +421,12 @@ def update_language_combo(self):
 
     def update_domain_combo(self):
         combo = self.domain_combo
-        current_domain = combo.currentText()
         allkeys = set(self.allinfo_local) | set(self.allinfo_remote)
         domains = {self._parse_info(key).domain for key in allkeys}
-        domains.discard("core")
-        domains = sorted(domains)
-        combo.clear()
-        combo.addItem(self.ALL_DOMAINS)
-        combo.addItems(domains)
-        combo.setCurrentText(current_domain)
+        domains -= {None, "sc"}
+        if domains:
+            combo.addItems(sorted(domains))
+            combo.addItem(self.ALL_DOMAINS_LABEL)
 
     def update_model(self):
         allkeys = set(self.allinfo_local) | set(self.allinfo_remote)
@@ -456,8 +464,17 @@ def update_model(self):
             model.appendRow(row)
 
             # for settings do not use os.path.join (Windows separator is different)
-            if "/".join(file_path) == self.selected_id:
+            if file_path[-1] == self.selected_id:
                 current_index = i
+                self.domain = datainfo.domain
+                combo = self.domain_combo
+                if self.domain == GENERAL_DOMAIN:
+                    combo.setCurrentIndex(0)
+                elif self.domain == ALL_DOMAINS:
+                    combo.setCurrentIndex(combo.count() - 1)
+                else:
+                    combo.setCurrentText(self.domain)
+                self._on_domain_changed()
 
         return model, current_index
 
@@ -471,8 +488,13 @@ def _on_language_changed(self):
 
     def _on_domain_changed(self):
         combo = self.domain_combo
-        domain = "core" if combo.currentIndex() == 0 else combo.currentText()
-        self.view.model().setDomain(domain)
+        if combo.currentIndex() == combo.count() - 1:
+            self.domain = ALL_DOMAINS
+        elif combo.currentIndex() == 0:
+            self.domain = GENERAL_DOMAIN
+        else:
+            self.domain = combo.currentText()
+        self.view.model().setDomain(self.domain)
 
     @Slot(object)
     def __set_index(self, f):
@@ -578,7 +600,7 @@ def __on_selection(self):
             text = description_html(di)
             self.descriptionlabel.setText(text)
             # for settings do not use os.path.join (Windows separator is different)
-            self.selected_id = "/".join(di.file_path)
+            self.selected_id = di.file_path[-1]
         else:
             self.descriptionlabel.setText("")
             self.selected_id = None
@@ -695,10 +717,13 @@ def load_data(path):
         return Orange.data.Table(path)
 
     @classmethod
-    def migrate_settings(cls, settings, _):
-        # until including 3.36.0 selected dataset was saved with \ on Windows
-        if "selected_id" in settings and isinstance(settings["selected_id"], str):
-            settings["selected_id"] = settings["selected_id"].replace("\\", "/")
+    def migrate_settings(cls, settings, version: Optional[int] = None):
+        selected_id = settings.get("selected_id")
+        if isinstance(selected_id, str):
+            # until including 3.36.0 selected dataset was saved with \ on Windows
+            selected_id = selected_id.replace("\\", "/")
+            if version is None or version < 2:
+                settings["selected_id"] = selected_id.split("/")[-1]
 
 
 class FutureWatcher(QObject):

Orange/widgets/data/tests/test_owdatasets.py

@@ -1,11 +1,13 @@
+import time
 import unittest
 from unittest.mock import patch, Mock
 
 import requests
 
 from AnyQt.QtCore import QItemSelectionModel, Qt
 
-from Orange.widgets.data.owdatasets import OWDataSets, Namespace as DSNamespace
+from Orange.widgets.data.owdatasets import OWDataSets, Namespace as DSNamespace, \
+    GENERAL_DOMAIN, ALL_DOMAINS
 from Orange.widgets.tests.base import WidgetTest
 
 
@@ -65,23 +67,59 @@ def test_filtering(self):
     @patch("Orange.widgets.data.owdatasets.list_remote",
            Mock(side_effect=requests.exceptions.ConnectionError))
     @patch("Orange.widgets.data.owdatasets.list_local",
-           Mock(return_value={('core', 'foo.tab'): {"domain": "core"},
-                              ('core', 'bar.tab'): {"domain": "edu"}}))
+           Mock(return_value={('core', 'foo.tab'): {"domain": None},
+                              ('edu', 'bar.tab'): {"domain": "edu"}}))
     @patch("Orange.widgets.data.owdatasets.log", Mock())
     def test_filtering_by_domain(self):
         w = self.create_widget(OWDataSets)  # type: OWDataSets
         model = w.view.model()
-        model.setDomain(None)
+        model.setDomain(GENERAL_DOMAIN)
+        self.wait_until_stop_blocking(w)
+        self.assertEqual(model.rowCount(), 1)
+
+        model.setDomain(ALL_DOMAINS)
         self.wait_until_stop_blocking(w)
         self.assertEqual(model.rowCount(), 2)
 
         model.setDomain("edu")
         self.assertEqual(model.rowCount(), 1)
         self.assertEqual(model.index(0, 0).data(Qt.UserRole).title, "bar.tab")
 
-        model.setDomain("core")
-        self.assertEqual(model.rowCount(), 1)
-        self.assertEqual(model.index(0, 0).data(Qt.UserRole).title, "foo.tab")
+        model.setDomain("baz")
+        self.assertEqual(model.rowCount(), 0)
+
+    @patch("Orange.widgets.data.owdatasets.list_local",
+           Mock(return_value={('core', 'foo.tab'): {"domain": None},
+                              ('core', 'bar.tab'): {"domain": "edu"}}))
+    @patch("Orange.widgets.data.owdatasets.log", Mock())
+    @patch("Orange.widgets.data.owdatasets.OWDataSets.commit", Mock())
+    def test_change_domain(self):
+        def wait_and_return(_):
+            time.sleep(0.2)
+            return {('core', 'foo.tab'): {"domain": "edu"},
+                    ('core', 'bar.tab'): {"domain": "edu"}}
+        with patch("Orange.widgets.data.owdatasets.list_remote",
+                   new=wait_and_return):
+            self.widget = w = self.create_widget(OWDataSets,
+                                   stored_settings={"selected_id": "bar.tab",
+                                                    "domain": "edu"})
+            self.wait_until_stop_blocking()
+            self.assertEqual(w.selected_id, "bar.tab")
+            self.assertEqual(w.domain_combo.currentText(), "edu")
+
+            self.widget = w = self.create_widget(OWDataSets,
+                                   stored_settings={"selected_id": "foo.tab",
+                                                    "domain": "(core)"})
+            self.wait_until_stop_blocking()
+            self.assertEqual(w.selected_id, "foo.tab")
+            self.assertEqual(w.domain_combo.currentText(), "edu")
+
+            self.widget = w = self.create_widget(OWDataSets,
+                                   stored_settings={"selected_id": "bar.tab",
+                                                    "domain": "(core)"})
+            self.wait_until_stop_blocking()
+            self.assertEqual(w.selected_id, "bar.tab")
+            self.assertEqual(w.domain_combo.currentText(), "edu")
 
     @patch("Orange.widgets.data.owdatasets.list_remote",
            Mock(side_effect=requests.exceptions.ConnectionError))
@@ -151,25 +189,7 @@ def test_download_iris(self):
         # select the only dataset
         sel_type = QItemSelectionModel.ClearAndSelect | QItemSelectionModel.Rows
         w.view.selectionModel().select(w.view.model().index(0, 0), sel_type)
-        self.assertEqual(w.selected_id, "core/iris.tab")
-        w.commit()
-        iris = self.get_output(w.Outputs.data, w)
-        self.assertEqual(len(iris), 150)
-
-    @patch("Orange.widgets.data.owdatasets.list_remote",
-           Mock(return_value={('dir1', 'dir2', 'foo.tab'): {}}))
-    @patch("Orange.widgets.data.owdatasets.list_local",
-           Mock(return_value={}))
-    @patch("Orange.widgets.data.owdatasets.ensure_local",
-           Mock(return_value="iris.tab"))
-    @WidgetTest.skipNonEnglish
-    def test_download_multidir(self):
-        w = self.create_widget(OWDataSets)  # type: OWDataSets
-        self.wait_until_stop_blocking(w)
-        # select the only dataset
-        sel_type = QItemSelectionModel.ClearAndSelect | QItemSelectionModel.Rows
-        w.view.selectionModel().select(w.view.model().index(0, 0), sel_type)
-        self.assertEqual(w.selected_id, "dir1/dir2/foo.tab")
+        self.assertEqual(w.selected_id, "iris.tab")
         w.commit()
         iris = self.get_output(w.Outputs.data, w)
         self.assertEqual(len(iris), 150)
@@ -197,11 +217,15 @@ def test_migrate_selected_id(self):
 
         settings = {"selected_id": "dir1\\bar"}
         OWDataSets.migrate_settings(settings, 0)
-        self.assertEqual(settings["selected_id"], "dir1/bar")
+        self.assertEqual(settings["selected_id"], "bar")
 
         settings = {"selected_id": "dir1/bar"}
         OWDataSets.migrate_settings(settings, 0)
-        self.assertEqual(settings["selected_id"], "dir1/bar")
+        self.assertEqual(settings["selected_id"], "bar")
+
+        settings = {"selected_id": "bar"}
+        OWDataSets.migrate_settings(settings, 0)
+        self.assertEqual(settings["selected_id"], "bar")
 
 
 if __name__ == "__main__":

i18n/si/msgs.jaml

@@ -5166,7 +5166,6 @@ widgets/data/owdatasets.py:
     class `Namespace`:
         def `__init__`:
             English: false
-            core: false
     class `OWDataSets`:
         Datasets: Zbirke podatkov
         Load a dataset from an online repository: Naloži podatke s spletnega skladišča.
@@ -5177,7 +5176,8 @@ widgets/data/owdatasets.py:
         datasets: false
         English: Slovenščina
         All Languages: (Vsi jeziki)
-        (general): (splošno)
+        (General): (Splošno)
+        (Show all): (Prikaži vse)
         islocal: false
         label: false
         title: false
@@ -5204,22 +5204,19 @@ widgets/data/owdatasets.py:
             _header_index: false
             Search for data set ...: Poišči podatke ...
             'Show data sets in ': 'Prikaži zbirke podatkov v jeziku '
-            'Domain: ': 'Področje: '
+            Domain:: Področje:
             Press Return or double-click to send: Pritisni Enter ali dvoklikni za izbor
             Description: Opis
             splitter_state: false
             Initializing: Zaganjam
         def `_parse_info`:
             version: false
         def `update_domain_combo`:
-            core: false
+            sc: false
         def `update_model`:
             ' ': false
             '  ': false
             ', ': false
-            /: false
-        def `_on_domain_changed`:
-            core: false
         def `__set_index`:
             Error while fetching updated index: false
         def `set_model`:
@@ -5231,8 +5228,6 @@ widgets/data/owdatasets.py:
         def `__update_cached_state`:
             ' ': false
             '  ': false
-        def `__on_selection`:
-            /: false
         def `commit`:
             Fetching...: Pobiram...
         def `__commit_complete`: