Merge branch 'master' of github.com:opengisch/QgisModelBaker into opt…

…imizestrattopping

QgisModelBaker/gui/panel/log_panel.py

@@ -19,7 +19,7 @@
 
 import logging
 
-from PyQt5.QtWidgets import QGridLayout
+from PyQt5.QtWidgets import QGridLayout, QProgressBar
 from qgis.core import Qgis
 from qgis.gui import QgsMessageBar
 from qgis.PyQt.QtCore import QSize, Qt
@@ -40,9 +40,17 @@ def __init__(self, parent=None):
         self.txtStdout.setLayout(QGridLayout())
         self.txtStdout.layout().setContentsMargins(0, 0, 0, 0)
         self.txtStdout.layout().addWidget(self.bar, 0, 0, Qt.AlignTop)
+        self.scrollbar = self.txtStdout.verticalScrollBar()
+
+        self.busy_bar = QProgressBar()
+        self.busy_bar.setRange(0, 0)
+        self.busy_bar.setTextVisible(True)
+        self.busy_bar.setVisible(False)
 
         layout = QGridLayout()
         layout.addWidget(self.txtStdout)
+        layout.addWidget(self.busy_bar)
+
         self.setLayout(layout)
 
     def sizeHint(self):

QgisModelBaker/gui/panel/pg_config_panel.py

@@ -19,7 +19,7 @@
 import logging
 from enum import IntEnum
 
-from qgis.PyQt.QtCore import Qt, QTimer
+from qgis.PyQt.QtCore import Qt, QThread, QTimer
 
 import QgisModelBaker.libs.modelbaker.libs.pgserviceparser as pgserviceparser
 import QgisModelBaker.libs.modelbaker.utils.db_utils as db_utils
@@ -69,6 +69,9 @@ def __init__(self, parent, db_action_type):
         self._fill_schema_combo_box_timer.setSingleShot(True)
         self._fill_schema_combo_box_timer.timeout.connect(self._fill_schema_combo_box)
 
+        self._read_pg_schemas_task = ReadPgSchemasTask(self)
+        self._read_pg_schemas_task.finished.connect(self._read_pg_schemas_task_finished)
+
         from QgisModelBaker.libs.modelbaker.iliwrapper.ili2dbconfig import (
             BaseConfiguration,
         )
@@ -109,6 +112,10 @@ def __init__(self, parent, db_action_type):
         self.pg_database_line_edit.textChanged.connect(self._fields_modified)
         self.pg_schema_combo_box.currentTextChanged.connect(self.notify_fields_modified)
 
+        self.pg_auth_settings.usernameChanged.connect(self._fields_modified)
+        self.pg_auth_settings.passwordChanged.connect(self._fields_modified)
+        self.pg_auth_settings.configIdChanged.connect(self._fields_modified)
+
         # Fill pg_services combo box
         self.pg_service_combo_box.addItem(self.tr("None"), None)
         for service in pgserviceparser.service_names():
@@ -171,6 +178,14 @@ def __init__(self, parent, db_action_type):
 
         self._show_panel()
 
+    def __del__(self):
+        # Make sure the refresh schemas task is finished to avoid crashes
+        try:
+            if self._read_pg_schemas_task:
+                self._read_pg_schemas_task.wait()
+        except RuntimeError:
+            pass
+
     def _show_panel(self):
 
         self._fill_schema_combo_box()
@@ -348,7 +363,7 @@ def _pg_service_combo_box_changed(self):
                     index, PgConfigPanel._SERVICE_COMBOBOX_ROLE.DATABASE
                 )
             )
-            self.pg_schema_combo_box.setText(
+            self.pg_schema_combo_box.setCurrentText(
                 self.pg_service_combo_box.itemData(
                     index, PgConfigPanel._SERVICE_COMBOBOX_ROLE.DBSCHEMA
                 )
@@ -459,27 +474,24 @@ def _keep_custom_settings(self):
         )
 
     def _fields_modified(self):
-
         self._fill_schema_combo_box_timer.start(self.REFRESH_SCHEMAS_TIMEOUT_MS)
 
         self.notify_fields_modified.emit()
 
     def _fill_schema_combo_box(self):
 
-        configuration = Ili2DbCommandConfiguration()
+        if self._read_pg_schemas_task.isRunning():
+            self._fill_schema_combo_box_timer.start()
+            return
 
-        mode = DbIliMode.pg
+        configuration = Ili2DbCommandConfiguration()
         self.get_fields(configuration)
+        configuration.tool = DbIliMode.pg
 
-        configuration.tool = mode
-        configuration.db_ili_version = db_utils.db_ili_version(configuration)
-
-        db_connector = db_utils.get_db_connector(configuration)
-        if not db_connector:
-            logging.warning("Refresh schema list connection error")
-            return
+        self._read_pg_schemas_task.configuration_changed(configuration)
 
-        schemas = db_connector.get_schemas()
+    def _read_pg_schemas_task_finished(self):
+        schemas = self._read_pg_schemas_task.schemas
 
         AUTO_ADDED_SCHEMA = "auto_added_schema"
 
@@ -497,3 +509,30 @@ def _fill_schema_combo_box(self):
         currentTextIndex = self.pg_schema_combo_box.findText(currentText)
         if currentTextIndex > -1:
             self.pg_schema_combo_box.setCurrentIndex(currentTextIndex)
+
+
+class ReadPgSchemasTask(QThread):
+    def __init__(self, parent):
+        super().__init__(parent)
+
+        self.schemas = []
+        self._configuration = None
+
+    def configuration_changed(self, configuration):
+        self._configuration = configuration
+        self.start()
+
+    def run(self):
+
+        try:
+            db_connector = db_utils.get_db_connector(self._configuration)
+            if not db_connector:
+                logging.warning("Refresh schema list connection error")
+                self.schemas = []
+                return
+
+            self.schemas = db_connector.get_schemas()
+
+        except Exception as exception:
+            logging.warning(f"Refresh schema list error: {exception}")
+            self.schemas = []

QgisModelBaker/gui/topping_wizard/models_page.py

@@ -67,21 +67,27 @@ def validatePage(self) -> bool:
                 self.tr("No models set."),
                 gui_utils.LogColor.COLOR_SUCCESS,
             )
+        self.topping_wizard.topping.preferred_datasource = (
+            self.source_combobox.currentData()
+        )
         return super().validatePage()
 
     def _refresh(self):
-        self._load_available_models()
+        self._load_available_models_and_sources()
         self.models_model.check_entries(self.topping_wizard.topping.models)
 
-    def _load_available_models(self):
+    def _load_available_models_and_sources(self):
         """
         Collects all the available sources in the project and makes the models_model to refresh accordingly.
         """
         checked_identificators = []
         db_connectors = []
+        sources = set()
+
         for layer in QgsProject.instance().mapLayers().values():
             if layer.type() == QgsMapLayer.VectorLayer:
                 source_provider = layer.dataProvider()
+                self._append_possible_sources(sources, source_provider)
                 schema_identificator = (
                     db_utils.get_schema_identificator_from_sourceprovider(
                         source_provider
@@ -101,3 +107,32 @@ def _load_available_models(self):
                         if db_connector:
                             db_connectors.append(db_connector)
         self.models_model.refresh_model(db_connectors)
+
+        self._refresh_source_combobox(sources)
+
+    def _append_possible_sources(self, sources, provider):
+        if provider.name() == "postgres":
+            sources.add("pg")
+        elif provider.name() == "mssql":
+            sources.add("mssql")
+        elif provider.name() == "ogr" and provider.storageType() == "GPKG":
+            sources.add("gpkg")
+
+    def _refresh_source_combobox(self, sources):
+        display_map = {"pg": "PostGIS", "gpkg": "GeoPackage", "mssql": "MSSQL"}
+
+        self.source_combobox.clear()
+        for source in list(sources):
+            self.source_combobox.addItem(display_map[source], source)
+
+        none_text = self.tr("No source defined (allow all)")
+
+        self.source_combobox.addItem(none_text, None)
+
+        if len(sources) == 1:
+            # it's clear which one we take
+            self.source_combobox.setCurrentText(display_map[list(sources)[0]])
+        else:
+            # no sources or it's not clear which one
+            self.source_combobox.setCurrentText(none_text)
+        self.source_combobox.setEnabled(bool(sources))

QgisModelBaker/gui/topping_wizard/referencedata_page.py

@@ -27,8 +27,8 @@
 import QgisModelBaker.utils.gui_utils as gui_utils
 from QgisModelBaker.libs.modelbaker.iliwrapper.ilicache import (
     IliDataCache,
+    IliDataFileCompleterDelegate,
     IliDataItemModel,
-    MetaConfigCompleterDelegate,
 )
 from QgisModelBaker.libs.modelbaker.utils.qt_utils import (
     FileValidator,
@@ -76,7 +76,7 @@ def __init__(self, parent, title):
             self.topping_wizard.log_panel.show_message
         )
 
-        self.ilireferencedata_delegate = MetaConfigCompleterDelegate()
+        self.ilireferencedata_delegate = IliDataFileCompleterDelegate()
         self.input_line_edit.setPlaceholderText(
             self.tr("[Search referenced data files from Repositories or Local System]")
         )

QgisModelBaker/gui/topping_wizard/topping_wizard.py

@@ -101,7 +101,7 @@ def _current_page_title(self, id):
         if id == ToppingWizardPageIds.Target:
             return self.tr("Target Folder Selection")
         elif id == ToppingWizardPageIds.Models:
-            return self.tr("Model Selection")
+            return self.tr("Model and Source Selection")
         elif id == ToppingWizardPageIds.Layers:
             return self.tr("Layer Configuration")
         elif id == ToppingWizardPageIds.Additives:

QgisModelBaker/gui/workflow_wizard/default_baskets_page.py

@@ -52,9 +52,6 @@ def isComplete(self):
 
     def setComplete(self, complete):
         self.is_complete = complete
-        self.create_default_baskets_button.setDisabled(complete)
-        self.skip_button.setDisabled(complete)
-        self.baskets_panel.setDisabled(complete)
         self.completeChanged.emit()
 
     def nextId(self):
@@ -88,6 +85,9 @@ def _create_default_baskets(self):
             self.progress_bar.setFormat(self.tr("Default baskets created!"))
             self.progress_bar.setValue(100)
             self.setStyleSheet(gui_utils.SUCCESS_STYLE)
+            self.create_default_baskets_button.setDisabled(True)
+            self.skip_button.setDisabled(True)
+            self.baskets_panel.setDisabled(True)
             self.setComplete(True)
         else:
             self.workflow_wizard.log_panel.print_info(message)
@@ -103,4 +103,7 @@ def _skip(self):
         self.progress_bar.setFormat(self.tr("SKIPPED"))
         self.progress_bar.setTextVisible(True)
         self.setStyleSheet(gui_utils.INACTIVE_STYLE)
+        self.create_default_baskets_button.setDisabled(True)
+        self.skip_button.setDisabled(True)
+        self.baskets_panel.setDisabled(True)
         self.setComplete(True)

QgisModelBaker/gui/workflow_wizard/import_data_configuration_page.py

@@ -36,8 +36,8 @@
 import QgisModelBaker.utils.gui_utils as gui_utils
 from QgisModelBaker.gui.dataset_manager import DatasetManagerDialog
 from QgisModelBaker.libs.modelbaker.iliwrapper.ilicache import (
+    IliDataFileCompleterDelegate,
     IliDataItemModel,
-    MetaConfigCompleterDelegate,
 )
 from QgisModelBaker.utils.globals import CATALOGUE_DATASETNAME, DEFAULT_DATASETNAME
 from QgisModelBaker.utils.gui_utils import CheckDelegate, LogColor
@@ -118,7 +118,7 @@ def __init__(self, parent, title):
         self.workflow_wizard.ilireferencedatacache.file_download_failed.connect(
             self._on_referencedata_failed
         )
-        self.ilireferencedata_delegate = MetaConfigCompleterDelegate()
+        self.ilireferencedata_delegate = IliDataFileCompleterDelegate()
         self.ilireferencedata_line_edit.setPlaceholderText(
             self.tr("[Search referenced data files from UsabILIty Hub]")
         )

QgisModelBaker/gui/workflow_wizard/import_schema_configuration_page.py

@@ -27,10 +27,11 @@
 from qgis.PyQt.QtWidgets import QCompleter, QWizardPage
 
 from QgisModelBaker.gui.ili2db_options import Ili2dbOptionsDialog
+from QgisModelBaker.libs.modelbaker.iliwrapper.globals import DbIliMode
 from QgisModelBaker.libs.modelbaker.iliwrapper.ilicache import (
     IliDataCache,
+    IliDataFileCompleterDelegate,
     IliDataItemModel,
-    MetaConfigCompleterDelegate,
 )
 from QgisModelBaker.utils import gui_utils
 from QgisModelBaker.utils.globals import CRS_PATTERNS
@@ -71,7 +72,7 @@ def __init__(self, parent, title):
         self.ilimetaconfigcache = IliDataCache(
             self.workflow_wizard.import_schema_configuration.base_configuration
         )
-        self.metaconfig_delegate = MetaConfigCompleterDelegate()
+        self.metaconfig_delegate = IliDataFileCompleterDelegate()
         self.metaconfig = configparser.ConfigParser()
         self.current_models = []
         self.current_metaconfig_id = None
@@ -158,8 +159,6 @@ def _update_models_dependent_info(self):
         - Calls update of ilireferencedata cache to load referenced
         """
         model_list = self.model_list_view.model().checked_models()
-        if set(model_list) == set(self.current_models):
-            return
         self.current_models = model_list
         for pattern, crs in CRS_PATTERNS.items():
             if re.search(pattern, ", ".join(model_list)):
@@ -173,6 +172,11 @@ def _update_ilimetaconfigcache(self):
         self.ilimetaconfigcache = IliDataCache(
             self.workflow_wizard.import_schema_configuration.base_configuration,
             models=";".join(self.model_list_view.model().checked_models()),
+            datasources=["pg"]
+            if (self.workflow_wizard.import_schema_configuration.tool & DbIliMode.pg)
+            else ["gpkg"]
+            if (self.workflow_wizard.import_schema_configuration.tool & DbIliMode.gpkg)
+            else None,
         )
         self.ilimetaconfigcache.file_download_succeeded.connect(
             lambda dataset_id, path: self._on_metaconfig_received(path)
@@ -181,6 +185,7 @@ def _update_ilimetaconfigcache(self):
         self.ilimetaconfigcache.model_refreshed.connect(
             self._update_metaconfig_completer
         )
+        self.workflow_wizard.busy(self, True, self.tr("Refresh repository data..."))
         self._refresh_ili_metaconfig_cache()
 
     def _update_ilireferencedatacache(self):
@@ -285,6 +290,7 @@ def _update_metaconfig_completer(self, rows):
         completer.popup().setItemDelegate(self.metaconfig_delegate)
         self.ili_metaconfig_line_edit.setCompleter(completer)
         self.ili_metaconfig_line_edit.setEnabled(bool(rows))
+        self.workflow_wizard.busy(self, False)
 
     def _on_metaconfig_completer_activated(self, text=None):
         self._clean_metaconfig()
@@ -311,8 +317,9 @@ def _on_metaconfig_completer_activated(self, text=None):
                     self.ilimetaconfigcache.model.data(model_index, Qt.DisplayRole),
                     metaconfig_id,
                     self.ilimetaconfigcache.model.data(
-                        model_index, int(IliDataItemModel.Roles.SHORT_DESCRIPTION) or ""
-                    ),
+                        model_index, int(IliDataItemModel.Roles.SHORT_DESCRIPTION)
+                    )
+                    or "",
                 )
             )
             self.metaconfig_file_info_label.setStyleSheet("color: #341d5c")
@@ -429,6 +436,7 @@ def _load_crs_from_metaconfig(self, ili2db_metaconfig):
         self._crs_changed()
 
     def _load_metaconfig(self):
+        self.workflow_wizard.busy(self, True, "Load metaconfiguration...")
         # load ili2db parameters to the GUI
         if "ch.ehi.ili2db" in self.metaconfig.sections():
             self.workflow_wizard.log_panel.print_info(
@@ -566,3 +574,5 @@ def _load_metaconfig(self):
                             LogColor.COLOR_TOPPING,
                         )
             self.workflow_wizard.refresh_import_models()
+
+        self.workflow_wizard.busy(self, False)