Skip to content

Commit

Permalink
Move provider/datasource search rules into single object, refactor
Browse files Browse the repository at this point in the history
  • Loading branch information
Johannes Kröger committed Jul 21, 2023
1 parent af16ccf commit b70a2ca
Show file tree
Hide file tree
Showing 3 changed files with 144 additions and 157 deletions.
2 changes: 1 addition & 1 deletion datasources/Dataservices/datasource_distributor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"ArcGisMapServer",
"ArcGisFeatureServer",
"GeoNode",
] # must match the names in InterfaceHandler.populate_data_source_tree's data_source_list
] # must match the names in data source provider's DATA_SOURCE_SEARCH_LOCATIONS. TODO re-use a single rules object!

def import_data_sources(
source_qgis_ini_file: str,
Expand Down
233 changes: 132 additions & 101 deletions datasources/Dataservices/datasource_provider.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,68 +7,131 @@
from qgis.core import Qgis, QgsMessageLog


# TODO document these!
# TODO document these! can we directly integrate them below somewhere?
SERVICE_NAME_REGEX = compile(r'\\(.*?)\\')
GPKG_SERVICE_NAME_REGEX = compile(r'\\(.+).\\')


def gather_data_source_connections(ini_path: str, compile_string: str) -> list[str]:
"""Returns a list of data source connection names matching the search string in the INI file.
Args:
ini_path (str): Path to the INI file to read
compile_string (str): Regex for searching corresponding data sources in INI file
Returns:
list[str]: List of found data source connections
"""
ini_parser = RawConfigParser()
ini_parser.optionxform = str # str = case-sensitive option names
ini_parser.read(ini_path)

try:
# the connections are inside the qgis section # TODO not always true, we need to check other places too!
section = ini_parser['qgis']
except KeyError:
return None

data_source_connections = []
search_string = compile(compile_string)
for key in section:
if search_string.search(key):
source_name_raw = search(SERVICE_NAME_REGEX, key)
# TODO why this replacement here? ->
source_name = source_name_raw.group(0).replace("\\", "")
# TODO "Bing VirtualEarth 💩" is not rendered well, also fails to import to other profile...
source_name = unquote(source_name, 'latin-1') # needed for e.g. %20 in connection names
data_source_connections.append(source_name)

return data_source_connections

def get_data_sources_tree(ini_path: str, compile_string: str, tree_name: str, make_checkable: bool) -> QTreeWidgetItem:
"""Returns a tree of checkable items for all data sources matching the search string in the INI file.
"""
"providername-ish": [ # a list of searching rules, not just one, because qgis changed between versions
{
"section": "section_to_search", # INI section in which to search for regex matches
"regex": "<°((^(-<", # regex to search for in the keys of the section
},
]
# TODO document the versions of QGIS that are using a specific rule
"""
DATA_SOURCE_SEARCH_LOCATIONS = {
"GeoPackage": [
{
"section": "providers",
"regex": "^ogr.GPKG.connections.*path",
},
],
"SpatiaLite": [
{
"section": "SpatiaLite",
"regex": "^connections.*sqlitepath",
},
],
"PostgreSQL": [
{
"section": "PostgreSQL",
"regex": "^connections.*host",
},
],
"MSSQL": [
{
"section": "MSSQL",
"regex": "^connections.*host",
},
],
"DB2": [
{
"section": "DB2",
"regex": "^connections.*host",
},
],
"Oracle": [
{
"section": "Oracle",
"regex": "^connections.*host",
},
],
"Vector-Tile": [
{
"section": "qgis",
"regex": "^connections-vector-tile.*url",
},
],
"WMS": [
{
"section": "qgis",
"regex": "^connections-wms.*url",
},
],
"WFS": [
{
"section": "qgis",
"regex": "^connections-wfs.*url",
},
],
"WCS": [
{
"section": "qgis",
"regex": "^connections-wcs.*url",
},
],
"XYZ": [
{
"section": "qgis",
"regex": "^connections-xyz.*url",
},
],
"ArcGisMapServer": [
{
"section": "qgis",
"regex": "^connections-arcgismapserver.*url",
},
],
"ArcGisFeatureServer": [
{
"section": "qgis",
"regex": "^connections-arcgisfeatureserver.*url",
},
],
# TODO GeoNode was a core plugin once TODO document?
"GeoNode": [
{
"section": "qgis",
"regex": "^connections-geonode.*url",
},
],
}


def get_data_sources_tree(ini_path: str, provider: str, make_checkable: bool) -> QTreeWidgetItem:
"""Returns a tree of checkable items for all data sources of the specified provider in the INI file.
The tree contains a checkable item per data source found.
Args:
ini_path (str): Path to the INI file to read
compile_string (str): Regex for searching corresponding data sources in INI file
tree_name (str): Name of the parent tree item (e.g. "WMS")
provider (str): Name of the provider to gather data sources for
make_checkable (bool): Flag to indicate if items should be checkable
Returns:
QTreeWidgetItem: Tree widget item representing the data sources or None if none were found
"""
data_source_connections = gather_data_source_connections(ini_path, compile_string)
data_source_connections = gather_data_source_connections(ini_path, provider)
if not data_source_connections:
QgsMessageLog.logMessage(f"- 0 {tree_name} connections found", "Profile Manager", Qgis.Info)
QgsMessageLog.logMessage(f"- 0 {provider} connections found", "Profile Manager", Qgis.Info)
return None
else:
QgsMessageLog.logMessage(
f"- {len(data_source_connections)} {tree_name} connections found", "Profile Manager", Qgis.Info
f"- {len(data_source_connections)} {provider} connections found", "Profile Manager", Qgis.Info
)

tree_root_item = QTreeWidgetItem([tree_name])
tree_root_item = QTreeWidgetItem([provider])
if make_checkable:
tree_root_item.setFlags(tree_root_item.flags() | Qt.ItemIsTristate | Qt.ItemIsUserCheckable)

Expand All @@ -83,83 +146,51 @@ def get_data_sources_tree(ini_path: str, compile_string: str, tree_name: str, ma
tree_root_item.addChildren(data_source_items)
return tree_root_item

def gather_db_data_source_connections(ini_path: str, section: str, compile_string: str) -> list[str]:
"""Returns a list of DB data source connection names matching the search string in the INI file.
def gather_data_source_connections(ini_path: str, provider: str) -> list[str]:
"""Returns the names of all data source connections of the specified provider in the INI file.
Args:
ini_path (str): Path to the INI file to read
section (str): The section of the INI file to search through
compile_string (str): Regex for searching corresponding data sources in INI section
provider (str): Name of the provider to gather connections of
Returns:
list[str]: List of found data source connections
list[str]: Names of the found data source connections
Raises:
NotImplementedError: If the provider name is not (yet) known here
"""
# TODO unify with gather_data_source_connections
search_rules = DATA_SOURCE_SEARCH_LOCATIONS.get(provider)
if not search_rules:
raise NotImplementedError(f"Unknown provider: {provider}")

# TODO make iterating if more than 1 rule was found
# TODO how to handle multiple finds? deduplicate?
section_to_search = search_rules[0]["section"]
regex = search_rules[0]["regex"]

ini_parser = RawConfigParser()
ini_parser.optionxform = str # str = case-sensitive option names
ini_parser.read(ini_path)

try:
section = ini_parser[section]
section = ini_parser[section_to_search]
except KeyError:
return None

db_data_source_connections = []
search_string = compile(compile_string)
data_source_connections = []
regex_pattern = compile(regex)
for key in section:
if search_string.search(key):
# ugly hack to use the section="providers" to check if we are looking for GeoPackages just
# to drop the tree_name parameter, but for now this will do...
# TODO use some better logic, i.e. specify the connection type at the caller and store the regex/logic HERE!
if section == "providers":
if regex_pattern.search(key):
if provider == "GeoPackage": # TODO move this logic/condition into the rules if possible?
source_name_raw = search(GPKG_SERVICE_NAME_REGEX, key)
source_name = source_name_raw.group(0).replace("\\GPKG\\connections\\", "").replace("\\", "")
else:
source_name_raw = search(SERVICE_NAME_REGEX, key)
source_name = source_name_raw.group(0).replace("\\", "")
# TODO what are the replacements needed for?!

# TODO see get_data_sources_tree
source_name = unquote(source_name, 'latin-1')
db_data_source_connections.append(source_name)
# TODO what are the replacements needed for?!

return db_data_source_connections

def get_db_sources_tree(ini_path, compile_string, tree_name, section, make_checkable):
"""Returns a tree of checkable items for all data sources matching the search string in the INI file.
The tree contains a checkable item per data source found.
Args:
ini_path (str): Path to the INI file to read
compile_string (str): Regex for searching corresponding data sources in INI file
section (str): The section of the INI file to search through
tree_name (str): Name of the parent tree item (e.g. "WMS")
make_checkable (bool): Flag to indicate if items should be checkable
Returns:
QTreeWidgetItem: Tree widget item representing the data sources or None if none were found
"""
data_source_connections = gather_db_data_source_connections(ini_path, section, compile_string)
if not data_source_connections:
QgsMessageLog.logMessage(f"- 0 {tree_name} connections found", "Profile Manager", Qgis.Info)
return None
else:
QgsMessageLog.logMessage(
f"- {len(data_source_connections)} {tree_name} connections found", "Profile Manager", Qgis.Info
)

tree_root_item = QTreeWidgetItem([tree_name])
if make_checkable:
tree_root_item.setFlags(tree_root_item.flags() | Qt.ItemIsTristate | Qt.ItemIsUserCheckable)

data_source_items = []
for data_source_connection in data_source_connections:
data_source_item = QTreeWidgetItem([data_source_connection])
if make_checkable:
data_source_item.setFlags(data_source_item.flags() | Qt.ItemIsUserCheckable)
data_source_item.setCheckState(0, Qt.Unchecked)
data_source_items.append(data_source_item)
# TODO "Bing VirtualEarth 💩" is not rendered well, also fails to import to other profile...
source_name = unquote(source_name, 'latin-1') # needed for e.g. %20 in connection names
data_source_connections.append(source_name)

tree_root_item.addChildren(data_source_items)
return tree_root_item
return data_source_connections
66 changes: 11 additions & 55 deletions userInterface/interface_handler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from qgis.PyQt.QtWidgets import QDialog, QListWidgetItem
from qgis.core import Qgis, QgsApplication, QgsMessageLog

from ..datasources.Dataservices.datasource_provider import get_data_sources_tree, get_db_sources_tree
from ..datasources.Dataservices.datasource_provider import get_data_sources_tree, DATA_SOURCE_SEARCH_LOCATIONS


class InterfaceHandler(QDialog):
Expand Down Expand Up @@ -35,70 +35,26 @@ def populate_data_source_tree(self, profile_name, populating_source_profile):
target_ini_path = ini_paths["target"]

# collect data source tree items from ini file
# WARNING:
# The "tree_name"s must match the connections-* lines in the INI (in lowercase) as they are used for lookup
# later in the DatasourceDistributor! E.g. "Vector-Tile" will be used for "connections-vector-tile ...".
# FIXME Use a better structure with a clear separation of names for the GUI and strings to lookup in the INI.
data_source_list = [
get_db_sources_tree(
target_ini_path, '^ogr.GPKG.connections.*path', "GeoPackage", "providers", make_checkable=populating_source_profile
),
get_db_sources_tree(
target_ini_path, '^connections.*sqlitepath', "SpatiaLite", "SpatiaLite", make_checkable=populating_source_profile
),
get_db_sources_tree(
target_ini_path, '^connections.*host', "PostgreSQL", "PostgreSQL", make_checkable=populating_source_profile
),
get_db_sources_tree(
target_ini_path, '^connections.*host', "MSSQL", "MSSQL", make_checkable=populating_source_profile
),
get_db_sources_tree(
target_ini_path, '^connections.*host', "DB2", "DB2", make_checkable=populating_source_profile
),
get_db_sources_tree(
target_ini_path, '^connections.*host', "Oracle", "Oracle", make_checkable=populating_source_profile
),
get_data_sources_tree(
target_ini_path, '^connections-vector-tile.*url', "Vector-Tile", make_checkable=populating_source_profile
),
get_data_sources_tree(
target_ini_path, '^connections-wms.*url', "WMS", make_checkable=populating_source_profile
),
get_data_sources_tree(
target_ini_path, '^connections-wfs.*url', "WFS", make_checkable=populating_source_profile
),
get_data_sources_tree(
target_ini_path, '^connections-wcs.*url', "WCS", make_checkable=populating_source_profile
),
get_data_sources_tree(
target_ini_path, '^connections-xyz.*url', "XYZ", make_checkable=populating_source_profile
),
get_data_sources_tree(
target_ini_path, '^connections-arcgismapserver.*url', "ArcGisMapServer", make_checkable=populating_source_profile
),
get_data_sources_tree(
target_ini_path, '^connections-arcgisfeatureserver.*url', "ArcGisFeatureServer", make_checkable=populating_source_profile
),
get_data_sources_tree(
target_ini_path, '^connections-geonode.*url', "GeoNode", make_checkable=populating_source_profile
),
]
data_source_list = []
for provider in DATA_SOURCE_SEARCH_LOCATIONS.keys():
tree_root_item = get_data_sources_tree(target_ini_path, provider, make_checkable=populating_source_profile)
if tree_root_item:
data_source_list.append(tree_root_item)
QgsMessageLog.logMessage(
f"Scanning profile '{profile_name}' for data source connections: Done!", "Profile Manager", Qgis.Info
)

# populate tree
if populating_source_profile:
self.dlg.treeWidgetSource.clear()
self.dlg.treeWidgetSource.setHeaderLabel(self.tr("Source Profile: {}").format(profile_name))
for dataSource in data_source_list:
if dataSource is not None:
self.dlg.treeWidgetSource.addTopLevelItem(dataSource)
for tree_root_item in data_source_list:
self.dlg.treeWidgetSource.addTopLevelItem(tree_root_item)
else:
self.dlg.treeWidgetTarget.clear()
self.dlg.treeWidgetTarget.setHeaderLabel(self.tr("Target Profile: {}").format(profile_name))
for dataSource in data_source_list:
if dataSource is not None:
self.dlg.treeWidgetTarget.addTopLevelItem(dataSource)
for tree_root_item in data_source_list:
self.dlg.treeWidgetTarget.addTopLevelItem(tree_root_item)

def populate_profile_listings(self):
"""Populates the main list as well as the comboboxes with available profile names.
Expand Down

0 comments on commit b70a2ca

Please sign in to comment.