From 6afd7d61a0acc8b5f1357dfdf50564521bb78d45 Mon Sep 17 00:00:00 2001 From: signedav Date: Fri, 14 Jul 2023 09:49:44 +0200 Subject: [PATCH 01/34] testdata (ili-models) for the use cases --- tests/testdata/ilimodels/Bauplanung_V1_1.ili | 74 +++++++++++++++++++ .../ilimodels/Kantonale_Bauplanung_V1_1.ili | 54 ++++++++++++++ .../Polymorphic_Ortsplanung_V1_1.ili | 63 ++++++++++++++++ 3 files changed, 191 insertions(+) create mode 100644 tests/testdata/ilimodels/Bauplanung_V1_1.ili create mode 100644 tests/testdata/ilimodels/Kantonale_Bauplanung_V1_1.ili create mode 100644 tests/testdata/ilimodels/Polymorphic_Ortsplanung_V1_1.ili diff --git a/tests/testdata/ilimodels/Bauplanung_V1_1.ili b/tests/testdata/ilimodels/Bauplanung_V1_1.ili new file mode 100644 index 0000000..63f804b --- /dev/null +++ b/tests/testdata/ilimodels/Bauplanung_V1_1.ili @@ -0,0 +1,74 @@ +INTERLIS 2.3; + +/* Bauplanung as national model importing the infrastrukture used for using geometry types and connectiong to strasse. +Having STRUCTURE and a many-to-many ASSOCIATION */ +MODEL Bauplanung_V1_1 (en) AT "https://modelbaker.ch" VERSION "2023-07-13" = + IMPORTS Infrastruktur_V1; + + DOMAIN + !! a top level structure used everywhere an not extended + STRUCTURE Bauart = + Bezeichnung : MANDATORY TEXT; + END Bauart; + + TOPIC Konstruktionen = + OID AS INTERLIS.UUIDOID; + DEPENDS ON Infrastruktur_V1.Strassen; + + DOMAIN + STRUCTURE Material = + Bezeichnung : MANDATORY TEXT; + END Material; + + CLASS Gebaeude = + Name : TEXT*99; + Materialien: BAG {1..*} OF Material; + Geometrie : MANDATORY Infrastruktur_V1.CHSurface; + Test : Bauplanung_V1_1.Bauart; + END Gebaeude; + + /*new many-to-many association*/ + ASSOCIATION Strassen_Gebaeude = + Strasse (EXTERNAL) -- {0..*} Infrastruktur_V1.Strassen.Strasse; + Gebaeude -- {0..*} Gebaeude; + END Strassen_Gebaeude; + + END Konstruktionen; + + TOPIC Natur = + CLASS Park = + Geometrie : MANDATORY Infrastruktur_V1.CHSurface; + END Park; + + CLASS Tierart = + Spezies: TEXT; + END Tierart; + + /*multiple Tierarten can live in a park */ + ASSOCIATION Park_Tierart = + Park -- {1..1} Park; + Tierart -- {1..*} Tierart; + END Park_Tierart; + + CLASS Brutstelle = + Geometrie : MANDATORY Infrastruktur_V1.CHSurface; + END Brutstelle; + + /*a Tierart can have multiple brutstellen */ + ASSOCIATION Brutstelle_Tierart = + Fauna -- {1..1} Tierart; + Brutstelle -- {1..*} Brutstelle; + END Brutstelle_Tierart; + + CLASS Buntbrache = + Geometrie : MANDATORY Infrastruktur_V1.CHSurface; + END Buntbrache; + + CLASS Feld = + Name : TEXT; + Geometrie : MANDATORY Infrastruktur_V1.CHSurface; + END Feld; + + END Natur; + +END Bauplanung_V1_1. \ No newline at end of file diff --git a/tests/testdata/ilimodels/Kantonale_Bauplanung_V1_1.ili b/tests/testdata/ilimodels/Kantonale_Bauplanung_V1_1.ili new file mode 100644 index 0000000..258fdc9 --- /dev/null +++ b/tests/testdata/ilimodels/Kantonale_Bauplanung_V1_1.ili @@ -0,0 +1,54 @@ +INTERLIS 2.3; + +/* Extended Bauplanung as cantonal model importing national model. +Extending STRUCTURE and a many-to-many ASSOCIATION */ +MODEL Kantonale_Bauplanung_V1_1 (en) AT "https://modelbaker.ch" VERSION "2023-07-13" = + IMPORTS Bauplanung_V1_1, Infrastruktur_V1; + + TOPIC Konstruktionen EXTENDS Bauplanung_V1_1.Konstruktionen = + OID AS INTERLIS.UUIDOID; + DEPENDS ON Infrastruktur_V1.Strassen; + + DOMAIN + STRUCTURE Material (EXTENDED) = + Details : MANDATORY TEXT; + END Material; + + CLASS Gebaeude (EXTENDED) = + DetailMaterialien: BAG {1..*} OF Material; + END Gebaeude; + + /*extension of many-to-many association */ + ASSOCIATION Strassen_Gebaeude (EXTENDED) = + Strasse (EXTENDED,EXTERNAL) -- {1..*} Infrastruktur_V1.Strassen.Strasse; + Gebaeude (EXTENDED) -- {1..*} Gebaeude; + END Strassen_Gebaeude; + + END Konstruktionen; + + TOPIC Natur EXTENDS Bauplanung_V1_1.Natur = + + CLASS Park (EXTENDED) = + Details: TEXT; + END Park; + + CLASS Tierart (EXTENDED) = + Details: TEXT; + END Tierart; + + !! Extension with another name + CLASS KantonaleBuntbrache EXTENDS Buntbrache= + Gemeinde: TEXT; + END KantonaleBuntbrache; + + !! Multiple extensions + CLASS Kartoffelfeld EXTENDS Feld = + Anzahl: 1 .. 1000; + END Kartoffelfeld; + CLASS Sonnenblumenfeld EXTENDS Feld = + Art: TEXT; + END Sonnenblumenfeld; + + END Natur; + +END Kantonale_Bauplanung_V1_1. \ No newline at end of file diff --git a/tests/testdata/ilimodels/Polymorphic_Ortsplanung_V1_1.ili b/tests/testdata/ilimodels/Polymorphic_Ortsplanung_V1_1.ili new file mode 100644 index 0000000..0e0033f --- /dev/null +++ b/tests/testdata/ilimodels/Polymorphic_Ortsplanung_V1_1.ili @@ -0,0 +1,63 @@ +INTERLIS 2.3; + +/* Model to test several types of extended classes */ +MODEL Polymorphic_Ortsplanung_V1_1 (en) AT "https://modelbaker.ch" VERSION "2023-07-13" = + IMPORTS Ortsplanung_V1_1,Infrastruktur_V1; + + TOPIC Gewerbe EXTENDS Ortsplanung_V1_1.Konstruktionen = + OID AS INTERLIS.UUIDOID; + DEPENDS ON Infrastruktur_V1.Strassen; + + CLASS Gebaeude (EXTENDED) = + Firma : TEXT*99; + END Gebaeude; + + END Gewerbe; + + TOPIC Freizeit EXTENDS Ortsplanung_V1_1.Konstruktionen = + OID AS INTERLIS.UUIDOID; + DEPENDS ON Infrastruktur_V1.Strassen; + + CLASS Gebaeude (EXTENDED) = + Unterhaltungsart : TEXT*99; + END Gebaeude; + + END Freizeit; + + TOPIC IndustrieGewerbe EXTENDS Polymorphic_Ortsplanung_V1_1.Gewerbe = + OID AS INTERLIS.UUIDOID; + DEPENDS ON Infrastruktur_V1.Strassen; + + CLASS Gebaeude (EXTENDED) = + Sektion : TEXT*99; + END Gebaeude; + + END IndustrieGewerbe; + + TOPIC Hallen EXTENDS Ortsplanung_V1_1.Konstruktionen = + OID AS INTERLIS.UUIDOID; + DEPENDS ON Infrastruktur_V1.Strassen; + + CLASS Gebaeude (EXTENDED) = + Groesse : TEXT*99; + END Gebaeude; + + CLASS Markthalle EXTENDS Gebaeude = + Standplaetze : 1 .. 100; + END Markthalle; + + CLASS Turnhalle (ABSTRACT) EXTENDS Gebaeude = + Bodenbelag :TEXT*99; + END Turnhalle; + + CLASS TurnhalleTyp1 EXTENDS Turnhalle = + AnzahlKoerbe : 1 .. 8; + END TurnhalleTyp1; + + CLASS TurnhalleTyp2 EXTENDS Turnhalle = + AnzahlTore : 1 .. 8; + END TurnhalleTyp2; + + END Hallen; + +END Polymorphic_Ortsplanung_V1_1. \ No newline at end of file From cddc71cfda6bf77bf20716edc5051a837337c3a1 Mon Sep 17 00:00:00 2001 From: signedav Date: Sun, 16 Jul 2023 10:51:40 +0200 Subject: [PATCH 02/34] Approach for relevance of tables on extended models in GPKG --- modelbaker/dataobjects/layers.py | 1 + modelbaker/dbconnector/db_connector.py | 1 + modelbaker/dbconnector/gpkg_connector.py | 40 ++++++++++++++++++++++-- modelbaker/generator/generator.py | 7 ++++- 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/modelbaker/dataobjects/layers.py b/modelbaker/dataobjects/layers.py index a5f49f3..ebe9ae9 100644 --- a/modelbaker/dataobjects/layers.py +++ b/modelbaker/dataobjects/layers.py @@ -54,6 +54,7 @@ def __init__( is_basket_table=False, is_dataset_table=False, ili_name=None, + is_relevant=True, definitionfile=None, qmlstylefile=None, styles={}, diff --git a/modelbaker/dbconnector/db_connector.py b/modelbaker/dbconnector/db_connector.py index 15bb128..131c3ff 100644 --- a/modelbaker/dbconnector/db_connector.py +++ b/modelbaker/dbconnector/db_connector.py @@ -76,6 +76,7 @@ def get_tables_info(self): extent [a string: "xmin;ymin;xmax;ymax"] table_alias model + relevance """ return [] diff --git a/modelbaker/dbconnector/gpkg_connector.py b/modelbaker/dbconnector/gpkg_connector.py index 1d4be20..f8b4fae 100644 --- a/modelbaker/dbconnector/gpkg_connector.py +++ b/modelbaker/dbconnector/gpkg_connector.py @@ -140,7 +140,44 @@ def _get_tables_info(self): GROUP BY tablename ) as coord_decimals, substr(c.iliname, 0, instr(c.iliname, '.')) AS model, - attrs.sqlname as attribute_name, """ + attrs.sqlname as attribute_name, + {relevance_field},""".format( + relevance_field="""CASE WHEN c.iliname IN ( + -- used to get the class names from the full names + WITH names AS ( + WITH class_level_name AS( + WITH topic_level_name AS ( + SELECT + thisClass as fullname, + substr(thisClass, 0, instr(thisClass, '.')) as model, + substr(ltrim(thisClass,substr(thisClass, 0, instr(thisClass, '.'))),2) as topicclass + FROM T_ILI2DB_INHERITANCE + ) + SELECT *, ltrim(topicclass,substr(topicclass, 0, instr(topicclass, '.'))) as class_with_dot + FROM topic_level_name + ) select fullname, model, topicclass, substr(class_with_dot, instr(class_with_dot,'.')+1) as class + FROM class_level_name + ) + SELECT i.baseClass as base + FROM T_ILI2DB_INHERITANCE i + LEFT JOIN names extend_names + ON thisClass = extend_names.fullname + LEFT JOIN names base_names + ON baseClass = base_names.fullname + -- it's extended + WHERE baseClass IS NOT NULL + -- in a different model + AND base_names.model != extend_names.model + AND ( + -- with the same name + base_names.class = extend_names.class + OR + -- multiple times in the same extended model + (SELECT COUNT(baseClass) FROM T_ILI2DB_INHERITANCE JOIN names extend_names ON thisClass = extend_names.fullname WHERE baseClass = i.baseClass GROUP BY baseClass, extend_names.model)>1 + ) + ) + THEN FALSE ELSE TRUE END AS relevance""" + ) interlis_joins = """LEFT JOIN T_ILI2DB_TABLE_PROP p ON p.tablename = s.name AND p.tag = 'ch.ehi.ili2db.tableKind' @@ -151,7 +188,6 @@ def _get_tables_info(self): ON s.name == c.sqlname LEFT JOIN t_ili2db_attrname attrs ON c.iliname = attrs.iliname """ - try: cursor.execute( """ diff --git a/modelbaker/generator/generator.py b/modelbaker/generator/generator.py index 312c707..69876b4 100644 --- a/modelbaker/generator/generator.py +++ b/modelbaker/generator/generator.py @@ -47,6 +47,7 @@ def __init__( parent=None, mgmt_uri=None, consider_basket_handling=False, + optimize_extended=True #TRUE for-relevance-tests: should be false after ): """ Creates a new Generator objects. @@ -68,6 +69,7 @@ def __init__( self._db_connector.stdout.connect(self.print_info) self._db_connector.new_message.connect(self.append_print_message) self.basket_handling = consider_basket_handling and self.get_basket_handling() + self.optimize_extended = optimize_extended self._additional_ignored_layers = ( [] @@ -137,6 +139,8 @@ def layers(self, filter_layer_list=[]): record.get("tablename") == self._db_connector.dataset_table_name ) + is_relevant = bool(record.get("relevance")) if self.optimize_extended else True + alias = record["table_alias"] if "table_alias" in record else None if not alias: short_name = None @@ -185,7 +189,7 @@ def layers(self, filter_layer_list=[]): + match.group(1).split(".")[-1] + ")" ) - alias = short_name + alias = short_name if is_relevant else f"{short_name} !IRRELEVANT!" #for-relevance-tests display_expression = "" if is_basket_table: @@ -224,6 +228,7 @@ def layers(self, filter_layer_list=[]): is_basket_table, is_dataset_table, record.get("ili_name"), + is_relevant ) # Configure fields for current table From 179719105f925e131ddcaa53826f868b345fa8cc Mon Sep 17 00:00:00 2001 From: signedav Date: Fri, 11 Aug 2023 15:48:44 +0200 Subject: [PATCH 03/34] another bit --- modelbaker/generator/generator.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/modelbaker/generator/generator.py b/modelbaker/generator/generator.py index 69876b4..e31529c 100644 --- a/modelbaker/generator/generator.py +++ b/modelbaker/generator/generator.py @@ -17,6 +17,7 @@ ***************************************************************************/ """ import re +from enum import Enum from qgis.core import QgsApplication, QgsRelation, QgsWkbTypes from qgis.PyQt.QtCore import QCoreApplication, QLocale, QObject, pyqtSignal @@ -37,6 +38,11 @@ class Generator(QObject): stdout = pyqtSignal(str) new_message = pyqtSignal(int, str) + class OptimizeStrategy(Enum): + NONE = 0 + RENAME = 1 + HIDE = 2 + def __init__( self, tool, @@ -47,7 +53,7 @@ def __init__( parent=None, mgmt_uri=None, consider_basket_handling=False, - optimize_extended=True #TRUE for-relevance-tests: should be false after + optimize_strategy=Generator.OptimizeStrategy.HIDE #HIDE for-relevance-tests: should be NONE after ): """ Creates a new Generator objects. @@ -69,7 +75,7 @@ def __init__( self._db_connector.stdout.connect(self.print_info) self._db_connector.new_message.connect(self.append_print_message) self.basket_handling = consider_basket_handling and self.get_basket_handling() - self.optimize_extended = optimize_extended + self.optimize_strategy = optimize_strategy self._additional_ignored_layers = ( [] @@ -139,7 +145,7 @@ def layers(self, filter_layer_list=[]): record.get("tablename") == self._db_connector.dataset_table_name ) - is_relevant = bool(record.get("relevance")) if self.optimize_extended else True + is_relevant = bool(record.get("relevance")) if self.optimize_strategy != Generator.OptimizeStrategy.NONE else True alias = record["table_alias"] if "table_alias" in record else None if not alias: From 9c532a5055561b9b9e07601fc8c647b91de4cdc6 Mon Sep 17 00:00:00 2001 From: signedav Date: Fri, 11 Aug 2023 17:11:47 +0200 Subject: [PATCH 04/34] Append topic prefix to all ambiguous layer based on a an INTERLIS model. --- modelbaker/generator/generator.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/modelbaker/generator/generator.py b/modelbaker/generator/generator.py index e31529c..9601f11 100644 --- a/modelbaker/generator/generator.py +++ b/modelbaker/generator/generator.py @@ -163,7 +163,7 @@ def layers(self, filter_layer_list=[]): table_appearance_count[record["tablename"]] > 1 and "geometry_column" in record ): - # multiple layers for this table - append geometry column to name + # table loaded multiple times (because of multiple geometry columns) - append geometry column to name (for PG source layers) fields_info = self.get_fields_info(record["tablename"]) for field_info in fields_info: if field_info["column_name"] == record["geometry_column"]: @@ -188,7 +188,7 @@ def layers(self, filter_layer_list=[]): if match.group(0) == match.group(1): short_name = match.group(1).split(".")[-1] else: - # additional brackets in the the name - extended layer in geopackage + # table does not fit in match group (on multigeometry in geopackage they are named with "...OriginalName.GeometryColumn (OriginalName)" so we want the GeometryColumn appended in brackets short_name = ( match.group(1).split(".")[-2] + " (" @@ -364,6 +364,14 @@ def layers(self, filter_layer_list=[]): layers.append(layer) + #rename ambiguous layers with topic prefix + ambiguous_aliases = [l.alias for l in layers if layers.count(l.alias)>1] + for layer in layers: + if layer.alias in ambiguous_aliases: + if layer.ili_name: + if layer.ili_name.count(".") > 1: + layer.alias = layer.ili_name.split('.')[1] + "_" +layer.alias + self.print_messages() return layers From adf6f8c7e5d27cf3d260240d5aca09eb10755306 Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 14 Aug 2023 11:28:49 +0200 Subject: [PATCH 05/34] rename ambiguous layer alias in two steps - find duplicates and add the topic and . as prefix and when still there are duplicates add model and . as prefix --- modelbaker/generator/generator.py | 18 ++++++++++++++---- .../ilimodels/Polymorphic_Ortsplanung_V1_1.ili | 9 +++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/modelbaker/generator/generator.py b/modelbaker/generator/generator.py index 9601f11..372bbd7 100644 --- a/modelbaker/generator/generator.py +++ b/modelbaker/generator/generator.py @@ -53,7 +53,7 @@ def __init__( parent=None, mgmt_uri=None, consider_basket_handling=False, - optimize_strategy=Generator.OptimizeStrategy.HIDE #HIDE for-relevance-tests: should be NONE after + optimize_strategy=2 #HIDE for-relevance-tests: should be NONE after ): """ Creates a new Generator objects. @@ -195,7 +195,7 @@ def layers(self, filter_layer_list=[]): + match.group(1).split(".")[-1] + ")" ) - alias = short_name if is_relevant else f"{short_name} !IRRELEVANT!" #for-relevance-tests + alias = short_name #for-relevance-tests if is_relevant else f"{short_name} !IRRELEVANT!" #for-relevance-tests display_expression = "" if is_basket_table: @@ -365,12 +365,22 @@ def layers(self, filter_layer_list=[]): layers.append(layer) #rename ambiguous layers with topic prefix - ambiguous_aliases = [l.alias for l in layers if layers.count(l.alias)>1] + aliases = [l.alias for l in layers] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] for layer in layers: if layer.alias in ambiguous_aliases: if layer.ili_name: if layer.ili_name.count(".") > 1: - layer.alias = layer.ili_name.split('.')[1] + "_" +layer.alias + layer.alias = layer.ili_name.split('.')[1] + "." +layer.alias + + #rename ambiguous layers with models prefix + aliases = [l.alias for l in layers] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + for layer in layers: + if layer.alias in ambiguous_aliases: + if layer.ili_name: + if layer.ili_name.count(".") > 1: + layer.alias = layer.ili_name.split('.')[0] + "." +layer.alias self.print_messages() diff --git a/tests/testdata/ilimodels/Polymorphic_Ortsplanung_V1_1.ili b/tests/testdata/ilimodels/Polymorphic_Ortsplanung_V1_1.ili index 0e0033f..dbc466f 100644 --- a/tests/testdata/ilimodels/Polymorphic_Ortsplanung_V1_1.ili +++ b/tests/testdata/ilimodels/Polymorphic_Ortsplanung_V1_1.ili @@ -60,4 +60,13 @@ MODEL Polymorphic_Ortsplanung_V1_1 (en) AT "https://modelbaker.ch" VERSION "2023 END Hallen; + !! An indipendend topic just having "coincidentally" the same name and the same named class (but no extensions involved here) + TOPIC Konstruktionen = + OID AS INTERLIS.UUIDOID; + + CLASS Gebaeude = + Name: TEXT; + END Gebaeude; + END Konstruktionen; + END Polymorphic_Ortsplanung_V1_1. \ No newline at end of file From 1f2f4c5f5303a08fe5378c7fc0ed1dd07cf718e9 Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 14 Aug 2023 12:43:40 +0200 Subject: [PATCH 06/34] fix tests and not concern the order of error messages --- tests/test_validate.py | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/tests/test_validate.py b/tests/test_validate.py index b247eeb..7f2d5b9 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -517,32 +517,20 @@ def test_validate_exportmodels_gpkg(self): result_model.reload() assert result_model.rowCount() == 3 - expected_staedtische_gewerbe_error = ( - "Needs an ethical evaluation (EthischeBeurteilung)" - ) - expected_kantonale_ortsplanung_error = ( - "Beschreibung and/or Referenzcode must be defined." - ) - expected_staedtische_ortsplanung_error = "Beschreibung needed when top secret." - - assert ( - result_model.index(0, 0).data( + expected_error_messages = [ + #staedtisches gewerbe + "Needs an ethical evaluation (EthischeBeurteilung)", + #kantonale ortsplanung + "Beschreibung and/or Referenzcode must be defined.", + #staedtische ortsplanung + "Beschreibung needed when top secret." + ] + + error_messages = [ result_model.index(i, 0).data( int(ilivalidator.ValidationResultModel.Roles.MESSAGE) - ) - == expected_staedtische_gewerbe_error - ) - assert ( - result_model.index(1, 0).data( - int(ilivalidator.ValidationResultModel.Roles.MESSAGE) - ) - == expected_kantonale_ortsplanung_error - ) - assert ( - result_model.index(2, 0).data( - int(ilivalidator.ValidationResultModel.Roles.MESSAGE) - ) - == expected_staedtische_ortsplanung_error - ) + ) for i in range(3) ] + + assert set(error_messages) == set(expected_error_messages) # Validate at cantonal level (setting --exportModels KantonaleOrtsplanung_V1_1) # means validation will fail - we encouter the constraint failure from Kantonale_Ortsplanung_V1_1 From 164337744e2f131b312f5c31694ff93ef329d58f Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 14 Aug 2023 12:52:25 +0200 Subject: [PATCH 07/34] tests for unique layer naming --- tests/test_projectgen.py | 112 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/tests/test_projectgen.py b/tests/test_projectgen.py index b931472..a967b37 100644 --- a/tests/test_projectgen.py +++ b/tests/test_projectgen.py @@ -57,6 +57,118 @@ def setUpClass(cls): """Run before all tests.""" cls.basetestpath = tempfile.mkdtemp() + def test_layernaming_postgis(self): + # purpose is to eliminate ambiguous layer names by adding suffix topic (if needed) and model (if needed) + importer = iliimporter.Importer() + importer.tool = DbIliMode.ili2pg + importer.configuration = iliimporter_config(importer.tool) + importer.configuration.ilifile = testdata_path("ilimodels/Polymorphic_Ortsplanung_V1_1.ili") + importer.configuration.ilimodels = "Polymorphic_Ortsplanung_V1_1" + importer.configuration.dbschema = "layernaming_{:%Y%m%d%H%M%S%f}".format( + datetime.datetime.now() + ) + + importer.configuration.srs_code = 2056 + importer.configuration.inheritance = "smart2" + importer.configuration.create_basket_col = True + importer.stdout.connect(self.print_info) + importer.stderr.connect(self.print_error) + assert importer.run() == iliimporter.Importer.SUCCESS + + generator = Generator( + DbIliMode.ili2pg, + get_pg_connection_string(), + importer.configuration.inheritance, + importer.configuration.dbschema, + ) + + available_layers = generator.layers() + aliases = [l.alias for l in available_layers] + + # are there no ambiguous layer aliases? + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + assert len(ambiguous_aliases) == 0 + + # are the layers named correctly + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + assert set(aliases) == set(expected_aliases) + + + def test_layernaming_geopackage(self): + # purpose is to eliminate ambiguous layer names by adding suffix topic (if needed) and model (if needed) + importer = iliimporter.Importer() + importer.tool = DbIliMode.ili2gpkg + importer.configuration = iliimporter_config(importer.tool) + importer.configuration.ilifile = testdata_path("ilimodels/Polymorphic_Ortsplanung_V1_1.ili") + importer.configuration.ilimodels = "Polymorphic_Ortsplanung_V1_1" + importer.configuration.dbfile = os.path.join( + self.basetestpath, + "tmp_layernaming_{:%Y%m%d%H%M%S%f}.gpkg".format(datetime.datetime.now()), + ) + importer.configuration.srs_code = 2056 + importer.configuration.inheritance = "smart2" + importer.configuration.create_basket_col = True + importer.stdout.connect(self.print_info) + importer.stderr.connect(self.print_error) + assert importer.run() == iliimporter.Importer.SUCCESS + + config_manager = GpkgCommandConfigManager(importer.configuration) + uri = config_manager.get_uri() + + generator = Generator(DbIliMode.ili2gpkg, uri, "smart2") + + available_layers = generator.layers() + aliases = [l.alias for l in available_layers] + + # are there no ambiguous layer aliases? + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + assert len(ambiguous_aliases) == 0 + + # are the layers named correctly + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + assert set(aliases) == set(expected_aliases) + + def test_layernaming_mssql(self): + # purpose is to eliminate ambiguous layer names by adding suffix topic (if needed) and model (if needed) + importer = iliimporter.Importer() + importer.tool = DbIliMode.ili2mssql + importer.configuration = iliimporter_config(importer.tool) + importer.configuration.ilifile = testdata_path("ilimodels/Polymorphic_Ortsplanung_V1_1.ili") + importer.configuration.ilimodels = "Polymorphic_Ortsplanung_V1_1" + importer.configuration.dbschema = "layernaming_{:%Y%m%d%H%M%S%f}".format( + datetime.datetime.now() + ) + importer.configuration.srs_code = 2056 + importer.configuration.inheritance = "smart2" + importer.configuration.create_basket_col = True + importer.stdout.connect(self.print_info) + importer.stderr.connect(self.print_error) + + uri = "DRIVER={drv};SERVER={server};DATABASE={db};UID={uid};PWD={pwd}".format( + drv="{ODBC Driver 17 for SQL Server}", + server=importer.configuration.dbhost, + db=importer.configuration.database, + uid=importer.configuration.dbusr, + pwd=importer.configuration.dbpwd, + ) + + assert importer.run() == iliimporter.Importer.SUCCESS + + generator = Generator( + DbIliMode.ili2mssql, uri, "smart2", importer.configuration.dbschema + ) + + available_layers = generator.layers() + aliases = [l.alias for l in available_layers] + + # are there no ambiguous layer aliases? + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + assert len(ambiguous_aliases) == 0 + + # are the layers named correctly + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + assert set(aliases) == set(expected_aliases) + def test_ili2db3_kbs_postgis(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2pg From a49fa84f19c6a95533c6a7b51121329362148ab8 Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 14 Aug 2023 14:13:52 +0200 Subject: [PATCH 08/34] tests according to the model checking: proper layer naming, if layer is relevant in current project --- README.md | 4 +- modelbaker/dataobjects/layers.py | 2 + modelbaker/generator/generator.py | 4 +- tests/test_projectgen.py | 112 ----- .../test_projectgen_extension_optimization.py | 449 ++++++++++++++++++ 5 files changed, 456 insertions(+), 115 deletions(-) create mode 100644 tests/test_projectgen_extension_optimization.py diff --git a/README.md b/README.md index 0a11ec5..78f7ff7 100644 --- a/README.md +++ b/README.md @@ -91,8 +91,10 @@ Is enforced with pre-commit. To use, make: ``` pip install pre-commit pre-commit install -```00 +``` + And to run it over all the files (with infile changes): + ``` pre-commit run --color=always --all-file ``` diff --git a/modelbaker/dataobjects/layers.py b/modelbaker/dataobjects/layers.py index ebe9ae9..38685b1 100644 --- a/modelbaker/dataobjects/layers.py +++ b/modelbaker/dataobjects/layers.py @@ -99,6 +99,8 @@ def __init__( self.model_topic_name = ( f"{self.ili_name.split('.')[0]}.{self.ili_name.split('.')[1]}" ) + + self.is_relevant = is_relevant self.definitionfile = definitionfile self.qmlstylefile = qmlstylefile diff --git a/modelbaker/generator/generator.py b/modelbaker/generator/generator.py index 372bbd7..0db3858 100644 --- a/modelbaker/generator/generator.py +++ b/modelbaker/generator/generator.py @@ -145,7 +145,7 @@ def layers(self, filter_layer_list=[]): record.get("tablename") == self._db_connector.dataset_table_name ) - is_relevant = bool(record.get("relevance")) if self.optimize_strategy != Generator.OptimizeStrategy.NONE else True + is_relevant = bool(record.get("relevance")) # it can be not relevant and still be displayed (in case of NONE) if self.optimize_strategy != Generator.OptimizeStrategy.NONE else True alias = record["table_alias"] if "table_alias" in record else None if not alias: @@ -195,7 +195,7 @@ def layers(self, filter_layer_list=[]): + match.group(1).split(".")[-1] + ")" ) - alias = short_name #for-relevance-tests if is_relevant else f"{short_name} !IRRELEVANT!" #for-relevance-tests + alias = short_name #for-relevance-tests if is_relevant else f"{short_name} !IRRELEVANT!" display_expression = "" if is_basket_table: diff --git a/tests/test_projectgen.py b/tests/test_projectgen.py index a967b37..b931472 100644 --- a/tests/test_projectgen.py +++ b/tests/test_projectgen.py @@ -57,118 +57,6 @@ def setUpClass(cls): """Run before all tests.""" cls.basetestpath = tempfile.mkdtemp() - def test_layernaming_postgis(self): - # purpose is to eliminate ambiguous layer names by adding suffix topic (if needed) and model (if needed) - importer = iliimporter.Importer() - importer.tool = DbIliMode.ili2pg - importer.configuration = iliimporter_config(importer.tool) - importer.configuration.ilifile = testdata_path("ilimodels/Polymorphic_Ortsplanung_V1_1.ili") - importer.configuration.ilimodels = "Polymorphic_Ortsplanung_V1_1" - importer.configuration.dbschema = "layernaming_{:%Y%m%d%H%M%S%f}".format( - datetime.datetime.now() - ) - - importer.configuration.srs_code = 2056 - importer.configuration.inheritance = "smart2" - importer.configuration.create_basket_col = True - importer.stdout.connect(self.print_info) - importer.stderr.connect(self.print_error) - assert importer.run() == iliimporter.Importer.SUCCESS - - generator = Generator( - DbIliMode.ili2pg, - get_pg_connection_string(), - importer.configuration.inheritance, - importer.configuration.dbschema, - ) - - available_layers = generator.layers() - aliases = [l.alias for l in available_layers] - - # are there no ambiguous layer aliases? - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] - assert len(ambiguous_aliases) == 0 - - # are the layers named correctly - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] - assert set(aliases) == set(expected_aliases) - - - def test_layernaming_geopackage(self): - # purpose is to eliminate ambiguous layer names by adding suffix topic (if needed) and model (if needed) - importer = iliimporter.Importer() - importer.tool = DbIliMode.ili2gpkg - importer.configuration = iliimporter_config(importer.tool) - importer.configuration.ilifile = testdata_path("ilimodels/Polymorphic_Ortsplanung_V1_1.ili") - importer.configuration.ilimodels = "Polymorphic_Ortsplanung_V1_1" - importer.configuration.dbfile = os.path.join( - self.basetestpath, - "tmp_layernaming_{:%Y%m%d%H%M%S%f}.gpkg".format(datetime.datetime.now()), - ) - importer.configuration.srs_code = 2056 - importer.configuration.inheritance = "smart2" - importer.configuration.create_basket_col = True - importer.stdout.connect(self.print_info) - importer.stderr.connect(self.print_error) - assert importer.run() == iliimporter.Importer.SUCCESS - - config_manager = GpkgCommandConfigManager(importer.configuration) - uri = config_manager.get_uri() - - generator = Generator(DbIliMode.ili2gpkg, uri, "smart2") - - available_layers = generator.layers() - aliases = [l.alias for l in available_layers] - - # are there no ambiguous layer aliases? - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] - assert len(ambiguous_aliases) == 0 - - # are the layers named correctly - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] - assert set(aliases) == set(expected_aliases) - - def test_layernaming_mssql(self): - # purpose is to eliminate ambiguous layer names by adding suffix topic (if needed) and model (if needed) - importer = iliimporter.Importer() - importer.tool = DbIliMode.ili2mssql - importer.configuration = iliimporter_config(importer.tool) - importer.configuration.ilifile = testdata_path("ilimodels/Polymorphic_Ortsplanung_V1_1.ili") - importer.configuration.ilimodels = "Polymorphic_Ortsplanung_V1_1" - importer.configuration.dbschema = "layernaming_{:%Y%m%d%H%M%S%f}".format( - datetime.datetime.now() - ) - importer.configuration.srs_code = 2056 - importer.configuration.inheritance = "smart2" - importer.configuration.create_basket_col = True - importer.stdout.connect(self.print_info) - importer.stderr.connect(self.print_error) - - uri = "DRIVER={drv};SERVER={server};DATABASE={db};UID={uid};PWD={pwd}".format( - drv="{ODBC Driver 17 for SQL Server}", - server=importer.configuration.dbhost, - db=importer.configuration.database, - uid=importer.configuration.dbusr, - pwd=importer.configuration.dbpwd, - ) - - assert importer.run() == iliimporter.Importer.SUCCESS - - generator = Generator( - DbIliMode.ili2mssql, uri, "smart2", importer.configuration.dbschema - ) - - available_layers = generator.layers() - aliases = [l.alias for l in available_layers] - - # are there no ambiguous layer aliases? - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] - assert len(ambiguous_aliases) == 0 - - # are the layers named correctly - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] - assert set(aliases) == set(expected_aliases) - def test_ili2db3_kbs_postgis(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2pg diff --git a/tests/test_projectgen_extension_optimization.py b/tests/test_projectgen_extension_optimization.py new file mode 100644 index 0000000..5f6f717 --- /dev/null +++ b/tests/test_projectgen_extension_optimization.py @@ -0,0 +1,449 @@ +""" +/*************************************************************************** + ------------------- + begin : 14.08.2023 + git sha : :%H$ + copyright : (C) 2023 by Dave Signer + email : david@opengis.ch + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" + +import configparser +import datetime +import logging +import os +import pathlib +import shutil +import tempfile +from decimal import Decimal + +import yaml +from qgis.core import Qgis, QgsEditFormConfig, QgsProject, QgsRelation +from qgis.PyQt.QtCore import QEventLoop, Qt, QTimer +from qgis.testing import start_app, unittest + +from modelbaker.dataobjects.project import Project +from modelbaker.db_factory.gpkg_command_config_manager import GpkgCommandConfigManager +from modelbaker.generator.generator import Generator +from modelbaker.iliwrapper import iliimporter +from modelbaker.iliwrapper.globals import DbIliMode +from modelbaker.iliwrapper.ilicache import ( + IliDataCache, + IliDataItemModel, + IliToppingFileCache, + IliToppingFileItemModel, +) +from tests.utils import get_pg_connection_string, iliimporter_config, testdata_path + +CATALOGUE_DATASETNAME = "Catset" + +start_app() + +test_path = pathlib.Path(__file__).parent.absolute() + + +class TestProjectExtOptimization(unittest.TestCase): + @classmethod + def setUpClass(cls): + """Run before all tests.""" + cls.basetestpath = tempfile.mkdtemp() + + ''' + Those tests check if: + - no ambiguous layers exists - they are all named properly and unique + - irrelevant layers are detected (according to the assumptions below) + - irrelevant layers are handled according the the chosen strategy: hidden or grouped + - relations (and their widgets) are handled according to the stategy (not created when hidden, not in the forms used when grouped) + + Assumption: + Since it appears almost impossible to care for all the cases, I need to make some assumptions what mostly would be the case. + - When you extend a base class with the same name, you intend to "replace" it, otherwise you would rename it. + - When you extend a base class multiple times (what you do with different names) then you intend to "replace" it. + - Exception for the two cases above: When you extended the class it in the same model but another topic (because if you intent to "replace" it, you would have made it ABSTRACT) + + This is tested with three use cases: + - Polymorphic_Ortsplanung_V1_1 containing several topics extending the same class + - Staedtische_Ortsplanung_V1_1 containing several extention levels on the same class + - Bauplanung_V1_1 containing structures and extending assocciations + ''' + + + def test_staedtische_postgis(self): + importer = iliimporter.Importer() + importer.tool = DbIliMode.ili2pg + importer.configuration = iliimporter_config(importer.tool) + importer.configuration.ilifile = testdata_path("ilimodels/Staedtische_Ortsplanung_V1_1.ili") + importer.configuration.ilimodels = "Staedtische_Ortsplanung_V1_1" + importer.configuration.dbschema = "optimal_staedtische_{:%Y%m%d%H%M%S%f}".format( + datetime.datetime.now() + ) + + importer.configuration.srs_code = 2056 + importer.configuration.inheritance = "smart2" + importer.configuration.create_basket_col = True + importer.stdout.connect(self.print_info) + importer.stderr.connect(self.print_error) + assert importer.run() == iliimporter.Importer.SUCCESS + + generator = Generator( + DbIliMode.ili2pg, + get_pg_connection_string(), + importer.configuration.inheritance, + importer.configuration.dbschema, + ) + + available_layers = generator.layers() + aliases = [l.alias for l in available_layers] + irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + + # check no ambiguous layers exists + assert len(ambiguous_aliases) == 0 + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + assert set(aliases) == set(expected_aliases) + + # irrelevant layers are detected + assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = ['dsaf.Konstruktionen.Gebaeude'] + assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + + + def test_staedtische_geopackage(self): + importer = iliimporter.Importer() + importer.tool = DbIliMode.ili2gpkg + importer.configuration = iliimporter_config(importer.tool) + importer.configuration.ilifile = testdata_path("ilimodels/Staedtische_Ortsplanung_V1_1.ili") + importer.configuration.ilimodels = "Staedtische_Ortsplanung_V1_1" + importer.configuration.dbfile = os.path.join( + self.basetestpath, + "tmp_optimal_staedtische_{:%Y%m%d%H%M%S%f}.gpkg".format(datetime.datetime.now()), + ) + importer.configuration.srs_code = 2056 + importer.configuration.inheritance = "smart2" + importer.configuration.create_basket_col = True + importer.stdout.connect(self.print_info) + importer.stderr.connect(self.print_error) + assert importer.run() == iliimporter.Importer.SUCCESS + + config_manager = GpkgCommandConfigManager(importer.configuration) + uri = config_manager.get_uri() + + generator = Generator(DbIliMode.ili2gpkg, uri, "smart2") + + available_layers = generator.layers() + aliases = [l.alias for l in available_layers] + irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + + # check no ambiguous layers exists + assert len(ambiguous_aliases) == 0 + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + assert set(aliases) == set(expected_aliases) + + # irrelevant layers are detected + assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = ['asdf.Konstruktionen.Gebaeude'] + assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + + def test_staedtische_mssql(self): + importer = iliimporter.Importer() + importer.tool = DbIliMode.ili2mssql + importer.configuration = iliimporter_config(importer.tool) + importer.configuration.ilifile = testdata_path("ilimodels/Staedtische_Ortsplanung_V1_1.ili") + importer.configuration.ilimodels = "Staedtische_Ortsplanung_V1_1" + importer.configuration.dbschema = "optimal_staedtische_{:%Y%m%d%H%M%S%f}".format( + datetime.datetime.now() + ) + importer.configuration.srs_code = 2056 + importer.configuration.inheritance = "smart2" + importer.configuration.create_basket_col = True + importer.stdout.connect(self.print_info) + importer.stderr.connect(self.print_error) + + uri = "DRIVER={drv};SERVER={server};DATABASE={db};UID={uid};PWD={pwd}".format( + drv="{ODBC Driver 17 for SQL Server}", + server=importer.configuration.dbhost, + db=importer.configuration.database, + uid=importer.configuration.dbusr, + pwd=importer.configuration.dbpwd, + ) + + assert importer.run() == iliimporter.Importer.SUCCESS + + generator = Generator( + DbIliMode.ili2mssql, uri, "smart2", importer.configuration.dbschema + ) + + available_layers = generator.layers() + aliases = [l.alias for l in available_layers] + irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + + # check no ambiguous layers exists + assert len(ambiguous_aliases) == 0 + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + assert set(aliases) == set(expected_aliases) + + # irrelevant layers are detected + assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = ['asdffasd.Konstruktionen.Gebaeude'] + assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + + def test_polymorphic_postgis(self): + importer = iliimporter.Importer() + importer.tool = DbIliMode.ili2pg + importer.configuration = iliimporter_config(importer.tool) + importer.configuration.ilifile = testdata_path("ilimodels/Polymorphic_Ortsplanung_V1_1.ili") + importer.configuration.ilimodels = "Polymorphic_Ortsplanung_V1_1" + importer.configuration.dbschema = "optimal_polymorph_{:%Y%m%d%H%M%S%f}".format( + datetime.datetime.now() + ) + + importer.configuration.srs_code = 2056 + importer.configuration.inheritance = "smart2" + importer.configuration.create_basket_col = True + importer.stdout.connect(self.print_info) + importer.stderr.connect(self.print_error) + assert importer.run() == iliimporter.Importer.SUCCESS + + generator = Generator( + DbIliMode.ili2pg, + get_pg_connection_string(), + importer.configuration.inheritance, + importer.configuration.dbschema, + ) + + available_layers = generator.layers() + aliases = [l.alias for l in available_layers] + irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + + # check no ambiguous layers exists + assert len(ambiguous_aliases) == 0 + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + assert set(aliases) == set(expected_aliases) + + # irrelevant layers are detected + assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = ['Ortsplanung_V1_1.Konstruktionen.Gebaeude'] + assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + + def test_polymorphic_geopackage(self): + importer = iliimporter.Importer() + importer.tool = DbIliMode.ili2gpkg + importer.configuration = iliimporter_config(importer.tool) + importer.configuration.ilifile = testdata_path("ilimodels/Polymorphic_Ortsplanung_V1_1.ili") + importer.configuration.ilimodels = "Polymorphic_Ortsplanung_V1_1" + importer.configuration.dbfile = os.path.join( + self.basetestpath, + "tmp_optimal_polymorph_{:%Y%m%d%H%M%S%f}.gpkg".format(datetime.datetime.now()), + ) + importer.configuration.srs_code = 2056 + importer.configuration.inheritance = "smart2" + importer.configuration.create_basket_col = True + importer.stdout.connect(self.print_info) + importer.stderr.connect(self.print_error) + assert importer.run() == iliimporter.Importer.SUCCESS + + config_manager = GpkgCommandConfigManager(importer.configuration) + uri = config_manager.get_uri() + + generator = Generator(DbIliMode.ili2gpkg, uri, "smart2") + + available_layers = generator.layers() + aliases = [l.alias for l in available_layers] + irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + + # check no ambiguous layers exists + assert len(ambiguous_aliases) == 0 + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + assert set(aliases) == set(expected_aliases) + + # irrelevant layers are detected + assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = ['Ortsplanung_V1_1.Konstruktionen.Gebaeude'] + assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + + def test_polymorphic_mssql(self): + importer = iliimporter.Importer() + importer.tool = DbIliMode.ili2mssql + importer.configuration = iliimporter_config(importer.tool) + importer.configuration.ilifile = testdata_path("ilimodels/Polymorphic_Ortsplanung_V1_1.ili") + importer.configuration.ilimodels = "Polymorphic_Ortsplanung_V1_1" + importer.configuration.dbschema = "optimal_polymorph_{:%Y%m%d%H%M%S%f}".format( + datetime.datetime.now() + ) + importer.configuration.srs_code = 2056 + importer.configuration.inheritance = "smart2" + importer.configuration.create_basket_col = True + importer.stdout.connect(self.print_info) + importer.stderr.connect(self.print_error) + + uri = "DRIVER={drv};SERVER={server};DATABASE={db};UID={uid};PWD={pwd}".format( + drv="{ODBC Driver 17 for SQL Server}", + server=importer.configuration.dbhost, + db=importer.configuration.database, + uid=importer.configuration.dbusr, + pwd=importer.configuration.dbpwd, + ) + + assert importer.run() == iliimporter.Importer.SUCCESS + + generator = Generator( + DbIliMode.ili2mssql, uri, "smart2", importer.configuration.dbschema + ) + + available_layers = generator.layers() + aliases = [l.alias for l in available_layers] + irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + + # check no ambiguous layers exists + assert len(ambiguous_aliases) == 0 + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + assert set(aliases) == set(expected_aliases) + + # irrelevant layers are detected + assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = ['Ortsplanung_V1_1.Konstruktionen.Gebaeude'] + assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + + def test_baustruct_postgis(self): + importer = iliimporter.Importer() + importer.tool = DbIliMode.ili2pg + importer.configuration = iliimporter_config(importer.tool) + importer.configuration.ilifile = testdata_path("ilimodels/Bauplanung_V1_1.ili") + importer.configuration.ilimodels = "Bauplanung_V1_1" + importer.configuration.dbschema = "optimal_baustruct_{:%Y%m%d%H%M%S%f}".format( + datetime.datetime.now() + ) + + importer.configuration.srs_code = 2056 + importer.configuration.inheritance = "smart2" + importer.configuration.create_basket_col = True + importer.stdout.connect(self.print_info) + importer.stderr.connect(self.print_error) + assert importer.run() == iliimporter.Importer.SUCCESS + + generator = Generator( + DbIliMode.ili2pg, + get_pg_connection_string(), + importer.configuration.inheritance, + importer.configuration.dbschema, + ) + + available_layers = generator.layers() + aliases = [l.alias for l in available_layers] + irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + + # check no ambiguous layers exists + assert len(ambiguous_aliases) == 0 + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + assert set(aliases) == set(expected_aliases) + + # irrelevant layers are detected + assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = ['asf.Konstruktionen.Gebaeude'] + assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + + + def test_baustruct_geopackage(self): + importer = iliimporter.Importer() + importer.tool = DbIliMode.ili2gpkg + importer.configuration = iliimporter_config(importer.tool) + importer.configuration.ilifile = testdata_path("ilimodels/Bauplanung_V1_1.ili") + importer.configuration.ilimodels = "Bauplanung_V1_1" + importer.configuration.dbfile = os.path.join( + self.basetestpath, + "tmp_optimal_baustruct_{:%Y%m%d%H%M%S%f}.gpkg".format(datetime.datetime.now()), + ) + importer.configuration.srs_code = 2056 + importer.configuration.inheritance = "smart2" + importer.configuration.create_basket_col = True + importer.stdout.connect(self.print_info) + importer.stderr.connect(self.print_error) + assert importer.run() == iliimporter.Importer.SUCCESS + + config_manager = GpkgCommandConfigManager(importer.configuration) + uri = config_manager.get_uri() + + generator = Generator(DbIliMode.ili2gpkg, uri, "smart2") + + available_layers = generator.layers() + aliases = [l.alias for l in available_layers] + irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + + # check no ambiguous layers exists + assert len(ambiguous_aliases) == 0 + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + assert set(aliases) == set(expected_aliases) + + # irrelevant layers are detected + assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = ['sdaf.Konstruktionen.Gebaeude'] + assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + + def test_baustruct_mssql(self): + importer = iliimporter.Importer() + importer.tool = DbIliMode.ili2mssql + importer.configuration = iliimporter_config(importer.tool) + importer.configuration.ilifile = testdata_path("ilimodels/Bauplanung_V1_1.ili") + importer.configuration.ilimodels = "Bauplanung_V1_1" + importer.configuration.dbschema = "optimal_baustruct_{:%Y%m%d%H%M%S%f}".format( + datetime.datetime.now() + ) + importer.configuration.srs_code = 2056 + importer.configuration.inheritance = "smart2" + importer.configuration.create_basket_col = True + importer.stdout.connect(self.print_info) + importer.stderr.connect(self.print_error) + + uri = "DRIVER={drv};SERVER={server};DATABASE={db};UID={uid};PWD={pwd}".format( + drv="{ODBC Driver 17 for SQL Server}", + server=importer.configuration.dbhost, + db=importer.configuration.database, + uid=importer.configuration.dbusr, + pwd=importer.configuration.dbpwd, + ) + + assert importer.run() == iliimporter.Importer.SUCCESS + + generator = Generator( + DbIliMode.ili2mssql, uri, "smart2", importer.configuration.dbschema + ) + + available_layers = generator.layers() + aliases = [l.alias for l in available_layers] + irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + + # check no ambiguous layers exists + assert len(ambiguous_aliases) == 0 + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + assert set(aliases) == set(expected_aliases) + + # irrelevant layers are detected + assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = ['asdfafs.Konstruktionen.Gebaeude'] + assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + + def print_info(self, text): + logging.info(text) + + def print_error(self, text): + logging.error(text) + + def tearDown(self): + QgsProject.instance().removeAllMapLayers() \ No newline at end of file From efc9746a1d152850ebb2e59893c349b8dc890857 Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 14 Aug 2023 17:03:56 +0200 Subject: [PATCH 09/34] concern optimize strategies on extended models by grouping or hiding unneeded layers --- modelbaker/generator/generator.py | 126 +++++++++++++----- .../test_projectgen_extension_optimization.py | 91 +++++++------ 2 files changed, 143 insertions(+), 74 deletions(-) diff --git a/modelbaker/generator/generator.py b/modelbaker/generator/generator.py index 0db3858..6a0f4b6 100644 --- a/modelbaker/generator/generator.py +++ b/modelbaker/generator/generator.py @@ -40,7 +40,7 @@ class Generator(QObject): class OptimizeStrategy(Enum): NONE = 0 - RENAME = 1 + GROUP = 1 HIDE = 2 def __init__( @@ -53,7 +53,7 @@ def __init__( parent=None, mgmt_uri=None, consider_basket_handling=False, - optimize_strategy=2 #HIDE for-relevance-tests: should be NONE after + optimize_strategy = None, ): """ Creates a new Generator objects. @@ -75,7 +75,7 @@ def __init__( self._db_connector.stdout.connect(self.print_info) self._db_connector.new_message.connect(self.append_print_message) self.basket_handling = consider_basket_handling and self.get_basket_handling() - self.optimize_strategy = optimize_strategy + self.optimize_strategy = optimize_strategy if optimize_strategy is not None else self.OptimizeStrategy.NONE self._additional_ignored_layers = ( [] @@ -607,35 +607,18 @@ def legend( if node: legend.append(node) else: - tables = LegendGroup(QCoreApplication.translate("LegendGroup", "tables")) - domains = LegendGroup(QCoreApplication.translate("LegendGroup", "domains")) - domains.expanded = False - system = LegendGroup(QCoreApplication.translate("LegendGroup", "system")) - system.expanded = False - - point_layers = [] - line_layers = [] - polygon_layers = [] - - for layer in layers: - if layer.geometry_column: - geometry_type = QgsWkbTypes.geometryType(layer.wkb_type) - if geometry_type == QgsWkbTypes.PointGeometry: - point_layers.append(layer) - elif geometry_type == QgsWkbTypes.LineGeometry: - line_layers.append(layer) - elif geometry_type == QgsWkbTypes.PolygonGeometry: - polygon_layers.append(layer) - else: - if layer.is_domain: - domains.append(layer) - elif layer.name in [ - self._db_connector.basket_table_name, - self._db_connector.dataset_table_name, - ]: - system.append(layer) + irrelevant_layers = [] + relevant_layers = [] + if self.optimize_strategy == self.OptimizeStrategy.NONE: + relevant_layers = layers + else: + for layer in layers: + if layer.is_relevant: + relevant_layers.append(layer) else: - tables.append(layer) + irrelevant_layers.append(layer) + + point_layers, line_layers, polygon_layers, domain_layers, table_layers, system_layers = self._separated_legend_layers(relevant_layers) for l in polygon_layers: legend.append(l) @@ -644,15 +627,90 @@ def legend( for l in point_layers: legend.append(l) - if not tables.is_empty(): + # create groups + if len(table_layers): + tables = LegendGroup(QCoreApplication.translate("LegendGroup", "tables")) + for layer in table_layers: + tables.append(layer) legend.append(tables) - if not domains.is_empty(): + if len(domain_layers): + domains = LegendGroup(QCoreApplication.translate("LegendGroup", "domains")) + domains.expanded = False + for layer in domain_layers: + domains.append(layer) legend.append(domains) - if not system.is_empty(): + if len(system_layers): + system = LegendGroup(QCoreApplication.translate("LegendGroup", "system")) + system.expanded = False + for layer in system_layers: + system.append(layer) legend.append(system) + + # when the irrelevant layers should be grouped (but visible), we make the structure for them and append it to a group + if self.optimize_strategy == self.OptimizeStrategy.GROUP: + point_layers, line_layers, polygon_layers, domain_layers, table_layers, system_layers = self._separated_legend_layers(irrelevant_layers) + + # create base group + base_group = LegendGroup(QCoreApplication.translate("LegendGroup", "base layers")) + base_group.expanded = False + base_group.checked = False + + for l in polygon_layers: + base_group.append(l) + for l in line_layers: + base_group.append(l) + for l in point_layers: + base_group.append(l) + + # create groups + if len(table_layers): + tables = LegendGroup(QCoreApplication.translate("LegendGroup", "base tables")) + for layer in table_layers: + tables.append(layer) + base_group.append(tables) + if len(domain_layers): + domains = LegendGroup(QCoreApplication.translate("LegendGroup", "base domains")) + domains.expanded = False + for layer in domain_layers: + domains.append(layer) + base_group.append(domains) + + legend.append(base_group) return legend + def _separated_legend_layers(self, layers): + domain_layers = [] + table_layers = [] + system_layers = [] + + point_layers = [] + line_layers = [] + polygon_layers = [] + + for layer in layers: + if layer.geometry_column: + geometry_type = QgsWkbTypes.geometryType(layer.wkb_type) + if geometry_type == QgsWkbTypes.PointGeometry: + point_layers.append(layer) + elif geometry_type == QgsWkbTypes.LineGeometry: + line_layers.append(layer) + elif geometry_type == QgsWkbTypes.PolygonGeometry: + polygon_layers.append(layer) + else: + if layer.is_domain: + domain_layers.append(layer) + elif layer.name in [ + self._db_connector.basket_table_name, + self._db_connector.dataset_table_name, + ]: + system_layers.append(layer) + else: + table_layers.append(layer) + + return point_layers, line_layers, polygon_layers, domain_layers, table_layers, system_layers + + def resolved_layouts( self, layouts={}, diff --git a/tests/test_projectgen_extension_optimization.py b/tests/test_projectgen_extension_optimization.py index 5f6f717..6963425 100644 --- a/tests/test_projectgen_extension_optimization.py +++ b/tests/test_projectgen_extension_optimization.py @@ -77,7 +77,7 @@ def setUpClass(cls): ''' - def test_staedtische_postgis(self): + def test_extopt_staedtische_postgis(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2pg importer.configuration = iliimporter_config(importer.tool) @@ -108,16 +108,16 @@ def test_staedtische_postgis(self): # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gebaeude_StadtFirma', 'Gewerbe.Gebaeude', 'Gewerbe_V1.Firmen.Firma', 'Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Staedtisches_Gewerbe_V1.Firmen.Firma', 'Strasse'] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected - assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ['dsaf.Konstruktionen.Gebaeude'] - assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + #todo assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = ['Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Gewerbe_V1.Firmen.Firma'] + #todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - def test_staedtische_geopackage(self): + def test_extopt_staedtische_geopackage(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2gpkg importer.configuration = iliimporter_config(importer.tool) @@ -146,15 +146,24 @@ def test_staedtische_geopackage(self): # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gebaeude_StadtFirma', 'Gewerbe.Gebaeude', 'Gewerbe_V1.Firmen.Firma', 'Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Staedtisches_Gewerbe_V1.Firmen.Firma', 'Strasse'] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ['asdf.Konstruktionen.Gebaeude'] + expected_irrelevant_layer_ilinames = ['Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Gewerbe_V1.Firmen.Firma'] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - def test_staedtische_mssql(self): + project = Project() + project.layers = available_layers + project.relations = relations + project.legend = legend + project.post_generate() + + qgis_project = QgsProject.instance() + project.create(None, qgis_project) + + def test_extopt_staedtische_mssql(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2mssql importer.configuration = iliimporter_config(importer.tool) @@ -190,15 +199,15 @@ def test_staedtische_mssql(self): # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gebaeude_StadtFirma', 'Gewerbe.Gebaeude', 'Gewerbe_V1.Firmen.Firma', 'Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Staedtisches_Gewerbe_V1.Firmen.Firma', 'Strasse'] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected - assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ['asdffasd.Konstruktionen.Gebaeude'] - assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + #todo assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = ['Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Gewerbe_V1.Firmen.Firma'] + #todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - def test_polymorphic_postgis(self): + def test_extopt_polymorphic_postgis(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2pg importer.configuration = iliimporter_config(importer.tool) @@ -225,6 +234,7 @@ def test_polymorphic_postgis(self): available_layers = generator.layers() aliases = [l.alias for l in available_layers] irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] + print(irrelevant_layer_ilinames) ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] # check no ambiguous layers exists @@ -233,11 +243,11 @@ def test_polymorphic_postgis(self): assert set(aliases) == set(expected_aliases) # irrelevant layers are detected - assert len(irrelevant_layer_ilinames) > 0 + #todo assert len(irrelevant_layer_ilinames) > 0 expected_irrelevant_layer_ilinames = ['Ortsplanung_V1_1.Konstruktionen.Gebaeude'] - assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + #todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - def test_polymorphic_geopackage(self): + def test_extopt_polymorphic_geopackage(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2gpkg importer.configuration = iliimporter_config(importer.tool) @@ -262,6 +272,7 @@ def test_polymorphic_geopackage(self): available_layers = generator.layers() aliases = [l.alias for l in available_layers] irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] + print(irrelevant_layer_ilinames) ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] # check no ambiguous layers exists @@ -274,7 +285,7 @@ def test_polymorphic_geopackage(self): expected_irrelevant_layer_ilinames = ['Ortsplanung_V1_1.Konstruktionen.Gebaeude'] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - def test_polymorphic_mssql(self): + def test_extopt_polymorphic_mssql(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2mssql importer.configuration = iliimporter_config(importer.tool) @@ -305,7 +316,7 @@ def test_polymorphic_mssql(self): available_layers = generator.layers() aliases = [l.alias for l in available_layers] - irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] + irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant] ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] # check no ambiguous layers exists @@ -314,16 +325,16 @@ def test_polymorphic_mssql(self): assert set(aliases) == set(expected_aliases) # irrelevant layers are detected - assert len(irrelevant_layer_ilinames) > 0 + #todo assert len(irrelevant_layer_ilinames) > 0 expected_irrelevant_layer_ilinames = ['Ortsplanung_V1_1.Konstruktionen.Gebaeude'] - assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + #todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - def test_baustruct_postgis(self): + def test_extopt_baustruct_postgis(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2pg importer.configuration = iliimporter_config(importer.tool) - importer.configuration.ilifile = testdata_path("ilimodels/Bauplanung_V1_1.ili") - importer.configuration.ilimodels = "Bauplanung_V1_1" + importer.configuration.ilifile = testdata_path("ilimodels/Kantonale_Bauplanung_V1_1.ili") + importer.configuration.ilimodels = "Kantonale_Bauplanung_V1_1" importer.configuration.dbschema = "optimal_baustruct_{:%Y%m%d%H%M%S%f}".format( datetime.datetime.now() ) @@ -349,21 +360,21 @@ def test_baustruct_postgis(self): # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + expected_aliases = ["Bauart", "Bauplanung_V1_1.Konstruktionen.Gebaeude", "Bauplanung_V1_1.Konstruktionen.Material", "Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", "Bauplanung_V1_1.Natur.Park", "Bauplanung_V1_1.Natur.Tierart", "Brutstelle", "Buntbrache", "Feld", "KantonaleBuntbrache", "Kantonale_Bauplanung_V1_1.Konstruktionen.Gebaeude", "Kantonale_Bauplanung_V1_1.Konstruktionen.Material", "Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", "Kantonale_Bauplanung_V1_1.Natur.Park", "Kantonale_Bauplanung_V1_1.Natur.Tierart", "Kartoffelfeld", "Sonnenblumenfeld", "Strasse"] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected - assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ['asf.Konstruktionen.Gebaeude'] - assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + #todo assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = ["Bauplanung_V1_1.Konstruktionen.Material", "Bauplanung_V1_1.Konstruktionen.Gebaeude", "Bauplanung_V1_1.Natur.Tierart", "Bauplanung_V1_1.Natur.Park"] + #todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - def test_baustruct_geopackage(self): + def test_extopt_baustruct_geopackage(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2gpkg importer.configuration = iliimporter_config(importer.tool) - importer.configuration.ilifile = testdata_path("ilimodels/Bauplanung_V1_1.ili") - importer.configuration.ilimodels = "Bauplanung_V1_1" + importer.configuration.ilifile = testdata_path("ilimodels/Kantonale_Bauplanung_V1_1.ili") + importer.configuration.ilimodels = "Kantonale_Bauplanung_V1_1" importer.configuration.dbfile = os.path.join( self.basetestpath, "tmp_optimal_baustruct_{:%Y%m%d%H%M%S%f}.gpkg".format(datetime.datetime.now()), @@ -387,20 +398,20 @@ def test_baustruct_geopackage(self): # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + expected_aliases = ["Bauart", "Bauplanung_V1_1.Konstruktionen.Gebaeude", "Bauplanung_V1_1.Konstruktionen.Material", "Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", "Bauplanung_V1_1.Natur.Park", "Bauplanung_V1_1.Natur.Tierart", "Brutstelle", "Buntbrache", "Feld", "KantonaleBuntbrache", "Kantonale_Bauplanung_V1_1.Konstruktionen.Gebaeude", "Kantonale_Bauplanung_V1_1.Konstruktionen.Material", "Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", "Kantonale_Bauplanung_V1_1.Natur.Park", "Kantonale_Bauplanung_V1_1.Natur.Tierart", "Kartoffelfeld", "Sonnenblumenfeld", "Strasse"] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ['sdaf.Konstruktionen.Gebaeude'] + expected_irrelevant_layer_ilinames = ["Bauplanung_V1_1.Konstruktionen.Material", "Bauplanung_V1_1.Konstruktionen.Gebaeude", "Bauplanung_V1_1.Natur.Tierart", "Bauplanung_V1_1.Natur.Park"] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - def test_baustruct_mssql(self): + def test_extopt_baustruct_mssql(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2mssql importer.configuration = iliimporter_config(importer.tool) - importer.configuration.ilifile = testdata_path("ilimodels/Bauplanung_V1_1.ili") - importer.configuration.ilimodels = "Bauplanung_V1_1" + importer.configuration.ilifile = testdata_path("ilimodels/Kantonale_Bauplanung_V1_1.ili") + importer.configuration.ilimodels = "Kantonale_Bauplanung_V1_1" importer.configuration.dbschema = "optimal_baustruct_{:%Y%m%d%H%M%S%f}".format( datetime.datetime.now() ) @@ -431,13 +442,13 @@ def test_baustruct_mssql(self): # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + expected_aliases = ["Bauart", "Bauplanung_V1_1.Konstruktionen.Gebaeude", "Bauplanung_V1_1.Konstruktionen.Material", "Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", "Bauplanung_V1_1.Natur.Park", "Bauplanung_V1_1.Natur.Tierart", "Brutstelle", "Buntbrache", "Feld", "KantonaleBuntbrache", "Kantonale_Bauplanung_V1_1.Konstruktionen.Gebaeude", "Kantonale_Bauplanung_V1_1.Konstruktionen.Material", "Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", "Kantonale_Bauplanung_V1_1.Natur.Park", "Kantonale_Bauplanung_V1_1.Natur.Tierart", "Kartoffelfeld", "Sonnenblumenfeld", "Strasse"] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected - assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ['asdfafs.Konstruktionen.Gebaeude'] - assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + #todo assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = ["Bauplanung_V1_1.Konstruktionen.Material", "Bauplanung_V1_1.Konstruktionen.Gebaeude", "Bauplanung_V1_1.Natur.Tierart", "Bauplanung_V1_1.Natur.Park"] + #todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) def print_info(self, text): logging.info(text) From 92b9fa0b0b02f91e64630bdb451d885f6ee49438 Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 14 Aug 2023 17:29:58 +0200 Subject: [PATCH 10/34] remove duplicate code with function --- modelbaker/generator/generator.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/modelbaker/generator/generator.py b/modelbaker/generator/generator.py index 6a0f4b6..71d4369 100644 --- a/modelbaker/generator/generator.py +++ b/modelbaker/generator/generator.py @@ -364,27 +364,25 @@ def layers(self, filter_layer_list=[]): layers.append(layer) - #rename ambiguous layers with topic prefix - aliases = [l.alias for l in layers] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] - for layer in layers: - if layer.alias in ambiguous_aliases: - if layer.ili_name: - if layer.ili_name.count(".") > 1: - layer.alias = layer.ili_name.split('.')[1] + "." +layer.alias + #append topic name to ambiguous layers + self._rename_ambiguous_layers(layers) + #append moel name to still ambiguous layers + self._rename_ambiguous_layers(layers, second_pass=True) + + self.print_messages() - #rename ambiguous layers with models prefix - aliases = [l.alias for l in layers] + return layers + + def _rename_ambiguous_layers(self, layers, second_pass = False): + #rename ambiguous layers with topic (on not second_pass) or model (on second_pass) prefix + #on irrelevant layers only if we don't ride OptimizeStrategy.HIDE + aliases = [l.alias for l in layers if l.is_relevant or self.optimize_strategy != self.OptimizeStrategy.HIDE] ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] for layer in layers: if layer.alias in ambiguous_aliases: if layer.ili_name: if layer.ili_name.count(".") > 1: - layer.alias = layer.ili_name.split('.')[0] + "." +layer.alias - - self.print_messages() - - return layers + layer.alias = layer.ili_name.split('.')[(0 if second_pass else 1)] + "." +layer.alias def relations(self, layers, filter_layer_list=[]): relations_info = self.get_relations_info(filter_layer_list) From fca236163af6e09b32472687d750f91331a4fb2d Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 28 Aug 2023 08:17:49 +0200 Subject: [PATCH 11/34] bump to ili2db 4.11.1 --- modelbaker/iliwrapper/ili2dbtools.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modelbaker/iliwrapper/ili2dbtools.py b/modelbaker/iliwrapper/ili2dbtools.py index 073e79b..6a6f61b 100644 --- a/modelbaker/iliwrapper/ili2dbtools.py +++ b/modelbaker/iliwrapper/ili2dbtools.py @@ -25,17 +25,17 @@ def get_tool_version(tool, db_ili_version): if db_ili_version == 3: return "3.11.3" else: - return "4.10.0" + return "4.11.1" elif tool == DbIliMode.ili2pg: if db_ili_version == 3: return "3.11.2" else: - return "4.10.0" + return "4.11.1" elif tool == DbIliMode.ili2mssql: if db_ili_version == 3: return "3.12.2" else: - return "4.10.0" + return "4.11.1" return "0" From 1a0cf9961aacb5b14a326e99153683d175bc72f2 Mon Sep 17 00:00:00 2001 From: signedav Date: Thu, 7 Sep 2023 17:03:49 +0200 Subject: [PATCH 12/34] make OptimizeStrategy global and consider it for Relations --- modelbaker/dataobjects/layers.py | 6 ++++++ modelbaker/dataobjects/project.py | 4 +++- modelbaker/generator/generator.py | 22 +++++++++++----------- modelbaker/utils/globals.py | 6 ++++++ 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/modelbaker/dataobjects/layers.py b/modelbaker/dataobjects/layers.py index 38685b1..16828c3 100644 --- a/modelbaker/dataobjects/layers.py +++ b/modelbaker/dataobjects/layers.py @@ -33,6 +33,7 @@ from ..generator.config import IGNORED_FIELDNAMES from .form import Form, FormFieldWidget, FormRelationWidget, FormTab +from ..utils.globals import OptimizeStrategy class Layer: @@ -253,6 +254,8 @@ def post_generate(self, project): relations_to_add = [] for relation in project.relations: if relation.referenced_layer == self: + if not relation.referencing_layer.is_relevant and project.optimize_strategy == OptimizeStrategy.GROUP: + continue # 1:n relation will be added only if does not point to a pure link table if ( @@ -269,6 +272,9 @@ def post_generate(self, project): if nm_relation.referenced_layer == self: continue + if not nm_relation.referenced_layer.is_relevant and project.optimize_strategy == OptimizeStrategy.GROUP: + continue + # relations to the same table with different geometries should not be added if ( self.srid diff --git a/modelbaker/dataobjects/project.py b/modelbaker/dataobjects/project.py index 81a8f42..3f3ca04 100644 --- a/modelbaker/dataobjects/project.py +++ b/modelbaker/dataobjects/project.py @@ -36,6 +36,7 @@ from .layers import Layer from .legend import LegendGroup from .relations import Relation +from ..utils.globals import OptimizeStrategy ENUM_THIS_CLASS_COLUMN = "thisclass" @@ -43,7 +44,7 @@ class Project(QObject): layer_added = pyqtSignal(str) - def __init__(self, auto_transaction=True, evaluate_default_values=True, context={}): + def __init__(self, auto_transaction=True, evaluate_default_values=True, context={}, optimize_strategy = OptimizeStrategy.NONE): QObject.__init__(self) self.crs = None self.name = "Not set" @@ -57,6 +58,7 @@ def __init__(self, auto_transaction=True, evaluate_default_values=True, context= self.layouts = {} self.mapthemes = {} self.context = context + self.optimize_strategy = optimize_strategy # {Layer_class_name: {dbattribute: {Layer_class, cardinality, Layer_domain, key_field, value_field]} self.bags_of_enum = dict() diff --git a/modelbaker/generator/generator.py b/modelbaker/generator/generator.py index 71d4369..4d53919 100644 --- a/modelbaker/generator/generator.py +++ b/modelbaker/generator/generator.py @@ -28,6 +28,7 @@ from ..dataobjects.relations import Relation from ..db_factory.db_simple_factory import DbSimpleFactory from ..utils.qt_utils import slugify +from ..utils.globals import OptimizeStrategy from .config import BASKET_FIELDNAMES, IGNORED_FIELDNAMES, READONLY_FIELDNAMES from .domain_relations_generator import DomainRelationGenerator @@ -38,11 +39,6 @@ class Generator(QObject): stdout = pyqtSignal(str) new_message = pyqtSignal(int, str) - class OptimizeStrategy(Enum): - NONE = 0 - GROUP = 1 - HIDE = 2 - def __init__( self, tool, @@ -53,7 +49,7 @@ def __init__( parent=None, mgmt_uri=None, consider_basket_handling=False, - optimize_strategy = None, + optimize_strategy = OptimizeStrategy.NONE, ): """ Creates a new Generator objects. @@ -75,7 +71,7 @@ def __init__( self._db_connector.stdout.connect(self.print_info) self._db_connector.new_message.connect(self.append_print_message) self.basket_handling = consider_basket_handling and self.get_basket_handling() - self.optimize_strategy = optimize_strategy if optimize_strategy is not None else self.OptimizeStrategy.NONE + self.optimize_strategy = optimize_strategy self._additional_ignored_layers = ( [] @@ -145,7 +141,7 @@ def layers(self, filter_layer_list=[]): record.get("tablename") == self._db_connector.dataset_table_name ) - is_relevant = bool(record.get("relevance")) # it can be not relevant and still be displayed (in case of NONE) if self.optimize_strategy != Generator.OptimizeStrategy.NONE else True + is_relevant = bool(record.get("relevance")) # it can be not relevant and still be displayed (in case of NONE) if self.optimize_strategy != OptimizeStrategy.NONE else True alias = record["table_alias"] if "table_alias" in record else None if not alias: @@ -376,7 +372,7 @@ def layers(self, filter_layer_list=[]): def _rename_ambiguous_layers(self, layers, second_pass = False): #rename ambiguous layers with topic (on not second_pass) or model (on second_pass) prefix #on irrelevant layers only if we don't ride OptimizeStrategy.HIDE - aliases = [l.alias for l in layers if l.is_relevant or self.optimize_strategy != self.OptimizeStrategy.HIDE] + aliases = [l.alias for l in layers if l.is_relevant or self.optimize_strategy != OptimizeStrategy.HIDE] ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] for layer in layers: if layer.alias in ambiguous_aliases: @@ -403,7 +399,11 @@ def relations(self, layers, filter_layer_list=[]): and record["referenced_table"] in layer_map.keys() ): for referencing_layer in layer_map[record["referencing_table"]]: + if not referencing_layer.is_relevant and self.optimize_strategy == OptimizeStrategy.HIDE: + continue for referenced_layer in layer_map[record["referenced_table"]]: + if not referenced_layer.is_relevant and self.optimize_strategy == OptimizeStrategy.HIDE: + continue relation = Relation() relation.referencing_layer = referencing_layer relation.referenced_layer = referenced_layer @@ -607,7 +607,7 @@ def legend( else: irrelevant_layers = [] relevant_layers = [] - if self.optimize_strategy == self.OptimizeStrategy.NONE: + if self.optimize_strategy == OptimizeStrategy.NONE: relevant_layers = layers else: for layer in layers: @@ -645,7 +645,7 @@ def legend( legend.append(system) # when the irrelevant layers should be grouped (but visible), we make the structure for them and append it to a group - if self.optimize_strategy == self.OptimizeStrategy.GROUP: + if self.optimize_strategy == OptimizeStrategy.GROUP: point_layers, line_layers, polygon_layers, domain_layers, table_layers, system_layers = self._separated_legend_layers(irrelevant_layers) # create base group diff --git a/modelbaker/utils/globals.py b/modelbaker/utils/globals.py index 9a881ae..e0bf788 100644 --- a/modelbaker/utils/globals.py +++ b/modelbaker/utils/globals.py @@ -25,3 +25,9 @@ class DbActionType(Enum): GENERATE = 1 IMPORT_DATA = 2 EXPORT = 3 + +class OptimizeStrategy(Enum): + """Defines the strategy that should be used for extended models.""" + NONE = 0 + GROUP = 1 + HIDE = 2 \ No newline at end of file From ca838b90ed90d99e3907a51eadfc01c80195678b Mon Sep 17 00:00:00 2001 From: signedav Date: Fri, 8 Sep 2023 22:58:45 +0200 Subject: [PATCH 13/34] gpkg tests --- modelbaker/generator/generator.py | 2 +- .../test_projectgen_extension_optimization.py | 515 +++++++++++++++++- 2 files changed, 506 insertions(+), 11 deletions(-) diff --git a/modelbaker/generator/generator.py b/modelbaker/generator/generator.py index 4d53919..0f26aa8 100644 --- a/modelbaker/generator/generator.py +++ b/modelbaker/generator/generator.py @@ -141,7 +141,7 @@ def layers(self, filter_layer_list=[]): record.get("tablename") == self._db_connector.dataset_table_name ) - is_relevant = bool(record.get("relevance")) # it can be not relevant and still be displayed (in case of NONE) if self.optimize_strategy != OptimizeStrategy.NONE else True + is_relevant = bool(record.get("relevance")) # it can be not relevant and still be displayed (in case of NONE) alias = record["table_alias"] if "table_alias" in record else None if not alias: diff --git a/tests/test_projectgen_extension_optimization.py b/tests/test_projectgen_extension_optimization.py index 6963425..4052f0e 100644 --- a/tests/test_projectgen_extension_optimization.py +++ b/tests/test_projectgen_extension_optimization.py @@ -27,10 +27,9 @@ from decimal import Decimal import yaml -from qgis.core import Qgis, QgsEditFormConfig, QgsProject, QgsRelation +from qgis.core import Qgis, QgsEditFormConfig, QgsProject, QgsRelation, QgsLayerTreeLayer from qgis.PyQt.QtCore import QEventLoop, Qt, QTimer from qgis.testing import start_app, unittest - from modelbaker.dataobjects.project import Project from modelbaker.db_factory.gpkg_command_config_manager import GpkgCommandConfigManager from modelbaker.generator.generator import Generator @@ -42,6 +41,8 @@ IliToppingFileCache, IliToppingFileItemModel, ) + +from modelbaker.utils.globals import OptimizeStrategy from tests.utils import get_pg_connection_string, iliimporter_config, testdata_path CATALOGUE_DATASETNAME = "Catset" @@ -137,9 +138,88 @@ def test_extopt_staedtische_geopackage(self): config_manager = GpkgCommandConfigManager(importer.configuration) uri = config_manager.get_uri() - generator = Generator(DbIliMode.ili2gpkg, uri, "smart2") + ### 1. OptimizeStrategy.NONE ### + strategy = OptimizeStrategy.NONE + + generator = Generator(tool=DbIliMode.ili2gpkg, uri=uri, inheritance="smart2", optimize_strategy = strategy ) + + available_layers = generator.layers() + relations, _ = generator.relations(available_layers) + legend = generator.legend(available_layers) + + aliases = [l.alias for l in available_layers] + irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + + # check no ambiguous layers exists + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gebaeude_StadtFirma', 'Gewerbe.Gebaeude', 'Gewerbe_V1.Firmen.Firma', 'Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Staedtisches_Gewerbe_V1.Firmen.Firma', 'Strasse'] + assert set(aliases) == set(expected_aliases) + + # irrelevant layers are detected + assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = ['Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Gewerbe_V1.Firmen.Firma'] + assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + + project = Project(optimize_strategy = strategy ) + project.layers = available_layers + project.relations = relations + project.legend = legend + project.post_generate() + + qgis_project = QgsProject.instance() + project.create(None, qgis_project) + + # check layertree + root = qgis_project.layerTreeRoot() + assert root is not None + + all_layers = root.findLayers() + assert len(all_layers) == 9 + + geometry_layers = set([l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer)]) + assert geometry_layers == {'Strasse', 'Freizeit.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Gewerbe.Gebaeude'} + + tables_layers = set([l.name() for l in root.findGroup('tables').children()]) + assert tables_layers == {'Staedtisches_Gewerbe_V1.Firmen.Firma', 'Gewerbe_V1.Firmen.Firma', 'Gebaeude_StadtFirma', 'BesitzerIn'} + + # check relations - all are there + relations = list(qgis_project.relationManager().relations().values()) + assert len(relations) == 10 + + # strasse should have relation editors to all layers (4/4) + count = 0 + for layer in project.layers: + if layer.layer.name() == "Strasse": + efc = layer.layer.editFormConfig() + # one general and four relation editors + assert len(efc.tabs()) == 5 + for tab in efc.tabs(): + if tab.name() == "stadtscng_v1_1freizeit_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "stadtscng_v1_1gewerbe_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "kantnl_ng_v1_1konstruktionen_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "gebaeude": + count += 1 + assert len(tab.children()) == 1 + # should find 4 + assert count == 4 + + QgsProject.instance().clear() + + ### 2. OptimizeStrategy.GROUP ### + strategy = OptimizeStrategy.GROUP + + generator = Generator(tool=DbIliMode.ili2gpkg, uri=uri, inheritance="smart2", optimize_strategy = strategy ) available_layers = generator.layers() + relations, _ = generator.relations(available_layers) + legend = generator.legend(available_layers) + aliases = [l.alias for l in available_layers] irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] @@ -154,7 +234,7 @@ def test_extopt_staedtische_geopackage(self): expected_irrelevant_layer_ilinames = ['Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Gewerbe_V1.Firmen.Firma'] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - project = Project() + project = Project(optimize_strategy = strategy ) project.layers = available_layers project.relations = relations project.legend = legend @@ -163,6 +243,123 @@ def test_extopt_staedtische_geopackage(self): qgis_project = QgsProject.instance() project.create(None, qgis_project) + # check layertree + root = qgis_project.layerTreeRoot() + assert root is not None + + all_layers = root.findLayers() + assert len(all_layers) == 9 + + geometry_layers = set([l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer)]) + assert geometry_layers == {'Strasse', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude'} + + tables_layers = set([l.name() for l in root.findGroup('tables').children()]) + assert tables_layers == {'Staedtisches_Gewerbe_V1.Firmen.Firma', 'Gebaeude_StadtFirma', 'BesitzerIn'} + + base_group = root.findGroup('base layers') + assert base_group + + grouped_base_layers = set([l.name() for l in base_group.children() if isinstance(l, QgsLayerTreeLayer)]) + + assert grouped_base_layers == {'Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude'} + + # check relations - all are there + relations = list(qgis_project.relationManager().relations().values()) + assert len(relations) == 10 + + # strasse should only have relation editors to relevant layers (2/4) + count = 0 + for layer in project.layers: + if layer.layer.name() == "Strasse": + efc = layer.layer.editFormConfig() + # one general and two relation editors + assert len(efc.tabs()) == 3 + for tab in efc.tabs(): + if tab.name() == "stadtscng_v1_1freizeit_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "stadtscng_v1_1gewerbe_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "kantnl_ng_v1_1konstruktionen_gebaeude": #should not happen + count += 1 + if tab.name() == "gebaeude": #should not happen + count += 1 + # should find only 2 + assert count == 2 + + QgsProject.instance().clear() + + ### 3. OptimizeStrategy.HIDE ### + strategy = OptimizeStrategy.HIDE + + generator = Generator(tool=DbIliMode.ili2gpkg, uri=uri, inheritance="smart2" ,optimize_strategy = strategy ) + + available_layers = generator.layers() + relations, _ = generator.relations(available_layers) + legend = generator.legend(available_layers) + + aliases = [l.alias for l in available_layers] + irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + + # ambiguous aliases exist, since we don't rename them when they are hidden anyway + assert len(ambiguous_aliases) == 4 + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gebaeude_StadtFirma', 'Gewerbe.Gebaeude', 'Konstruktionen.Gebaeude', 'Firma', 'Strasse'] + assert set(aliases) == set(expected_aliases) + + # irrelevant layers are detected + assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = ['Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Gewerbe_V1.Firmen.Firma'] + assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + + project = Project(optimize_strategy = strategy ) + project.layers = available_layers + project.relations = relations + project.legend = legend + project.post_generate() + + qgis_project = QgsProject.instance() + project.create(None, qgis_project) + + # check layertree + root = qgis_project.layerTreeRoot() + assert root is not None + + all_layers = root.findLayers() + assert len(all_layers) == 6 + + geometry_layers = set([l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer)]) + assert geometry_layers == {'Strasse', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude'} + + tables_layers = set([l.name() for l in root.findGroup('tables').children()]) + assert tables_layers == {'Firma', 'Gebaeude_StadtFirma', 'BesitzerIn'} + + # check relations - only 13 are here + relations = list(qgis_project.relationManager().relations().values()) + assert len(relations) == 6 + + # strasse should only have relation editors to relevant layers (2/4) + count = 0 + for layer in project.layers: + if layer.layer.name() == "Strasse": + efc = layer.layer.editFormConfig() + # one general and two relation editors + assert len(efc.tabs()) == 3 + for tab in efc.tabs(): + if tab.name() == "stadtscng_v1_1freizeit_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "stadtscng_v1_1gewerbe_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "kantnl_ng_v1_1konstruktionen_gebaeude": #should not happen + count += 1 + if tab.name() == "gebaeude": #should not happen + count += 1 + # should find only 2 + assert count == 2 + def test_extopt_staedtische_mssql(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2mssql @@ -234,7 +431,6 @@ def test_extopt_polymorphic_postgis(self): available_layers = generator.layers() aliases = [l.alias for l in available_layers] irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] - print(irrelevant_layer_ilinames) ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] # check no ambiguous layers exists @@ -267,12 +463,103 @@ def test_extopt_polymorphic_geopackage(self): config_manager = GpkgCommandConfigManager(importer.configuration) uri = config_manager.get_uri() - generator = Generator(DbIliMode.ili2gpkg, uri, "smart2") + ### 1. OptimizeStrategy.NONE ### + strategy = OptimizeStrategy.NONE + + generator = Generator(tool=DbIliMode.ili2gpkg, uri=uri, inheritance="smart2" ,optimize_strategy = strategy ) + + available_layers = generator.layers() + relations, _ = generator.relations(available_layers) + legend = generator.legend(available_layers) + + aliases = [l.alias for l in available_layers] + irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + + # check no ambiguous layers exists + assert len(ambiguous_aliases) == 0 + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + assert set(aliases) == set(expected_aliases) + + # irrelevant layers are detected + assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = ['Ortsplanung_V1_1.Konstruktionen.Gebaeude'] + assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + + project = Project(optimize_strategy = strategy ) + project.layers = available_layers + project.relations = relations + project.legend = legend + project.post_generate() + + qgis_project = QgsProject.instance() + project.create(None, qgis_project) + + # check layertree + root = qgis_project.layerTreeRoot() + assert root is not None + + all_layers = root.findLayers() + assert len(all_layers) == 11 + + geometry_layers = set([l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer)]) + assert geometry_layers == {'IndustrieGewerbe.Gebaeude', 'Strasse', 'Freizeit.Gebaeude', 'Markthalle', 'TurnhalleTyp2', 'Hallen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'TurnhalleTyp1', 'Gewerbe.Gebaeude'} + + tables_layers = set([l.name() for l in root.findGroup('tables').children()]) + assert tables_layers == {'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'BesitzerIn'} + + # check relations - all are there + relations = list(qgis_project.relationManager().relations().values()) + assert len(relations) == 16 + + # strasse should have relation editors to all layers (8/8) + count = 0 + for layer in project.layers: + if layer.layer.name() == "Strasse": + efc = layer.layer.editFormConfig() + # one general and four relation editors + assert len(efc.tabs()) == 9 + for tab in efc.tabs(): + if tab.name() == "gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "polymrpng_v1_1gewerbe_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "polymrpng_v1_1freizeit_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "polymrpng_v1_1industriegewerbe_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "polymrpng_v1_1hallen_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "markthalle": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "turnhalletyp1": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "turnhalletyp2": + count += 1 + assert len(tab.children()) == 1 + # should find 8 + assert count == 8 + + QgsProject.instance().clear() + + ### 2. OptimizeStrategy.GROUP ### + strategy = OptimizeStrategy.GROUP + + generator = Generator(tool=DbIliMode.ili2gpkg, uri=uri, inheritance="smart2" ,optimize_strategy = strategy ) available_layers = generator.layers() + relations, _ = generator.relations(available_layers) + legend = generator.legend(available_layers) + aliases = [l.alias for l in available_layers] irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] - print(irrelevant_layer_ilinames) ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] # check no ambiguous layers exists @@ -285,6 +572,162 @@ def test_extopt_polymorphic_geopackage(self): expected_irrelevant_layer_ilinames = ['Ortsplanung_V1_1.Konstruktionen.Gebaeude'] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + project = Project(optimize_strategy = strategy ) + project.layers = available_layers + project.relations = relations + project.legend = legend + project.post_generate() + + qgis_project = QgsProject.instance() + project.create(None, qgis_project) + + # check layertree + root = qgis_project.layerTreeRoot() + assert root is not None + + all_layers = root.findLayers() + assert len(all_layers) == 11 + + geometry_layers = set([l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer)]) + assert geometry_layers == {'IndustrieGewerbe.Gebaeude', 'Strasse', 'Freizeit.Gebaeude', 'Markthalle', 'TurnhalleTyp2', 'Hallen.Gebaeude', 'TurnhalleTyp1', 'Gewerbe.Gebaeude'} + + tables_layers = set([l.name() for l in root.findGroup('tables').children()]) + assert tables_layers == {'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'BesitzerIn'} + + base_group = root.findGroup('base layers') + assert base_group + + grouped_base_layers = set([l.name() for l in base_group.children() if isinstance(l, QgsLayerTreeLayer)]) + + assert grouped_base_layers == {'Ortsplanung_V1_1.Konstruktionen.Gebaeude'} + + # check relations - all are there + relations = list(qgis_project.relationManager().relations().values()) + assert len(relations) == 16 + + # strasse should have relation editors to all layers (8/8) + count = 0 + for layer in project.layers: + if layer.layer.name() == "Strasse": + efc = layer.layer.editFormConfig() + # one general and four relation editors + assert len(efc.tabs()) == 8 + for tab in efc.tabs(): + if tab.name() == "gebaeude": # this should not happen + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "polymrpng_v1_1gewerbe_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "polymrpng_v1_1freizeit_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "polymrpng_v1_1industriegewerbe_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "polymrpng_v1_1hallen_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "markthalle": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "turnhalletyp1": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "turnhalletyp2": + count += 1 + assert len(tab.children()) == 1 + # should find 7 + assert count == 7 + + QgsProject.instance().clear() + + ### 3. OptimizeStrategy.HIDE ### + strategy = OptimizeStrategy.HIDE + + generator = Generator(tool=DbIliMode.ili2gpkg, uri=uri, inheritance="smart2" ,optimize_strategy = strategy ) + + available_layers = generator.layers() + relations, _ = generator.relations(available_layers) + legend = generator.legend(available_layers) + + aliases = [l.alias for l in available_layers] + irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + + # ambiguous aliases exist, since we don't rename them when they are hidden anyway + assert len(ambiguous_aliases) == 2 + expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + assert set(aliases) == set(expected_aliases) + + # irrelevant layers are detected + assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = ['Ortsplanung_V1_1.Konstruktionen.Gebaeude'] + assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + + project = Project(optimize_strategy = strategy ) + project.layers = available_layers + project.relations = relations + project.legend = legend + project.post_generate() + + qgis_project = QgsProject.instance() + project.create(None, qgis_project) + + # check layertree + root = qgis_project.layerTreeRoot() + assert root is not None + + all_layers = root.findLayers() + assert len(all_layers) == 10 + + geometry_layers = set([l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer)]) + assert geometry_layers == {'IndustrieGewerbe.Gebaeude', 'Strasse', 'Freizeit.Gebaeude', 'Markthalle', 'TurnhalleTyp2', 'Hallen.Gebaeude', 'TurnhalleTyp1', 'Gewerbe.Gebaeude'} + + tables_layers = set([l.name() for l in root.findGroup('tables').children()]) + assert tables_layers == {'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'BesitzerIn'} + + # check relations - all are there + relations = list(qgis_project.relationManager().relations().values()) + assert len(relations) == 14 + + # strasse should have relation editors to all layers (8/8) + count = 0 + for layer in project.layers: + if layer.layer.name() == "Strasse": + efc = layer.layer.editFormConfig() + # one general and four relation editors + assert len(efc.tabs()) == 8 + for tab in efc.tabs(): + if tab.name() == "gebaeude": # this should not happen + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "polymrpng_v1_1gewerbe_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "polymrpng_v1_1freizeit_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "polymrpng_v1_1industriegewerbe_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "polymrpng_v1_1hallen_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "markthalle": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "turnhalletyp1": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "turnhalletyp2": + count += 1 + assert len(tab.children()) == 1 + # should find 7 + assert count == 7 + + QgsProject.instance().clear() + def test_extopt_polymorphic_mssql(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2mssql @@ -389,23 +832,75 @@ def test_extopt_baustruct_geopackage(self): config_manager = GpkgCommandConfigManager(importer.configuration) uri = config_manager.get_uri() - generator = Generator(DbIliMode.ili2gpkg, uri, "smart2") + ### 1. OptimizeStrategy.NONE ### + strategy = OptimizeStrategy.NONE + + generator = Generator(tool=DbIliMode.ili2gpkg, uri=uri, inheritance="smart2",optimize_strategy = strategy ) available_layers = generator.layers() + relations, _ = generator.relations(available_layers) + legend = generator.legend(available_layers) + aliases = [l.alias for l in available_layers] irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 - expected_aliases = ["Bauart", "Bauplanung_V1_1.Konstruktionen.Gebaeude", "Bauplanung_V1_1.Konstruktionen.Material", "Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", "Bauplanung_V1_1.Natur.Park", "Bauplanung_V1_1.Natur.Tierart", "Brutstelle", "Buntbrache", "Feld", "KantonaleBuntbrache", "Kantonale_Bauplanung_V1_1.Konstruktionen.Gebaeude", "Kantonale_Bauplanung_V1_1.Konstruktionen.Material", "Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", "Kantonale_Bauplanung_V1_1.Natur.Park", "Kantonale_Bauplanung_V1_1.Natur.Tierart", "Kartoffelfeld", "Sonnenblumenfeld", "Strasse"] + expected_aliases = ['Strasse', 'Sonnenblumenfeld', 'Kartoffelfeld', 'KantonaleBuntbrache', 'Kantonale_Bauplanung_V1_1.Natur.Park', 'Kantonale_Bauplanung_V1_1.Konstruktionen.Gebaeude', 'Feld', 'Buntbrache', 'Brutstelle', 'Bauplanung_V1_1.Natur.Park', 'Bauplanung_V1_1.Konstruktionen.Gebaeude', 'Kantonale_Bauplanung_V1_1.Natur.Tierart', 'Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude', 'Kantonale_Bauplanung_V1_1.Konstruktionen.Material', 'Bauplanung_V1_1.Natur.Tierart', 'Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude', 'Bauplanung_V1_1.Konstruktionen.Material', 'Bauart'] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ["Bauplanung_V1_1.Konstruktionen.Material", "Bauplanung_V1_1.Konstruktionen.Gebaeude", "Bauplanung_V1_1.Natur.Tierart", "Bauplanung_V1_1.Natur.Park"] + expected_irrelevant_layer_ilinames = ['Ortsplanung_V1_1.Konstruktionen.Gebaeude'] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + project = Project(optimize_strategy = strategy ) + project.layers = available_layers + project.relations = relations + project.legend = legend + project.post_generate() + + qgis_project = QgsProject.instance() + project.create(None, qgis_project) + + # check layertree + root = qgis_project.layerTreeRoot() + assert root is not None + + all_layers = root.findLayers() + assert len(all_layers) == 18 + + geometry_layers = set([l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer)]) + assert geometry_layers == {'Kantonale_Bauplanung_V1_1.Konstruktionen.Gebaeude', 'Kantonale_Bauplanung_V1_1.Natur.Park', 'Strasse', 'Bauplanung_V1_1.Natur.Park', 'Sonnenblumenfeld', 'Buntbrache', 'Brutstelle', 'KantonaleBuntbrache', 'Feld', 'Kartoffelfeld', 'Bauplanung_V1_1.Konstruktionen.Gebaeude'} + + tables_layers = set([l.name() for l in root.findGroup('tables').children()]) + assert tables_layers == {'Bauart', 'Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude', 'Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude', 'Kantonale_Bauplanung_V1_1.Natur.Tierart', 'Bauplanung_V1_1.Konstruktionen.Material', 'Kantonale_Bauplanung_V1_1.Konstruktionen.Material', 'Bauplanung_V1_1.Natur.Tierart'} + + # check relations - all are there + relations = list(qgis_project.relationManager().relations().values()) + assert len(relations) == 18 + + # strasse should have relation editors to all layers (2/2) + # for-relevance-tests hier werden diese relation editoren nicht ins formular geschoben. Ich weiss nicht weshalb... + count = 0 + for layer in project.layers: + if layer.layer.name() == "Strasse": + efc = layer.layer.editFormConfig() + # one general and four relation editors + assert len(efc.tabs()) == 9 + for tab in efc.tabs(): + if tab.name() == "gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "polymrpng_v1_1gewerbe_gebaeude": + count += 1 + assert len(tab.children()) == 1 + # should find 2 + assert count == 2 + + QgsProject.instance().clear() + def test_extopt_baustruct_mssql(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2mssql From 6ad125ddf9f9ae16c5cf69b3ada760fe6fd4f9ab Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 11 Sep 2023 21:25:03 +0200 Subject: [PATCH 14/34] black style them --- modelbaker/dataobjects/layers.py | 16 +- modelbaker/dataobjects/project.py | 10 +- modelbaker/dbconnector/gpkg_connector.py | 14 +- modelbaker/generator/generator.py | 105 ++- modelbaker/utils/globals.py | 4 +- tests/test_gpkg_pk.py | 6 +- tests/test_ilicache.py | 322 +++++---- tests/test_projectgen.py | 210 +++--- .../test_projectgen_extension_optimization.py | 661 +++++++++++++----- tests/test_validate.py | 15 +- tests/testdata/ilimodels/Bauplanung_V1_1.ili | 12 +- .../ilimodels/Kantonale_Bauplanung_V1_1.ili | 12 +- .../Polymorphic_Ortsplanung_V1_1.ili | 6 +- 13 files changed, 889 insertions(+), 504 deletions(-) diff --git a/modelbaker/dataobjects/layers.py b/modelbaker/dataobjects/layers.py index 16828c3..70c76e2 100644 --- a/modelbaker/dataobjects/layers.py +++ b/modelbaker/dataobjects/layers.py @@ -32,8 +32,8 @@ from qgis.PyQt.QtCore import QCoreApplication, QSettings from ..generator.config import IGNORED_FIELDNAMES -from .form import Form, FormFieldWidget, FormRelationWidget, FormTab from ..utils.globals import OptimizeStrategy +from .form import Form, FormFieldWidget, FormRelationWidget, FormTab class Layer: @@ -100,7 +100,7 @@ def __init__( self.model_topic_name = ( f"{self.ili_name.split('.')[0]}.{self.ili_name.split('.')[1]}" ) - + self.is_relevant = is_relevant self.definitionfile = definitionfile @@ -254,7 +254,10 @@ def post_generate(self, project): relations_to_add = [] for relation in project.relations: if relation.referenced_layer == self: - if not relation.referencing_layer.is_relevant and project.optimize_strategy == OptimizeStrategy.GROUP: + if ( + not relation.referencing_layer.is_relevant + and project.optimize_strategy == OptimizeStrategy.GROUP + ): continue # 1:n relation will be added only if does not point to a pure link table @@ -272,9 +275,12 @@ def post_generate(self, project): if nm_relation.referenced_layer == self: continue - if not nm_relation.referenced_layer.is_relevant and project.optimize_strategy == OptimizeStrategy.GROUP: + if ( + not nm_relation.referenced_layer.is_relevant + and project.optimize_strategy == OptimizeStrategy.GROUP + ): continue - + # relations to the same table with different geometries should not be added if ( self.srid diff --git a/modelbaker/dataobjects/project.py b/modelbaker/dataobjects/project.py index 3f3ca04..2694afb 100644 --- a/modelbaker/dataobjects/project.py +++ b/modelbaker/dataobjects/project.py @@ -33,10 +33,10 @@ from qgis.PyQt.QtCore import QObject, pyqtSignal from qgis.PyQt.QtXml import QDomDocument +from ..utils.globals import OptimizeStrategy from .layers import Layer from .legend import LegendGroup from .relations import Relation -from ..utils.globals import OptimizeStrategy ENUM_THIS_CLASS_COLUMN = "thisclass" @@ -44,7 +44,13 @@ class Project(QObject): layer_added = pyqtSignal(str) - def __init__(self, auto_transaction=True, evaluate_default_values=True, context={}, optimize_strategy = OptimizeStrategy.NONE): + def __init__( + self, + auto_transaction=True, + evaluate_default_values=True, + context={}, + optimize_strategy=OptimizeStrategy.NONE, + ): QObject.__init__(self) self.crs = None self.name = "Not set" diff --git a/modelbaker/dbconnector/gpkg_connector.py b/modelbaker/dbconnector/gpkg_connector.py index f8b4fae..d62798a 100644 --- a/modelbaker/dbconnector/gpkg_connector.py +++ b/modelbaker/dbconnector/gpkg_connector.py @@ -140,14 +140,14 @@ def _get_tables_info(self): GROUP BY tablename ) as coord_decimals, substr(c.iliname, 0, instr(c.iliname, '.')) AS model, - attrs.sqlname as attribute_name, + attrs.sqlname as attribute_name, {relevance_field},""".format( - relevance_field="""CASE WHEN c.iliname IN ( + relevance_field="""CASE WHEN c.iliname IN ( -- used to get the class names from the full names WITH names AS ( WITH class_level_name AS( WITH topic_level_name AS ( - SELECT + SELECT thisClass as fullname, substr(thisClass, 0, instr(thisClass, '.')) as model, substr(ltrim(thisClass,substr(thisClass, 0, instr(thisClass, '.'))),2) as topicclass @@ -168,16 +168,16 @@ def _get_tables_info(self): WHERE baseClass IS NOT NULL -- in a different model AND base_names.model != extend_names.model - AND ( + AND ( -- with the same name base_names.class = extend_names.class - OR + OR -- multiple times in the same extended model (SELECT COUNT(baseClass) FROM T_ILI2DB_INHERITANCE JOIN names extend_names ON thisClass = extend_names.fullname WHERE baseClass = i.baseClass GROUP BY baseClass, extend_names.model)>1 ) - ) + ) THEN FALSE ELSE TRUE END AS relevance""" - ) + ) interlis_joins = """LEFT JOIN T_ILI2DB_TABLE_PROP p ON p.tablename = s.name AND p.tag = 'ch.ehi.ili2db.tableKind' diff --git a/modelbaker/generator/generator.py b/modelbaker/generator/generator.py index 0f26aa8..a1a0549 100644 --- a/modelbaker/generator/generator.py +++ b/modelbaker/generator/generator.py @@ -17,7 +17,6 @@ ***************************************************************************/ """ import re -from enum import Enum from qgis.core import QgsApplication, QgsRelation, QgsWkbTypes from qgis.PyQt.QtCore import QCoreApplication, QLocale, QObject, pyqtSignal @@ -27,8 +26,8 @@ from ..dataobjects.legend import LegendGroup from ..dataobjects.relations import Relation from ..db_factory.db_simple_factory import DbSimpleFactory -from ..utils.qt_utils import slugify from ..utils.globals import OptimizeStrategy +from ..utils.qt_utils import slugify from .config import BASKET_FIELDNAMES, IGNORED_FIELDNAMES, READONLY_FIELDNAMES from .domain_relations_generator import DomainRelationGenerator @@ -49,7 +48,7 @@ def __init__( parent=None, mgmt_uri=None, consider_basket_handling=False, - optimize_strategy = OptimizeStrategy.NONE, + optimize_strategy=OptimizeStrategy.NONE, ): """ Creates a new Generator objects. @@ -141,7 +140,9 @@ def layers(self, filter_layer_list=[]): record.get("tablename") == self._db_connector.dataset_table_name ) - is_relevant = bool(record.get("relevance")) # it can be not relevant and still be displayed (in case of NONE) + is_relevant = bool( + record.get("relevance") + ) # it can be not relevant and still be displayed (in case of NONE) alias = record["table_alias"] if "table_alias" in record else None if not alias: @@ -191,7 +192,7 @@ def layers(self, filter_layer_list=[]): + match.group(1).split(".")[-1] + ")" ) - alias = short_name #for-relevance-tests if is_relevant else f"{short_name} !IRRELEVANT!" + alias = short_name # for-relevance-tests if is_relevant else f"{short_name} !IRRELEVANT!" display_expression = "" if is_basket_table: @@ -230,7 +231,7 @@ def layers(self, filter_layer_list=[]): is_basket_table, is_dataset_table, record.get("ili_name"), - is_relevant + is_relevant, ) # Configure fields for current table @@ -360,25 +361,33 @@ def layers(self, filter_layer_list=[]): layers.append(layer) - #append topic name to ambiguous layers + # append topic name to ambiguous layers self._rename_ambiguous_layers(layers) - #append moel name to still ambiguous layers + # append moel name to still ambiguous layers self._rename_ambiguous_layers(layers, second_pass=True) self.print_messages() return layers - def _rename_ambiguous_layers(self, layers, second_pass = False): - #rename ambiguous layers with topic (on not second_pass) or model (on second_pass) prefix - #on irrelevant layers only if we don't ride OptimizeStrategy.HIDE - aliases = [l.alias for l in layers if l.is_relevant or self.optimize_strategy != OptimizeStrategy.HIDE] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + def _rename_ambiguous_layers(self, layers, second_pass=False): + # rename ambiguous layers with topic (on not second_pass) or model (on second_pass) prefix + # on irrelevant layers only if we don't ride OptimizeStrategy.HIDE + aliases = [ + l.alias + for l in layers + if l.is_relevant or self.optimize_strategy != OptimizeStrategy.HIDE + ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] for layer in layers: if layer.alias in ambiguous_aliases: if layer.ili_name: if layer.ili_name.count(".") > 1: - layer.alias = layer.ili_name.split('.')[(0 if second_pass else 1)] + "." +layer.alias + layer.alias = ( + layer.ili_name.split(".")[(0 if second_pass else 1)] + + "." + + layer.alias + ) def relations(self, layers, filter_layer_list=[]): relations_info = self.get_relations_info(filter_layer_list) @@ -399,10 +408,16 @@ def relations(self, layers, filter_layer_list=[]): and record["referenced_table"] in layer_map.keys() ): for referencing_layer in layer_map[record["referencing_table"]]: - if not referencing_layer.is_relevant and self.optimize_strategy == OptimizeStrategy.HIDE: + if ( + not referencing_layer.is_relevant + and self.optimize_strategy == OptimizeStrategy.HIDE + ): continue for referenced_layer in layer_map[record["referenced_table"]]: - if not referenced_layer.is_relevant and self.optimize_strategy == OptimizeStrategy.HIDE: + if ( + not referenced_layer.is_relevant + and self.optimize_strategy == OptimizeStrategy.HIDE + ): continue relation = Relation() relation.referencing_layer = referencing_layer @@ -608,7 +623,7 @@ def legend( irrelevant_layers = [] relevant_layers = [] if self.optimize_strategy == OptimizeStrategy.NONE: - relevant_layers = layers + relevant_layers = layers else: for layer in layers: if layer.is_relevant: @@ -616,7 +631,14 @@ def legend( else: irrelevant_layers.append(layer) - point_layers, line_layers, polygon_layers, domain_layers, table_layers, system_layers = self._separated_legend_layers(relevant_layers) + ( + point_layers, + line_layers, + polygon_layers, + domain_layers, + table_layers, + system_layers, + ) = self._separated_legend_layers(relevant_layers) for l in polygon_layers: legend.append(l) @@ -627,29 +649,44 @@ def legend( # create groups if len(table_layers): - tables = LegendGroup(QCoreApplication.translate("LegendGroup", "tables")) + tables = LegendGroup( + QCoreApplication.translate("LegendGroup", "tables") + ) for layer in table_layers: tables.append(layer) legend.append(tables) if len(domain_layers): - domains = LegendGroup(QCoreApplication.translate("LegendGroup", "domains")) + domains = LegendGroup( + QCoreApplication.translate("LegendGroup", "domains") + ) domains.expanded = False for layer in domain_layers: domains.append(layer) legend.append(domains) if len(system_layers): - system = LegendGroup(QCoreApplication.translate("LegendGroup", "system")) + system = LegendGroup( + QCoreApplication.translate("LegendGroup", "system") + ) system.expanded = False for layer in system_layers: system.append(layer) legend.append(system) - + # when the irrelevant layers should be grouped (but visible), we make the structure for them and append it to a group if self.optimize_strategy == OptimizeStrategy.GROUP: - point_layers, line_layers, polygon_layers, domain_layers, table_layers, system_layers = self._separated_legend_layers(irrelevant_layers) + ( + point_layers, + line_layers, + polygon_layers, + domain_layers, + table_layers, + system_layers, + ) = self._separated_legend_layers(irrelevant_layers) # create base group - base_group = LegendGroup(QCoreApplication.translate("LegendGroup", "base layers")) + base_group = LegendGroup( + QCoreApplication.translate("LegendGroup", "base layers") + ) base_group.expanded = False base_group.checked = False @@ -662,17 +699,21 @@ def legend( # create groups if len(table_layers): - tables = LegendGroup(QCoreApplication.translate("LegendGroup", "base tables")) + tables = LegendGroup( + QCoreApplication.translate("LegendGroup", "base tables") + ) for layer in table_layers: tables.append(layer) base_group.append(tables) if len(domain_layers): - domains = LegendGroup(QCoreApplication.translate("LegendGroup", "base domains")) + domains = LegendGroup( + QCoreApplication.translate("LegendGroup", "base domains") + ) domains.expanded = False for layer in domain_layers: domains.append(layer) base_group.append(domains) - + legend.append(base_group) return legend @@ -706,8 +747,14 @@ def _separated_legend_layers(self, layers): else: table_layers.append(layer) - return point_layers, line_layers, polygon_layers, domain_layers, table_layers, system_layers - + return ( + point_layers, + line_layers, + polygon_layers, + domain_layers, + table_layers, + system_layers, + ) def resolved_layouts( self, diff --git a/modelbaker/utils/globals.py b/modelbaker/utils/globals.py index e0bf788..c31e2d7 100644 --- a/modelbaker/utils/globals.py +++ b/modelbaker/utils/globals.py @@ -26,8 +26,10 @@ class DbActionType(Enum): IMPORT_DATA = 2 EXPORT = 3 + class OptimizeStrategy(Enum): """Defines the strategy that should be used for extended models.""" + NONE = 0 GROUP = 1 - HIDE = 2 \ No newline at end of file + HIDE = 2 diff --git a/tests/test_gpkg_pk.py b/tests/test_gpkg_pk.py index 053bf38..30ff5e8 100644 --- a/tests/test_gpkg_pk.py +++ b/tests/test_gpkg_pk.py @@ -154,9 +154,9 @@ def test_gpkg_primary_key(self): project.create(None, qgis_project) punktobjekt_layer = next( - layer - for layer in available_layers - if layer.name == "erholungsinfrastruktur_punktobjekt" + layer + for layer in available_layers + if layer.name == "erholungsinfrastruktur_punktobjekt" ) scopes = QgsExpressionContextUtils.globalProjectLayerScopes( diff --git a/tests/test_ilicache.py b/tests/test_ilicache.py index 82c37c0..bbfc457 100644 --- a/tests/test_ilicache.py +++ b/tests/test_ilicache.py @@ -75,70 +75,70 @@ def test_ilimodels_xml_parser_23(self): e["name"] for e in next(elem for elem in ilicache.repositories.values()) } expected_models = { - "IliRepository09", - "IliRepository09", - "IliSite09", - "IlisMeta07", - "AbstractSymbology", - "CoordSys", - "RoadsExdm2ben_10", - "RoadsExdm2ien_10", - "RoadsExgm2ien_10", - "StandardSymbology", - "Time", - "Units", - "AbstractSymbology", - "CoordSys", - "RoadsExdm2ben", - "RoadsExdm2ien", - "RoadsExgm2ien", - "StandardSymbology", - "Time", - "Units", - "GM03Core", - "GM03Comprehensive", - "CodeISO", - "GM03_2Core", - "GM03_2Comprehensive", - "CodeISO", - "GM03_2_1Core", - "GM03_2_1Comprehensive", - "IlisMeta07", - "Units", - "IlisMeta07", - "IliRepository09", - "StandardSymbology", - "StandardSymbology", - "GM03Core", - "GM03_2Core", - "GM03_2_1Core", - "CoordSys", - "INTERLIS_ext", - "StandardSymbology", - "INTERLIS_ext", - "IliVErrors", - "RoadsExdm2ien_10", - "RoadsExdm2ien", - "CoordSys", - "StandardSymbology", - "Units", - "Time", - "Time", - "Base", - "Base_f", - "SIA405_Base", - "SIA405_Base_f", - "Base_LV95", - "Base_f_MN95", - "SIA405_Base_LV95", - "SIA405_Base_f_MN95", - "Math", - "Text", - "DatasetIdx16", - "AbstractSymbology", - "AbstractSymbology", - "Time", - "IliRepository20", + "IliRepository09", + "IliRepository09", + "IliSite09", + "IlisMeta07", + "AbstractSymbology", + "CoordSys", + "RoadsExdm2ben_10", + "RoadsExdm2ien_10", + "RoadsExgm2ien_10", + "StandardSymbology", + "Time", + "Units", + "AbstractSymbology", + "CoordSys", + "RoadsExdm2ben", + "RoadsExdm2ien", + "RoadsExgm2ien", + "StandardSymbology", + "Time", + "Units", + "GM03Core", + "GM03Comprehensive", + "CodeISO", + "GM03_2Core", + "GM03_2Comprehensive", + "CodeISO", + "GM03_2_1Core", + "GM03_2_1Comprehensive", + "IlisMeta07", + "Units", + "IlisMeta07", + "IliRepository09", + "StandardSymbology", + "StandardSymbology", + "GM03Core", + "GM03_2Core", + "GM03_2_1Core", + "CoordSys", + "INTERLIS_ext", + "StandardSymbology", + "INTERLIS_ext", + "IliVErrors", + "RoadsExdm2ien_10", + "RoadsExdm2ien", + "CoordSys", + "StandardSymbology", + "Units", + "Time", + "Time", + "Base", + "Base_f", + "SIA405_Base", + "SIA405_Base_f", + "Base_LV95", + "Base_f_MN95", + "SIA405_Base_LV95", + "SIA405_Base_f_MN95", + "Math", + "Text", + "DatasetIdx16", + "AbstractSymbology", + "AbstractSymbology", + "Time", + "IliRepository20", } assert models == expected_models @@ -154,14 +154,14 @@ def test_ilimodels_xml_parser_24(self): e["name"] for e in next(elem for elem in ilicache.repositories.values()) } expected_models = { - "AbstractSymbology", - "CoordSys", - "RoadsExdm2ben", - "RoadsExdm2ien", - "RoadsExgm2ien", - "StandardSymbology", - "Time", - "Units", + "AbstractSymbology", + "CoordSys", + "RoadsExdm2ben", + "RoadsExdm2ien", + "RoadsExgm2ien", + "StandardSymbology", + "Time", + "Units", } assert models == expected_models @@ -178,44 +178,44 @@ def test_ilimodels_xml_parser_24_local_files(self): in ilicache.repositories.keys() ) models = { - model["name"] - for model in [ - e for values in ilicache.repositories.values() for e in values - ] + model["name"] + for model in [ + e for values in ilicache.repositories.values() for e in values + ] } expected_models_of_ilimodels_xml = { - "AbstractSymbology", - "CoordSys", - "RoadsExdm2ben", - "RoadsExdm2ien", - "RoadsExgm2ien", - "StandardSymbology", - "Time", - "Units", + "AbstractSymbology", + "CoordSys", + "RoadsExdm2ben", + "RoadsExdm2ien", + "RoadsExgm2ien", + "StandardSymbology", + "Time", + "Units", } expected_models_of_local_ili_files = { - "KbS_Basis_V1_4", - "KbS_LV03_V1_4", - "KbS_LV95_V1_4", - "RoadsSimple", - "Abfallsammelstellen_ZEBA_LV03_V1", - "Abfallsammelstellen_ZEBA_LV95_V1", - "DictionariesCH_V1", - "GeometryCHLV03_V1", - "S", - "AdministrativeUnitsCH_V1", - "CHAdminCodes_V1", - "Localisation_V1", - "LE", - "LocalisationCH_V1", - "Dictionaries_V1", - "CatalogueObjectTrees_V1", - "AdministrativeUnits_V1", - "GeometryCHLV95_V1", - "InternationalCodes_V1", - "CatalogueObjects_V1", - "PlanerischerGewaesserschutz_LV95_V1_1", - "PlanerischerGewaesserschutz_LV03_V1_1", + "KbS_Basis_V1_4", + "KbS_LV03_V1_4", + "KbS_LV95_V1_4", + "RoadsSimple", + "Abfallsammelstellen_ZEBA_LV03_V1", + "Abfallsammelstellen_ZEBA_LV95_V1", + "DictionariesCH_V1", + "GeometryCHLV03_V1", + "S", + "AdministrativeUnitsCH_V1", + "CHAdminCodes_V1", + "Localisation_V1", + "LE", + "LocalisationCH_V1", + "Dictionaries_V1", + "CatalogueObjectTrees_V1", + "AdministrativeUnits_V1", + "GeometryCHLV95_V1", + "InternationalCodes_V1", + "CatalogueObjects_V1", + "PlanerischerGewaesserschutz_LV95_V1_1", + "PlanerischerGewaesserschutz_LV03_V1_1", } assert models == set.union( @@ -238,44 +238,44 @@ def test_ilimodels_xml_parser_24_local_repo_local_files(self): in ilicache.repositories.keys() ) models = { - model["name"] - for model in [ - e for values in ilicache.repositories.values() for e in values - ] + model["name"] + for model in [ + e for values in ilicache.repositories.values() for e in values + ] } expected_models_of_ilimodels_xml = { - "AbstractSymbology", - "CoordSys", - "RoadsExdm2ben", - "RoadsExdm2ien", - "RoadsExgm2ien", - "StandardSymbology", - "Time", - "Units", + "AbstractSymbology", + "CoordSys", + "RoadsExdm2ben", + "RoadsExdm2ien", + "RoadsExgm2ien", + "StandardSymbology", + "Time", + "Units", } expected_models_of_local_ili_files = { - "KbS_Basis_V1_4", - "KbS_LV03_V1_4", - "KbS_LV95_V1_4", - "RoadsSimple", - "Abfallsammelstellen_ZEBA_LV03_V1", - "Abfallsammelstellen_ZEBA_LV95_V1", - "DictionariesCH_V1", - "GeometryCHLV03_V1", - "S", - "AdministrativeUnitsCH_V1", - "CHAdminCodes_V1", - "Localisation_V1", - "LE", - "LocalisationCH_V1", - "Dictionaries_V1", - "CatalogueObjectTrees_V1", - "AdministrativeUnits_V1", - "GeometryCHLV95_V1", - "InternationalCodes_V1", - "CatalogueObjects_V1", - "PlanerischerGewaesserschutz_LV95_V1_1", - "PlanerischerGewaesserschutz_LV03_V1_1", + "KbS_Basis_V1_4", + "KbS_LV03_V1_4", + "KbS_LV95_V1_4", + "RoadsSimple", + "Abfallsammelstellen_ZEBA_LV03_V1", + "Abfallsammelstellen_ZEBA_LV95_V1", + "DictionariesCH_V1", + "GeometryCHLV03_V1", + "S", + "AdministrativeUnitsCH_V1", + "CHAdminCodes_V1", + "Localisation_V1", + "LE", + "LocalisationCH_V1", + "Dictionaries_V1", + "CatalogueObjectTrees_V1", + "AdministrativeUnits_V1", + "GeometryCHLV95_V1", + "InternationalCodes_V1", + "CatalogueObjects_V1", + "PlanerischerGewaesserschutz_LV95_V1_1", + "PlanerischerGewaesserschutz_LV03_V1_1", } assert models == set.union( expected_models_of_ilimodels_xml, expected_models_of_local_ili_files @@ -293,8 +293,8 @@ def test_ilidata_xml_parser_metaconfig_kbs(self): ) assert "test_repo" in ilimetaconfigcache.repositories.keys() metaconfigs = { - e["id"] - for e in next(elem for elem in ilimetaconfigcache.repositories.values()) + e["id"] + for e in next(elem for elem in ilimetaconfigcache.repositories.values()) } expected_metaconfigs = { "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0-technical", @@ -376,8 +376,8 @@ def test_ilidata_xml_parser_local_repo_metaconfig(self): ) metaconfigs = { - e["id"] - for e in next(elem for elem in ilimetaconfigcache.repositories.values()) + e["id"] + for e in next(elem for elem in ilimetaconfigcache.repositories.values()) } expected_metaconfigs = { "ch.opengis.ili.config.PlanerischerGewaesserschutz_config", @@ -446,10 +446,8 @@ def test_ilidata_xml_parser_linkedmodels(self): ) assert "test_repo" in ilireferencedatacache.repositories.keys() referencedata = { - e["id"] - for e in next( - elem for elem in ilireferencedatacache.repositories.values() - ) + e["id"] + for e in next(elem for elem in ilireferencedatacache.repositories.values()) } expected_referencedata = { "ch.opengis.ili.catalogue.PlanerischerGewaesserschutz_Codetexte_V1_1", @@ -501,10 +499,8 @@ def test_ilidata_xml_parser_local_repo_linkedmodels(self): ) referencedata = { - e["id"] - for e in next( - elem for elem in ilireferencedatacache.repositories.values() - ) + e["id"] + for e in next(elem for elem in ilireferencedatacache.repositories.values()) } expected_referencedata = { "ch.opengis.ili.catalogue.PlanerischerGewaesserschutz_Codetexte_V1_1", @@ -556,10 +552,8 @@ def test_ilidata_xml_parser_toppingfiles(self): ) assert "test_repo" in ilitoppingfilecache.repositories.keys() files = { - e["relative_file_path"] - for e in next( - elem for elem in ilitoppingfilecache.repositories.values() - ) + e["relative_file_path"] + for e in next(elem for elem in ilitoppingfilecache.repositories.values()) } expected_files = { "layerstyle/opengisch_KbS_LV95_V1_4_005_parzellenidentifikation.qml", @@ -635,10 +629,8 @@ def test_ilidata_xml_parser_24_local_repo_toppingfiles(self): assert "local_files" in ilitoppingfilecache.repositories.keys() files = { - e["relative_file_path"] - for e in next( - elem for elem in ilitoppingfilecache.repositories.values() - ) + e["relative_file_path"] + for e in next(elem for elem in ilitoppingfilecache.repositories.values()) } expected_files = { "layerstyle/opengisch_KbS_LV95_V1_4_005_parzellenidentifikation.qml", @@ -673,10 +665,8 @@ def test_ilidata_xml_parser_local_repo_local_toppingfiles(self): assert "local_files" in ilitoppingfilecache.repositories.keys() files = { - e["relative_file_path"] - for e in next( - elem for elem in ilitoppingfilecache.repositories.values() - ) + e["relative_file_path"] + for e in next(elem for elem in ilitoppingfilecache.repositories.values()) } expected_files = { "layerstyle/opengisch_KbS_LV95_V1_4_005_parzellenidentifikation.qml", @@ -715,8 +705,8 @@ def test_ilidata_xml_parser_invalid(self): ) assert "test_repo" in ilimetaconfigcache.repositories.keys() metaconfigs = { - e["id"] - for e in next(elem for elem in ilimetaconfigcache.repositories.values()) + e["id"] + for e in next(elem for elem in ilimetaconfigcache.repositories.values()) } # not finding invalid metaconfig but the one with none as id expected_metaconfigs = {None, "ch.opengis.ili.config.valid"} diff --git a/tests/test_projectgen.py b/tests/test_projectgen.py index b931472..5fe3bce 100644 --- a/tests/test_projectgen.py +++ b/tests/test_projectgen.py @@ -103,24 +103,24 @@ def test_ili2db3_kbs_postgis(self): tabs = edit_form_config.tabs() fields = {field.name() for field in tabs[0].children()} assert fields == { - "letzteanpassung", - "zustaendigkeitkataster", - "geo_lage_polygon", - "inbetrieb", - "ersteintrag", - "bemerkung_en", - "bemerkung_rm", - "katasternummer", - "bemerkung_it", - "nachsorge", - "url_kbs_auszug", - "url_standort", - "statusaltlv", - "bemerkung_fr", - "standorttyp", - "bemerkung", - "geo_lage_punkt", - "bemerkung_de", + "letzteanpassung", + "zustaendigkeitkataster", + "geo_lage_polygon", + "inbetrieb", + "ersteintrag", + "bemerkung_en", + "bemerkung_rm", + "katasternummer", + "bemerkung_it", + "nachsorge", + "url_kbs_auszug", + "url_standort", + "statusaltlv", + "bemerkung_fr", + "standorttyp", + "bemerkung", + "geo_lage_punkt", + "bemerkung_de", } if Qgis.QGIS_VERSION_INT >= 31600: @@ -222,24 +222,24 @@ def test_kbs_postgis(self): tabs = edit_form_config.tabs() fields = {field.name() for field in tabs[0].children()} assert fields == { - "letzteanpassung", - "zustaendigkeitkataster", - "geo_lage_polygon", - "inbetrieb", - "ersteintrag", - "katasternummer", - "nachsorge", - "url_kbs_auszug", - "url_standort", - "statusaltlv", - "standorttyp", - "bemerkung", - "bemerkung_de", - "bemerkung_fr", - "bemerkung_rm", - "bemerkung_it", - "bemerkung_en", - "geo_lage_punkt", + "letzteanpassung", + "zustaendigkeitkataster", + "geo_lage_polygon", + "inbetrieb", + "ersteintrag", + "katasternummer", + "nachsorge", + "url_kbs_auszug", + "url_standort", + "statusaltlv", + "standorttyp", + "bemerkung", + "bemerkung_de", + "bemerkung_fr", + "bemerkung_rm", + "bemerkung_it", + "bemerkung_en", + "geo_lage_punkt", } if Qgis.QGIS_VERSION_INT >= 31600: @@ -340,23 +340,23 @@ def test_ili2db3_kbs_geopackage(self): tabs = edit_form_config.tabs() fields = {field.name() for field in tabs[0].children()} assert fields == { - "letzteanpassung", - "zustaendigkeitkataster", - "geo_lage_polygon", - "inbetrieb", - "ersteintrag", - "bemerkung_en", - "bemerkung_rm", - "katasternummer", - "bemerkung_it", - "nachsorge", - "url_kbs_auszug", - "url_standort", - "statusaltlv", - "bemerkung_fr", - "standorttyp", - "bemerkung", - "bemerkung_de", + "letzteanpassung", + "zustaendigkeitkataster", + "geo_lage_polygon", + "inbetrieb", + "ersteintrag", + "bemerkung_en", + "bemerkung_rm", + "katasternummer", + "bemerkung_it", + "nachsorge", + "url_kbs_auszug", + "url_standort", + "statusaltlv", + "bemerkung_fr", + "standorttyp", + "bemerkung", + "bemerkung_de", } if Qgis.QGIS_VERSION_INT >= 31600: @@ -380,22 +380,22 @@ def test_ili2db3_kbs_geopackage(self): assert count == 1 assert { - "statusaltlv", - "multilingualtext", - "untersmassn", - "multilingualmtext", - "languagecode_iso639_1", - "deponietyp", - "zustaendigkeitkataster", - "standorttyp", - "localisedtext", - "localisedmtext", - "belasteter_standort", - "deponietyp_", - "egrid_", - "untersmassn_", - "parzellenidentifikation", - "belasteter_standort_geo_lage_punkt", + "statusaltlv", + "multilingualtext", + "untersmassn", + "multilingualmtext", + "languagecode_iso639_1", + "deponietyp", + "zustaendigkeitkataster", + "standorttyp", + "localisedtext", + "localisedmtext", + "belasteter_standort", + "deponietyp_", + "egrid_", + "untersmassn_", + "parzellenidentifikation", + "belasteter_standort_geo_lage_punkt", } == {layer.name for layer in available_layers} def test_kbs_geopackage(self): @@ -441,23 +441,23 @@ def test_kbs_geopackage(self): tabs = edit_form_config.tabs() fields = {field.name() for field in tabs[0].children()} assert fields == { - "letzteanpassung", - "zustaendigkeitkataster", - "geo_lage_polygon", - "inbetrieb", - "ersteintrag", - "katasternummer", - "nachsorge", - "url_kbs_auszug", - "url_standort", - "statusaltlv", - "standorttyp", - "bemerkung", - "bemerkung_de", - "bemerkung_fr", - "bemerkung_rm", - "bemerkung_it", - "bemerkung_en", + "letzteanpassung", + "zustaendigkeitkataster", + "geo_lage_polygon", + "inbetrieb", + "ersteintrag", + "katasternummer", + "nachsorge", + "url_kbs_auszug", + "url_standort", + "statusaltlv", + "standorttyp", + "bemerkung", + "bemerkung_de", + "bemerkung_fr", + "bemerkung_rm", + "bemerkung_it", + "bemerkung_en", } if Qgis.QGIS_VERSION_INT >= 31600: @@ -481,22 +481,22 @@ def test_kbs_geopackage(self): assert count == 1 assert { - "statusaltlv", - "multilingualtext", - "untersmassn", - "multilingualmtext", - "languagecode_iso639_1", - "deponietyp", - "zustaendigkeitkataster", - "standorttyp", - "localisedtext", - "localisedmtext", - "belasteter_standort", - "deponietyp_", - "egrid_", - "untersmassn_", - "parzellenidentifikation", - "belasteter_standort_geo_lage_punkt", + "statusaltlv", + "multilingualtext", + "untersmassn", + "multilingualmtext", + "languagecode_iso639_1", + "deponietyp", + "zustaendigkeitkataster", + "standorttyp", + "localisedtext", + "localisedmtext", + "belasteter_standort", + "deponietyp_", + "egrid_", + "untersmassn_", + "parzellenidentifikation", + "belasteter_standort_geo_lage_punkt", } == {layer.name for layer in available_layers} def test_naturschutz_postgis(self): @@ -3996,9 +3996,9 @@ def test_unit(self): available_layers = generator.layers() infra_po = next( - layer - for layer in available_layers - if layer.name == "erholungsinfrastruktur_punktobjekt" + layer + for layer in available_layers + if layer.name == "erholungsinfrastruktur_punktobjekt" ) naechste_kontrolle = next( field for field in infra_po.fields if field.name == "naechste_kontrolle" diff --git a/tests/test_projectgen_extension_optimization.py b/tests/test_projectgen_extension_optimization.py index 4052f0e..54ca1e3 100644 --- a/tests/test_projectgen_extension_optimization.py +++ b/tests/test_projectgen_extension_optimization.py @@ -17,31 +17,20 @@ ***************************************************************************/ """ -import configparser import datetime import logging import os import pathlib -import shutil import tempfile -from decimal import Decimal -import yaml -from qgis.core import Qgis, QgsEditFormConfig, QgsProject, QgsRelation, QgsLayerTreeLayer -from qgis.PyQt.QtCore import QEventLoop, Qt, QTimer +from qgis.core import QgsLayerTreeLayer, QgsProject from qgis.testing import start_app, unittest + from modelbaker.dataobjects.project import Project from modelbaker.db_factory.gpkg_command_config_manager import GpkgCommandConfigManager from modelbaker.generator.generator import Generator from modelbaker.iliwrapper import iliimporter from modelbaker.iliwrapper.globals import DbIliMode -from modelbaker.iliwrapper.ilicache import ( - IliDataCache, - IliDataItemModel, - IliToppingFileCache, - IliToppingFileItemModel, -) - from modelbaker.utils.globals import OptimizeStrategy from tests.utils import get_pg_connection_string, iliimporter_config, testdata_path @@ -58,13 +47,13 @@ def setUpClass(cls): """Run before all tests.""" cls.basetestpath = tempfile.mkdtemp() - ''' + """ Those tests check if: - no ambiguous layers exists - they are all named properly and unique - irrelevant layers are detected (according to the assumptions below) - irrelevant layers are handled according the the chosen strategy: hidden or grouped - relations (and their widgets) are handled according to the stategy (not created when hidden, not in the forms used when grouped) - + Assumption: Since it appears almost impossible to care for all the cases, I need to make some assumptions what mostly would be the case. - When you extend a base class with the same name, you intend to "replace" it, otherwise you would rename it. @@ -75,17 +64,18 @@ def setUpClass(cls): - Polymorphic_Ortsplanung_V1_1 containing several topics extending the same class - Staedtische_Ortsplanung_V1_1 containing several extention levels on the same class - Bauplanung_V1_1 containing structures and extending assocciations - ''' - + """ def test_extopt_staedtische_postgis(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2pg importer.configuration = iliimporter_config(importer.tool) - importer.configuration.ilifile = testdata_path("ilimodels/Staedtische_Ortsplanung_V1_1.ili") + importer.configuration.ilifile = testdata_path( + "ilimodels/Staedtische_Ortsplanung_V1_1.ili" + ) importer.configuration.ilimodels = "Staedtische_Ortsplanung_V1_1" - importer.configuration.dbschema = "optimal_staedtische_{:%Y%m%d%H%M%S%f}".format( - datetime.datetime.now() + importer.configuration.dbschema = ( + "optimal_staedtische_{:%Y%m%d%H%M%S%f}".format(datetime.datetime.now()) ) importer.configuration.srs_code = 2056 @@ -104,29 +94,41 @@ def test_extopt_staedtische_postgis(self): available_layers = generator.layers() aliases = [l.alias for l in available_layers] - irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + [l.ili_name for l in available_layers if not l.is_relevant] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gebaeude_StadtFirma', 'Gewerbe.Gebaeude', 'Gewerbe_V1.Firmen.Firma', 'Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Staedtisches_Gewerbe_V1.Firmen.Firma', 'Strasse'] + expected_aliases = [ + "BesitzerIn", + "Freizeit.Gebaeude", + "Gebaeude_StadtFirma", + "Gewerbe.Gebaeude", + "Gewerbe_V1.Firmen.Firma", + "Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Staedtisches_Gewerbe_V1.Firmen.Firma", + "Strasse", + ] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected - #todo assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ['Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Gewerbe_V1.Firmen.Firma'] - #todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + # todo assert len(irrelevant_layer_ilinames) > 0 + # todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - def test_extopt_staedtische_geopackage(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2gpkg importer.configuration = iliimporter_config(importer.tool) - importer.configuration.ilifile = testdata_path("ilimodels/Staedtische_Ortsplanung_V1_1.ili") + importer.configuration.ilifile = testdata_path( + "ilimodels/Staedtische_Ortsplanung_V1_1.ili" + ) importer.configuration.ilimodels = "Staedtische_Ortsplanung_V1_1" importer.configuration.dbfile = os.path.join( self.basetestpath, - "tmp_optimal_staedtische_{:%Y%m%d%H%M%S%f}.gpkg".format(datetime.datetime.now()), + "tmp_optimal_staedtische_{:%Y%m%d%H%M%S%f}.gpkg".format( + datetime.datetime.now() + ), ) importer.configuration.srs_code = 2056 importer.configuration.inheritance = "smart2" @@ -141,26 +143,47 @@ def test_extopt_staedtische_geopackage(self): ### 1. OptimizeStrategy.NONE ### strategy = OptimizeStrategy.NONE - generator = Generator(tool=DbIliMode.ili2gpkg, uri=uri, inheritance="smart2", optimize_strategy = strategy ) + generator = Generator( + tool=DbIliMode.ili2gpkg, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) available_layers = generator.layers() relations, _ = generator.relations(available_layers) legend = generator.legend(available_layers) - + aliases = [l.alias for l in available_layers] - irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + irrelevant_layer_ilinames = [ + l.ili_name for l in available_layers if not l.is_relevant + ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] # check no ambiguous layers exists - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gebaeude_StadtFirma', 'Gewerbe.Gebaeude', 'Gewerbe_V1.Firmen.Firma', 'Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Staedtisches_Gewerbe_V1.Firmen.Firma', 'Strasse'] + expected_aliases = [ + "BesitzerIn", + "Freizeit.Gebaeude", + "Gebaeude_StadtFirma", + "Gewerbe.Gebaeude", + "Gewerbe_V1.Firmen.Firma", + "Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Staedtisches_Gewerbe_V1.Firmen.Firma", + "Strasse", + ] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ['Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Gewerbe_V1.Firmen.Firma'] + expected_irrelevant_layer_ilinames = [ + "Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Gewerbe_V1.Firmen.Firma", + ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - project = Project(optimize_strategy = strategy ) + project = Project(optimize_strategy=strategy) project.layers = available_layers project.relations = relations project.legend = legend @@ -176,11 +199,24 @@ def test_extopt_staedtische_geopackage(self): all_layers = root.findLayers() assert len(all_layers) == 9 - geometry_layers = set([l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer)]) - assert geometry_layers == {'Strasse', 'Freizeit.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Gewerbe.Gebaeude'} - - tables_layers = set([l.name() for l in root.findGroup('tables').children()]) - assert tables_layers == {'Staedtisches_Gewerbe_V1.Firmen.Firma', 'Gewerbe_V1.Firmen.Firma', 'Gebaeude_StadtFirma', 'BesitzerIn'} + geometry_layers = { + l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer) + } + assert geometry_layers == { + "Strasse", + "Freizeit.Gebaeude", + "Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Gewerbe.Gebaeude", + } + + tables_layers = {l.name() for l in root.findGroup("tables").children()} + assert tables_layers == { + "Staedtisches_Gewerbe_V1.Firmen.Firma", + "Gewerbe_V1.Firmen.Firma", + "Gebaeude_StadtFirma", + "BesitzerIn", + } # check relations - all are there relations = list(qgis_project.relationManager().relations().values()) @@ -214,27 +250,48 @@ def test_extopt_staedtische_geopackage(self): ### 2. OptimizeStrategy.GROUP ### strategy = OptimizeStrategy.GROUP - generator = Generator(tool=DbIliMode.ili2gpkg, uri=uri, inheritance="smart2", optimize_strategy = strategy ) + generator = Generator( + tool=DbIliMode.ili2gpkg, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) available_layers = generator.layers() relations, _ = generator.relations(available_layers) legend = generator.legend(available_layers) - + aliases = [l.alias for l in available_layers] - irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + irrelevant_layer_ilinames = [ + l.ili_name for l in available_layers if not l.is_relevant + ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gebaeude_StadtFirma', 'Gewerbe.Gebaeude', 'Gewerbe_V1.Firmen.Firma', 'Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Staedtisches_Gewerbe_V1.Firmen.Firma', 'Strasse'] + expected_aliases = [ + "BesitzerIn", + "Freizeit.Gebaeude", + "Gebaeude_StadtFirma", + "Gewerbe.Gebaeude", + "Gewerbe_V1.Firmen.Firma", + "Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Staedtisches_Gewerbe_V1.Firmen.Firma", + "Strasse", + ] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ['Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Gewerbe_V1.Firmen.Firma'] + expected_irrelevant_layer_ilinames = [ + "Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Gewerbe_V1.Firmen.Firma", + ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - project = Project(optimize_strategy = strategy ) + project = Project(optimize_strategy=strategy) project.layers = available_layers project.relations = relations project.legend = legend @@ -250,18 +307,29 @@ def test_extopt_staedtische_geopackage(self): all_layers = root.findLayers() assert len(all_layers) == 9 - geometry_layers = set([l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer)]) - assert geometry_layers == {'Strasse', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude'} + geometry_layers = { + l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer) + } + assert geometry_layers == {"Strasse", "Freizeit.Gebaeude", "Gewerbe.Gebaeude"} - tables_layers = set([l.name() for l in root.findGroup('tables').children()]) - assert tables_layers == {'Staedtisches_Gewerbe_V1.Firmen.Firma', 'Gebaeude_StadtFirma', 'BesitzerIn'} + tables_layers = {l.name() for l in root.findGroup("tables").children()} + assert tables_layers == { + "Staedtisches_Gewerbe_V1.Firmen.Firma", + "Gebaeude_StadtFirma", + "BesitzerIn", + } - base_group = root.findGroup('base layers') + base_group = root.findGroup("base layers") assert base_group - grouped_base_layers = set([l.name() for l in base_group.children() if isinstance(l, QgsLayerTreeLayer)]) + grouped_base_layers = { + l.name() for l in base_group.children() if isinstance(l, QgsLayerTreeLayer) + } - assert grouped_base_layers == {'Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude'} + assert grouped_base_layers == { + "Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Ortsplanung_V1_1.Konstruktionen.Gebaeude", + } # check relations - all are there relations = list(qgis_project.relationManager().relations().values()) @@ -273,7 +341,7 @@ def test_extopt_staedtische_geopackage(self): if layer.layer.name() == "Strasse": efc = layer.layer.editFormConfig() # one general and two relation editors - assert len(efc.tabs()) == 3 + assert len(efc.tabs()) == 3 for tab in efc.tabs(): if tab.name() == "stadtscng_v1_1freizeit_gebaeude": count += 1 @@ -281,9 +349,11 @@ def test_extopt_staedtische_geopackage(self): if tab.name() == "stadtscng_v1_1gewerbe_gebaeude": count += 1 assert len(tab.children()) == 1 - if tab.name() == "kantnl_ng_v1_1konstruktionen_gebaeude": #should not happen + if ( + tab.name() == "kantnl_ng_v1_1konstruktionen_gebaeude" + ): # should not happen count += 1 - if tab.name() == "gebaeude": #should not happen + if tab.name() == "gebaeude": # should not happen count += 1 # should find only 2 assert count == 2 @@ -293,27 +363,46 @@ def test_extopt_staedtische_geopackage(self): ### 3. OptimizeStrategy.HIDE ### strategy = OptimizeStrategy.HIDE - generator = Generator(tool=DbIliMode.ili2gpkg, uri=uri, inheritance="smart2" ,optimize_strategy = strategy ) + generator = Generator( + tool=DbIliMode.ili2gpkg, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) available_layers = generator.layers() relations, _ = generator.relations(available_layers) legend = generator.legend(available_layers) - + aliases = [l.alias for l in available_layers] - irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + irrelevant_layer_ilinames = [ + l.ili_name for l in available_layers if not l.is_relevant + ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] # ambiguous aliases exist, since we don't rename them when they are hidden anyway assert len(ambiguous_aliases) == 4 - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gebaeude_StadtFirma', 'Gewerbe.Gebaeude', 'Konstruktionen.Gebaeude', 'Firma', 'Strasse'] + expected_aliases = [ + "BesitzerIn", + "Freizeit.Gebaeude", + "Gebaeude_StadtFirma", + "Gewerbe.Gebaeude", + "Konstruktionen.Gebaeude", + "Firma", + "Strasse", + ] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ['Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Gewerbe_V1.Firmen.Firma'] + expected_irrelevant_layer_ilinames = [ + "Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Gewerbe_V1.Firmen.Firma", + ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - project = Project(optimize_strategy = strategy ) + project = Project(optimize_strategy=strategy) project.layers = available_layers project.relations = relations project.legend = legend @@ -329,11 +418,13 @@ def test_extopt_staedtische_geopackage(self): all_layers = root.findLayers() assert len(all_layers) == 6 - geometry_layers = set([l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer)]) - assert geometry_layers == {'Strasse', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude'} + geometry_layers = { + l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer) + } + assert geometry_layers == {"Strasse", "Freizeit.Gebaeude", "Gewerbe.Gebaeude"} - tables_layers = set([l.name() for l in root.findGroup('tables').children()]) - assert tables_layers == {'Firma', 'Gebaeude_StadtFirma', 'BesitzerIn'} + tables_layers = {l.name() for l in root.findGroup("tables").children()} + assert tables_layers == {"Firma", "Gebaeude_StadtFirma", "BesitzerIn"} # check relations - only 13 are here relations = list(qgis_project.relationManager().relations().values()) @@ -345,7 +436,7 @@ def test_extopt_staedtische_geopackage(self): if layer.layer.name() == "Strasse": efc = layer.layer.editFormConfig() # one general and two relation editors - assert len(efc.tabs()) == 3 + assert len(efc.tabs()) == 3 for tab in efc.tabs(): if tab.name() == "stadtscng_v1_1freizeit_gebaeude": count += 1 @@ -353,9 +444,11 @@ def test_extopt_staedtische_geopackage(self): if tab.name() == "stadtscng_v1_1gewerbe_gebaeude": count += 1 assert len(tab.children()) == 1 - if tab.name() == "kantnl_ng_v1_1konstruktionen_gebaeude": #should not happen + if ( + tab.name() == "kantnl_ng_v1_1konstruktionen_gebaeude" + ): # should not happen count += 1 - if tab.name() == "gebaeude": #should not happen + if tab.name() == "gebaeude": # should not happen count += 1 # should find only 2 assert count == 2 @@ -364,10 +457,12 @@ def test_extopt_staedtische_mssql(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2mssql importer.configuration = iliimporter_config(importer.tool) - importer.configuration.ilifile = testdata_path("ilimodels/Staedtische_Ortsplanung_V1_1.ili") + importer.configuration.ilifile = testdata_path( + "ilimodels/Staedtische_Ortsplanung_V1_1.ili" + ) importer.configuration.ilimodels = "Staedtische_Ortsplanung_V1_1" - importer.configuration.dbschema = "optimal_staedtische_{:%Y%m%d%H%M%S%f}".format( - datetime.datetime.now() + importer.configuration.dbschema = ( + "optimal_staedtische_{:%Y%m%d%H%M%S%f}".format(datetime.datetime.now()) ) importer.configuration.srs_code = 2056 importer.configuration.inheritance = "smart2" @@ -391,24 +486,35 @@ def test_extopt_staedtische_mssql(self): available_layers = generator.layers() aliases = [l.alias for l in available_layers] - irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + [l.ili_name for l in available_layers if not l.is_relevant] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gebaeude_StadtFirma', 'Gewerbe.Gebaeude', 'Gewerbe_V1.Firmen.Firma', 'Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Staedtisches_Gewerbe_V1.Firmen.Firma', 'Strasse'] + expected_aliases = [ + "BesitzerIn", + "Freizeit.Gebaeude", + "Gebaeude_StadtFirma", + "Gewerbe.Gebaeude", + "Gewerbe_V1.Firmen.Firma", + "Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Staedtisches_Gewerbe_V1.Firmen.Firma", + "Strasse", + ] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected - #todo assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ['Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Gewerbe_V1.Firmen.Firma'] - #todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + # todo assert len(irrelevant_layer_ilinames) > 0 + # todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) def test_extopt_polymorphic_postgis(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2pg importer.configuration = iliimporter_config(importer.tool) - importer.configuration.ilifile = testdata_path("ilimodels/Polymorphic_Ortsplanung_V1_1.ili") + importer.configuration.ilifile = testdata_path( + "ilimodels/Polymorphic_Ortsplanung_V1_1.ili" + ) importer.configuration.ilimodels = "Polymorphic_Ortsplanung_V1_1" importer.configuration.dbschema = "optimal_polymorph_{:%Y%m%d%H%M%S%f}".format( datetime.datetime.now() @@ -430,28 +536,43 @@ def test_extopt_polymorphic_postgis(self): available_layers = generator.layers() aliases = [l.alias for l in available_layers] - irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + [l.ili_name for l in available_layers if not l.is_relevant] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + expected_aliases = [ + "BesitzerIn", + "Freizeit.Gebaeude", + "Gewerbe.Gebaeude", + "Hallen.Gebaeude", + "IndustrieGewerbe.Gebaeude", + "Markthalle", + "Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Strasse", + "TurnhalleTyp1", + "TurnhalleTyp2", + ] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected - #todo assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ['Ortsplanung_V1_1.Konstruktionen.Gebaeude'] - #todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + # todo assert len(irrelevant_layer_ilinames) > 0 + # todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) def test_extopt_polymorphic_geopackage(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2gpkg importer.configuration = iliimporter_config(importer.tool) - importer.configuration.ilifile = testdata_path("ilimodels/Polymorphic_Ortsplanung_V1_1.ili") + importer.configuration.ilifile = testdata_path( + "ilimodels/Polymorphic_Ortsplanung_V1_1.ili" + ) importer.configuration.ilimodels = "Polymorphic_Ortsplanung_V1_1" importer.configuration.dbfile = os.path.join( self.basetestpath, - "tmp_optimal_polymorph_{:%Y%m%d%H%M%S%f}.gpkg".format(datetime.datetime.now()), + "tmp_optimal_polymorph_{:%Y%m%d%H%M%S%f}.gpkg".format( + datetime.datetime.now() + ), ) importer.configuration.srs_code = 2056 importer.configuration.inheritance = "smart2" @@ -466,27 +587,48 @@ def test_extopt_polymorphic_geopackage(self): ### 1. OptimizeStrategy.NONE ### strategy = OptimizeStrategy.NONE - generator = Generator(tool=DbIliMode.ili2gpkg, uri=uri, inheritance="smart2" ,optimize_strategy = strategy ) + generator = Generator( + tool=DbIliMode.ili2gpkg, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) available_layers = generator.layers() relations, _ = generator.relations(available_layers) legend = generator.legend(available_layers) aliases = [l.alias for l in available_layers] - irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + irrelevant_layer_ilinames = [ + l.ili_name for l in available_layers if not l.is_relevant + ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + expected_aliases = [ + "BesitzerIn", + "Freizeit.Gebaeude", + "Gewerbe.Gebaeude", + "Hallen.Gebaeude", + "IndustrieGewerbe.Gebaeude", + "Markthalle", + "Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Strasse", + "TurnhalleTyp1", + "TurnhalleTyp2", + ] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ['Ortsplanung_V1_1.Konstruktionen.Gebaeude'] + expected_irrelevant_layer_ilinames = [ + "Ortsplanung_V1_1.Konstruktionen.Gebaeude" + ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - project = Project(optimize_strategy = strategy ) + project = Project(optimize_strategy=strategy) project.layers = available_layers project.relations = relations project.legend = legend @@ -502,11 +644,26 @@ def test_extopt_polymorphic_geopackage(self): all_layers = root.findLayers() assert len(all_layers) == 11 - geometry_layers = set([l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer)]) - assert geometry_layers == {'IndustrieGewerbe.Gebaeude', 'Strasse', 'Freizeit.Gebaeude', 'Markthalle', 'TurnhalleTyp2', 'Hallen.Gebaeude', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'TurnhalleTyp1', 'Gewerbe.Gebaeude'} - - tables_layers = set([l.name() for l in root.findGroup('tables').children()]) - assert tables_layers == {'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'BesitzerIn'} + geometry_layers = { + l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer) + } + assert geometry_layers == { + "IndustrieGewerbe.Gebaeude", + "Strasse", + "Freizeit.Gebaeude", + "Markthalle", + "TurnhalleTyp2", + "Hallen.Gebaeude", + "Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "TurnhalleTyp1", + "Gewerbe.Gebaeude", + } + + tables_layers = {l.name() for l in root.findGroup("tables").children()} + assert tables_layers == { + "Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "BesitzerIn", + } # check relations - all are there relations = list(qgis_project.relationManager().relations().values()) @@ -552,27 +709,48 @@ def test_extopt_polymorphic_geopackage(self): ### 2. OptimizeStrategy.GROUP ### strategy = OptimizeStrategy.GROUP - generator = Generator(tool=DbIliMode.ili2gpkg, uri=uri, inheritance="smart2" ,optimize_strategy = strategy ) + generator = Generator( + tool=DbIliMode.ili2gpkg, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) available_layers = generator.layers() relations, _ = generator.relations(available_layers) legend = generator.legend(available_layers) aliases = [l.alias for l in available_layers] - irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + irrelevant_layer_ilinames = [ + l.ili_name for l in available_layers if not l.is_relevant + ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + expected_aliases = [ + "BesitzerIn", + "Freizeit.Gebaeude", + "Gewerbe.Gebaeude", + "Hallen.Gebaeude", + "IndustrieGewerbe.Gebaeude", + "Markthalle", + "Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Strasse", + "TurnhalleTyp1", + "TurnhalleTyp2", + ] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ['Ortsplanung_V1_1.Konstruktionen.Gebaeude'] + expected_irrelevant_layer_ilinames = [ + "Ortsplanung_V1_1.Konstruktionen.Gebaeude" + ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - project = Project(optimize_strategy = strategy ) + project = Project(optimize_strategy=strategy) project.layers = available_layers project.relations = relations project.legend = legend @@ -588,18 +766,34 @@ def test_extopt_polymorphic_geopackage(self): all_layers = root.findLayers() assert len(all_layers) == 11 - geometry_layers = set([l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer)]) - assert geometry_layers == {'IndustrieGewerbe.Gebaeude', 'Strasse', 'Freizeit.Gebaeude', 'Markthalle', 'TurnhalleTyp2', 'Hallen.Gebaeude', 'TurnhalleTyp1', 'Gewerbe.Gebaeude'} - - tables_layers = set([l.name() for l in root.findGroup('tables').children()]) - assert tables_layers == {'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'BesitzerIn'} - - base_group = root.findGroup('base layers') + geometry_layers = { + l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer) + } + assert geometry_layers == { + "IndustrieGewerbe.Gebaeude", + "Strasse", + "Freizeit.Gebaeude", + "Markthalle", + "TurnhalleTyp2", + "Hallen.Gebaeude", + "TurnhalleTyp1", + "Gewerbe.Gebaeude", + } + + tables_layers = {l.name() for l in root.findGroup("tables").children()} + assert tables_layers == { + "Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "BesitzerIn", + } + + base_group = root.findGroup("base layers") assert base_group - grouped_base_layers = set([l.name() for l in base_group.children() if isinstance(l, QgsLayerTreeLayer)]) + grouped_base_layers = { + l.name() for l in base_group.children() if isinstance(l, QgsLayerTreeLayer) + } - assert grouped_base_layers == {'Ortsplanung_V1_1.Konstruktionen.Gebaeude'} + assert grouped_base_layers == {"Ortsplanung_V1_1.Konstruktionen.Gebaeude"} # check relations - all are there relations = list(qgis_project.relationManager().relations().values()) @@ -613,7 +807,7 @@ def test_extopt_polymorphic_geopackage(self): # one general and four relation editors assert len(efc.tabs()) == 8 for tab in efc.tabs(): - if tab.name() == "gebaeude": # this should not happen + if tab.name() == "gebaeude": # this should not happen count += 1 assert len(tab.children()) == 1 if tab.name() == "polymrpng_v1_1gewerbe_gebaeude": @@ -645,27 +839,47 @@ def test_extopt_polymorphic_geopackage(self): ### 3. OptimizeStrategy.HIDE ### strategy = OptimizeStrategy.HIDE - generator = Generator(tool=DbIliMode.ili2gpkg, uri=uri, inheritance="smart2" ,optimize_strategy = strategy ) + generator = Generator( + tool=DbIliMode.ili2gpkg, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) available_layers = generator.layers() relations, _ = generator.relations(available_layers) legend = generator.legend(available_layers) aliases = [l.alias for l in available_layers] - irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + irrelevant_layer_ilinames = [ + l.ili_name for l in available_layers if not l.is_relevant + ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] # ambiguous aliases exist, since we don't rename them when they are hidden anyway assert len(ambiguous_aliases) == 2 - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + expected_aliases = [ + "BesitzerIn", + "Freizeit.Gebaeude", + "Gewerbe.Gebaeude", + "Hallen.Gebaeude", + "IndustrieGewerbe.Gebaeude", + "Markthalle", + "Konstruktionen.Gebaeude", + "Strasse", + "TurnhalleTyp1", + "TurnhalleTyp2", + ] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ['Ortsplanung_V1_1.Konstruktionen.Gebaeude'] + expected_irrelevant_layer_ilinames = [ + "Ortsplanung_V1_1.Konstruktionen.Gebaeude" + ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - project = Project(optimize_strategy = strategy ) + project = Project(optimize_strategy=strategy) project.layers = available_layers project.relations = relations project.legend = legend @@ -681,11 +895,22 @@ def test_extopt_polymorphic_geopackage(self): all_layers = root.findLayers() assert len(all_layers) == 10 - geometry_layers = set([l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer)]) - assert geometry_layers == {'IndustrieGewerbe.Gebaeude', 'Strasse', 'Freizeit.Gebaeude', 'Markthalle', 'TurnhalleTyp2', 'Hallen.Gebaeude', 'TurnhalleTyp1', 'Gewerbe.Gebaeude'} - - tables_layers = set([l.name() for l in root.findGroup('tables').children()]) - assert tables_layers == {'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'BesitzerIn'} + geometry_layers = { + l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer) + } + assert geometry_layers == { + "IndustrieGewerbe.Gebaeude", + "Strasse", + "Freizeit.Gebaeude", + "Markthalle", + "TurnhalleTyp2", + "Hallen.Gebaeude", + "TurnhalleTyp1", + "Gewerbe.Gebaeude", + } + + tables_layers = {l.name() for l in root.findGroup("tables").children()} + assert tables_layers == {"Konstruktionen.Gebaeude", "BesitzerIn"} # check relations - all are there relations = list(qgis_project.relationManager().relations().values()) @@ -699,7 +924,7 @@ def test_extopt_polymorphic_geopackage(self): # one general and four relation editors assert len(efc.tabs()) == 8 for tab in efc.tabs(): - if tab.name() == "gebaeude": # this should not happen + if tab.name() == "gebaeude": # this should not happen count += 1 assert len(tab.children()) == 1 if tab.name() == "polymrpng_v1_1gewerbe_gebaeude": @@ -732,7 +957,9 @@ def test_extopt_polymorphic_mssql(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2mssql importer.configuration = iliimporter_config(importer.tool) - importer.configuration.ilifile = testdata_path("ilimodels/Polymorphic_Ortsplanung_V1_1.ili") + importer.configuration.ilifile = testdata_path( + "ilimodels/Polymorphic_Ortsplanung_V1_1.ili" + ) importer.configuration.ilimodels = "Polymorphic_Ortsplanung_V1_1" importer.configuration.dbschema = "optimal_polymorph_{:%Y%m%d%H%M%S%f}".format( datetime.datetime.now() @@ -759,24 +986,37 @@ def test_extopt_polymorphic_mssql(self): available_layers = generator.layers() aliases = [l.alias for l in available_layers] - irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + [l.ili_name for l in available_layers if not l.is_relevant] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 - expected_aliases = ['BesitzerIn', 'Freizeit.Gebaeude', 'Gewerbe.Gebaeude', 'Hallen.Gebaeude', 'IndustrieGewerbe.Gebaeude', 'Markthalle', 'Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude', 'Strasse', 'TurnhalleTyp1', 'TurnhalleTyp2'] + expected_aliases = [ + "BesitzerIn", + "Freizeit.Gebaeude", + "Gewerbe.Gebaeude", + "Hallen.Gebaeude", + "IndustrieGewerbe.Gebaeude", + "Markthalle", + "Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude", + "Strasse", + "TurnhalleTyp1", + "TurnhalleTyp2", + ] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected - #todo assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ['Ortsplanung_V1_1.Konstruktionen.Gebaeude'] - #todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + # todo assert len(irrelevant_layer_ilinames) > 0 + # todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) def test_extopt_baustruct_postgis(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2pg importer.configuration = iliimporter_config(importer.tool) - importer.configuration.ilifile = testdata_path("ilimodels/Kantonale_Bauplanung_V1_1.ili") + importer.configuration.ilifile = testdata_path( + "ilimodels/Kantonale_Bauplanung_V1_1.ili" + ) importer.configuration.ilimodels = "Kantonale_Bauplanung_V1_1" importer.configuration.dbschema = "optimal_baustruct_{:%Y%m%d%H%M%S%f}".format( datetime.datetime.now() @@ -798,29 +1038,50 @@ def test_extopt_baustruct_postgis(self): available_layers = generator.layers() aliases = [l.alias for l in available_layers] - irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + [l.ili_name for l in available_layers if not l.is_relevant] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 - expected_aliases = ["Bauart", "Bauplanung_V1_1.Konstruktionen.Gebaeude", "Bauplanung_V1_1.Konstruktionen.Material", "Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", "Bauplanung_V1_1.Natur.Park", "Bauplanung_V1_1.Natur.Tierart", "Brutstelle", "Buntbrache", "Feld", "KantonaleBuntbrache", "Kantonale_Bauplanung_V1_1.Konstruktionen.Gebaeude", "Kantonale_Bauplanung_V1_1.Konstruktionen.Material", "Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", "Kantonale_Bauplanung_V1_1.Natur.Park", "Kantonale_Bauplanung_V1_1.Natur.Tierart", "Kartoffelfeld", "Sonnenblumenfeld", "Strasse"] + expected_aliases = [ + "Bauart", + "Bauplanung_V1_1.Konstruktionen.Gebaeude", + "Bauplanung_V1_1.Konstruktionen.Material", + "Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", + "Bauplanung_V1_1.Natur.Park", + "Bauplanung_V1_1.Natur.Tierart", + "Brutstelle", + "Buntbrache", + "Feld", + "KantonaleBuntbrache", + "Kantonale_Bauplanung_V1_1.Konstruktionen.Gebaeude", + "Kantonale_Bauplanung_V1_1.Konstruktionen.Material", + "Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", + "Kantonale_Bauplanung_V1_1.Natur.Park", + "Kantonale_Bauplanung_V1_1.Natur.Tierart", + "Kartoffelfeld", + "Sonnenblumenfeld", + "Strasse", + ] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected - #todo assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ["Bauplanung_V1_1.Konstruktionen.Material", "Bauplanung_V1_1.Konstruktionen.Gebaeude", "Bauplanung_V1_1.Natur.Tierart", "Bauplanung_V1_1.Natur.Park"] - #todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + # todo assert len(irrelevant_layer_ilinames) > 0 + # todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - def test_extopt_baustruct_geopackage(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2gpkg importer.configuration = iliimporter_config(importer.tool) - importer.configuration.ilifile = testdata_path("ilimodels/Kantonale_Bauplanung_V1_1.ili") + importer.configuration.ilifile = testdata_path( + "ilimodels/Kantonale_Bauplanung_V1_1.ili" + ) importer.configuration.ilimodels = "Kantonale_Bauplanung_V1_1" importer.configuration.dbfile = os.path.join( self.basetestpath, - "tmp_optimal_baustruct_{:%Y%m%d%H%M%S%f}.gpkg".format(datetime.datetime.now()), + "tmp_optimal_baustruct_{:%Y%m%d%H%M%S%f}.gpkg".format( + datetime.datetime.now() + ), ) importer.configuration.srs_code = 2056 importer.configuration.inheritance = "smart2" @@ -835,27 +1096,55 @@ def test_extopt_baustruct_geopackage(self): ### 1. OptimizeStrategy.NONE ### strategy = OptimizeStrategy.NONE - generator = Generator(tool=DbIliMode.ili2gpkg, uri=uri, inheritance="smart2",optimize_strategy = strategy ) + generator = Generator( + tool=DbIliMode.ili2gpkg, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) available_layers = generator.layers() relations, _ = generator.relations(available_layers) legend = generator.legend(available_layers) aliases = [l.alias for l in available_layers] - irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + irrelevant_layer_ilinames = [ + l.ili_name for l in available_layers if not l.is_relevant + ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 - expected_aliases = ['Strasse', 'Sonnenblumenfeld', 'Kartoffelfeld', 'KantonaleBuntbrache', 'Kantonale_Bauplanung_V1_1.Natur.Park', 'Kantonale_Bauplanung_V1_1.Konstruktionen.Gebaeude', 'Feld', 'Buntbrache', 'Brutstelle', 'Bauplanung_V1_1.Natur.Park', 'Bauplanung_V1_1.Konstruktionen.Gebaeude', 'Kantonale_Bauplanung_V1_1.Natur.Tierart', 'Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude', 'Kantonale_Bauplanung_V1_1.Konstruktionen.Material', 'Bauplanung_V1_1.Natur.Tierart', 'Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude', 'Bauplanung_V1_1.Konstruktionen.Material', 'Bauart'] + expected_aliases = [ + "Strasse", + "Sonnenblumenfeld", + "Kartoffelfeld", + "KantonaleBuntbrache", + "Kantonale_Bauplanung_V1_1.Natur.Park", + "Kantonale_Bauplanung_V1_1.Konstruktionen.Gebaeude", + "Feld", + "Buntbrache", + "Brutstelle", + "Bauplanung_V1_1.Natur.Park", + "Bauplanung_V1_1.Konstruktionen.Gebaeude", + "Kantonale_Bauplanung_V1_1.Natur.Tierart", + "Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", + "Kantonale_Bauplanung_V1_1.Konstruktionen.Material", + "Bauplanung_V1_1.Natur.Tierart", + "Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", + "Bauplanung_V1_1.Konstruktionen.Material", + "Bauart", + ] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ['Ortsplanung_V1_1.Konstruktionen.Gebaeude'] + expected_irrelevant_layer_ilinames = [ + "Ortsplanung_V1_1.Konstruktionen.Gebaeude" + ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - project = Project(optimize_strategy = strategy ) + project = Project(optimize_strategy=strategy) project.layers = available_layers project.relations = relations project.legend = legend @@ -871,11 +1160,33 @@ def test_extopt_baustruct_geopackage(self): all_layers = root.findLayers() assert len(all_layers) == 18 - geometry_layers = set([l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer)]) - assert geometry_layers == {'Kantonale_Bauplanung_V1_1.Konstruktionen.Gebaeude', 'Kantonale_Bauplanung_V1_1.Natur.Park', 'Strasse', 'Bauplanung_V1_1.Natur.Park', 'Sonnenblumenfeld', 'Buntbrache', 'Brutstelle', 'KantonaleBuntbrache', 'Feld', 'Kartoffelfeld', 'Bauplanung_V1_1.Konstruktionen.Gebaeude'} - - tables_layers = set([l.name() for l in root.findGroup('tables').children()]) - assert tables_layers == {'Bauart', 'Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude', 'Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude', 'Kantonale_Bauplanung_V1_1.Natur.Tierart', 'Bauplanung_V1_1.Konstruktionen.Material', 'Kantonale_Bauplanung_V1_1.Konstruktionen.Material', 'Bauplanung_V1_1.Natur.Tierart'} + geometry_layers = { + l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer) + } + assert geometry_layers == { + "Kantonale_Bauplanung_V1_1.Konstruktionen.Gebaeude", + "Kantonale_Bauplanung_V1_1.Natur.Park", + "Strasse", + "Bauplanung_V1_1.Natur.Park", + "Sonnenblumenfeld", + "Buntbrache", + "Brutstelle", + "KantonaleBuntbrache", + "Feld", + "Kartoffelfeld", + "Bauplanung_V1_1.Konstruktionen.Gebaeude", + } + + tables_layers = {l.name() for l in root.findGroup("tables").children()} + assert tables_layers == { + "Bauart", + "Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", + "Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", + "Kantonale_Bauplanung_V1_1.Natur.Tierart", + "Bauplanung_V1_1.Konstruktionen.Material", + "Kantonale_Bauplanung_V1_1.Konstruktionen.Material", + "Bauplanung_V1_1.Natur.Tierart", + } # check relations - all are there relations = list(qgis_project.relationManager().relations().values()) @@ -905,7 +1216,9 @@ def test_extopt_baustruct_mssql(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2mssql importer.configuration = iliimporter_config(importer.tool) - importer.configuration.ilifile = testdata_path("ilimodels/Kantonale_Bauplanung_V1_1.ili") + importer.configuration.ilifile = testdata_path( + "ilimodels/Kantonale_Bauplanung_V1_1.ili" + ) importer.configuration.ilimodels = "Kantonale_Bauplanung_V1_1" importer.configuration.dbschema = "optimal_baustruct_{:%Y%m%d%H%M%S%f}".format( datetime.datetime.now() @@ -932,18 +1245,36 @@ def test_extopt_baustruct_mssql(self): available_layers = generator.layers() aliases = [l.alias for l in available_layers] - irrelevant_layer_ilinames = [l.ili_name for l in available_layers if not l.is_relevant ] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias)>1] + [l.ili_name for l in available_layers if not l.is_relevant] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 - expected_aliases = ["Bauart", "Bauplanung_V1_1.Konstruktionen.Gebaeude", "Bauplanung_V1_1.Konstruktionen.Material", "Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", "Bauplanung_V1_1.Natur.Park", "Bauplanung_V1_1.Natur.Tierart", "Brutstelle", "Buntbrache", "Feld", "KantonaleBuntbrache", "Kantonale_Bauplanung_V1_1.Konstruktionen.Gebaeude", "Kantonale_Bauplanung_V1_1.Konstruktionen.Material", "Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", "Kantonale_Bauplanung_V1_1.Natur.Park", "Kantonale_Bauplanung_V1_1.Natur.Tierart", "Kartoffelfeld", "Sonnenblumenfeld", "Strasse"] + expected_aliases = [ + "Bauart", + "Bauplanung_V1_1.Konstruktionen.Gebaeude", + "Bauplanung_V1_1.Konstruktionen.Material", + "Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", + "Bauplanung_V1_1.Natur.Park", + "Bauplanung_V1_1.Natur.Tierart", + "Brutstelle", + "Buntbrache", + "Feld", + "KantonaleBuntbrache", + "Kantonale_Bauplanung_V1_1.Konstruktionen.Gebaeude", + "Kantonale_Bauplanung_V1_1.Konstruktionen.Material", + "Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", + "Kantonale_Bauplanung_V1_1.Natur.Park", + "Kantonale_Bauplanung_V1_1.Natur.Tierart", + "Kartoffelfeld", + "Sonnenblumenfeld", + "Strasse", + ] assert set(aliases) == set(expected_aliases) # irrelevant layers are detected - #todo assert len(irrelevant_layer_ilinames) > 0 - expected_irrelevant_layer_ilinames = ["Bauplanung_V1_1.Konstruktionen.Material", "Bauplanung_V1_1.Konstruktionen.Gebaeude", "Bauplanung_V1_1.Natur.Tierart", "Bauplanung_V1_1.Natur.Park"] - #todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + # todo assert len(irrelevant_layer_ilinames) > 0 + # todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) def print_info(self, text): logging.info(text) @@ -952,4 +1283,4 @@ def print_error(self, text): logging.error(text) def tearDown(self): - QgsProject.instance().removeAllMapLayers() \ No newline at end of file + QgsProject.instance().removeAllMapLayers() diff --git a/tests/test_validate.py b/tests/test_validate.py index 7f2d5b9..4fe901b 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -518,17 +518,20 @@ def test_validate_exportmodels_gpkg(self): assert result_model.rowCount() == 3 expected_error_messages = [ - #staedtisches gewerbe + # staedtisches gewerbe "Needs an ethical evaluation (EthischeBeurteilung)", - #kantonale ortsplanung + # kantonale ortsplanung "Beschreibung and/or Referenzcode must be defined.", - #staedtische ortsplanung - "Beschreibung needed when top secret." + # staedtische ortsplanung + "Beschreibung needed when top secret.", ] - error_messages = [ result_model.index(i, 0).data( + error_messages = [ + result_model.index(i, 0).data( int(ilivalidator.ValidationResultModel.Roles.MESSAGE) - ) for i in range(3) ] + ) + for i in range(3) + ] assert set(error_messages) == set(expected_error_messages) diff --git a/tests/testdata/ilimodels/Bauplanung_V1_1.ili b/tests/testdata/ilimodels/Bauplanung_V1_1.ili index 63f804b..0224bf2 100644 --- a/tests/testdata/ilimodels/Bauplanung_V1_1.ili +++ b/tests/testdata/ilimodels/Bauplanung_V1_1.ili @@ -25,8 +25,8 @@ MODEL Bauplanung_V1_1 (en) AT "https://modelbaker.ch" VERSION "2023-07-13" = Materialien: BAG {1..*} OF Material; Geometrie : MANDATORY Infrastruktur_V1.CHSurface; Test : Bauplanung_V1_1.Bauart; - END Gebaeude; - + END Gebaeude; + /*new many-to-many association*/ ASSOCIATION Strassen_Gebaeude = Strasse (EXTERNAL) -- {0..*} Infrastruktur_V1.Strassen.Strasse; @@ -35,7 +35,7 @@ MODEL Bauplanung_V1_1 (en) AT "https://modelbaker.ch" VERSION "2023-07-13" = END Konstruktionen; - TOPIC Natur = + TOPIC Natur = CLASS Park = Geometrie : MANDATORY Infrastruktur_V1.CHSurface; END Park; @@ -60,15 +60,15 @@ MODEL Bauplanung_V1_1 (en) AT "https://modelbaker.ch" VERSION "2023-07-13" = Brutstelle -- {1..*} Brutstelle; END Brutstelle_Tierart; - CLASS Buntbrache = + CLASS Buntbrache = Geometrie : MANDATORY Infrastruktur_V1.CHSurface; END Buntbrache; - CLASS Feld = + CLASS Feld = Name : TEXT; Geometrie : MANDATORY Infrastruktur_V1.CHSurface; END Feld; END Natur; -END Bauplanung_V1_1. \ No newline at end of file +END Bauplanung_V1_1. diff --git a/tests/testdata/ilimodels/Kantonale_Bauplanung_V1_1.ili b/tests/testdata/ilimodels/Kantonale_Bauplanung_V1_1.ili index 258fdc9..289b653 100644 --- a/tests/testdata/ilimodels/Kantonale_Bauplanung_V1_1.ili +++ b/tests/testdata/ilimodels/Kantonale_Bauplanung_V1_1.ili @@ -26,7 +26,7 @@ MODEL Kantonale_Bauplanung_V1_1 (en) AT "https://modelbaker.ch" VERSION "2023-07 END Konstruktionen; - TOPIC Natur EXTENDS Bauplanung_V1_1.Natur = + TOPIC Natur EXTENDS Bauplanung_V1_1.Natur = CLASS Park (EXTENDED) = Details: TEXT; @@ -37,18 +37,18 @@ MODEL Kantonale_Bauplanung_V1_1 (en) AT "https://modelbaker.ch" VERSION "2023-07 END Tierart; !! Extension with another name - CLASS KantonaleBuntbrache EXTENDS Buntbrache= + CLASS KantonaleBuntbrache EXTENDS Buntbrache= Gemeinde: TEXT; END KantonaleBuntbrache; !! Multiple extensions - CLASS Kartoffelfeld EXTENDS Feld = + CLASS Kartoffelfeld EXTENDS Feld = Anzahl: 1 .. 1000; END Kartoffelfeld; - CLASS Sonnenblumenfeld EXTENDS Feld = + CLASS Sonnenblumenfeld EXTENDS Feld = Art: TEXT; END Sonnenblumenfeld; END Natur; - -END Kantonale_Bauplanung_V1_1. \ No newline at end of file + +END Kantonale_Bauplanung_V1_1. diff --git a/tests/testdata/ilimodels/Polymorphic_Ortsplanung_V1_1.ili b/tests/testdata/ilimodels/Polymorphic_Ortsplanung_V1_1.ili index dbc466f..4820fbb 100644 --- a/tests/testdata/ilimodels/Polymorphic_Ortsplanung_V1_1.ili +++ b/tests/testdata/ilimodels/Polymorphic_Ortsplanung_V1_1.ili @@ -3,7 +3,7 @@ INTERLIS 2.3; /* Model to test several types of extended classes */ MODEL Polymorphic_Ortsplanung_V1_1 (en) AT "https://modelbaker.ch" VERSION "2023-07-13" = IMPORTS Ortsplanung_V1_1,Infrastruktur_V1; - + TOPIC Gewerbe EXTENDS Ortsplanung_V1_1.Konstruktionen = OID AS INTERLIS.UUIDOID; DEPENDS ON Infrastruktur_V1.Strassen; @@ -22,7 +22,7 @@ MODEL Polymorphic_Ortsplanung_V1_1 (en) AT "https://modelbaker.ch" VERSION "2023 Unterhaltungsart : TEXT*99; END Gebaeude; - END Freizeit; + END Freizeit; TOPIC IndustrieGewerbe EXTENDS Polymorphic_Ortsplanung_V1_1.Gewerbe = OID AS INTERLIS.UUIDOID; @@ -69,4 +69,4 @@ MODEL Polymorphic_Ortsplanung_V1_1 (en) AT "https://modelbaker.ch" VERSION "2023 END Gebaeude; END Konstruktionen; -END Polymorphic_Ortsplanung_V1_1. \ No newline at end of file +END Polymorphic_Ortsplanung_V1_1. From 24cd5a86ecd73fd3dd26096d89fb72c252feaf3e Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 11 Sep 2023 22:43:55 +0200 Subject: [PATCH 15/34] AFAIK it does not make to chech if the 'parent' layer has the same srid - this only leads to the situation that many-to-many with geometry layers cannot be added. This fixes https://github.com/opengisch/QgisModelBaker/issues/826 But we do need to consider, that talbes can be purelinking tables with additional links to basket-table. This has not been considered before. --- modelbaker/dataobjects/layers.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/modelbaker/dataobjects/layers.py b/modelbaker/dataobjects/layers.py index 70c76e2..96ec2e9 100644 --- a/modelbaker/dataobjects/layers.py +++ b/modelbaker/dataobjects/layers.py @@ -31,7 +31,7 @@ ) from qgis.PyQt.QtCore import QCoreApplication, QSettings -from ..generator.config import IGNORED_FIELDNAMES +from ..generator.config import BASKET_FIELDNAMES, IGNORED_FIELDNAMES from ..utils.globals import OptimizeStrategy from .form import Form, FormFieldWidget, FormRelationWidget, FormTab @@ -281,13 +281,6 @@ def post_generate(self, project): ): continue - # relations to the same table with different geometries should not be added - if ( - self.srid - and nm_relation.referenced_layer.srid == self.srid - ): - continue - if nm_relation.referenced_layer.is_basket_table: continue @@ -347,7 +340,7 @@ def isPureLinkTable(self, project): remaining_fields = set() for field in self.fields: - if field.name not in IGNORED_FIELDNAMES: + if field.name not in IGNORED_FIELDNAMES + BASKET_FIELDNAMES: remaining_fields.add(field.name) # Remove all fields that are referencing fields From ceb41527e4442e66cbd37c656aa8578ed8defe26 Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 11 Sep 2023 22:46:26 +0200 Subject: [PATCH 16/34] finalize tests for GPKG --- .../test_projectgen_extension_optimization.py | 277 +++++++++++++++++- 1 file changed, 269 insertions(+), 8 deletions(-) diff --git a/tests/test_projectgen_extension_optimization.py b/tests/test_projectgen_extension_optimization.py index 54ca1e3..f1ff658 100644 --- a/tests/test_projectgen_extension_optimization.py +++ b/tests/test_projectgen_extension_optimization.py @@ -1125,14 +1125,14 @@ def test_extopt_baustruct_geopackage(self): "Feld", "Buntbrache", "Brutstelle", - "Bauplanung_V1_1.Natur.Park", - "Bauplanung_V1_1.Konstruktionen.Gebaeude", "Kantonale_Bauplanung_V1_1.Natur.Tierart", "Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", "Kantonale_Bauplanung_V1_1.Konstruktionen.Material", - "Bauplanung_V1_1.Natur.Tierart", + "Bauplanung_V1_1.Natur.Park", + "Bauplanung_V1_1.Konstruktionen.Gebaeude", "Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", "Bauplanung_V1_1.Konstruktionen.Material", + "Bauplanung_V1_1.Natur.Tierart", "Bauart", ] assert set(aliases) == set(expected_aliases) @@ -1140,7 +1140,12 @@ def test_extopt_baustruct_geopackage(self): # irrelevant layers are detected assert len(irrelevant_layer_ilinames) > 0 expected_irrelevant_layer_ilinames = [ - "Ortsplanung_V1_1.Konstruktionen.Gebaeude" + "Bauplanung_V1_1.Natur.Park", + "Bauplanung_V1_1.Konstruktionen.Gebaeude", + "Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", + "Bauplanung_V1_1.Konstruktionen.Material", + "Bauplanung_V1_1.Natur.Tierart", + "Bauplanung_V1_1.Natur.Feld", # because extended multiple times ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) @@ -1199,16 +1204,272 @@ def test_extopt_baustruct_geopackage(self): if layer.layer.name() == "Strasse": efc = layer.layer.editFormConfig() # one general and four relation editors - assert len(efc.tabs()) == 9 + assert len(efc.tabs()) == 4 for tab in efc.tabs(): + print(f"-------- {tab.name()}") + if tab.name() == "kantnl_ng_v1_1konstruktionen_gebaeude": + count += 1 + assert len(tab.children()) == 1 if tab.name() == "gebaeude": count += 1 assert len(tab.children()) == 1 - if tab.name() == "polymrpng_v1_1gewerbe_gebaeude": + # should find 3 (one times gebaeude and two times kantnl_ng_v1_1konstruktionen_gebaeude because it's extended) + assert count == 3 + + QgsProject.instance().clear() + + ### 2. OptimizeStrategy.GROUP ### + strategy = OptimizeStrategy.GROUP + + generator = Generator( + tool=DbIliMode.ili2gpkg, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) + + available_layers = generator.layers() + relations, _ = generator.relations(available_layers) + legend = generator.legend(available_layers) + + aliases = [l.alias for l in available_layers] + irrelevant_layer_ilinames = [ + l.ili_name for l in available_layers if not l.is_relevant + ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] + + # check no ambiguous layers exists + assert len(ambiguous_aliases) == 0 + expected_aliases = [ + "Strasse", + "Sonnenblumenfeld", + "Kartoffelfeld", + "KantonaleBuntbrache", + "Kantonale_Bauplanung_V1_1.Natur.Park", + "Kantonale_Bauplanung_V1_1.Konstruktionen.Gebaeude", + "Feld", + "Buntbrache", + "Brutstelle", + "Kantonale_Bauplanung_V1_1.Natur.Tierart", + "Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", + "Kantonale_Bauplanung_V1_1.Konstruktionen.Material", + "Bauplanung_V1_1.Natur.Park", + "Bauplanung_V1_1.Konstruktionen.Gebaeude", + "Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", + "Bauplanung_V1_1.Konstruktionen.Material", + "Bauplanung_V1_1.Natur.Tierart", + "Bauart", + ] + assert set(aliases) == set(expected_aliases) + + # irrelevant layers are detected + assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = [ + "Bauplanung_V1_1.Natur.Park", + "Bauplanung_V1_1.Konstruktionen.Gebaeude", + "Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", + "Bauplanung_V1_1.Konstruktionen.Material", + "Bauplanung_V1_1.Natur.Tierart", + "Bauplanung_V1_1.Natur.Feld", # because extended multiple times + ] + assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + + project = Project(optimize_strategy=strategy) + project.layers = available_layers + project.relations = relations + project.legend = legend + project.post_generate() + + qgis_project = QgsProject.instance() + project.create(None, qgis_project) + + # check layertree + root = qgis_project.layerTreeRoot() + assert root is not None + + all_layers = root.findLayers() + assert len(all_layers) == 18 + + geometry_layers = { + l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer) + } + assert geometry_layers == { + "Kantonale_Bauplanung_V1_1.Konstruktionen.Gebaeude", + "Kantonale_Bauplanung_V1_1.Natur.Park", + "Strasse", + "Sonnenblumenfeld", + "Buntbrache", + "Brutstelle", + "KantonaleBuntbrache", + "Kartoffelfeld", + } + + tables_layers = {l.name() for l in root.findGroup("tables").children()} + assert tables_layers == { + "Bauart", + "Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", + "Kantonale_Bauplanung_V1_1.Natur.Tierart", + "Kantonale_Bauplanung_V1_1.Konstruktionen.Material", + } + + base_group = root.findGroup("base layers") + assert base_group + + grouped_base_layers = { + l.name() for l in base_group.children() if isinstance(l, QgsLayerTreeLayer) + } + + assert grouped_base_layers == { + "Bauplanung_V1_1.Natur.Park", + "Bauplanung_V1_1.Konstruktionen.Gebaeude", + "Feld", + } + + base_table_group = base_group.findGroup("base tables") + assert base_table_group + + grouped_base_tables = { + l.name() + for l in base_table_group.children() + if isinstance(l, QgsLayerTreeLayer) + } + + assert grouped_base_tables == { + "Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", + "Bauplanung_V1_1.Konstruktionen.Material", + "Bauplanung_V1_1.Natur.Tierart", + } + + # check relations - all are there + relations = list(qgis_project.relationManager().relations().values()) + assert len(relations) == 18 + + # strasse should have relation editors to all layers (3/2) - since extended relation it's two times + count = 0 + for layer in project.layers: + if layer.layer.name() == "Strasse": + efc = layer.layer.editFormConfig() + # one general and four relation editors + assert len(efc.tabs()) == 2 + for tab in efc.tabs(): + print(f"-------- {tab.name()}") + if tab.name() == "kantnl_ng_v1_1konstruktionen_gebaeude": count += 1 assert len(tab.children()) == 1 - # should find 2 - assert count == 2 + if tab.name() == "gebaeude": # this should not happen + count += 1 + assert len(tab.children()) == 1 + # should find 1 (one times kantnl_ng_v1_1konstruktionen_gebaeude) + assert count == 1 + + QgsProject.instance().clear() + + ### 3. OptimizeStrategy.HIDE ### + strategy = OptimizeStrategy.HIDE + + generator = Generator( + tool=DbIliMode.ili2gpkg, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) + + available_layers = generator.layers() + relations, _ = generator.relations(available_layers) + legend = generator.legend(available_layers) + + aliases = [l.alias for l in available_layers] + irrelevant_layer_ilinames = [ + l.ili_name for l in available_layers if not l.is_relevant + ] + ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] + + # ambiguous aliases exist, since we don't rename them when they are hidden anyway + assert len(ambiguous_aliases) == 10 + expected_aliases = [ + "Strasse", + "Sonnenblumenfeld", + "Kartoffelfeld", + "KantonaleBuntbrache", + "Park", + "Gebaeude", + "Buntbrache", + "Brutstelle", + "Tierart", + "Strassen_Gebaeude", + "Material", + "Bauart", + "Feld", + ] + assert set(aliases) == set(expected_aliases) + + # irrelevant layers are detected + assert len(irrelevant_layer_ilinames) > 0 + expected_irrelevant_layer_ilinames = [ + "Bauplanung_V1_1.Natur.Park", + "Bauplanung_V1_1.Konstruktionen.Gebaeude", + "Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", + "Bauplanung_V1_1.Konstruktionen.Material", + "Bauplanung_V1_1.Natur.Tierart", + "Bauplanung_V1_1.Natur.Feld", # because extended multiple times + ] + assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + + project = Project(optimize_strategy=strategy) + project.layers = available_layers + project.relations = relations + project.legend = legend + project.post_generate() + + qgis_project = QgsProject.instance() + project.create(None, qgis_project) + + # check layertree + root = qgis_project.layerTreeRoot() + assert root is not None + + all_layers = root.findLayers() + assert len(all_layers) == 12 + + geometry_layers = { + l.name() for l in root.children() if isinstance(l, QgsLayerTreeLayer) + } + assert geometry_layers == { + "Gebaeude", + "Park", + "Strasse", + "Park", + "Sonnenblumenfeld", + "Buntbrache", + "Brutstelle", + "KantonaleBuntbrache", + "Kartoffelfeld", + } + + tables_layers = {l.name() for l in root.findGroup("tables").children()} + assert tables_layers == {"Bauart", "Strassen_Gebaeude", "Tierart", "Material"} + + # check relations - all are there + relations = list(qgis_project.relationManager().relations().values()) + assert len(relations) == 7 + + # strasse should have relation editors to only relevant layers + count = 0 + for layer in project.layers: + if layer.layer.name() == "Strasse": + efc = layer.layer.editFormConfig() + # one general and four relation editors + assert len(efc.tabs()) == 2 + for tab in efc.tabs(): + print(f"-------- {tab.name()}") + if tab.name() == "kantnl_ng_v1_1konstruktionen_gebaeude": + count += 1 + assert len(tab.children()) == 1 + if tab.name() == "gebaeude": # this should not happen + count += 1 + assert len(tab.children()) == 1 + # should find 1 (one times kantnl_ng_v1_1konstruktionen_gebaeude) + assert count == 1 QgsProject.instance().clear() From 5fe5d2f1944cb5b0d1f6ad004c65a2da04b13fa2 Mon Sep 17 00:00:00 2001 From: signedav Date: Tue, 12 Sep 2023 22:23:21 +0200 Subject: [PATCH 17/34] tests for mssql and postgres --- .../test_projectgen_extension_optimization.py | 669 ++++++++++-------- 1 file changed, 367 insertions(+), 302 deletions(-) diff --git a/tests/test_projectgen_extension_optimization.py b/tests/test_projectgen_extension_optimization.py index f1ff658..7762568 100644 --- a/tests/test_projectgen_extension_optimization.py +++ b/tests/test_projectgen_extension_optimization.py @@ -85,36 +85,44 @@ def test_extopt_staedtische_postgis(self): importer.stderr.connect(self.print_error) assert importer.run() == iliimporter.Importer.SUCCESS + ### 1. OptimizeStrategy.NONE ### + strategy = OptimizeStrategy.NONE + generator = Generator( - DbIliMode.ili2pg, - get_pg_connection_string(), - importer.configuration.inheritance, - importer.configuration.dbschema, + tool=DbIliMode.ili2pg, + uri=get_pg_connection_string(), + inheritance="smart2", + schema=importer.configuration.dbschema, + optimize_strategy=strategy, ) - available_layers = generator.layers() - aliases = [l.alias for l in available_layers] - [l.ili_name for l in available_layers if not l.is_relevant] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] + self._extopt_staedtische_none(generator, strategy) - # check no ambiguous layers exists - assert len(ambiguous_aliases) == 0 - expected_aliases = [ - "BesitzerIn", - "Freizeit.Gebaeude", - "Gebaeude_StadtFirma", - "Gewerbe.Gebaeude", - "Gewerbe_V1.Firmen.Firma", - "Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude", - "Ortsplanung_V1_1.Konstruktionen.Gebaeude", - "Staedtisches_Gewerbe_V1.Firmen.Firma", - "Strasse", - ] - assert set(aliases) == set(expected_aliases) + ### 2. OptimizeStrategy.GROUP ### + strategy = OptimizeStrategy.GROUP - # irrelevant layers are detected - # todo assert len(irrelevant_layer_ilinames) > 0 - # todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + generator = Generator( + tool=DbIliMode.ili2pg, + uri=get_pg_connection_string(), + inheritance="smart2", + schema=importer.configuration.dbschema, + optimize_strategy=strategy, + ) + + self._extopt_staedtische_group(generator, strategy) + + ### 3. OptimizeStrategy.HIDE ### + strategy = OptimizeStrategy.HIDE + + generator = Generator( + tool=DbIliMode.ili2pg, + uri=get_pg_connection_string(), + inheritance="smart2", + schema=importer.configuration.dbschema, + optimize_strategy=strategy, + ) + + self._extopt_staedtische_hide(generator, strategy) def test_extopt_staedtische_geopackage(self): importer = iliimporter.Importer() @@ -150,6 +158,98 @@ def test_extopt_staedtische_geopackage(self): optimize_strategy=strategy, ) + self._extopt_staedtische_none(generator, strategy) + + ### 2. OptimizeStrategy.GROUP ### + strategy = OptimizeStrategy.GROUP + + generator = Generator( + tool=DbIliMode.ili2gpkg, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) + + self._extopt_staedtische_group(generator, strategy) + + ### 3. OptimizeStrategy.HIDE ### + strategy = OptimizeStrategy.HIDE + + generator = Generator( + tool=DbIliMode.ili2gpkg, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) + + self._extopt_staedtische_hide(generator, strategy) + + def test_extopt_staedtische_mssql(self): + return # todo + importer = iliimporter.Importer() + importer.tool = DbIliMode.ili2mssql + importer.configuration = iliimporter_config(importer.tool) + importer.configuration.ilifile = testdata_path( + "ilimodels/Staedtische_Ortsplanung_V1_1.ili" + ) + importer.configuration.ilimodels = "Staedtische_Ortsplanung_V1_1" + importer.configuration.dbschema = ( + "optimal_staedtische_{:%Y%m%d%H%M%S%f}".format(datetime.datetime.now()) + ) + importer.configuration.srs_code = 2056 + importer.configuration.inheritance = "smart2" + importer.configuration.create_basket_col = True + importer.stdout.connect(self.print_info) + importer.stderr.connect(self.print_error) + + uri = "DRIVER={drv};SERVER={server};DATABASE={db};UID={uid};PWD={pwd}".format( + drv="{ODBC Driver 17 for SQL Server}", + server=importer.configuration.dbhost, + db=importer.configuration.database, + uid=importer.configuration.dbusr, + pwd=importer.configuration.dbpwd, + ) + + assert importer.run() == iliimporter.Importer.SUCCESS + + ### 1. OptimizeStrategy.NONE ### + strategy = OptimizeStrategy.NONE + + generator = Generator( + tool=DbIliMode.ili2mssql, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) + + self._extopt_staedtische_none(generator, strategy) + + ### 2. OptimizeStrategy.GROUP ### + strategy = OptimizeStrategy.GROUP + + generator = Generator( + tool=DbIliMode.ili2mssql, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) + + self._extopt_staedtische_group(generator, strategy) + + ### 3. OptimizeStrategy.HIDE ### + strategy = OptimizeStrategy.HIDE + + generator = Generator( + tool=DbIliMode.ili2mssql, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) + + self._extopt_staedtische_hide(generator, strategy) + + def _extopt_staedtische_none(self, generator, strategy): + available_layers = generator.layers() relations, _ = generator.relations(available_layers) legend = generator.legend(available_layers) @@ -158,7 +258,7 @@ def test_extopt_staedtische_geopackage(self): irrelevant_layer_ilinames = [ l.ili_name for l in available_layers if not l.is_relevant ] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] + [alias for alias in aliases if aliases.count(alias) > 1] # check no ambiguous layers exists expected_aliases = [ @@ -247,16 +347,7 @@ def test_extopt_staedtische_geopackage(self): QgsProject.instance().clear() - ### 2. OptimizeStrategy.GROUP ### - strategy = OptimizeStrategy.GROUP - - generator = Generator( - tool=DbIliMode.ili2gpkg, - uri=uri, - inheritance="smart2", - optimize_strategy=strategy, - ) - + def _extopt_staedtische_group(self, generator, strategy): available_layers = generator.layers() relations, _ = generator.relations(available_layers) legend = generator.legend(available_layers) @@ -360,15 +451,7 @@ def test_extopt_staedtische_geopackage(self): QgsProject.instance().clear() - ### 3. OptimizeStrategy.HIDE ### - strategy = OptimizeStrategy.HIDE - - generator = Generator( - tool=DbIliMode.ili2gpkg, - uri=uri, - inheritance="smart2", - optimize_strategy=strategy, - ) + def _extopt_staedtische_hide(self, generator, strategy): available_layers = generator.layers() relations, _ = generator.relations(available_layers) @@ -453,73 +536,78 @@ def test_extopt_staedtische_geopackage(self): # should find only 2 assert count == 2 - def test_extopt_staedtische_mssql(self): + def test_extopt_polymorphic_postgis(self): importer = iliimporter.Importer() - importer.tool = DbIliMode.ili2mssql + importer.tool = DbIliMode.ili2pg importer.configuration = iliimporter_config(importer.tool) importer.configuration.ilifile = testdata_path( - "ilimodels/Staedtische_Ortsplanung_V1_1.ili" + "ilimodels/Polymorphic_Ortsplanung_V1_1.ili" ) - importer.configuration.ilimodels = "Staedtische_Ortsplanung_V1_1" - importer.configuration.dbschema = ( - "optimal_staedtische_{:%Y%m%d%H%M%S%f}".format(datetime.datetime.now()) + importer.configuration.ilimodels = "Polymorphic_Ortsplanung_V1_1" + importer.configuration.dbschema = "optimal_polymorph_{:%Y%m%d%H%M%S%f}".format( + datetime.datetime.now() ) + importer.configuration.srs_code = 2056 importer.configuration.inheritance = "smart2" importer.configuration.create_basket_col = True importer.stdout.connect(self.print_info) importer.stderr.connect(self.print_error) + assert importer.run() == iliimporter.Importer.SUCCESS - uri = "DRIVER={drv};SERVER={server};DATABASE={db};UID={uid};PWD={pwd}".format( - drv="{ODBC Driver 17 for SQL Server}", - server=importer.configuration.dbhost, - db=importer.configuration.database, - uid=importer.configuration.dbusr, - pwd=importer.configuration.dbpwd, + ### 1. OptimizeStrategy.NONE ### + strategy = OptimizeStrategy.NONE + + generator = Generator( + tool=DbIliMode.ili2pg, + uri=get_pg_connection_string(), + schema=importer.configuration.dbschema, + inheritance="smart2", + optimize_strategy=strategy, ) - assert importer.run() == iliimporter.Importer.SUCCESS + self._extopt_polymorphic_none(generator, strategy) + + ### 2. OptimizeStrategy.GROUP ### + strategy = OptimizeStrategy.GROUP generator = Generator( - DbIliMode.ili2mssql, uri, "smart2", importer.configuration.dbschema + tool=DbIliMode.ili2pg, + uri=get_pg_connection_string(), + schema=importer.configuration.dbschema, + inheritance="smart2", + optimize_strategy=strategy, ) - available_layers = generator.layers() - aliases = [l.alias for l in available_layers] - [l.ili_name for l in available_layers if not l.is_relevant] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] + self._extopt_polymorphic_group(generator, strategy) - # check no ambiguous layers exists - assert len(ambiguous_aliases) == 0 - expected_aliases = [ - "BesitzerIn", - "Freizeit.Gebaeude", - "Gebaeude_StadtFirma", - "Gewerbe.Gebaeude", - "Gewerbe_V1.Firmen.Firma", - "Kantonale_Ortsplanung_V1_1.Konstruktionen.Gebaeude", - "Ortsplanung_V1_1.Konstruktionen.Gebaeude", - "Staedtisches_Gewerbe_V1.Firmen.Firma", - "Strasse", - ] - assert set(aliases) == set(expected_aliases) + ### 3. OptimizeStrategy.HIDE ### + strategy = OptimizeStrategy.HIDE - # irrelevant layers are detected - # todo assert len(irrelevant_layer_ilinames) > 0 - # todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + generator = Generator( + tool=DbIliMode.ili2pg, + uri=get_pg_connection_string(), + schema=importer.configuration.dbschema, + inheritance="smart2", + optimize_strategy=strategy, + ) - def test_extopt_polymorphic_postgis(self): + self._extopt_polymorphic_hide(generator, strategy) + + def test_extopt_polymorphic_geopackage(self): importer = iliimporter.Importer() - importer.tool = DbIliMode.ili2pg + importer.tool = DbIliMode.ili2gpkg importer.configuration = iliimporter_config(importer.tool) importer.configuration.ilifile = testdata_path( "ilimodels/Polymorphic_Ortsplanung_V1_1.ili" ) importer.configuration.ilimodels = "Polymorphic_Ortsplanung_V1_1" - importer.configuration.dbschema = "optimal_polymorph_{:%Y%m%d%H%M%S%f}".format( - datetime.datetime.now() + importer.configuration.dbfile = os.path.join( + self.basetestpath, + "tmp_optimal_polymorph_{:%Y%m%d%H%M%S%f}.gpkg".format( + datetime.datetime.now() + ), ) - importer.configuration.srs_code = 2056 importer.configuration.inheritance = "smart2" importer.configuration.create_basket_col = True @@ -527,73 +615,110 @@ def test_extopt_polymorphic_postgis(self): importer.stderr.connect(self.print_error) assert importer.run() == iliimporter.Importer.SUCCESS + config_manager = GpkgCommandConfigManager(importer.configuration) + uri = config_manager.get_uri() + + ### 1. OptimizeStrategy.NONE ### + strategy = OptimizeStrategy.NONE + generator = Generator( - DbIliMode.ili2pg, - get_pg_connection_string(), - importer.configuration.inheritance, - importer.configuration.dbschema, + tool=DbIliMode.ili2gpkg, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, ) - available_layers = generator.layers() - aliases = [l.alias for l in available_layers] - [l.ili_name for l in available_layers if not l.is_relevant] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] + self._extopt_polymorphic_none(generator, strategy) - # check no ambiguous layers exists - assert len(ambiguous_aliases) == 0 - expected_aliases = [ - "BesitzerIn", - "Freizeit.Gebaeude", - "Gewerbe.Gebaeude", - "Hallen.Gebaeude", - "IndustrieGewerbe.Gebaeude", - "Markthalle", - "Ortsplanung_V1_1.Konstruktionen.Gebaeude", - "Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude", - "Strasse", - "TurnhalleTyp1", - "TurnhalleTyp2", - ] - assert set(aliases) == set(expected_aliases) + ### 2. OptimizeStrategy.GROUP ### + strategy = OptimizeStrategy.GROUP - # irrelevant layers are detected - # todo assert len(irrelevant_layer_ilinames) > 0 - # todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + generator = Generator( + tool=DbIliMode.ili2gpkg, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) - def test_extopt_polymorphic_geopackage(self): + self._extopt_polymorphic_group(generator, strategy) + + ### 3. OptimizeStrategy.HIDE ### + strategy = OptimizeStrategy.HIDE + + generator = Generator( + tool=DbIliMode.ili2gpkg, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) + + self._extopt_polymorphic_hide(generator, strategy) + + def test_extopt_polymorphic_mssql(self): + return # todo importer = iliimporter.Importer() - importer.tool = DbIliMode.ili2gpkg + importer.tool = DbIliMode.ili2mssql importer.configuration = iliimporter_config(importer.tool) importer.configuration.ilifile = testdata_path( "ilimodels/Polymorphic_Ortsplanung_V1_1.ili" ) importer.configuration.ilimodels = "Polymorphic_Ortsplanung_V1_1" - importer.configuration.dbfile = os.path.join( - self.basetestpath, - "tmp_optimal_polymorph_{:%Y%m%d%H%M%S%f}.gpkg".format( - datetime.datetime.now() - ), + importer.configuration.dbschema = "optimal_polymorph_{:%Y%m%d%H%M%S%f}".format( + datetime.datetime.now() ) importer.configuration.srs_code = 2056 importer.configuration.inheritance = "smart2" importer.configuration.create_basket_col = True importer.stdout.connect(self.print_info) importer.stderr.connect(self.print_error) - assert importer.run() == iliimporter.Importer.SUCCESS - config_manager = GpkgCommandConfigManager(importer.configuration) - uri = config_manager.get_uri() + uri = "DRIVER={drv};SERVER={server};DATABASE={db};UID={uid};PWD={pwd}".format( + drv="{ODBC Driver 17 for SQL Server}", + server=importer.configuration.dbhost, + db=importer.configuration.database, + uid=importer.configuration.dbusr, + pwd=importer.configuration.dbpwd, + ) + + assert importer.run() == iliimporter.Importer.SUCCESS ### 1. OptimizeStrategy.NONE ### strategy = OptimizeStrategy.NONE generator = Generator( - tool=DbIliMode.ili2gpkg, + tool=DbIliMode.ili2mssql, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) + + self._extopt_polymorphic_none(generator, strategy) + + ### 2. OptimizeStrategy.GROUP ### + strategy = OptimizeStrategy.GROUP + + generator = Generator( + tool=DbIliMode.ili2mssql, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) + + self._extopt_polymorphic_group(generator, strategy) + + ### 3. OptimizeStrategy.HIDE ### + strategy = OptimizeStrategy.HIDE + + generator = Generator( + tool=DbIliMode.ili2mssql, uri=uri, inheritance="smart2", optimize_strategy=strategy, ) + self._extopt_polymorphic_hide(generator, strategy) + + def _extopt_polymorphic_none(self, generator, strategy): available_layers = generator.layers() relations, _ = generator.relations(available_layers) legend = generator.legend(available_layers) @@ -706,15 +831,7 @@ def test_extopt_polymorphic_geopackage(self): QgsProject.instance().clear() - ### 2. OptimizeStrategy.GROUP ### - strategy = OptimizeStrategy.GROUP - - generator = Generator( - tool=DbIliMode.ili2gpkg, - uri=uri, - inheritance="smart2", - optimize_strategy=strategy, - ) + def _extopt_polymorphic_group(self, generator, strategy): available_layers = generator.layers() relations, _ = generator.relations(available_layers) @@ -836,15 +953,7 @@ def test_extopt_polymorphic_geopackage(self): QgsProject.instance().clear() - ### 3. OptimizeStrategy.HIDE ### - strategy = OptimizeStrategy.HIDE - - generator = Generator( - tool=DbIliMode.ili2gpkg, - uri=uri, - inheritance="smart2", - optimize_strategy=strategy, - ) + def _extopt_polymorphic_hide(self, generator, strategy): available_layers = generator.layers() relations, _ = generator.relations(available_layers) @@ -953,75 +1062,83 @@ def test_extopt_polymorphic_geopackage(self): QgsProject.instance().clear() - def test_extopt_polymorphic_mssql(self): + def test_extopt_baustruct_postgis(self): importer = iliimporter.Importer() - importer.tool = DbIliMode.ili2mssql + importer.tool = DbIliMode.ili2pg importer.configuration = iliimporter_config(importer.tool) importer.configuration.ilifile = testdata_path( - "ilimodels/Polymorphic_Ortsplanung_V1_1.ili" + "ilimodels/Kantonale_Bauplanung_V1_1.ili" ) - importer.configuration.ilimodels = "Polymorphic_Ortsplanung_V1_1" - importer.configuration.dbschema = "optimal_polymorph_{:%Y%m%d%H%M%S%f}".format( - datetime.datetime.now() + importer.configuration.ilimodels = "Kantonale_Bauplanung_V1_1" + importer.configuration.dbfile = os.path.join( + self.basetestpath, + "tmp_optimal_baustruct_{:%Y%m%d%H%M%S%f}.gpkg".format( + datetime.datetime.now() + ), ) importer.configuration.srs_code = 2056 importer.configuration.inheritance = "smart2" importer.configuration.create_basket_col = True importer.stdout.connect(self.print_info) importer.stderr.connect(self.print_error) + assert importer.run() == iliimporter.Importer.SUCCESS - uri = "DRIVER={drv};SERVER={server};DATABASE={db};UID={uid};PWD={pwd}".format( - drv="{ODBC Driver 17 for SQL Server}", - server=importer.configuration.dbhost, - db=importer.configuration.database, - uid=importer.configuration.dbusr, - pwd=importer.configuration.dbpwd, + config_manager = GpkgCommandConfigManager(importer.configuration) + config_manager.get_uri() + + ### 1. OptimizeStrategy.NONE ### + strategy = OptimizeStrategy.NONE + + generator = Generator( + tool=DbIliMode.ili2pg, + uri=get_pg_connection_string(), + inheritance="smart2", + schema=importer.configuration.dbschema, + optimize_strategy=strategy, ) - assert importer.run() == iliimporter.Importer.SUCCESS + self._extopt_baustruct_none(generator, strategy) + + ### 2. OptimizeStrategy.GROUP ### + strategy = OptimizeStrategy.GROUP generator = Generator( - DbIliMode.ili2mssql, uri, "smart2", importer.configuration.dbschema + tool=DbIliMode.ili2pg, + uri=get_pg_connection_string(), + inheritance="smart2", + schema=importer.configuration.dbschema, + optimize_strategy=strategy, ) - available_layers = generator.layers() - aliases = [l.alias for l in available_layers] - [l.ili_name for l in available_layers if not l.is_relevant] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] + self._extopt_baustruct_group(generator, strategy) - # check no ambiguous layers exists - assert len(ambiguous_aliases) == 0 - expected_aliases = [ - "BesitzerIn", - "Freizeit.Gebaeude", - "Gewerbe.Gebaeude", - "Hallen.Gebaeude", - "IndustrieGewerbe.Gebaeude", - "Markthalle", - "Ortsplanung_V1_1.Konstruktionen.Gebaeude", - "Polymorphic_Ortsplanung_V1_1.Konstruktionen.Gebaeude", - "Strasse", - "TurnhalleTyp1", - "TurnhalleTyp2", - ] - assert set(aliases) == set(expected_aliases) + ### 3. OptimizeStrategy.HIDE ### + strategy = OptimizeStrategy.HIDE - # irrelevant layers are detected - # todo assert len(irrelevant_layer_ilinames) > 0 - # todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + generator = Generator( + tool=DbIliMode.ili2pg, + uri=get_pg_connection_string(), + inheritance="smart2", + schema=importer.configuration.dbschema, + optimize_strategy=strategy, + ) - def test_extopt_baustruct_postgis(self): + self._extopt_baustruct_hide(generator, strategy) + + def test_extopt_baustruct_geopackage(self): importer = iliimporter.Importer() - importer.tool = DbIliMode.ili2pg + importer.tool = DbIliMode.ili2gpkg importer.configuration = iliimporter_config(importer.tool) importer.configuration.ilifile = testdata_path( "ilimodels/Kantonale_Bauplanung_V1_1.ili" ) importer.configuration.ilimodels = "Kantonale_Bauplanung_V1_1" - importer.configuration.dbschema = "optimal_baustruct_{:%Y%m%d%H%M%S%f}".format( - datetime.datetime.now() + importer.configuration.dbfile = os.path.join( + self.basetestpath, + "tmp_optimal_baustruct_{:%Y%m%d%H%M%S%f}.gpkg".format( + datetime.datetime.now() + ), ) - importer.configuration.srs_code = 2056 importer.configuration.inheritance = "smart2" importer.configuration.create_basket_col = True @@ -1029,49 +1146,49 @@ def test_extopt_baustruct_postgis(self): importer.stderr.connect(self.print_error) assert importer.run() == iliimporter.Importer.SUCCESS + config_manager = GpkgCommandConfigManager(importer.configuration) + uri = config_manager.get_uri() + + ### 1. OptimizeStrategy.NONE ### + strategy = OptimizeStrategy.NONE + generator = Generator( - DbIliMode.ili2pg, - get_pg_connection_string(), - importer.configuration.inheritance, - importer.configuration.dbschema, + tool=DbIliMode.ili2gpkg, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, ) - available_layers = generator.layers() - aliases = [l.alias for l in available_layers] - [l.ili_name for l in available_layers if not l.is_relevant] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] + self._extopt_baustruct_none(generator, strategy) - # check no ambiguous layers exists - assert len(ambiguous_aliases) == 0 - expected_aliases = [ - "Bauart", - "Bauplanung_V1_1.Konstruktionen.Gebaeude", - "Bauplanung_V1_1.Konstruktionen.Material", - "Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", - "Bauplanung_V1_1.Natur.Park", - "Bauplanung_V1_1.Natur.Tierart", - "Brutstelle", - "Buntbrache", - "Feld", - "KantonaleBuntbrache", - "Kantonale_Bauplanung_V1_1.Konstruktionen.Gebaeude", - "Kantonale_Bauplanung_V1_1.Konstruktionen.Material", - "Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", - "Kantonale_Bauplanung_V1_1.Natur.Park", - "Kantonale_Bauplanung_V1_1.Natur.Tierart", - "Kartoffelfeld", - "Sonnenblumenfeld", - "Strasse", - ] - assert set(aliases) == set(expected_aliases) + ### 2. OptimizeStrategy.GROUP ### + strategy = OptimizeStrategy.GROUP - # irrelevant layers are detected - # todo assert len(irrelevant_layer_ilinames) > 0 - # todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + generator = Generator( + tool=DbIliMode.ili2gpkg, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) - def test_extopt_baustruct_geopackage(self): + self._extopt_baustruct_group(generator, strategy) + + ### 3. OptimizeStrategy.HIDE ### + strategy = OptimizeStrategy.HIDE + + generator = Generator( + tool=DbIliMode.ili2gpkg, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) + + self._extopt_baustruct_hide(generator, strategy) + + def test_extopt_baustruct_mssql(self): + return # todo importer = iliimporter.Importer() - importer.tool = DbIliMode.ili2gpkg + importer.tool = DbIliMode.ili2mssql importer.configuration = iliimporter_config(importer.tool) importer.configuration.ilifile = testdata_path( "ilimodels/Kantonale_Bauplanung_V1_1.ili" @@ -1097,12 +1214,40 @@ def test_extopt_baustruct_geopackage(self): strategy = OptimizeStrategy.NONE generator = Generator( - tool=DbIliMode.ili2gpkg, + tool=DbIliMode.ili2mssql, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) + + self._extopt_baustruct_none(generator, strategy) + + ### 2. OptimizeStrategy.GROUP ### + strategy = OptimizeStrategy.GROUP + + generator = Generator( + tool=DbIliMode.ili2mssql, + uri=uri, + inheritance="smart2", + optimize_strategy=strategy, + ) + + self._extopt_baustruct_group(generator, strategy) + + ### 3. OptimizeStrategy.HIDE ### + strategy = OptimizeStrategy.HIDE + + generator = Generator( + tool=DbIliMode.ili2mssql, uri=uri, inheritance="smart2", optimize_strategy=strategy, ) + self._extopt_baustruct_hide(generator, strategy) + + def _extopt_baustruct_none(self, generator, strategy): + available_layers = generator.layers() relations, _ = generator.relations(available_layers) legend = generator.legend(available_layers) @@ -1218,15 +1363,7 @@ def test_extopt_baustruct_geopackage(self): QgsProject.instance().clear() - ### 2. OptimizeStrategy.GROUP ### - strategy = OptimizeStrategy.GROUP - - generator = Generator( - tool=DbIliMode.ili2gpkg, - uri=uri, - inheritance="smart2", - optimize_strategy=strategy, - ) + def _extopt_baustruct_group(self, generator, strategy): available_layers = generator.layers() relations, _ = generator.relations(available_layers) @@ -1364,15 +1501,7 @@ def test_extopt_baustruct_geopackage(self): QgsProject.instance().clear() - ### 3. OptimizeStrategy.HIDE ### - strategy = OptimizeStrategy.HIDE - - generator = Generator( - tool=DbIliMode.ili2gpkg, - uri=uri, - inheritance="smart2", - optimize_strategy=strategy, - ) + def _extopt_baustruct_hide(self, generator, strategy): available_layers = generator.layers() relations, _ = generator.relations(available_layers) @@ -1473,70 +1602,6 @@ def test_extopt_baustruct_geopackage(self): QgsProject.instance().clear() - def test_extopt_baustruct_mssql(self): - importer = iliimporter.Importer() - importer.tool = DbIliMode.ili2mssql - importer.configuration = iliimporter_config(importer.tool) - importer.configuration.ilifile = testdata_path( - "ilimodels/Kantonale_Bauplanung_V1_1.ili" - ) - importer.configuration.ilimodels = "Kantonale_Bauplanung_V1_1" - importer.configuration.dbschema = "optimal_baustruct_{:%Y%m%d%H%M%S%f}".format( - datetime.datetime.now() - ) - importer.configuration.srs_code = 2056 - importer.configuration.inheritance = "smart2" - importer.configuration.create_basket_col = True - importer.stdout.connect(self.print_info) - importer.stderr.connect(self.print_error) - - uri = "DRIVER={drv};SERVER={server};DATABASE={db};UID={uid};PWD={pwd}".format( - drv="{ODBC Driver 17 for SQL Server}", - server=importer.configuration.dbhost, - db=importer.configuration.database, - uid=importer.configuration.dbusr, - pwd=importer.configuration.dbpwd, - ) - - assert importer.run() == iliimporter.Importer.SUCCESS - - generator = Generator( - DbIliMode.ili2mssql, uri, "smart2", importer.configuration.dbschema - ) - - available_layers = generator.layers() - aliases = [l.alias for l in available_layers] - [l.ili_name for l in available_layers if not l.is_relevant] - ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] - - # check no ambiguous layers exists - assert len(ambiguous_aliases) == 0 - expected_aliases = [ - "Bauart", - "Bauplanung_V1_1.Konstruktionen.Gebaeude", - "Bauplanung_V1_1.Konstruktionen.Material", - "Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", - "Bauplanung_V1_1.Natur.Park", - "Bauplanung_V1_1.Natur.Tierart", - "Brutstelle", - "Buntbrache", - "Feld", - "KantonaleBuntbrache", - "Kantonale_Bauplanung_V1_1.Konstruktionen.Gebaeude", - "Kantonale_Bauplanung_V1_1.Konstruktionen.Material", - "Kantonale_Bauplanung_V1_1.Konstruktionen.Strassen_Gebaeude", - "Kantonale_Bauplanung_V1_1.Natur.Park", - "Kantonale_Bauplanung_V1_1.Natur.Tierart", - "Kartoffelfeld", - "Sonnenblumenfeld", - "Strasse", - ] - assert set(aliases) == set(expected_aliases) - - # irrelevant layers are detected - # todo assert len(irrelevant_layer_ilinames) > 0 - # todo assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - def print_info(self, text): logging.info(text) From af5df36301c0461449522cab6dc08ed0546594a5 Mon Sep 17 00:00:00 2001 From: signedav Date: Wed, 13 Sep 2023 08:38:57 +0200 Subject: [PATCH 18/34] support postgres for relevance --- modelbaker/dbconnector/gpkg_connector.py | 3 +- modelbaker/dbconnector/pg_connector.py | 43 +++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/modelbaker/dbconnector/gpkg_connector.py b/modelbaker/dbconnector/gpkg_connector.py index d62798a..49dedb1 100644 --- a/modelbaker/dbconnector/gpkg_connector.py +++ b/modelbaker/dbconnector/gpkg_connector.py @@ -155,7 +155,8 @@ def _get_tables_info(self): ) SELECT *, ltrim(topicclass,substr(topicclass, 0, instr(topicclass, '.'))) as class_with_dot FROM topic_level_name - ) select fullname, model, topicclass, substr(class_with_dot, instr(class_with_dot,'.')+1) as class + ) + SELECT fullname, model, topicclass, substr(class_with_dot, instr(class_with_dot,'.')+1) as class FROM class_level_name ) SELECT i.baseClass as base diff --git a/modelbaker/dbconnector/pg_connector.py b/modelbaker/dbconnector/pg_connector.py index 3c31fb4..2e27935 100644 --- a/modelbaker/dbconnector/pg_connector.py +++ b/modelbaker/dbconnector/pg_connector.py @@ -192,6 +192,45 @@ def get_tables_info(self): self.schema ) model_name = "left(c.iliname, strpos(c.iliname, '.')-1) AS model," + relevance = """ + CASE WHEN c.iliname IN ( + WITH names AS ( + WITH class_level_name AS( + WITH topic_level_name AS ( + SELECT + thisClass as fullname, + substring(thisClass from 1 for position('.' in thisClass)-1) as model, + substring(thisClass from position('.' in thisClass)+1) as topicclass + FROM {schema}.t_ili2db_inheritance + ) + SELECT *, ltrim(topicclass,substring(topicclass from 1 for position('.' in topicclass)-1)) as class_with_dot + FROM topic_level_name + ) + SELECT fullname, model, topicclass, substring(class_with_dot from position('.' in class_with_dot)+1) as class + FROM class_level_name + ) + SELECT i.baseClass as base + FROM {schema}.t_ili2db_inheritance i + LEFT JOIN names extend_names + ON thisClass = extend_names.fullname + LEFT JOIN names base_names + ON baseClass = base_names.fullname + -- it's extended + WHERE baseClass IS NOT NULL + -- in a different model + AND base_names.model != extend_names.model + AND ( + -- with the same name + base_names.class = extend_names.class + OR + -- multiple times in the same extended model + (SELECT COUNT(baseClass) FROM {schema}.t_ili2db_inheritance JOIN names extend_names ON thisClass = extend_names.fullname WHERE baseClass = i.baseClass GROUP BY baseClass, extend_names.model)>1 + ) + ) + THEN FALSE ELSE TRUE END AS relevance + """.format( + schema=self.schema + ) domain_left_join = """LEFT JOIN {}.t_ili2db_table_prop p ON p.tablename = tbls.tablename AND p.tag = 'ch.ehi.ili2db.tableKind'""".format( @@ -232,7 +271,8 @@ def get_tables_info(self): {attribute_name} {coord_decimals} g.type AS simple_type, - format_type(ga.atttypid, ga.atttypmod) as formatted_type + format_type(ga.atttypid, ga.atttypmod) as formatted_type, + {relevance} FROM pg_catalog.pg_tables tbls LEFT JOIN pg_index i ON i.indrelid = CONCAT(tbls.schemaname, '."', tbls.tablename, '"')::regclass @@ -257,6 +297,7 @@ def get_tables_info(self): ili_name=ili_name, extent=extent, coord_decimals=coord_decimals, + relevance=relevance, domain_left_join=domain_left_join, alias_left_join=alias_left_join, model_where=model_where, From d574228b8b0e8e0ee569c484b9ccacbf503bde3a Mon Sep 17 00:00:00 2001 From: signedav Date: Wed, 13 Sep 2023 15:24:48 +0200 Subject: [PATCH 19/34] use default is_relevant true --- modelbaker/generator/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelbaker/generator/generator.py b/modelbaker/generator/generator.py index a1a0549..dc175ca 100644 --- a/modelbaker/generator/generator.py +++ b/modelbaker/generator/generator.py @@ -141,7 +141,7 @@ def layers(self, filter_layer_list=[]): ) is_relevant = bool( - record.get("relevance") + record.get("relevance", True) ) # it can be not relevant and still be displayed (in case of NONE) alias = record["table_alias"] if "table_alias" in record else None From 672e3ef638610ea8bf0042abcea88ec4f864e918 Mon Sep 17 00:00:00 2001 From: signedav Date: Wed, 13 Sep 2023 15:25:16 +0200 Subject: [PATCH 20/34] pg/mssql support but mssql not complete --- modelbaker/dbconnector/mssql_connector.py | 33 ++++++ modelbaker/dbconnector/pg_connector.py | 56 +++++----- .../test_projectgen_extension_optimization.py | 101 +++++++++--------- 3 files changed, 112 insertions(+), 78 deletions(-) diff --git a/modelbaker/dbconnector/mssql_connector.py b/modelbaker/dbconnector/mssql_connector.py index 1282aea..de67b15 100644 --- a/modelbaker/dbconnector/mssql_connector.py +++ b/modelbaker/dbconnector/mssql_connector.py @@ -173,6 +173,39 @@ def get_tables_info(self): stmt += ln + " , tgeomtype.setting AS simple_type" stmt += ln + " , null AS formatted_type" stmt += ln + " , attrs.sqlname AS attribute_name" + stmt += ( + ln + + """ , CASE WHEN c.iliname IN ( + SELECT i.baseClass as base + FROM {schema}.t_ili2db_inheritance i + LEFT JOIN ( + SELECT fullname, model, topicclass, substring(topicclass, charindex('.',topicclass)+1,len(topicclass)) as class + FROM ( + SELECT + thisClass as fullname, + substring(thisClass, 1, charindex('.', thisClass)-1) as model, + substring(thisClass, charindex('.', thisClass)-1, len(thisClass)) as topicclass + FROM {schema}.t_ili2db_inheritance + ) + ) AS extend_names + ON thisClass = extend_names.fullname + LEFT JOIN names base_names + ON baseClass = base_names.fullname + -- it's extended + WHERE baseClass IS NOT NULL + -- in a different model + AND base_names.model != extend_names.model + AND ( + -- with the same name + base_names.class = extend_names.class + OR + -- multiple times in the same extended model + (SELECT COUNT(baseClass) FROM {schema}.t_ili2db_inheritance JOIN names extend_names ON thisClass = extend_names.fullname WHERE baseClass = i.baseClass GROUP BY baseClass, extend_names.model)>1 + ) + ) + THEN FALSE ELSE TRUE AS relevance + """ + ) stmt += ln + "FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS Tab" stmt += ln + "INNER JOIN INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE AS Col" stmt += ln + " ON Col.Constraint_Name = Tab.Constraint_Name" diff --git a/modelbaker/dbconnector/pg_connector.py b/modelbaker/dbconnector/pg_connector.py index 2e27935..f14f47a 100644 --- a/modelbaker/dbconnector/pg_connector.py +++ b/modelbaker/dbconnector/pg_connector.py @@ -195,37 +195,33 @@ def get_tables_info(self): relevance = """ CASE WHEN c.iliname IN ( WITH names AS ( - WITH class_level_name AS( - WITH topic_level_name AS ( - SELECT - thisClass as fullname, - substring(thisClass from 1 for position('.' in thisClass)-1) as model, - substring(thisClass from position('.' in thisClass)+1) as topicclass - FROM {schema}.t_ili2db_inheritance - ) - SELECT *, ltrim(topicclass,substring(topicclass from 1 for position('.' in topicclass)-1)) as class_with_dot - FROM topic_level_name - ) - SELECT fullname, model, topicclass, substring(class_with_dot from position('.' in class_with_dot)+1) as class - FROM class_level_name - ) - SELECT i.baseClass as base - FROM {schema}.t_ili2db_inheritance i - LEFT JOIN names extend_names - ON thisClass = extend_names.fullname - LEFT JOIN names base_names - ON baseClass = base_names.fullname - -- it's extended - WHERE baseClass IS NOT NULL - -- in a different model - AND base_names.model != extend_names.model - AND ( - -- with the same name - base_names.class = extend_names.class - OR - -- multiple times in the same extended model - (SELECT COUNT(baseClass) FROM {schema}.t_ili2db_inheritance JOIN names extend_names ON thisClass = extend_names.fullname WHERE baseClass = i.baseClass GROUP BY baseClass, extend_names.model)>1 + WITH topic_level_name AS ( + SELECT + thisClass as fullname, + substring(thisClass from 1 for position('.' in thisClass)-1) as model, + substring(thisClass from position('.' in thisClass)+1) as topicclass + FROM {schema}.t_ili2db_inheritance ) + SELECT fullname, model, topicclass, substring(topicclass, position('.' in topicclass)+1) as class + FROM topic_level_name + ) + SELECT i.baseClass as base + FROM {schema}.t_ili2db_inheritance i + LEFT JOIN names extend_names + ON thisClass = extend_names.fullname + LEFT JOIN names base_names + ON baseClass = base_names.fullname + -- it's extended + WHERE baseClass IS NOT NULL + -- in a different model + AND base_names.model != extend_names.model + AND ( + -- with the same name + base_names.class = extend_names.class + OR + -- multiple times in the same extended model + (SELECT COUNT(baseClass) FROM {schema}.t_ili2db_inheritance JOIN names extend_names ON thisClass = extend_names.fullname WHERE baseClass = i.baseClass GROUP BY baseClass, extend_names.model)>1 + ) ) THEN FALSE ELSE TRUE END AS relevance """.format( diff --git a/tests/test_projectgen_extension_optimization.py b/tests/test_projectgen_extension_optimization.py index 7762568..3ff1d26 100644 --- a/tests/test_projectgen_extension_optimization.py +++ b/tests/test_projectgen_extension_optimization.py @@ -91,7 +91,7 @@ def test_extopt_staedtische_postgis(self): generator = Generator( tool=DbIliMode.ili2pg, uri=get_pg_connection_string(), - inheritance="smart2", + inheritance=importer.configuration.inheritance, schema=importer.configuration.dbschema, optimize_strategy=strategy, ) @@ -104,7 +104,7 @@ def test_extopt_staedtische_postgis(self): generator = Generator( tool=DbIliMode.ili2pg, uri=get_pg_connection_string(), - inheritance="smart2", + inheritance=importer.configuration.inheritance, schema=importer.configuration.dbschema, optimize_strategy=strategy, ) @@ -117,7 +117,7 @@ def test_extopt_staedtische_postgis(self): generator = Generator( tool=DbIliMode.ili2pg, uri=get_pg_connection_string(), - inheritance="smart2", + inheritance=importer.configuration.inheritance, schema=importer.configuration.dbschema, optimize_strategy=strategy, ) @@ -154,7 +154,7 @@ def test_extopt_staedtische_geopackage(self): generator = Generator( tool=DbIliMode.ili2gpkg, uri=uri, - inheritance="smart2", + inheritance=importer.configuration.inheritance, optimize_strategy=strategy, ) @@ -166,7 +166,7 @@ def test_extopt_staedtische_geopackage(self): generator = Generator( tool=DbIliMode.ili2gpkg, uri=uri, - inheritance="smart2", + inheritance=importer.configuration.inheritance, optimize_strategy=strategy, ) @@ -178,14 +178,14 @@ def test_extopt_staedtische_geopackage(self): generator = Generator( tool=DbIliMode.ili2gpkg, uri=uri, - inheritance="smart2", + inheritance=importer.configuration.inheritance, optimize_strategy=strategy, ) self._extopt_staedtische_hide(generator, strategy) def test_extopt_staedtische_mssql(self): - return # todo + return # to do importer = iliimporter.Importer() importer.tool = DbIliMode.ili2mssql importer.configuration = iliimporter_config(importer.tool) @@ -218,7 +218,8 @@ def test_extopt_staedtische_mssql(self): generator = Generator( tool=DbIliMode.ili2mssql, uri=uri, - inheritance="smart2", + inheritance=importer.configuration.inheritance, + schema=importer.configuration.dbschema, optimize_strategy=strategy, ) @@ -230,7 +231,8 @@ def test_extopt_staedtische_mssql(self): generator = Generator( tool=DbIliMode.ili2mssql, uri=uri, - inheritance="smart2", + inheritance=importer.configuration.inheritance, + schema=importer.configuration.dbschema, optimize_strategy=strategy, ) @@ -242,7 +244,8 @@ def test_extopt_staedtische_mssql(self): generator = Generator( tool=DbIliMode.ili2mssql, uri=uri, - inheritance="smart2", + inheritance=importer.configuration.inheritance, + schema=importer.configuration.dbschema, optimize_strategy=strategy, ) @@ -562,7 +565,7 @@ def test_extopt_polymorphic_postgis(self): tool=DbIliMode.ili2pg, uri=get_pg_connection_string(), schema=importer.configuration.dbschema, - inheritance="smart2", + inheritance=importer.configuration.inheritance, optimize_strategy=strategy, ) @@ -575,7 +578,7 @@ def test_extopt_polymorphic_postgis(self): tool=DbIliMode.ili2pg, uri=get_pg_connection_string(), schema=importer.configuration.dbschema, - inheritance="smart2", + inheritance=importer.configuration.inheritance, optimize_strategy=strategy, ) @@ -588,7 +591,7 @@ def test_extopt_polymorphic_postgis(self): tool=DbIliMode.ili2pg, uri=get_pg_connection_string(), schema=importer.configuration.dbschema, - inheritance="smart2", + inheritance=importer.configuration.inheritance, optimize_strategy=strategy, ) @@ -624,7 +627,7 @@ def test_extopt_polymorphic_geopackage(self): generator = Generator( tool=DbIliMode.ili2gpkg, uri=uri, - inheritance="smart2", + inheritance=importer.configuration.inheritance, optimize_strategy=strategy, ) @@ -636,7 +639,7 @@ def test_extopt_polymorphic_geopackage(self): generator = Generator( tool=DbIliMode.ili2gpkg, uri=uri, - inheritance="smart2", + inheritance=importer.configuration.inheritance, optimize_strategy=strategy, ) @@ -648,14 +651,14 @@ def test_extopt_polymorphic_geopackage(self): generator = Generator( tool=DbIliMode.ili2gpkg, uri=uri, - inheritance="smart2", + inheritance=importer.configuration.inheritance, optimize_strategy=strategy, ) self._extopt_polymorphic_hide(generator, strategy) def test_extopt_polymorphic_mssql(self): - return # todo + return # to do importer = iliimporter.Importer() importer.tool = DbIliMode.ili2mssql importer.configuration = iliimporter_config(importer.tool) @@ -688,7 +691,8 @@ def test_extopt_polymorphic_mssql(self): generator = Generator( tool=DbIliMode.ili2mssql, uri=uri, - inheritance="smart2", + inheritance=importer.configuration.inheritance, + schema=importer.configuration.dbschema, optimize_strategy=strategy, ) @@ -700,7 +704,8 @@ def test_extopt_polymorphic_mssql(self): generator = Generator( tool=DbIliMode.ili2mssql, uri=uri, - inheritance="smart2", + inheritance=importer.configuration.inheritance, + schema=importer.configuration.dbschema, optimize_strategy=strategy, ) @@ -712,7 +717,8 @@ def test_extopt_polymorphic_mssql(self): generator = Generator( tool=DbIliMode.ili2mssql, uri=uri, - inheritance="smart2", + inheritance=importer.configuration.inheritance, + schema=importer.configuration.dbschema, optimize_strategy=strategy, ) @@ -1070,11 +1076,8 @@ def test_extopt_baustruct_postgis(self): "ilimodels/Kantonale_Bauplanung_V1_1.ili" ) importer.configuration.ilimodels = "Kantonale_Bauplanung_V1_1" - importer.configuration.dbfile = os.path.join( - self.basetestpath, - "tmp_optimal_baustruct_{:%Y%m%d%H%M%S%f}.gpkg".format( - datetime.datetime.now() - ), + importer.configuration.dbschema = "optimal_baustruct_{:%Y%m%d%H%M%S%f}".format( + datetime.datetime.now() ) importer.configuration.srs_code = 2056 importer.configuration.inheritance = "smart2" @@ -1083,16 +1086,13 @@ def test_extopt_baustruct_postgis(self): importer.stderr.connect(self.print_error) assert importer.run() == iliimporter.Importer.SUCCESS - config_manager = GpkgCommandConfigManager(importer.configuration) - config_manager.get_uri() - ### 1. OptimizeStrategy.NONE ### strategy = OptimizeStrategy.NONE generator = Generator( tool=DbIliMode.ili2pg, uri=get_pg_connection_string(), - inheritance="smart2", + inheritance=importer.configuration.inheritance, schema=importer.configuration.dbschema, optimize_strategy=strategy, ) @@ -1105,7 +1105,7 @@ def test_extopt_baustruct_postgis(self): generator = Generator( tool=DbIliMode.ili2pg, uri=get_pg_connection_string(), - inheritance="smart2", + inheritance=importer.configuration.inheritance, schema=importer.configuration.dbschema, optimize_strategy=strategy, ) @@ -1118,7 +1118,7 @@ def test_extopt_baustruct_postgis(self): generator = Generator( tool=DbIliMode.ili2pg, uri=get_pg_connection_string(), - inheritance="smart2", + inheritance=importer.configuration.inheritance, schema=importer.configuration.dbschema, optimize_strategy=strategy, ) @@ -1155,7 +1155,7 @@ def test_extopt_baustruct_geopackage(self): generator = Generator( tool=DbIliMode.ili2gpkg, uri=uri, - inheritance="smart2", + inheritance=importer.configuration.inheritance, optimize_strategy=strategy, ) @@ -1167,7 +1167,7 @@ def test_extopt_baustruct_geopackage(self): generator = Generator( tool=DbIliMode.ili2gpkg, uri=uri, - inheritance="smart2", + inheritance=importer.configuration.inheritance, optimize_strategy=strategy, ) @@ -1179,14 +1179,14 @@ def test_extopt_baustruct_geopackage(self): generator = Generator( tool=DbIliMode.ili2gpkg, uri=uri, - inheritance="smart2", + inheritance=importer.configuration.inheritance, optimize_strategy=strategy, ) self._extopt_baustruct_hide(generator, strategy) def test_extopt_baustruct_mssql(self): - return # todo + return # to do importer = iliimporter.Importer() importer.tool = DbIliMode.ili2mssql importer.configuration = iliimporter_config(importer.tool) @@ -1194,21 +1194,23 @@ def test_extopt_baustruct_mssql(self): "ilimodels/Kantonale_Bauplanung_V1_1.ili" ) importer.configuration.ilimodels = "Kantonale_Bauplanung_V1_1" - importer.configuration.dbfile = os.path.join( - self.basetestpath, - "tmp_optimal_baustruct_{:%Y%m%d%H%M%S%f}.gpkg".format( - datetime.datetime.now() - ), + importer.configuration.dbschema = "optimal_polymorph_{:%Y%m%d%H%M%S%f}".format( + datetime.datetime.now() ) importer.configuration.srs_code = 2056 importer.configuration.inheritance = "smart2" importer.configuration.create_basket_col = True importer.stdout.connect(self.print_info) importer.stderr.connect(self.print_error) - assert importer.run() == iliimporter.Importer.SUCCESS - config_manager = GpkgCommandConfigManager(importer.configuration) - uri = config_manager.get_uri() + uri = "DRIVER={drv};SERVER={server};DATABASE={db};UID={uid};PWD={pwd}".format( + drv="{ODBC Driver 17 for SQL Server}", + server=importer.configuration.dbhost, + db=importer.configuration.database, + uid=importer.configuration.dbusr, + pwd=importer.configuration.dbpwd, + ) + assert importer.run() == iliimporter.Importer.SUCCESS ### 1. OptimizeStrategy.NONE ### strategy = OptimizeStrategy.NONE @@ -1216,7 +1218,8 @@ def test_extopt_baustruct_mssql(self): generator = Generator( tool=DbIliMode.ili2mssql, uri=uri, - inheritance="smart2", + inheritance=importer.configuration.inheritance, + schema=importer.configuration.dbschema, optimize_strategy=strategy, ) @@ -1228,7 +1231,8 @@ def test_extopt_baustruct_mssql(self): generator = Generator( tool=DbIliMode.ili2mssql, uri=uri, - inheritance="smart2", + inheritance=importer.configuration.inheritance, + schema=importer.configuration.dbschema, optimize_strategy=strategy, ) @@ -1240,24 +1244,25 @@ def test_extopt_baustruct_mssql(self): generator = Generator( tool=DbIliMode.ili2mssql, uri=uri, - inheritance="smart2", + inheritance=importer.configuration.inheritance, + schema=importer.configuration.dbschema, optimize_strategy=strategy, ) self._extopt_baustruct_hide(generator, strategy) def _extopt_baustruct_none(self, generator, strategy): - available_layers = generator.layers() relations, _ = generator.relations(available_layers) legend = generator.legend(available_layers) - + print(available_layers) aliases = [l.alias for l in available_layers] irrelevant_layer_ilinames = [ l.ili_name for l in available_layers if not l.is_relevant ] ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] + print(aliases) # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 expected_aliases = [ From 8f78d7239a44b835d10234d59c4186ba35fd3682 Mon Sep 17 00:00:00 2001 From: signedav Date: Wed, 13 Sep 2023 17:58:09 +0200 Subject: [PATCH 21/34] Emitting relevant topics used for smart basket handling --- modelbaker/dbconnector/gpkg_connector.py | 36 ++++++++++++++++--- modelbaker/generator/generator.py | 5 ++- .../test_projectgen_extension_optimization.py | 6 ---- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/modelbaker/dbconnector/gpkg_connector.py b/modelbaker/dbconnector/gpkg_connector.py index 49dedb1..795ede8 100644 --- a/modelbaker/dbconnector/gpkg_connector.py +++ b/modelbaker/dbconnector/gpkg_connector.py @@ -141,7 +141,8 @@ def _get_tables_info(self): ) as coord_decimals, substr(c.iliname, 0, instr(c.iliname, '.')) AS model, attrs.sqlname as attribute_name, - {relevance_field},""".format( + {relevance_field}, + {relevant_topics}""".format( relevance_field="""CASE WHEN c.iliname IN ( -- used to get the class names from the full names WITH names AS ( @@ -177,7 +178,20 @@ def _get_tables_info(self): (SELECT COUNT(baseClass) FROM T_ILI2DB_INHERITANCE JOIN names extend_names ON thisClass = extend_names.fullname WHERE baseClass = i.baseClass GROUP BY baseClass, extend_names.model)>1 ) ) - THEN FALSE ELSE TRUE END AS relevance""" + THEN FALSE ELSE TRUE END AS relevance""", + # relevant topics are emitted by going recursively through the inheritance table. If something of this topic where the current class is located has been extended, it gets the top extended topics as a list + relevant_topics=""" + WITH RECURSIVE children(level, childTopic, baseTopic) AS ( + SELECT 0 as level, substr( thisClass, 0, instr(substr( thisClass, instr(thisClass, '.')+1), '.')+instr(thisClass, '.')) as childTopic , substr( baseClass, 0, instr(substr( baseClass, instr(baseClass, '.')+1), '.')+instr(baseClass, '.')) as baseTopic + FROM T_ILI2DB_INHERITANCE + WHERE baseTopic = substr( c.iliname, 0, instr(substr( c.iliname, instr(c.iliname, '.')+1), '.')+instr(c.iliname, '.')) + UNION + SELECT children.level+1 as level, substr( inheritance.thisClass, 0, instr(substr( inheritance.thisClass, instr(inheritance.thisClass, '.')+1), '.')+instr(inheritance.thisClass, '.')) as childTopic , substr( inheritance.baseClass, 0, instr(substr( inheritance.baseClass, instr(baseClass, '.')+1), '.')+instr(inheritance.baseClass, '.')) as baseTopic FROM children + JOIN T_ILI2DB_INHERITANCE as inheritance ON substr( inheritance.baseClass, 0, instr(substr( inheritance.baseClass, instr(baseClass, '.')+1), '.')+instr(inheritance.baseClass, '.')) = children.childTopic + ) + SELECT group_concat(childTopic) FROM children WHERE level = (SELECT MAX(level) FROM children) + ) as relevant_topics, + """, ) interlis_joins = """LEFT JOIN T_ILI2DB_TABLE_PROP p ON p.tablename = s.name @@ -792,12 +806,26 @@ def get_topics_info(self): cursor.execute( """ SELECT DISTINCT substr(CN.IliName, 0, instr(CN.IliName, '.')) as model, - substr(substr(CN.IliName, instr(CN.IliName, '.')+1),0, instr(substr(CN.IliName, instr(CN.IliName, '.')+1),'.')) as topic + substr(substr(CN.IliName, instr(CN.IliName, '.')+1),0, instr(substr(CN.IliName, instr(CN.IliName, '.')+1),'.')) as topic, + {relevance} FROM T_ILI2DB_CLASSNAME as CN JOIN T_ILI2DB_TABLE_PROP as TP ON CN.sqlname = TP.tablename WHERE topic != '' and TP.setting != 'ENUM' - """ + """.format( + # it's relevant, when it's not extended + # relevance is emitted by going recursively through the inheritance table. If nothing on this topic is extended, it is relevant. Otherwise it's not. + relevance=""" + CASE WHEN (WITH RECURSIVE children(childTopic, baseTopic) AS ( + SELECT substr( thisClass, 0, instr(substr( thisClass, instr(thisClass, '.')+1), '.')+instr(thisClass, '.')) as childTopic , substr( baseClass, 0, instr(substr( baseClass, instr(baseClass, '.')+1), '.')+instr(baseClass, '.')) as baseTopic + FROM T_ILI2DB_INHERITANCE + WHERE baseTopic = substr( CN.IliName, 0, instr(substr( CN.IliName, instr(CN.IliName, '.')+1), '.')+instr(CN.IliName, '.')) -- model.topic + UNION + SELECT substr( inheritance.thisClass, 0, instr(substr( inheritance.thisClass, instr(inheritance.thisClass, '.')+1), '.')+instr(inheritance.thisClass, '.')) as childTopic , substr( inheritance.baseClass, 0, instr(substr( inheritance.baseClass, instr(baseClass, '.')+1), '.')+instr(inheritance.baseClass, '.')) as baseTopic FROM children + JOIN T_ILI2DB_INHERITANCE as inheritance ON substr( inheritance.baseClass, 0, instr(substr( inheritance.baseClass, instr(baseClass, '.')+1), '.')+instr(inheritance.baseClass, '.')) = children.childTopic + )SELECT count(childTopic) FROM children)>0 THEN FALSE ELSE TRUE END AS relevance + """ + ) ) contents = cursor.fetchall() cursor.close() diff --git a/modelbaker/generator/generator.py b/modelbaker/generator/generator.py index dc175ca..18baa33 100644 --- a/modelbaker/generator/generator.py +++ b/modelbaker/generator/generator.py @@ -144,6 +144,8 @@ def layers(self, filter_layer_list=[]): record.get("relevance", True) ) # it can be not relevant and still be displayed (in case of NONE) + relevant_topics = record.get("relevant_topics").split(",") + alias = record["table_alias"] if "table_alias" in record else None if not alias: short_name = None @@ -192,7 +194,7 @@ def layers(self, filter_layer_list=[]): + match.group(1).split(".")[-1] + ")" ) - alias = short_name # for-relevance-tests if is_relevant else f"{short_name} !IRRELEVANT!" + alias = short_name display_expression = "" if is_basket_table: @@ -232,6 +234,7 @@ def layers(self, filter_layer_list=[]): is_dataset_table, record.get("ili_name"), is_relevant, + relevant_topics, ) # Configure fields for current table diff --git a/tests/test_projectgen_extension_optimization.py b/tests/test_projectgen_extension_optimization.py index 3ff1d26..53040f5 100644 --- a/tests/test_projectgen_extension_optimization.py +++ b/tests/test_projectgen_extension_optimization.py @@ -1255,14 +1255,12 @@ def _extopt_baustruct_none(self, generator, strategy): available_layers = generator.layers() relations, _ = generator.relations(available_layers) legend = generator.legend(available_layers) - print(available_layers) aliases = [l.alias for l in available_layers] irrelevant_layer_ilinames = [ l.ili_name for l in available_layers if not l.is_relevant ] ambiguous_aliases = [alias for alias in aliases if aliases.count(alias) > 1] - print(aliases) # check no ambiguous layers exists assert len(ambiguous_aliases) == 0 expected_aliases = [ @@ -1348,7 +1346,6 @@ def _extopt_baustruct_none(self, generator, strategy): assert len(relations) == 18 # strasse should have relation editors to all layers (2/2) - # for-relevance-tests hier werden diese relation editoren nicht ins formular geschoben. Ich weiss nicht weshalb... count = 0 for layer in project.layers: if layer.layer.name() == "Strasse": @@ -1356,7 +1353,6 @@ def _extopt_baustruct_none(self, generator, strategy): # one general and four relation editors assert len(efc.tabs()) == 4 for tab in efc.tabs(): - print(f"-------- {tab.name()}") if tab.name() == "kantnl_ng_v1_1konstruktionen_gebaeude": count += 1 assert len(tab.children()) == 1 @@ -1494,7 +1490,6 @@ def _extopt_baustruct_group(self, generator, strategy): # one general and four relation editors assert len(efc.tabs()) == 2 for tab in efc.tabs(): - print(f"-------- {tab.name()}") if tab.name() == "kantnl_ng_v1_1konstruktionen_gebaeude": count += 1 assert len(tab.children()) == 1 @@ -1595,7 +1590,6 @@ def _extopt_baustruct_hide(self, generator, strategy): # one general and four relation editors assert len(efc.tabs()) == 2 for tab in efc.tabs(): - print(f"-------- {tab.name()}") if tab.name() == "kantnl_ng_v1_1konstruktionen_gebaeude": count += 1 assert len(tab.children()) == 1 From accb26795e61218ae893d2629c1c0c72d74024c1 Mon Sep 17 00:00:00 2001 From: signedav Date: Wed, 13 Sep 2023 18:06:29 +0200 Subject: [PATCH 22/34] relevant_topics in layer --- modelbaker/dataobjects/layers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modelbaker/dataobjects/layers.py b/modelbaker/dataobjects/layers.py index 96ec2e9..de15463 100644 --- a/modelbaker/dataobjects/layers.py +++ b/modelbaker/dataobjects/layers.py @@ -56,6 +56,7 @@ def __init__( is_dataset_table=False, ili_name=None, is_relevant=True, + relevant_topics=[], definitionfile=None, qmlstylefile=None, styles={}, @@ -102,6 +103,7 @@ def __init__( ) self.is_relevant = is_relevant + self.relevant_topics = relevant_topics self.definitionfile = definitionfile self.qmlstylefile = qmlstylefile @@ -127,6 +129,8 @@ def dump(self): definition["coordinateprecision"] = self.coordinate_precision definition["modeltopicname"] = self.model_topic_name definition["ili_name"] = self.ili_name + definition["is_relevant"] = self.is_relevant + definition["relevant_topics"] = self.relevant_topics definition["definitionfile"] = self.definitionfile definition["qmlstylefile"] = self.qmlstylefile definition["styles"] = self.styles @@ -145,6 +149,8 @@ def load(self, definition): self.coordinate_precision = definition["coordinateprecision"] self.model_topic_name = definition["modeltopicname"] self.ili_name = definition["ili_name"] + self.is_relevant = definition["is_relevant"] + self.relevant_topics = definition["relevant_topics"] self.definitionfile = definition["definitionfile"] self.qmlstylefile = definition["qmlstylefile"] self.styles = definition["styles"] From 4c212392865b035cebad6a760ac87dbdc381a433 Mon Sep 17 00:00:00 2001 From: signedav Date: Thu, 14 Sep 2023 17:33:25 +0200 Subject: [PATCH 23/34] alltopics and relevanttopics on gpkg finalized and failing and unfinished tests. --- modelbaker/dataobjects/layers.py | 6 +- modelbaker/dbconnector/gpkg_connector.py | 38 +++++--- modelbaker/generator/generator.py | 10 ++- .../test_projectgen_extension_optimization.py | 87 +++++++++++++++++++ 4 files changed, 129 insertions(+), 12 deletions(-) diff --git a/modelbaker/dataobjects/layers.py b/modelbaker/dataobjects/layers.py index de15463..ea6557d 100644 --- a/modelbaker/dataobjects/layers.py +++ b/modelbaker/dataobjects/layers.py @@ -56,7 +56,8 @@ def __init__( is_dataset_table=False, ili_name=None, is_relevant=True, - relevant_topics=[], + all_topics=[], # all the topics this class (or an instance of it) are located + relevant_topics=[], # the topics of the most extended instance of it only definitionfile=None, qmlstylefile=None, styles={}, @@ -103,6 +104,7 @@ def __init__( ) self.is_relevant = is_relevant + self.all_topics = all_topics self.relevant_topics = relevant_topics self.definitionfile = definitionfile @@ -130,6 +132,7 @@ def dump(self): definition["modeltopicname"] = self.model_topic_name definition["ili_name"] = self.ili_name definition["is_relevant"] = self.is_relevant + definition["all_topics"] = self.all_topics definition["relevant_topics"] = self.relevant_topics definition["definitionfile"] = self.definitionfile definition["qmlstylefile"] = self.qmlstylefile @@ -150,6 +153,7 @@ def load(self, definition): self.model_topic_name = definition["modeltopicname"] self.ili_name = definition["ili_name"] self.is_relevant = definition["is_relevant"] + self.all_topics = definition["all_topics"] self.relevant_topics = definition["relevant_topics"] self.definitionfile = definition["definitionfile"] self.qmlstylefile = definition["qmlstylefile"] diff --git a/modelbaker/dbconnector/gpkg_connector.py b/modelbaker/dbconnector/gpkg_connector.py index 795ede8..5ea6a49 100644 --- a/modelbaker/dbconnector/gpkg_connector.py +++ b/modelbaker/dbconnector/gpkg_connector.py @@ -142,7 +142,7 @@ def _get_tables_info(self): substr(c.iliname, 0, instr(c.iliname, '.')) AS model, attrs.sqlname as attribute_name, {relevance_field}, - {relevant_topics}""".format( + {topics},""".format( relevance_field="""CASE WHEN c.iliname IN ( -- used to get the class names from the full names WITH names AS ( @@ -179,19 +179,37 @@ def _get_tables_info(self): ) ) THEN FALSE ELSE TRUE END AS relevance""", - # relevant topics are emitted by going recursively through the inheritance table. If something of this topic where the current class is located has been extended, it gets the top extended topics as a list - relevant_topics=""" - WITH RECURSIVE children(level, childTopic, baseTopic) AS ( - SELECT 0 as level, substr( thisClass, 0, instr(substr( thisClass, instr(thisClass, '.')+1), '.')+instr(thisClass, '.')) as childTopic , substr( baseClass, 0, instr(substr( baseClass, instr(baseClass, '.')+1), '.')+instr(baseClass, '.')) as baseTopic + # topics - where this class or an instance of it is located - are emitted by going recursively through the inheritance table. + # if something of this topic where the current class is located has been extended, it gets the next child topic. + # the relevant topics for optimization are the ones that are not more extended (or in the very last class). + topics="""(SELECT group_concat(childTopic) FROM {topic_pedigree}) as all_topics, + (SELECT group_concat(childTopic) FROM {topic_pedigree} WHERE NOT is_a_base) as relevant_topics""".format( + topic_pedigree="""(WITH RECURSIVE children(is_a_base, childTopic, baseTopic) AS ( + SELECT + (CASE + WHEN substr( thisClass, 0, instr(substr( thisClass, instr(thisClass, '.')+1), '.')+instr(thisClass, '.')) IN (SELECT substr( i.baseClass, 0, instr(substr( i.baseClass, instr(i.baseClass, '.')+1), '.')+instr(i.baseClass, '.')) FROM T_ILI2DB_INHERITANCE i WHERE substr( i.thisClass, 0, instr(i.thisClass, '.') ) != substr( i.baseClass, 0, instr(i.baseClass, '.'))) + THEN TRUE + ELSE FALSE + END) AS is_a_base, + substr( thisClass, 0, instr(substr( thisClass, instr(thisClass, '.')+1), '.')+instr(thisClass, '.')) as childTopic, + substr( baseClass, 0, instr(substr( baseClass, instr(baseClass, '.')+1), '.')+instr(baseClass, '.')) as baseTopic FROM T_ILI2DB_INHERITANCE WHERE baseTopic = substr( c.iliname, 0, instr(substr( c.iliname, instr(c.iliname, '.')+1), '.')+instr(c.iliname, '.')) UNION - SELECT children.level+1 as level, substr( inheritance.thisClass, 0, instr(substr( inheritance.thisClass, instr(inheritance.thisClass, '.')+1), '.')+instr(inheritance.thisClass, '.')) as childTopic , substr( inheritance.baseClass, 0, instr(substr( inheritance.baseClass, instr(baseClass, '.')+1), '.')+instr(inheritance.baseClass, '.')) as baseTopic FROM children - JOIN T_ILI2DB_INHERITANCE as inheritance ON substr( inheritance.baseClass, 0, instr(substr( inheritance.baseClass, instr(baseClass, '.')+1), '.')+instr(inheritance.baseClass, '.')) = children.childTopic + SELECT + (CASE + WHEN substr( thisClass, 0, instr(substr( thisClass, instr(thisClass, '.')+1), '.')+instr(thisClass, '.')) IN (SELECT substr( i.baseClass, 0, instr(substr( i.baseClass, instr(i.baseClass, '.')+1), '.')+instr(i.baseClass, '.')) FROM T_ILI2DB_INHERITANCE i WHERE substr( i.thisClass, 0, instr(i.thisClass, '.')) != substr( i.baseClass, 0,instr(i.baseClass, '.'))) + THEN TRUE + ELSE FALSE + END) AS is_a_base, + substr( inheritance.thisClass, 0, instr(substr( inheritance.thisClass, instr(inheritance.thisClass, '.')+1), '.')+instr(inheritance.thisClass, '.')) as childTopic , + substr( inheritance.baseClass, 0, instr(substr( inheritance.baseClass, instr(baseClass, '.')+1), '.')+instr(inheritance.baseClass, '.')) as baseTopic FROM children + JOIN T_ILI2DB_INHERITANCE as inheritance + ON substr( inheritance.baseClass, 0, instr(substr( inheritance.baseClass, instr(inheritance.baseClass, '.')+1), '.')+instr(inheritance.baseClass, '.')) = children.childTopic -- when the childTopic is as well the baseTopic of another childTopic + WHERE substr( inheritance.thisClass, 0, instr(substr( inheritance.thisClass, instr(inheritance.thisClass, '.')+1), '.')+instr(inheritance.thisClass, '.')) != children.childTopic --break the recursion when the coming childTopic will be the same ) - SELECT group_concat(childTopic) FROM children WHERE level = (SELECT MAX(level) FROM children) - ) as relevant_topics, - """, + SELECT childTopic, baseTopic, is_a_base FROM children)""" + ), ) interlis_joins = """LEFT JOIN T_ILI2DB_TABLE_PROP p ON p.tablename = s.name diff --git a/modelbaker/generator/generator.py b/modelbaker/generator/generator.py index 18baa33..0e58284 100644 --- a/modelbaker/generator/generator.py +++ b/modelbaker/generator/generator.py @@ -144,7 +144,14 @@ def layers(self, filter_layer_list=[]): record.get("relevance", True) ) # it can be not relevant and still be displayed (in case of NONE) - relevant_topics = record.get("relevant_topics").split(",") + all_topics = ( + record.get("all_topics").split(",") if record.get("all_topics") else [] + ) + relevant_topics = ( + record.get("relevant_topics").split(",") + if record.get("relevant_topics") + else [] + ) alias = record["table_alias"] if "table_alias" in record else None if not alias: @@ -234,6 +241,7 @@ def layers(self, filter_layer_list=[]): is_dataset_table, record.get("ili_name"), is_relevant, + all_topics, relevant_topics, ) diff --git a/tests/test_projectgen_extension_optimization.py b/tests/test_projectgen_extension_optimization.py index 53040f5..5adb440 100644 --- a/tests/test_projectgen_extension_optimization.py +++ b/tests/test_projectgen_extension_optimization.py @@ -348,6 +348,36 @@ def _extopt_staedtische_none(self, generator, strategy): # should find 4 assert count == 4 + # check relevant topics + count = 0 + for layer in project.layers: + if layer.layer.name() == "Strasse": # Strasse from Infrastruktur_V1_1 + count += 1 + assert set(layer.all_topics) == {""} + assert set(layer.relevant_topics) == {""} + if layer.layer.name() == "BesitzerIn": # BesitzerIn from Ortsplanung_V1_1 + count += 1 + assert set(layer.all_topics) == { + "Kantonale_Ortsplanung_V1_1.Konstruktionen", + "Staedtische_Ortsplanung_V1_1.Freizeit", + "Staedtische_Ortsplanung_V1_1.Gewerbe", + } + assert set(layer.relevant_topics) == { + "Staedtische_Ortsplanung_V1_1.Freizeit", + "Staedtische_Ortsplanung_V1_1.Gewerbe", + } + if ( + layer.layer.name() == "Staedtisches_Gewerbe_V1.Firmen.Firma" + ): # Firma from Staedtisches_Gewerbe_V1 + count += 1 + assert set(layer.all_topics) == {""} + assert set(layer.relevant_topics) == {""} + if layer.layer.name() == "Gewerbe_V1.Firmen.Firma": # Firma from Gewerbe_V1 + count += 1 + assert set(layer.all_topics) == {"Staedtisches_Gewerbe_V1.Firmen"} + assert set(layer.relevant_topics) == {"Staedtisches_Gewerbe_V1.Firmen"} + assert count == 4 + QgsProject.instance().clear() def _extopt_staedtische_group(self, generator, strategy): @@ -450,8 +480,39 @@ def _extopt_staedtische_group(self, generator, strategy): if tab.name() == "gebaeude": # should not happen count += 1 # should find only 2 + assert count == 2 + # check relevant topics + count = 0 + for layer in project.layers: + if layer.layer.name() == "Strasse": # Strasse from Infrastruktur_V1_1 + count += 1 + assert set(layer.all_topics) == {""} + assert set(layer.relevant_topics) == {""} + if layer.layer.name() == "BesitzerIn": # BesitzerIn from Ortsplanung_V1_1 + count += 1 + assert set(layer.all_topics) == { + "Kantonale_Ortsplanung_V1_1.Konstruktionen", + "Staedtische_Ortsplanung_V1_1.Freizeit", + "Staedtische_Ortsplanung_V1_1.Gewerbe", + } + assert set(layer.relevant_topics) == { + "Staedtische_Ortsplanung_V1_1.Freizeit", + "Staedtische_Ortsplanung_V1_1.Gewerbe", + } + if ( + layer.layer.name() == "Staedtisches_Gewerbe_V1.Firmen.Firma" + ): # Firma from Staedtisches_Gewerbe_V1 + count += 1 + assert set(layer.all_topics) == {""} + assert set(layer.relevant_topics) == {""} + if layer.layer.name() == "Gewerbe_V1.Firmen.Firma": # Firma from Gewerbe_V1 + count += 1 + assert set(layer.all_topics) == {"Staedtisches_Gewerbe_V1.Firmen"} + assert set(layer.relevant_topics) == {"Staedtisches_Gewerbe_V1.Firmen"} + assert count == 4 + QgsProject.instance().clear() def _extopt_staedtische_hide(self, generator, strategy): @@ -539,6 +600,32 @@ def _extopt_staedtische_hide(self, generator, strategy): # should find only 2 assert count == 2 + # check relevant topics + count = 0 + for layer in project.layers: + if layer.layer.name() == "Strasse": # Strasse from Infrastruktur_V1_1 + count += 1 + assert set(layer.all_topics) == {""} + assert set(layer.relevant_topics) == {""} + if layer.layer.name() == "BesitzerIn": # BesitzerIn from Ortsplanung_V1_1 + count += 1 + assert set(layer.all_topics) == { + "Kantonale_Ortsplanung_V1_1.Konstruktionen", + "Staedtische_Ortsplanung_V1_1.Freizeit", + "Staedtische_Ortsplanung_V1_1.Gewerbe", + } + assert set(layer.relevant_topics) == { + "Staedtische_Ortsplanung_V1_1.Freizeit", + "Staedtische_Ortsplanung_V1_1.Gewerbe", + } + if layer.layer.name() == "Firma": # Firma from Staedtisches_Gewerbe_V1 + count += 1 + assert set(layer.all_topics) == {""} + assert set(layer.relevant_topics) == {""} + assert count == 3 + + QgsProject.instance().clear() + def test_extopt_polymorphic_postgis(self): importer = iliimporter.Importer() importer.tool = DbIliMode.ili2pg From 9d7c273089abafb60c3eba18b8dc65077ba123e4 Mon Sep 17 00:00:00 2001 From: signedav Date: Fri, 15 Sep 2023 15:23:48 +0200 Subject: [PATCH 24/34] relevant_topics postgres support and tests --- modelbaker/dbconnector/pg_connector.py | 59 ++- .../test_projectgen_extension_optimization.py | 439 ++++++++++++++---- 2 files changed, 410 insertions(+), 88 deletions(-) diff --git a/modelbaker/dbconnector/pg_connector.py b/modelbaker/dbconnector/pg_connector.py index f14f47a..d572cb2 100644 --- a/modelbaker/dbconnector/pg_connector.py +++ b/modelbaker/dbconnector/pg_connector.py @@ -202,7 +202,7 @@ def get_tables_info(self): substring(thisClass from position('.' in thisClass)+1) as topicclass FROM {schema}.t_ili2db_inheritance ) - SELECT fullname, model, topicclass, substring(topicclass, position('.' in topicclass)+1) as class + SELECT fullname, model, topicclass, substring(topicclass from position('.' in topicclass)+1) as class FROM topic_level_name ) SELECT i.baseClass as base @@ -223,10 +223,46 @@ def get_tables_info(self): (SELECT COUNT(baseClass) FROM {schema}.t_ili2db_inheritance JOIN names extend_names ON thisClass = extend_names.fullname WHERE baseClass = i.baseClass GROUP BY baseClass, extend_names.model)>1 ) ) - THEN FALSE ELSE TRUE END AS relevance + THEN FALSE ELSE TRUE END AS relevance, """.format( schema=self.schema ) + + # topics - where this class or an instance of it is located - are emitted by going recursively through the inheritance table. + # if something of this topic where the current class is located has been extended, it gets the next child topic. + # the relevant topics for optimization are the ones that are not more extended (or in the very last class). + topics = """(SELECT STRING_AGG(childTopic,',') FROM {topic_pedigree}) as all_topics, + (SELECT STRING_AGG(childTopic,',') FROM {topic_pedigree} WHERE NOT is_a_base) as relevant_topics""".format( + topic_pedigree="""(WITH RECURSIVE children(is_a_base, childTopic, baseTopic) AS ( + SELECT + (CASE + WHEN substring( thisClass from 1 for position('.' in substring( thisClass from position('.' in thisClass)+1))+position('.' in thisClass)-1) IN (SELECT substring( i.baseClass from 1 for position('.' in substring( i.baseClass from position('.' in i.baseClass)+1))+position('.' in i.baseClass)-1) FROM {schema}.T_ILI2DB_INHERITANCE i WHERE substring( i.thisClass from 1 for position('.' in i.thisClass) ) != substring( i.baseClass from 1 for position('.' in i.baseClass))) + THEN TRUE + ELSE FALSE + END) AS is_a_base, + substring( thisClass from 1 for position('.' in substring( thisClass, position('.' in thisClass)+1))+position('.' in thisClass)-1) as childTopic, + substring( baseClass from 1 for position('.' in substring( baseClass, position('.' in baseClass)+1))+position('.' in baseClass)-1) as baseTopic + FROM {schema}.T_ILI2DB_INHERITANCE + WHERE substring( baseClass from 1 for position('.' in substring( baseClass, position('.' in baseClass)+1))+position('.' in baseClass)-1) = substring( c.iliname from 1 for position('.' in substring( c.iliname from position('.' in c.iliname)+1))+position('.' in c.iliname)-1) + UNION + SELECT + (CASE + WHEN substring( thisClass from 1 for position('.' in substring( thisClass from position('.' in thisClass)+1))+position('.' in thisClass)-1) IN (SELECT substring( i.baseClass from 1 for position('.' in substring( i.baseClass from position('.' in i.baseClass)+1))+position('.' in i.baseClass)-1) FROM {schema}.T_ILI2DB_INHERITANCE i WHERE substring( i.thisClass from 1 for position('.' in i.thisClass)) != substring( i.baseClass from 1 for position('.' in i.baseClass))) + THEN TRUE + ELSE FALSE + END) AS is_a_base, + substring( inheritance.thisClass from 1 for position('.' in substring( inheritance.thisClass from position('.' in inheritance.thisClass)+1))+position('.' in inheritance.thisClass)-1) as childTopic , + substring( inheritance.baseClass from 1 for position('.' in substring( inheritance.baseClass from position('.' in baseClass)+1))+position('.' in inheritance.baseClass)-1) as baseTopic + FROM children + JOIN {schema}.T_ILI2DB_INHERITANCE as inheritance + ON substring( inheritance.baseClass from 1 for position('.' in substring( inheritance.baseClass from position('.' in inheritance.baseClass)+1))+position('.' in inheritance.baseClass)-1) = children.childTopic -- when the childTopic is as well the baseTopic of another childTopic + WHERE substring( inheritance.thisClass from 1 for position('.' in substring( inheritance.thisClass from position('.' in inheritance.thisClass)+1))+position('.' in inheritance.thisClass)-1) != children.childTopic --break the recursion when the coming childTopic will be the same + ) + SELECT childTopic, baseTopic, is_a_base FROM children) AS kiddies""".format( + schema=self.schema + ) + ) + domain_left_join = """LEFT JOIN {}.t_ili2db_table_prop p ON p.tablename = tbls.tablename AND p.tag = 'ch.ehi.ili2db.tableKind'""".format( @@ -269,6 +305,7 @@ def get_tables_info(self): g.type AS simple_type, format_type(ga.atttypid, ga.atttypmod) as formatted_type, {relevance} + {topics} FROM pg_catalog.pg_tables tbls LEFT JOIN pg_index i ON i.indrelid = CONCAT(tbls.schemaname, '."', tbls.tablename, '"')::regclass @@ -294,6 +331,7 @@ def get_tables_info(self): extent=extent, coord_decimals=coord_decimals, relevance=relevance, + topics=topics, domain_left_join=domain_left_join, alias_left_join=alias_left_join, model_where=model_where, @@ -931,13 +969,26 @@ def get_topics_info(self): cur.execute( """ SELECT DISTINCT (string_to_array(cn.iliname, '.'))[1] as model, - (string_to_array(cn.iliname, '.'))[2] as topic + (string_to_array(cn.iliname, '.'))[2] as topic, + {relevance} FROM {schema}.t_ili2db_classname as cn JOIN {schema}.t_ili2db_table_prop as tp ON cn.sqlname = tp.tablename WHERE array_length(string_to_array(cn.iliname, '.'),1) > 2 and tp.setting != 'ENUM' """.format( - schema=self.schema + schema=self.schema, + relevance=""" + CASE WHEN (WITH RECURSIVE children(childTopic, baseTopic) AS ( + SELECT substring( thisClass from 1 for position('.' in substring( thisClass from position('.' in thisClass)+1))+position('.' in thisClass)-1) as childTopic , substring( baseClass from 1 for position('.' in substring( baseClass from position('.' in baseClass)+1))+position('.' in baseClass)-1) as baseTopic + FROM {schema}.T_ILI2DB_INHERITANCE + WHERE substring( baseClass from 1 for position('.' in substring( baseClass from position('.' in baseClass)+1))+position('.' in baseClass)-1) = substring( CN.IliName from 1 for position('.' in substring( CN.IliName from position('.' in CN.IliName)+1))+position('.' in CN.IliName)-1) -- model.topic + UNION + SELECT substring( inheritance.thisClass from 1 for position('.' in substring( inheritance.thisClass from position('.' in inheritance.thisClass)+1))+position('.' in inheritance.thisClass)-1) as childTopic , substring( inheritance.baseClass from 1 for position('.' in substring( inheritance.baseClass from position('.' in baseClass)+1))+position('.' in inheritance.baseClass)-1) as baseTopic FROM children + JOIN {schema}.T_ILI2DB_INHERITANCE as inheritance ON substring( inheritance.baseClass from 1 for position('.' in substring( inheritance.baseClass from position('.' in baseClass)+1))+position('.' in inheritance.baseClass)-1) = children.childTopic + )SELECT count(childTopic) FROM children)>0 THEN FALSE ELSE TRUE END AS relevance + """.format( + schema=self.schema + ), ) ) return cur.fetchall() diff --git a/tests/test_projectgen_extension_optimization.py b/tests/test_projectgen_extension_optimization.py index 5adb440..a84c50b 100644 --- a/tests/test_projectgen_extension_optimization.py +++ b/tests/test_projectgen_extension_optimization.py @@ -286,6 +286,39 @@ def _extopt_staedtische_none(self, generator, strategy): ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + # check relevant topics + count = 0 + for layer in available_layers: + # Strasse from Infrastruktur_V1_1 + if layer.alias == "Strasse": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # BesitzerIn from Ortsplanung_V1_1 + if layer.alias == "BesitzerIn": + count += 1 + assert set(layer.all_topics) == { + "Kantonale_Ortsplanung_V1_1.Konstruktionen", + "Staedtische_Ortsplanung_V1_1.Freizeit", + "Staedtische_Ortsplanung_V1_1.Gewerbe", + } + assert set(layer.relevant_topics) == { + "Staedtische_Ortsplanung_V1_1.Freizeit", + "Staedtische_Ortsplanung_V1_1.Gewerbe", + } + # Firma from Staedtisches_Gewerbe_V1 + if layer.alias == "Staedtisches_Gewerbe_V1.Firmen.Firma": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # Firma from Gewerbe_V1 + if layer.alias == "Gewerbe_V1.Firmen.Firma": + count += 1 + print(layer.all_topics) + assert set(layer.all_topics) == {"Staedtisches_Gewerbe_V1.Firmen"} + assert set(layer.relevant_topics) == {"Staedtisches_Gewerbe_V1.Firmen"} + assert count == 4 + project = Project(optimize_strategy=strategy) project.layers = available_layers project.relations = relations @@ -348,36 +381,6 @@ def _extopt_staedtische_none(self, generator, strategy): # should find 4 assert count == 4 - # check relevant topics - count = 0 - for layer in project.layers: - if layer.layer.name() == "Strasse": # Strasse from Infrastruktur_V1_1 - count += 1 - assert set(layer.all_topics) == {""} - assert set(layer.relevant_topics) == {""} - if layer.layer.name() == "BesitzerIn": # BesitzerIn from Ortsplanung_V1_1 - count += 1 - assert set(layer.all_topics) == { - "Kantonale_Ortsplanung_V1_1.Konstruktionen", - "Staedtische_Ortsplanung_V1_1.Freizeit", - "Staedtische_Ortsplanung_V1_1.Gewerbe", - } - assert set(layer.relevant_topics) == { - "Staedtische_Ortsplanung_V1_1.Freizeit", - "Staedtische_Ortsplanung_V1_1.Gewerbe", - } - if ( - layer.layer.name() == "Staedtisches_Gewerbe_V1.Firmen.Firma" - ): # Firma from Staedtisches_Gewerbe_V1 - count += 1 - assert set(layer.all_topics) == {""} - assert set(layer.relevant_topics) == {""} - if layer.layer.name() == "Gewerbe_V1.Firmen.Firma": # Firma from Gewerbe_V1 - count += 1 - assert set(layer.all_topics) == {"Staedtisches_Gewerbe_V1.Firmen"} - assert set(layer.relevant_topics) == {"Staedtisches_Gewerbe_V1.Firmen"} - assert count == 4 - QgsProject.instance().clear() def _extopt_staedtische_group(self, generator, strategy): @@ -415,6 +418,38 @@ def _extopt_staedtische_group(self, generator, strategy): ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + # check relevant topics + count = 0 + for layer in available_layers: + # Strasse from Infrastruktur_V1_1 + if layer.alias == "Strasse": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # BesitzerIn from Ortsplanung_V1_1 + if layer.alias == "BesitzerIn": + count += 1 + assert set(layer.all_topics) == { + "Kantonale_Ortsplanung_V1_1.Konstruktionen", + "Staedtische_Ortsplanung_V1_1.Freizeit", + "Staedtische_Ortsplanung_V1_1.Gewerbe", + } + assert set(layer.relevant_topics) == { + "Staedtische_Ortsplanung_V1_1.Freizeit", + "Staedtische_Ortsplanung_V1_1.Gewerbe", + } + # Firma from Staedtisches_Gewerbe_V1 + if layer.alias == "Staedtisches_Gewerbe_V1.Firmen.Firma": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # Firma from Gewerbe_V1 + if layer.alias == "Gewerbe_V1.Firmen.Firma": + count += 1 + assert set(layer.all_topics) == {"Staedtisches_Gewerbe_V1.Firmen"} + assert set(layer.relevant_topics) == {"Staedtisches_Gewerbe_V1.Firmen"} + assert count == 4 + project = Project(optimize_strategy=strategy) project.layers = available_layers project.relations = relations @@ -483,36 +518,6 @@ def _extopt_staedtische_group(self, generator, strategy): assert count == 2 - # check relevant topics - count = 0 - for layer in project.layers: - if layer.layer.name() == "Strasse": # Strasse from Infrastruktur_V1_1 - count += 1 - assert set(layer.all_topics) == {""} - assert set(layer.relevant_topics) == {""} - if layer.layer.name() == "BesitzerIn": # BesitzerIn from Ortsplanung_V1_1 - count += 1 - assert set(layer.all_topics) == { - "Kantonale_Ortsplanung_V1_1.Konstruktionen", - "Staedtische_Ortsplanung_V1_1.Freizeit", - "Staedtische_Ortsplanung_V1_1.Gewerbe", - } - assert set(layer.relevant_topics) == { - "Staedtische_Ortsplanung_V1_1.Freizeit", - "Staedtische_Ortsplanung_V1_1.Gewerbe", - } - if ( - layer.layer.name() == "Staedtisches_Gewerbe_V1.Firmen.Firma" - ): # Firma from Staedtisches_Gewerbe_V1 - count += 1 - assert set(layer.all_topics) == {""} - assert set(layer.relevant_topics) == {""} - if layer.layer.name() == "Gewerbe_V1.Firmen.Firma": # Firma from Gewerbe_V1 - count += 1 - assert set(layer.all_topics) == {"Staedtisches_Gewerbe_V1.Firmen"} - assert set(layer.relevant_topics) == {"Staedtisches_Gewerbe_V1.Firmen"} - assert count == 4 - QgsProject.instance().clear() def _extopt_staedtische_hide(self, generator, strategy): @@ -549,6 +554,33 @@ def _extopt_staedtische_hide(self, generator, strategy): ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + # check relevant topics + count = 0 + for layer in available_layers: + # Strasse from Infrastruktur_V1_1 + if layer.alias == "Strasse": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # BesitzerIn from Ortsplanung_V1_1 + if layer.alias == "BesitzerIn": + count += 1 + assert set(layer.all_topics) == { + "Kantonale_Ortsplanung_V1_1.Konstruktionen", + "Staedtische_Ortsplanung_V1_1.Freizeit", + "Staedtische_Ortsplanung_V1_1.Gewerbe", + } + assert set(layer.relevant_topics) == { + "Staedtische_Ortsplanung_V1_1.Freizeit", + "Staedtische_Ortsplanung_V1_1.Gewerbe", + } + # Firma from Staedtisches_Gewerbe_V1 + if layer.alias == "Firma" and layer.is_relevant: + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + assert count == 3 + project = Project(optimize_strategy=strategy) project.layers = available_layers project.relations = relations @@ -600,30 +632,6 @@ def _extopt_staedtische_hide(self, generator, strategy): # should find only 2 assert count == 2 - # check relevant topics - count = 0 - for layer in project.layers: - if layer.layer.name() == "Strasse": # Strasse from Infrastruktur_V1_1 - count += 1 - assert set(layer.all_topics) == {""} - assert set(layer.relevant_topics) == {""} - if layer.layer.name() == "BesitzerIn": # BesitzerIn from Ortsplanung_V1_1 - count += 1 - assert set(layer.all_topics) == { - "Kantonale_Ortsplanung_V1_1.Konstruktionen", - "Staedtische_Ortsplanung_V1_1.Freizeit", - "Staedtische_Ortsplanung_V1_1.Gewerbe", - } - assert set(layer.relevant_topics) == { - "Staedtische_Ortsplanung_V1_1.Freizeit", - "Staedtische_Ortsplanung_V1_1.Gewerbe", - } - if layer.layer.name() == "Firma": # Firma from Staedtisches_Gewerbe_V1 - count += 1 - assert set(layer.all_topics) == {""} - assert set(layer.relevant_topics) == {""} - assert count == 3 - QgsProject.instance().clear() def test_extopt_polymorphic_postgis(self): @@ -846,6 +854,61 @@ def _extopt_polymorphic_none(self, generator, strategy): ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + # check relevant topics + count = 0 + for layer in available_layers: + # Strasse from Infrastruktur_V1_1 + if layer.alias == "Strasse": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # BesitzerIn from Ortsplanung_V1_1 + if layer.alias == "BesitzerIn": + count += 1 + assert set(layer.all_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + assert set(layer.relevant_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + # Gebaeude from Ortsplanung_V1_1 + if layer.alias == "Ortsplanung_V1_1.Konstruktionen.Gebaeude": + count += 1 + assert set(layer.all_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + assert set(layer.relevant_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + # Gebaeude from Polymorphic_Ortsplanung_V1_1.Gewerbe + if layer.alias == "Gewerbe.Gebaeude": + count += 1 + assert set(layer.all_topics) == { + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe" + } + assert set(layer.relevant_topics) == { + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe" + } + # Gebaeude from Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe + if layer.alias == "IndustrieGewerbe.Gebaeude": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + + assert count == 5 + project = Project(optimize_strategy=strategy) project.layers = available_layers project.relations = relations @@ -960,6 +1023,61 @@ def _extopt_polymorphic_group(self, generator, strategy): ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + # check relevant topics + count = 0 + for layer in available_layers: + # Strasse from Infrastruktur_V1_1 + if layer.alias == "Strasse": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # BesitzerIn from Ortsplanung_V1_1 + if layer.alias == "BesitzerIn": + count += 1 + assert set(layer.all_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + assert set(layer.relevant_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + # Gebaeude from Ortsplanung_V1_1 + if layer.alias == "Ortsplanung_V1_1.Konstruktionen.Gebaeude": + count += 1 + assert set(layer.all_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + assert set(layer.relevant_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + # Gebaeude from Polymorphic_Ortsplanung_V1_1.Gewerbe + if layer.alias == "Gewerbe.Gebaeude": + count += 1 + assert set(layer.all_topics) == { + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe" + } + assert set(layer.relevant_topics) == { + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe" + } + # Gebaeude from Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe + if layer.alias == "IndustrieGewerbe.Gebaeude": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + + assert count == 5 + project = Project(optimize_strategy=strategy) project.layers = available_layers project.relations = relations @@ -1081,6 +1199,66 @@ def _extopt_polymorphic_hide(self, generator, strategy): ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + # check relevant topics + count = 0 + for layer in available_layers: + # Strasse from Infrastruktur_V1_1 + if layer.alias == "Strasse": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # BesitzerIn from Ortsplanung_V1_1 + if layer.alias == "BesitzerIn": + count += 1 + assert set(layer.all_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + assert set(layer.relevant_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + # Gebaeude from Ortsplanung_V1_1 + if layer.alias == "Konstruktionen.Gebaeude" and not layer.is_relevant: + count += 1 + assert set(layer.all_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + assert set(layer.relevant_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + # Gebaeude from Polymorphic_Ortsplanung_V1_1 + if layer.alias == "Konstruktionen.Gebaeude" and layer.is_relevant: + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # Gebaeude from Polymorphic_Ortsplanung_V1_1.Gewerbe + if layer.alias == "Gewerbe.Gebaeude": + count += 1 + assert set(layer.all_topics) == { + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe" + } + assert set(layer.relevant_topics) == { + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe" + } + # Gebaeude from Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe + if layer.alias == "IndustrieGewerbe.Gebaeude": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + + assert count == 6 + project = Project(optimize_strategy=strategy) project.layers = available_layers project.relations = relations @@ -1384,6 +1562,37 @@ def _extopt_baustruct_none(self, generator, strategy): ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + # check relevant topics + count = 0 + for layer in available_layers: + # Strasse from Infrastruktur_V1_1 + if layer.alias == "Strasse": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # Park from Bauplanung_V1_1 + if layer.alias == "Bauplanung_V1_1.Natur.Park": + count += 1 + assert set(layer.all_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} + assert set(layer.relevant_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} + # Park from Kantonale_Bauplanung_V1_1 + if layer.alias == "Kantonale_Bauplanung_V1_1.Natur.Park": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # Feld from Bauplanung_V1_1 + if layer.alias == "Feld": + count += 1 + assert set(layer.all_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} + assert set(layer.relevant_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} + # Kartoffelfeld from Kantonale_Bauplanung_V1_1.Natur + if layer.alias == "Kartoffelfeld": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + + assert count == 5 + project = Project(optimize_strategy=strategy) project.layers = available_layers project.relations = relations @@ -1499,6 +1708,37 @@ def _extopt_baustruct_group(self, generator, strategy): ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + # check relevant topics + count = 0 + for layer in available_layers: + # Strasse from Infrastruktur_V1_1 + if layer.alias == "Strasse": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # Park from Bauplanung_V1_1 + if layer.alias == "Bauplanung_V1_1.Natur.Park": + count += 1 + assert set(layer.all_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} + assert set(layer.relevant_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} + # Park from Kantonale_Bauplanung_V1_1 + if layer.alias == "Kantonale_Bauplanung_V1_1.Natur.Park": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # Feld from Bauplanung_V1_1 + if layer.alias == "Feld": + count += 1 + assert set(layer.all_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} + assert set(layer.relevant_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} + # Kartoffelfeld from Kantonale_Bauplanung_V1_1.Natur + if layer.alias == "Kartoffelfeld": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + + assert count == 5 + project = Project(optimize_strategy=strategy) project.layers = available_layers project.relations = relations @@ -1631,6 +1871,37 @@ def _extopt_baustruct_hide(self, generator, strategy): ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) + # check relevant topics + count = 0 + for layer in available_layers: + # Strasse from Infrastruktur_V1_1 + if layer.alias == "Strasse": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # Park from Bauplanung_V1_1 + if layer.alias == "Park" and not layer.is_relevant: + count += 1 + assert set(layer.all_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} + assert set(layer.relevant_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} + # Park from Kantonale_Bauplanung_V1_1 + if layer.alias == "Park" and layer.is_relevant: + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # Feld from Bauplanung_V1_1 + if layer.alias == "Feld": + count += 1 + assert set(layer.all_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} + assert set(layer.relevant_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} + # Kartoffelfeld from Kantonale_Bauplanung_V1_1.Natur + if layer.alias == "Kartoffelfeld": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + + assert count == 5 + project = Project(optimize_strategy=strategy) project.layers = available_layers project.relations = relations From 4172a077fca26360049a81583e0ad2990e6528a2 Mon Sep 17 00:00:00 2001 From: signedav Date: Wed, 20 Sep 2023 17:18:46 +0200 Subject: [PATCH 25/34] Fixed error when model has multiple extensions within multiple models (so the subquery returned multiple entries), if one of them is bigger 1 its fulfilled --- modelbaker/dbconnector/gpkg_connector.py | 2 +- modelbaker/dbconnector/pg_connector.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modelbaker/dbconnector/gpkg_connector.py b/modelbaker/dbconnector/gpkg_connector.py index 5ea6a49..eedeb0f 100644 --- a/modelbaker/dbconnector/gpkg_connector.py +++ b/modelbaker/dbconnector/gpkg_connector.py @@ -175,7 +175,7 @@ def _get_tables_info(self): base_names.class = extend_names.class OR -- multiple times in the same extended model - (SELECT COUNT(baseClass) FROM T_ILI2DB_INHERITANCE JOIN names extend_names ON thisClass = extend_names.fullname WHERE baseClass = i.baseClass GROUP BY baseClass, extend_names.model)>1 + (SELECT MAX(count) FROM (SELECT COUNT(baseClass) AS count FROM T_ILI2DB_INHERITANCE JOIN names extend_names ON thisClass = extend_names.fullname WHERE baseClass = i.baseClass GROUP BY baseClass, extend_names.model) AS counts )>1 ) ) THEN FALSE ELSE TRUE END AS relevance""", diff --git a/modelbaker/dbconnector/pg_connector.py b/modelbaker/dbconnector/pg_connector.py index d572cb2..9109e6e 100644 --- a/modelbaker/dbconnector/pg_connector.py +++ b/modelbaker/dbconnector/pg_connector.py @@ -219,8 +219,8 @@ def get_tables_info(self): -- with the same name base_names.class = extend_names.class OR - -- multiple times in the same extended model - (SELECT COUNT(baseClass) FROM {schema}.t_ili2db_inheritance JOIN names extend_names ON thisClass = extend_names.fullname WHERE baseClass = i.baseClass GROUP BY baseClass, extend_names.model)>1 + -- multiple times in a same extended model + (SELECT MAX(count) FROM (SELECT COUNT(baseClass) AS count FROM {schema}.t_ili2db_inheritance JOIN names extend_names ON thisClass = extend_names.fullname WHERE baseClass = i.baseClass GROUP BY baseClass, extend_names.model) AS counts )>1 ) ) THEN FALSE ELSE TRUE END AS relevance, From 4a3d2e21020a12529707f212fe4c6564c99f6af1 Mon Sep 17 00:00:00 2001 From: signedav Date: Fri, 22 Sep 2023 16:23:34 +0200 Subject: [PATCH 26/34] mssql not done status --- modelbaker/dbconnector/mssql_connector.py | 60 +++++++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/modelbaker/dbconnector/mssql_connector.py b/modelbaker/dbconnector/mssql_connector.py index de67b15..d634183 100644 --- a/modelbaker/dbconnector/mssql_connector.py +++ b/modelbaker/dbconnector/mssql_connector.py @@ -200,12 +200,48 @@ def get_tables_info(self): base_names.class = extend_names.class OR -- multiple times in the same extended model - (SELECT COUNT(baseClass) FROM {schema}.t_ili2db_inheritance JOIN names extend_names ON thisClass = extend_names.fullname WHERE baseClass = i.baseClass GROUP BY baseClass, extend_names.model)>1 + (SELECT MAX(count) FROM (SELECT COUNT(baseClass) AS count FROM {schema}.t_ili2db_inheritance JOIN names extend_names ON thisClass = extend_names.fullname WHERE baseClass = i.baseClass GROUP BY baseClass, extend_names.model) AS counts )>1 ) ) - THEN FALSE ELSE TRUE AS relevance + THEN 0 ELSE 1 AS relevance """ ) + # topics - where this class or an instance of it is located - are emitted by going recursively through the inheritance table. + # if something of this topic where the current class is located has been extended, it gets the next child topic. + # the relevant topics for optimization are the ones that are not more extended (or in the very last class). + stmt += ( + ln + + """ ,(SELECT STRING_AGG(childTopic,',') FROM {topic_pedigree}) as all_topics + ,(SELECT STRING_AGG(childTopic,',') FROM {topic_pedigree} WHERE NOT is_a_base) as relevant_topics""".format( + topic_pedigree="""(WITH RECURSIVE children(is_a_base, childTopic, baseTopic) AS ( + SELECT + (CASE + WHEN substring( thisClass, 1, CHARINEX('.', substring( thisClass, CHARINEX('.', thisClass)+1))+CHARINEX('.', thisClass)-1) IN (SELECT substring( i.baseClass, 1, CHARINEX('.', substring( i.baseClass, CHARINEX('.', i.baseClass)+1))+CHARINEX('.', i.baseClass)-1) FROM {schema}.T_ILI2DB_INHERITANCE i WHERE substring( i.thisClass, 1, CHARINEX('.', i.thisClass) ) != substring( i.baseClass, 1, CHARINEX('.', i.baseClass))) + THEN 1 + ELSE 0 + END) AS is_a_base, + substring( thisClass, 1, CHARINEX('.', substring( thisClass, CHARINEX('.', thisClass)+1))+CHARINEX('.', thisClass)-1) as childTopic, + substring( baseClass, 1, CHARINEX('.', substring( baseClass, CHARINEX('.', baseClass)+1))+CHARINEX('.', baseClass)-1) as baseTopic + FROM {schema}.T_ILI2DB_INHERITANCE + WHERE substring( baseClass, 1, CHARINEX('.', substring( baseClass, CHARINEX('.', baseClass)+1))+CHARINEX('.', baseClass)-1) = substring( c.iliname, 1, CHARINEX('.', substring( c.iliname, CHARINEX('.', c.iliname)+1))+CHARINEX('.', c.iliname)-1) + UNION + SELECT + (CASE + WHEN substring( thisClass, 1, CHARINEX('.', substring( thisClass, CHARINEX('.', thisClass)+1))+CHARINEX('.', thisClass)-1) IN (SELECT substring( i.baseClass, 1, CHARINEX('.', substring( i.baseClass, CHARINEX('.', i.baseClass)+1))+CHARINEX('.', i.baseClass)-1) FROM {schema}.T_ILI2DB_INHERITANCE i WHERE substring( i.thisClass, 1, CHARINEX('.', i.thisClass)) != substring( i.baseClass, 1, CHARINEX('.', i.baseClass))) + THEN 1 + ELSE 0 + END) AS is_a_base, + substring( inheritance.thisClass, 1, CHARINEX('.', substring( inheritance.thisClass, CHARINEX('.', inheritance.thisClass)+1))+CHARINEX('.', inheritance.thisClass)-1) as childTopic , + substring( inheritance.baseClass, 1, CHARINEX('.', substring( inheritance.baseClass, CHARINEX('.', baseClass)+1))+CHARINEX('.', inheritance.baseClass)-1) as baseTopic + FROM children + JOIN {schema}.T_ILI2DB_INHERITANCE as inheritance + ON substring( inheritance.baseClass, 1, CHARINEX('.', substring( inheritance.baseClass, CHARINEX('.', inheritance.baseClass)+1))+CHARINEX('.', inheritance.baseClass)-1) = children.childTopic -- when the childTopic is as well the baseTopic of another childTopic + WHERE substring( inheritance.thisClass, 1, CHARINEX('.', substring( inheritance.thisClass, CHARINEX('.', inheritance.thisClass)+1))+CHARINEX('.', inheritance.thisClass)-1) != children.childTopic -- break the recursion when the coming childTopic will be the same + ) + SELECT childTopic, baseTopic, is_a_base FROM children) AS kiddies + """ + ) + ) stmt += ln + "FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS Tab" stmt += ln + "INNER JOIN INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE AS Col" stmt += ln + " ON Col.Constraint_Name = Tab.Constraint_Name" @@ -245,6 +281,9 @@ def get_tables_info(self): ) stmt = stmt.format(schema=self.schema) + print("\n\n\n\n\n\n***********************************\n\n\n\n") + print(stmt) + print("\n\n\n\n\n\n***********************************\n\n\n\n") if not metadata_exists: stmt = self._def_cursor(stmt) @@ -852,13 +891,26 @@ def get_topics_info(self): cur.execute( """ SELECT DISTINCT PARSENAME(cn.iliname,1) as model, - PARSENAME(cn.iliname,2) as topic + PARSENAME(cn.iliname,2) as topic, + {relevance} FROM {schema}.t_ili2db_classname as cn JOIN {schema}.t_ili2db_table_prop as tp ON cn.sqlname = tp.tablename WHERE PARSENAME(cn.iliname,3) != '' and tp.setting != 'ENUM' """.format( - schema=self.schema + schema=self.schema, + relevance=""" + CASE WHEN (WITH RECURSIVE children(childTopic, baseTopic) AS ( + SELECT substring( thisClass, 1, CHARINDEX('.', substring( thisClass, CHARINDEX('.', thisClass)+1))+CHARINDEX('.', thisClass)-1) as childTopic , substring( baseClass, 1, CHARINDEX('.', substring( baseClass, CHARINDEX('.', baseClass)+1))+CHARINDEX('.', baseClass)-1) as baseTopic + FROM {schema}.T_ILI2DB_INHERITANCE + WHERE substring( baseClass, 1, CHARINDEX('.', substring( baseClass, CHARINDEX('.', baseClass)+1))+CHARINDEX('.', baseClass)-1) = substring( CN.IliName, 1, CHARINDEX('.', substring( CN.IliName, CHARINDEX('.', CN.IliName)+1))+CHARINDEX('.', CN.IliName)-1) -- model.topic + UNION + SELECT substring( inheritance.thisClass, 1, CHARINDEX('.', substring( inheritance.thisClass, CHARINDEX('.', inheritance.thisClass)+1))+CHARINDEX('.', inheritance.thisClass)-1) as childTopic , substring( inheritance.baseClass, 1, CHARINDEX('.', substring( inheritance.baseClass, CHARINDEX('.', baseClass)+1))+CHARINDEX('.', inheritance.baseClass)-1) as baseTopic FROM children + JOIN {schema}.T_ILI2DB_INHERITANCE as inheritance ON substring( inheritance.baseClass, 1, CHARINDEX('.', substring( inheritance.baseClass, CHARINDEX('.', baseClass)+1))+CHARINDEX('.', inheritance.baseClass)-1) = children.childTopic + )SELECT count(childTopic) FROM children)>0 THEN 0 ELSE 1 END AS relevance + """.format( + schema=self.schema + ), ) ) result = self._get_dict_result(cur) From d38d74ec7613edbc299cbccfaf252d9912381f4e Mon Sep 17 00:00:00 2001 From: signedav Date: Fri, 22 Sep 2023 16:43:00 +0200 Subject: [PATCH 27/34] fixed mssql query --- modelbaker/dbconnector/mssql_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelbaker/dbconnector/mssql_connector.py b/modelbaker/dbconnector/mssql_connector.py index d634183..c877a06 100644 --- a/modelbaker/dbconnector/mssql_connector.py +++ b/modelbaker/dbconnector/mssql_connector.py @@ -184,7 +184,7 @@ def get_tables_info(self): SELECT thisClass as fullname, substring(thisClass, 1, charindex('.', thisClass)-1) as model, - substring(thisClass, charindex('.', thisClass)-1, len(thisClass)) as topicclass + substring(thisClass, charindex('.', thisClass)+1, len(thisClass)) as topicclass FROM {schema}.t_ili2db_inheritance ) ) AS extend_names From 231cc26707894b921b36905af01d4a35d9e7a24c Mon Sep 17 00:00:00 2001 From: signedav Date: Fri, 22 Sep 2023 16:44:56 +0200 Subject: [PATCH 28/34] Bump ili2db to 5.0.0 --- modelbaker/iliwrapper/ili2dbtools.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modelbaker/iliwrapper/ili2dbtools.py b/modelbaker/iliwrapper/ili2dbtools.py index 6a6f61b..ef559a4 100644 --- a/modelbaker/iliwrapper/ili2dbtools.py +++ b/modelbaker/iliwrapper/ili2dbtools.py @@ -25,17 +25,17 @@ def get_tool_version(tool, db_ili_version): if db_ili_version == 3: return "3.11.3" else: - return "4.11.1" + return "5.0.0" elif tool == DbIliMode.ili2pg: if db_ili_version == 3: return "3.11.2" else: - return "4.11.1" + return "5.0.0" elif tool == DbIliMode.ili2mssql: if db_ili_version == 3: return "3.12.2" else: - return "4.11.1" + return "5.0.0" return "0" From 6ecffa4d26974cb5b8551ef69f93f9c611eb855d Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 25 Sep 2023 13:51:50 +0200 Subject: [PATCH 29/34] Update modelbaker/generator/generator.py Co-authored-by: Damiano Lombardi --- modelbaker/generator/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelbaker/generator/generator.py b/modelbaker/generator/generator.py index bc2fea5..470face 100644 --- a/modelbaker/generator/generator.py +++ b/modelbaker/generator/generator.py @@ -374,7 +374,7 @@ def layers(self, filter_layer_list=[]): # append topic name to ambiguous layers self._rename_ambiguous_layers(layers) - # append moel name to still ambiguous layers + # append model name to still ambiguous layers self._rename_ambiguous_layers(layers, second_pass=True) self.print_messages() From acb2d27232d963af7fd484c68cbe03749db54608 Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 25 Sep 2023 23:49:06 +0200 Subject: [PATCH 30/34] remove print --- modelbaker/dbconnector/mssql_connector.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/modelbaker/dbconnector/mssql_connector.py b/modelbaker/dbconnector/mssql_connector.py index c877a06..d25b361 100644 --- a/modelbaker/dbconnector/mssql_connector.py +++ b/modelbaker/dbconnector/mssql_connector.py @@ -281,9 +281,6 @@ def get_tables_info(self): ) stmt = stmt.format(schema=self.schema) - print("\n\n\n\n\n\n***********************************\n\n\n\n") - print(stmt) - print("\n\n\n\n\n\n***********************************\n\n\n\n") if not metadata_exists: stmt = self._def_cursor(stmt) From 599e8d5017874f14bf8ad0944e745414f043607c Mon Sep 17 00:00:00 2001 From: signedav Date: Tue, 26 Sep 2023 07:58:30 +0200 Subject: [PATCH 31/34] fix missing assignement --- modelbaker/dbconnector/pg_connector.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modelbaker/dbconnector/pg_connector.py b/modelbaker/dbconnector/pg_connector.py index 9109e6e..fdb03fb 100644 --- a/modelbaker/dbconnector/pg_connector.py +++ b/modelbaker/dbconnector/pg_connector.py @@ -161,6 +161,8 @@ def get_tables_info(self): model_where = "" attribute_name = "" attribute_left_join = "" + relevance = "" + topics = "" if self.metadata_exists(): kind_settings_field = "p.setting AS kind_settings," From 254e9183d73729a07b50a12353cd573d3e160095 Mon Sep 17 00:00:00 2001 From: signedav Date: Tue, 26 Sep 2023 08:33:51 +0200 Subject: [PATCH 32/34] fixup --- modelbaker/dbconnector/pg_connector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modelbaker/dbconnector/pg_connector.py b/modelbaker/dbconnector/pg_connector.py index fdb03fb..18a2d39 100644 --- a/modelbaker/dbconnector/pg_connector.py +++ b/modelbaker/dbconnector/pg_connector.py @@ -234,7 +234,7 @@ def get_tables_info(self): # if something of this topic where the current class is located has been extended, it gets the next child topic. # the relevant topics for optimization are the ones that are not more extended (or in the very last class). topics = """(SELECT STRING_AGG(childTopic,',') FROM {topic_pedigree}) as all_topics, - (SELECT STRING_AGG(childTopic,',') FROM {topic_pedigree} WHERE NOT is_a_base) as relevant_topics""".format( + (SELECT STRING_AGG(childTopic,',') FROM {topic_pedigree} WHERE NOT is_a_base) as relevant_topics,""".format( topic_pedigree="""(WITH RECURSIVE children(is_a_base, childTopic, baseTopic) AS ( SELECT (CASE @@ -304,10 +304,10 @@ def get_tables_info(self): {extent} {attribute_name} {coord_decimals} - g.type AS simple_type, - format_type(ga.atttypid, ga.atttypmod) as formatted_type, {relevance} {topics} + g.type AS simple_type, + format_type(ga.atttypid, ga.atttypmod) as formatted_type FROM pg_catalog.pg_tables tbls LEFT JOIN pg_index i ON i.indrelid = CONCAT(tbls.schemaname, '."', tbls.tablename, '"')::regclass From b09991e543f83d615b3dc2576173d22ba635f92b Mon Sep 17 00:00:00 2001 From: signedav Date: Sun, 1 Oct 2023 22:39:22 +0200 Subject: [PATCH 33/34] finally working relevance part for mssql --- modelbaker/dbconnector/mssql_connector.py | 53 +- tests/README.md | 32 + .../test_projectgen_extension_optimization.py | 768 +++++++++--------- 3 files changed, 470 insertions(+), 383 deletions(-) diff --git a/modelbaker/dbconnector/mssql_connector.py b/modelbaker/dbconnector/mssql_connector.py index d25b361..f180c0c 100644 --- a/modelbaker/dbconnector/mssql_connector.py +++ b/modelbaker/dbconnector/mssql_connector.py @@ -179,17 +179,26 @@ def get_tables_info(self): SELECT i.baseClass as base FROM {schema}.t_ili2db_inheritance i LEFT JOIN ( - SELECT fullname, model, topicclass, substring(topicclass, charindex('.',topicclass)+1,len(topicclass)) as class + SELECT fullname, model, topicclass, substring(topicclass, charindex('.',topicclass)+1, len(topicclass)) as class FROM ( SELECT thisClass as fullname, substring(thisClass, 1, charindex('.', thisClass)-1) as model, substring(thisClass, charindex('.', thisClass)+1, len(thisClass)) as topicclass FROM {schema}.t_ili2db_inheritance - ) + ) AS extended ) AS extend_names ON thisClass = extend_names.fullname - LEFT JOIN names base_names + LEFT JOIN ( + SELECT fullname, model, topicclass, substring(topicclass, charindex('.', topicclass)+1, len(topicclass)) as class + FROM ( + SELECT + thisClass as fullname, + substring(thisClass, 1, charindex('.',thisClass)-1) as model, + substring(thisClass, charindex('.', thisClass)+1, len(thisClass)) as topicclass + FROM {schema}.t_ili2db_inheritance + ) AS topic_level_name + ) AS base_names ON baseClass = base_names.fullname -- it's extended WHERE baseClass IS NOT NULL @@ -200,48 +209,59 @@ def get_tables_info(self): base_names.class = extend_names.class OR -- multiple times in the same extended model - (SELECT MAX(count) FROM (SELECT COUNT(baseClass) AS count FROM {schema}.t_ili2db_inheritance JOIN names extend_names ON thisClass = extend_names.fullname WHERE baseClass = i.baseClass GROUP BY baseClass, extend_names.model) AS counts )>1 + (SELECT MAX(count) FROM (SELECT COUNT(baseClass) AS count FROM {schema}.t_ili2db_inheritance JOIN ( + SELECT fullname, model, topicclass, substring(topicclass, charindex('.', topicclass)+1, len(topicclass)) as class + FROM ( + SELECT + thisClass as fullname, + substring(thisClass, 1, charindex('.',thisClass)-1) as model, + substring(thisClass, charindex('.', thisClass)+1, len(thisClass)) as topicclass + FROM {schema}.t_ili2db_inheritance + ) AS topic_level_name + ) AS extend_names ON thisClass = extend_names.fullname WHERE baseClass = i.baseClass GROUP BY baseClass, extend_names.model) AS counts )>1 ) ) - THEN 0 ELSE 1 AS relevance + THEN 0 ELSE 1 END AS relevance """ ) # topics - where this class or an instance of it is located - are emitted by going recursively through the inheritance table. # if something of this topic where the current class is located has been extended, it gets the next child topic. # the relevant topics for optimization are the ones that are not more extended (or in the very last class). + ''' This part is not yet working stmt += ( ln + """ ,(SELECT STRING_AGG(childTopic,',') FROM {topic_pedigree}) as all_topics ,(SELECT STRING_AGG(childTopic,',') FROM {topic_pedigree} WHERE NOT is_a_base) as relevant_topics""".format( - topic_pedigree="""(WITH RECURSIVE children(is_a_base, childTopic, baseTopic) AS ( + topic_pedigree="""(WITH children(is_a_base, childTopic, baseTopic) AS ( SELECT (CASE - WHEN substring( thisClass, 1, CHARINEX('.', substring( thisClass, CHARINEX('.', thisClass)+1))+CHARINEX('.', thisClass)-1) IN (SELECT substring( i.baseClass, 1, CHARINEX('.', substring( i.baseClass, CHARINEX('.', i.baseClass)+1))+CHARINEX('.', i.baseClass)-1) FROM {schema}.T_ILI2DB_INHERITANCE i WHERE substring( i.thisClass, 1, CHARINEX('.', i.thisClass) ) != substring( i.baseClass, 1, CHARINEX('.', i.baseClass))) + WHEN substring( thisClass, 1, CHARINDEX('.', substring( thisClass, CHARINDEX('.', thisClass)+1, len(thisClass)))+CHARINDEX('.', thisClass)-1) IN (SELECT substring( i.baseClass, 1, CHARINDEX('.', substring( i.baseClass, CHARINDEX('.', i.baseClass)+1, len(i.baseClass)))+CHARINDEX('.', i.baseClass)-1) FROM {schema}.T_ILI2DB_INHERITANCE i WHERE substring( i.thisClass, 1, CHARINDEX('.', i.thisClass) ) != substring( i.baseClass, 1, CHARINDEX('.', i.baseClass))) THEN 1 ELSE 0 END) AS is_a_base, - substring( thisClass, 1, CHARINEX('.', substring( thisClass, CHARINEX('.', thisClass)+1))+CHARINEX('.', thisClass)-1) as childTopic, - substring( baseClass, 1, CHARINEX('.', substring( baseClass, CHARINEX('.', baseClass)+1))+CHARINEX('.', baseClass)-1) as baseTopic + substring( thisClass, 1, CHARINDEX('.', substring( thisClass, CHARINDEX('.', thisClass)+1, len(thisClass)))+CHARINDEX('.', thisClass)-1) as childTopic, + substring( baseClass, 1, CHARINDEX('.', substring( baseClass, CHARINDEX('.', baseClass)+1, len(baseClass)))+CHARINDEX('.', baseClass)-1) as baseTopic FROM {schema}.T_ILI2DB_INHERITANCE - WHERE substring( baseClass, 1, CHARINEX('.', substring( baseClass, CHARINEX('.', baseClass)+1))+CHARINEX('.', baseClass)-1) = substring( c.iliname, 1, CHARINEX('.', substring( c.iliname, CHARINEX('.', c.iliname)+1))+CHARINEX('.', c.iliname)-1) + WHERE substring( baseClass, 1, CHARINDEX('.', substring( baseClass, CHARINDEX('.', baseClass)+1, len(thisClass)))+CHARINDEX('.', baseClass)-1) = substring( c.iliname, 1, CHARINDEX('.', substring( c.iliname, CHARINDEX('.', c.iliname)+1, len(c.iliname)))+CHARINDEX('.', c.iliname)-1) UNION SELECT (CASE - WHEN substring( thisClass, 1, CHARINEX('.', substring( thisClass, CHARINEX('.', thisClass)+1))+CHARINEX('.', thisClass)-1) IN (SELECT substring( i.baseClass, 1, CHARINEX('.', substring( i.baseClass, CHARINEX('.', i.baseClass)+1))+CHARINEX('.', i.baseClass)-1) FROM {schema}.T_ILI2DB_INHERITANCE i WHERE substring( i.thisClass, 1, CHARINEX('.', i.thisClass)) != substring( i.baseClass, 1, CHARINEX('.', i.baseClass))) + WHEN substring( thisClass, 1, CHARINDEX('.', substring( thisClass, CHARINDEX('.', thisClass)+1, len(thisClass)))+CHARINDEX('.', thisClass)-1) IN (SELECT substring( i.baseClass, 1, CHARINDEX('.', substring( i.baseClass, CHARINDEX('.', i.baseClass)+1, len(i.baseClass)))+CHARINDEX('.', i.baseClass)-1) FROM {schema}.T_ILI2DB_INHERITANCE i WHERE substring( i.thisClass, 1, CHARINDEX('.', i.thisClass)) != substring( i.baseClass, 1, CHARINDEX('.', i.baseClass))) THEN 1 ELSE 0 END) AS is_a_base, - substring( inheritance.thisClass, 1, CHARINEX('.', substring( inheritance.thisClass, CHARINEX('.', inheritance.thisClass)+1))+CHARINEX('.', inheritance.thisClass)-1) as childTopic , - substring( inheritance.baseClass, 1, CHARINEX('.', substring( inheritance.baseClass, CHARINEX('.', baseClass)+1))+CHARINEX('.', inheritance.baseClass)-1) as baseTopic + substring( inheritance.thisClass, 1, CHARINDEX('.', substring( inheritance.thisClass, CHARINDEX('.', inheritance.thisClass)+1))+CHARINDEX('.', inheritance.thisClass)-1,len(inheritance.thisClass)) as childTopic , + substring( inheritance.baseClass, 1, CHARINDEX('.', substring( inheritance.baseClass, CHARINDEX('.', baseClass)+1))+CHARINDEX('.', inheritance.baseClass)-1,len(inheritance.baseClass)) as baseTopic FROM children JOIN {schema}.T_ILI2DB_INHERITANCE as inheritance - ON substring( inheritance.baseClass, 1, CHARINEX('.', substring( inheritance.baseClass, CHARINEX('.', inheritance.baseClass)+1))+CHARINEX('.', inheritance.baseClass)-1) = children.childTopic -- when the childTopic is as well the baseTopic of another childTopic - WHERE substring( inheritance.thisClass, 1, CHARINEX('.', substring( inheritance.thisClass, CHARINEX('.', inheritance.thisClass)+1))+CHARINEX('.', inheritance.thisClass)-1) != children.childTopic -- break the recursion when the coming childTopic will be the same + ON substring( inheritance.baseClass, 1, CHARINDEX('.', substring( inheritance.baseClass, CHARINDEX('.', inheritance.baseClass)+1))+CHARINDEX('.', inheritance.baseClass)-1, len(inheritance.thisClass)) = children.childTopic -- when the childTopic is as well the baseTopic of another childTopic + WHERE substring( inheritance.thisClass, 1, CHARINDEX('.', substring( inheritance.thisClass, CHARINDEX('.', inheritance.thisClass)+1))+CHARINDEX('.', inheritance.thisClass)-1, len(inheritance.thisClass)) != children.childTopic -- break the recursion when the coming childTopic will be the same ) SELECT childTopic, baseTopic, is_a_base FROM children) AS kiddies """ ) ) + ''' stmt += ln + "FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS Tab" stmt += ln + "INNER JOIN INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE AS Col" stmt += ln + " ON Col.Constraint_Name = Tab.Constraint_Name" @@ -285,6 +305,7 @@ def get_tables_info(self): stmt = self._def_cursor(stmt) cur = self.conn.cursor() + cur.execute(stmt) if cur.description: @@ -897,7 +918,7 @@ def get_topics_info(self): """.format( schema=self.schema, relevance=""" - CASE WHEN (WITH RECURSIVE children(childTopic, baseTopic) AS ( + CASE WHEN (WITH children(childTopic, baseTopic) AS ( SELECT substring( thisClass, 1, CHARINDEX('.', substring( thisClass, CHARINDEX('.', thisClass)+1))+CHARINDEX('.', thisClass)-1) as childTopic , substring( baseClass, 1, CHARINDEX('.', substring( baseClass, CHARINDEX('.', baseClass)+1))+CHARINDEX('.', baseClass)-1) as baseTopic FROM {schema}.T_ILI2DB_INHERITANCE WHERE substring( baseClass, 1, CHARINDEX('.', substring( baseClass, CHARINDEX('.', baseClass)+1))+CHARINDEX('.', baseClass)-1) = substring( CN.IliName, 1, CHARINDEX('.', substring( CN.IliName, CHARINDEX('.', CN.IliName)+1))+CHARINDEX('.', CN.IliName)-1) -- model.topic diff --git a/tests/README.md b/tests/README.md index 6e11ac2..78f7065 100644 --- a/tests/README.md +++ b/tests/README.md @@ -21,3 +21,35 @@ Run all test starting with ``test_array_` ```sh [...] run-tests.sh -k test_array_ [...] ``` + +## Testing with MSSQL + +These are dirty notes for the quickest way to test mssql queries manually in the way the tests work. + +1. Create a new dir. E.g. `.local_docker_test` +2. Copy the original docker-compose file from directory `.docker` and remove everything except the qgis and the mssql container. +3. Copy the original Dockerfile as well. Leaf it like it is... +4. Copy the original `run-docker-tests.sh` and remove everything except: + ```bash + set -e + + /usr/src/tests/testdata/mssql/setup-mssql.sh + ``` +5. Do the following: + ```bash + docker-compose up -d + docker exec -it local_docker_tests_qgis_1 bash + ``` + And then in the qgis docker: + ```bash + /usr/src/.local_docker_tests/run-docker-tests.sh; + java -jar /usr/src/modelbaker/iliwrapper/bin/ili2mssql-5.0.0/ili2mssql-5.0.0.jar --schemaimport --dbhost mssql --dbusr sa --dbpwd '' --dbdatabase gis --dbschema optimal_polymorph_manuel --coalesceCatalogueRef --createEnumTabs --createNumChecks --createUnique --createFk --createFkIdx --coalesceMultiSurface --coalesceMultiLine --coalesceMultiPoint --coalesceArray --beautifyEnumDispName --createGeomIdx --createMetaInfo --expandMultilingual --createTypeConstraint --createEnumTabsWithId --createTidCol --importTid --smart2Inheritance --strokeArcs --createBasketCol --defaultSrsAuth EPSG --defaultSrsCode 2056 --preScript NULL --postScript NULL --models Polymorphic_Ortsplanung_V1_1 --iliMetaAttrs NULL /usr/src/tests/testdata/ilimodels/Polymorphic_Ortsplanung_V1_1.ili + ``` + (Surely this could be done without this qgis container, but there you have everything for the set up already...) +6. Now connect (with eg. the VS Code extension) with these params: + ``` + localhost + 1433 + sa + '' + ``` diff --git a/tests/test_projectgen_extension_optimization.py b/tests/test_projectgen_extension_optimization.py index a84c50b..8ee67af 100644 --- a/tests/test_projectgen_extension_optimization.py +++ b/tests/test_projectgen_extension_optimization.py @@ -185,7 +185,7 @@ def test_extopt_staedtische_geopackage(self): self._extopt_staedtische_hide(generator, strategy) def test_extopt_staedtische_mssql(self): - return # to do + importer = iliimporter.Importer() importer.tool = DbIliMode.ili2mssql importer.configuration = iliimporter_config(importer.tool) @@ -223,7 +223,8 @@ def test_extopt_staedtische_mssql(self): optimize_strategy=strategy, ) - self._extopt_staedtische_none(generator, strategy) + # we skip the relation check since it's not supported in mssql + self._extopt_staedtische_none(generator, strategy, True) ### 2. OptimizeStrategy.GROUP ### strategy = OptimizeStrategy.GROUP @@ -236,7 +237,8 @@ def test_extopt_staedtische_mssql(self): optimize_strategy=strategy, ) - self._extopt_staedtische_group(generator, strategy) + # we skip the relation check since it's not supported in mssql + self._extopt_staedtische_group(generator, strategy, True) ### 3. OptimizeStrategy.HIDE ### strategy = OptimizeStrategy.HIDE @@ -249,9 +251,10 @@ def test_extopt_staedtische_mssql(self): optimize_strategy=strategy, ) - self._extopt_staedtische_hide(generator, strategy) + # we skip the relation check since it's not supported in mssql + self._extopt_staedtische_hide(generator, strategy, True) - def _extopt_staedtische_none(self, generator, strategy): + def _extopt_staedtische_none(self, generator, strategy, skip_topic_check=False): available_layers = generator.layers() relations, _ = generator.relations(available_layers) @@ -286,38 +289,41 @@ def _extopt_staedtische_none(self, generator, strategy): ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - # check relevant topics - count = 0 - for layer in available_layers: - # Strasse from Infrastruktur_V1_1 - if layer.alias == "Strasse": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - # BesitzerIn from Ortsplanung_V1_1 - if layer.alias == "BesitzerIn": - count += 1 - assert set(layer.all_topics) == { - "Kantonale_Ortsplanung_V1_1.Konstruktionen", - "Staedtische_Ortsplanung_V1_1.Freizeit", - "Staedtische_Ortsplanung_V1_1.Gewerbe", - } - assert set(layer.relevant_topics) == { - "Staedtische_Ortsplanung_V1_1.Freizeit", - "Staedtische_Ortsplanung_V1_1.Gewerbe", - } - # Firma from Staedtisches_Gewerbe_V1 - if layer.alias == "Staedtisches_Gewerbe_V1.Firmen.Firma": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - # Firma from Gewerbe_V1 - if layer.alias == "Gewerbe_V1.Firmen.Firma": - count += 1 - print(layer.all_topics) - assert set(layer.all_topics) == {"Staedtisches_Gewerbe_V1.Firmen"} - assert set(layer.relevant_topics) == {"Staedtisches_Gewerbe_V1.Firmen"} - assert count == 4 + if not skip_topic_check: + # check relevant topics + count = 0 + for layer in available_layers: + # Strasse from Infrastruktur_V1_1 + if layer.alias == "Strasse": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # BesitzerIn from Ortsplanung_V1_1 + if layer.alias == "BesitzerIn": + count += 1 + assert set(layer.all_topics) == { + "Kantonale_Ortsplanung_V1_1.Konstruktionen", + "Staedtische_Ortsplanung_V1_1.Freizeit", + "Staedtische_Ortsplanung_V1_1.Gewerbe", + } + assert set(layer.relevant_topics) == { + "Staedtische_Ortsplanung_V1_1.Freizeit", + "Staedtische_Ortsplanung_V1_1.Gewerbe", + } + # Firma from Staedtisches_Gewerbe_V1 + if layer.alias == "Staedtisches_Gewerbe_V1.Firmen.Firma": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # Firma from Gewerbe_V1 + if layer.alias == "Gewerbe_V1.Firmen.Firma": + count += 1 + print(layer.all_topics) + assert set(layer.all_topics) == {"Staedtisches_Gewerbe_V1.Firmen"} + assert set(layer.relevant_topics) == { + "Staedtisches_Gewerbe_V1.Firmen" + } + assert count == 4 project = Project(optimize_strategy=strategy) project.layers = available_layers @@ -383,7 +389,7 @@ def _extopt_staedtische_none(self, generator, strategy): QgsProject.instance().clear() - def _extopt_staedtische_group(self, generator, strategy): + def _extopt_staedtische_group(self, generator, strategy, skip_topic_check=False): available_layers = generator.layers() relations, _ = generator.relations(available_layers) legend = generator.legend(available_layers) @@ -418,37 +424,40 @@ def _extopt_staedtische_group(self, generator, strategy): ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - # check relevant topics - count = 0 - for layer in available_layers: - # Strasse from Infrastruktur_V1_1 - if layer.alias == "Strasse": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - # BesitzerIn from Ortsplanung_V1_1 - if layer.alias == "BesitzerIn": - count += 1 - assert set(layer.all_topics) == { - "Kantonale_Ortsplanung_V1_1.Konstruktionen", - "Staedtische_Ortsplanung_V1_1.Freizeit", - "Staedtische_Ortsplanung_V1_1.Gewerbe", - } - assert set(layer.relevant_topics) == { - "Staedtische_Ortsplanung_V1_1.Freizeit", - "Staedtische_Ortsplanung_V1_1.Gewerbe", - } - # Firma from Staedtisches_Gewerbe_V1 - if layer.alias == "Staedtisches_Gewerbe_V1.Firmen.Firma": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - # Firma from Gewerbe_V1 - if layer.alias == "Gewerbe_V1.Firmen.Firma": - count += 1 - assert set(layer.all_topics) == {"Staedtisches_Gewerbe_V1.Firmen"} - assert set(layer.relevant_topics) == {"Staedtisches_Gewerbe_V1.Firmen"} - assert count == 4 + if not skip_topic_check: + # check relevant topics + count = 0 + for layer in available_layers: + # Strasse from Infrastruktur_V1_1 + if layer.alias == "Strasse": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # BesitzerIn from Ortsplanung_V1_1 + if layer.alias == "BesitzerIn": + count += 1 + assert set(layer.all_topics) == { + "Kantonale_Ortsplanung_V1_1.Konstruktionen", + "Staedtische_Ortsplanung_V1_1.Freizeit", + "Staedtische_Ortsplanung_V1_1.Gewerbe", + } + assert set(layer.relevant_topics) == { + "Staedtische_Ortsplanung_V1_1.Freizeit", + "Staedtische_Ortsplanung_V1_1.Gewerbe", + } + # Firma from Staedtisches_Gewerbe_V1 + if layer.alias == "Staedtisches_Gewerbe_V1.Firmen.Firma": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # Firma from Gewerbe_V1 + if layer.alias == "Gewerbe_V1.Firmen.Firma": + count += 1 + assert set(layer.all_topics) == {"Staedtisches_Gewerbe_V1.Firmen"} + assert set(layer.relevant_topics) == { + "Staedtisches_Gewerbe_V1.Firmen" + } + assert count == 4 project = Project(optimize_strategy=strategy) project.layers = available_layers @@ -520,7 +529,7 @@ def _extopt_staedtische_group(self, generator, strategy): QgsProject.instance().clear() - def _extopt_staedtische_hide(self, generator, strategy): + def _extopt_staedtische_hide(self, generator, strategy, skip_topic_check=False): available_layers = generator.layers() relations, _ = generator.relations(available_layers) @@ -554,32 +563,33 @@ def _extopt_staedtische_hide(self, generator, strategy): ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - # check relevant topics - count = 0 - for layer in available_layers: - # Strasse from Infrastruktur_V1_1 - if layer.alias == "Strasse": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - # BesitzerIn from Ortsplanung_V1_1 - if layer.alias == "BesitzerIn": - count += 1 - assert set(layer.all_topics) == { - "Kantonale_Ortsplanung_V1_1.Konstruktionen", - "Staedtische_Ortsplanung_V1_1.Freizeit", - "Staedtische_Ortsplanung_V1_1.Gewerbe", - } - assert set(layer.relevant_topics) == { - "Staedtische_Ortsplanung_V1_1.Freizeit", - "Staedtische_Ortsplanung_V1_1.Gewerbe", - } - # Firma from Staedtisches_Gewerbe_V1 - if layer.alias == "Firma" and layer.is_relevant: - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - assert count == 3 + if not skip_topic_check: + # check relevant topics + count = 0 + for layer in available_layers: + # Strasse from Infrastruktur_V1_1 + if layer.alias == "Strasse": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # BesitzerIn from Ortsplanung_V1_1 + if layer.alias == "BesitzerIn": + count += 1 + assert set(layer.all_topics) == { + "Kantonale_Ortsplanung_V1_1.Konstruktionen", + "Staedtische_Ortsplanung_V1_1.Freizeit", + "Staedtische_Ortsplanung_V1_1.Gewerbe", + } + assert set(layer.relevant_topics) == { + "Staedtische_Ortsplanung_V1_1.Freizeit", + "Staedtische_Ortsplanung_V1_1.Gewerbe", + } + # Firma from Staedtisches_Gewerbe_V1 + if layer.alias == "Firma" and layer.is_relevant: + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + assert count == 3 project = Project(optimize_strategy=strategy) project.layers = available_layers @@ -753,7 +763,7 @@ def test_extopt_polymorphic_geopackage(self): self._extopt_polymorphic_hide(generator, strategy) def test_extopt_polymorphic_mssql(self): - return # to do + importer = iliimporter.Importer() importer.tool = DbIliMode.ili2mssql importer.configuration = iliimporter_config(importer.tool) @@ -791,7 +801,8 @@ def test_extopt_polymorphic_mssql(self): optimize_strategy=strategy, ) - self._extopt_polymorphic_none(generator, strategy) + # we skip the relation check since it's not supported in mssql + self._extopt_polymorphic_none(generator, strategy, True) ### 2. OptimizeStrategy.GROUP ### strategy = OptimizeStrategy.GROUP @@ -804,7 +815,8 @@ def test_extopt_polymorphic_mssql(self): optimize_strategy=strategy, ) - self._extopt_polymorphic_group(generator, strategy) + # we skip the relation check since it's not supported in mssql + self._extopt_polymorphic_group(generator, strategy, True) ### 3. OptimizeStrategy.HIDE ### strategy = OptimizeStrategy.HIDE @@ -817,9 +829,10 @@ def test_extopt_polymorphic_mssql(self): optimize_strategy=strategy, ) - self._extopt_polymorphic_hide(generator, strategy) + # we skip the relation check since it's not supported in mssql + self._extopt_polymorphic_hide(generator, strategy, True) - def _extopt_polymorphic_none(self, generator, strategy): + def _extopt_polymorphic_none(self, generator, strategy, skip_topic_check=False): available_layers = generator.layers() relations, _ = generator.relations(available_layers) legend = generator.legend(available_layers) @@ -854,60 +867,61 @@ def _extopt_polymorphic_none(self, generator, strategy): ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - # check relevant topics - count = 0 - for layer in available_layers: - # Strasse from Infrastruktur_V1_1 - if layer.alias == "Strasse": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - # BesitzerIn from Ortsplanung_V1_1 - if layer.alias == "BesitzerIn": - count += 1 - assert set(layer.all_topics) == { - "Polymorphic_Ortsplanung_V1_1.Gewerbe", - "Polymorphic_Ortsplanung_V1_1.Freizeit", - "Polymorphic_Ortsplanung_V1_1.Hallen", - "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", - } - assert set(layer.relevant_topics) == { - "Polymorphic_Ortsplanung_V1_1.Gewerbe", - "Polymorphic_Ortsplanung_V1_1.Freizeit", - "Polymorphic_Ortsplanung_V1_1.Hallen", - "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", - } - # Gebaeude from Ortsplanung_V1_1 - if layer.alias == "Ortsplanung_V1_1.Konstruktionen.Gebaeude": - count += 1 - assert set(layer.all_topics) == { - "Polymorphic_Ortsplanung_V1_1.Gewerbe", - "Polymorphic_Ortsplanung_V1_1.Freizeit", - "Polymorphic_Ortsplanung_V1_1.Hallen", - "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", - } - assert set(layer.relevant_topics) == { - "Polymorphic_Ortsplanung_V1_1.Gewerbe", - "Polymorphic_Ortsplanung_V1_1.Freizeit", - "Polymorphic_Ortsplanung_V1_1.Hallen", - "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", - } - # Gebaeude from Polymorphic_Ortsplanung_V1_1.Gewerbe - if layer.alias == "Gewerbe.Gebaeude": - count += 1 - assert set(layer.all_topics) == { - "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe" - } - assert set(layer.relevant_topics) == { - "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe" - } - # Gebaeude from Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe - if layer.alias == "IndustrieGewerbe.Gebaeude": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - - assert count == 5 + if not skip_topic_check: + # check relevant topics + count = 0 + for layer in available_layers: + # Strasse from Infrastruktur_V1_1 + if layer.alias == "Strasse": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # BesitzerIn from Ortsplanung_V1_1 + if layer.alias == "BesitzerIn": + count += 1 + assert set(layer.all_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + assert set(layer.relevant_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + # Gebaeude from Ortsplanung_V1_1 + if layer.alias == "Ortsplanung_V1_1.Konstruktionen.Gebaeude": + count += 1 + assert set(layer.all_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + assert set(layer.relevant_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + # Gebaeude from Polymorphic_Ortsplanung_V1_1.Gewerbe + if layer.alias == "Gewerbe.Gebaeude": + count += 1 + assert set(layer.all_topics) == { + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe" + } + assert set(layer.relevant_topics) == { + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe" + } + # Gebaeude from Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe + if layer.alias == "IndustrieGewerbe.Gebaeude": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + + assert count == 5 project = Project(optimize_strategy=strategy) project.layers = available_layers @@ -987,7 +1001,7 @@ def _extopt_polymorphic_none(self, generator, strategy): QgsProject.instance().clear() - def _extopt_polymorphic_group(self, generator, strategy): + def _extopt_polymorphic_group(self, generator, strategy, skip_topic_check=False): available_layers = generator.layers() relations, _ = generator.relations(available_layers) @@ -1023,60 +1037,61 @@ def _extopt_polymorphic_group(self, generator, strategy): ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - # check relevant topics - count = 0 - for layer in available_layers: - # Strasse from Infrastruktur_V1_1 - if layer.alias == "Strasse": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - # BesitzerIn from Ortsplanung_V1_1 - if layer.alias == "BesitzerIn": - count += 1 - assert set(layer.all_topics) == { - "Polymorphic_Ortsplanung_V1_1.Gewerbe", - "Polymorphic_Ortsplanung_V1_1.Freizeit", - "Polymorphic_Ortsplanung_V1_1.Hallen", - "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", - } - assert set(layer.relevant_topics) == { - "Polymorphic_Ortsplanung_V1_1.Gewerbe", - "Polymorphic_Ortsplanung_V1_1.Freizeit", - "Polymorphic_Ortsplanung_V1_1.Hallen", - "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", - } - # Gebaeude from Ortsplanung_V1_1 - if layer.alias == "Ortsplanung_V1_1.Konstruktionen.Gebaeude": - count += 1 - assert set(layer.all_topics) == { - "Polymorphic_Ortsplanung_V1_1.Gewerbe", - "Polymorphic_Ortsplanung_V1_1.Freizeit", - "Polymorphic_Ortsplanung_V1_1.Hallen", - "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", - } - assert set(layer.relevant_topics) == { - "Polymorphic_Ortsplanung_V1_1.Gewerbe", - "Polymorphic_Ortsplanung_V1_1.Freizeit", - "Polymorphic_Ortsplanung_V1_1.Hallen", - "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", - } - # Gebaeude from Polymorphic_Ortsplanung_V1_1.Gewerbe - if layer.alias == "Gewerbe.Gebaeude": - count += 1 - assert set(layer.all_topics) == { - "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe" - } - assert set(layer.relevant_topics) == { - "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe" - } - # Gebaeude from Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe - if layer.alias == "IndustrieGewerbe.Gebaeude": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - - assert count == 5 + if not skip_topic_check: + # check relevant topics + count = 0 + for layer in available_layers: + # Strasse from Infrastruktur_V1_1 + if layer.alias == "Strasse": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # BesitzerIn from Ortsplanung_V1_1 + if layer.alias == "BesitzerIn": + count += 1 + assert set(layer.all_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + assert set(layer.relevant_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + # Gebaeude from Ortsplanung_V1_1 + if layer.alias == "Ortsplanung_V1_1.Konstruktionen.Gebaeude": + count += 1 + assert set(layer.all_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + assert set(layer.relevant_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + # Gebaeude from Polymorphic_Ortsplanung_V1_1.Gewerbe + if layer.alias == "Gewerbe.Gebaeude": + count += 1 + assert set(layer.all_topics) == { + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe" + } + assert set(layer.relevant_topics) == { + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe" + } + # Gebaeude from Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe + if layer.alias == "IndustrieGewerbe.Gebaeude": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + + assert count == 5 project = Project(optimize_strategy=strategy) project.layers = available_layers @@ -1164,7 +1179,7 @@ def _extopt_polymorphic_group(self, generator, strategy): QgsProject.instance().clear() - def _extopt_polymorphic_hide(self, generator, strategy): + def _extopt_polymorphic_hide(self, generator, strategy, skip_topic_check=False): available_layers = generator.layers() relations, _ = generator.relations(available_layers) @@ -1199,65 +1214,66 @@ def _extopt_polymorphic_hide(self, generator, strategy): ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - # check relevant topics - count = 0 - for layer in available_layers: - # Strasse from Infrastruktur_V1_1 - if layer.alias == "Strasse": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - # BesitzerIn from Ortsplanung_V1_1 - if layer.alias == "BesitzerIn": - count += 1 - assert set(layer.all_topics) == { - "Polymorphic_Ortsplanung_V1_1.Gewerbe", - "Polymorphic_Ortsplanung_V1_1.Freizeit", - "Polymorphic_Ortsplanung_V1_1.Hallen", - "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", - } - assert set(layer.relevant_topics) == { - "Polymorphic_Ortsplanung_V1_1.Gewerbe", - "Polymorphic_Ortsplanung_V1_1.Freizeit", - "Polymorphic_Ortsplanung_V1_1.Hallen", - "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", - } - # Gebaeude from Ortsplanung_V1_1 - if layer.alias == "Konstruktionen.Gebaeude" and not layer.is_relevant: - count += 1 - assert set(layer.all_topics) == { - "Polymorphic_Ortsplanung_V1_1.Gewerbe", - "Polymorphic_Ortsplanung_V1_1.Freizeit", - "Polymorphic_Ortsplanung_V1_1.Hallen", - "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", - } - assert set(layer.relevant_topics) == { - "Polymorphic_Ortsplanung_V1_1.Gewerbe", - "Polymorphic_Ortsplanung_V1_1.Freizeit", - "Polymorphic_Ortsplanung_V1_1.Hallen", - "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", - } - # Gebaeude from Polymorphic_Ortsplanung_V1_1 - if layer.alias == "Konstruktionen.Gebaeude" and layer.is_relevant: - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - # Gebaeude from Polymorphic_Ortsplanung_V1_1.Gewerbe - if layer.alias == "Gewerbe.Gebaeude": - count += 1 - assert set(layer.all_topics) == { - "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe" - } - assert set(layer.relevant_topics) == { - "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe" - } - # Gebaeude from Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe - if layer.alias == "IndustrieGewerbe.Gebaeude": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - - assert count == 6 + if not skip_topic_check: + # check relevant topics + count = 0 + for layer in available_layers: + # Strasse from Infrastruktur_V1_1 + if layer.alias == "Strasse": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # BesitzerIn from Ortsplanung_V1_1 + if layer.alias == "BesitzerIn": + count += 1 + assert set(layer.all_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + assert set(layer.relevant_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + # Gebaeude from Ortsplanung_V1_1 + if layer.alias == "Konstruktionen.Gebaeude" and not layer.is_relevant: + count += 1 + assert set(layer.all_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + assert set(layer.relevant_topics) == { + "Polymorphic_Ortsplanung_V1_1.Gewerbe", + "Polymorphic_Ortsplanung_V1_1.Freizeit", + "Polymorphic_Ortsplanung_V1_1.Hallen", + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe", + } + # Gebaeude from Polymorphic_Ortsplanung_V1_1 + if layer.alias == "Konstruktionen.Gebaeude" and layer.is_relevant: + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # Gebaeude from Polymorphic_Ortsplanung_V1_1.Gewerbe + if layer.alias == "Gewerbe.Gebaeude": + count += 1 + assert set(layer.all_topics) == { + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe" + } + assert set(layer.relevant_topics) == { + "Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe" + } + # Gebaeude from Polymorphic_Ortsplanung_V1_1.IndustrieGewerbe + if layer.alias == "IndustrieGewerbe.Gebaeude": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + + assert count == 6 project = Project(optimize_strategy=strategy) project.layers = available_layers @@ -1451,7 +1467,7 @@ def test_extopt_baustruct_geopackage(self): self._extopt_baustruct_hide(generator, strategy) def test_extopt_baustruct_mssql(self): - return # to do + importer = iliimporter.Importer() importer.tool = DbIliMode.ili2mssql importer.configuration = iliimporter_config(importer.tool) @@ -1488,7 +1504,8 @@ def test_extopt_baustruct_mssql(self): optimize_strategy=strategy, ) - self._extopt_baustruct_none(generator, strategy) + # we skip the relation check since it's not supported in mssql + self._extopt_baustruct_none(generator, strategy, True) ### 2. OptimizeStrategy.GROUP ### strategy = OptimizeStrategy.GROUP @@ -1501,7 +1518,8 @@ def test_extopt_baustruct_mssql(self): optimize_strategy=strategy, ) - self._extopt_baustruct_group(generator, strategy) + # we skip the relation check since it's not supported in mssql + self._extopt_baustruct_group(generator, strategy, True) ### 3. OptimizeStrategy.HIDE ### strategy = OptimizeStrategy.HIDE @@ -1514,9 +1532,10 @@ def test_extopt_baustruct_mssql(self): optimize_strategy=strategy, ) - self._extopt_baustruct_hide(generator, strategy) + # we skip the relation check since it's not supported in mssql + self._extopt_baustruct_hide(generator, strategy, True) - def _extopt_baustruct_none(self, generator, strategy): + def _extopt_baustruct_none(self, generator, strategy, skip_topic_check=False): available_layers = generator.layers() relations, _ = generator.relations(available_layers) legend = generator.legend(available_layers) @@ -1562,36 +1581,41 @@ def _extopt_baustruct_none(self, generator, strategy): ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - # check relevant topics - count = 0 - for layer in available_layers: - # Strasse from Infrastruktur_V1_1 - if layer.alias == "Strasse": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - # Park from Bauplanung_V1_1 - if layer.alias == "Bauplanung_V1_1.Natur.Park": - count += 1 - assert set(layer.all_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} - assert set(layer.relevant_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} - # Park from Kantonale_Bauplanung_V1_1 - if layer.alias == "Kantonale_Bauplanung_V1_1.Natur.Park": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - # Feld from Bauplanung_V1_1 - if layer.alias == "Feld": - count += 1 - assert set(layer.all_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} - assert set(layer.relevant_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} - # Kartoffelfeld from Kantonale_Bauplanung_V1_1.Natur - if layer.alias == "Kartoffelfeld": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - - assert count == 5 + if not skip_topic_check: + # check relevant topics + count = 0 + for layer in available_layers: + # Strasse from Infrastruktur_V1_1 + if layer.alias == "Strasse": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # Park from Bauplanung_V1_1 + if layer.alias == "Bauplanung_V1_1.Natur.Park": + count += 1 + assert set(layer.all_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} + assert set(layer.relevant_topics) == { + "Kantonale_Bauplanung_V1_1.Natur" + } + # Park from Kantonale_Bauplanung_V1_1 + if layer.alias == "Kantonale_Bauplanung_V1_1.Natur.Park": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # Feld from Bauplanung_V1_1 + if layer.alias == "Feld": + count += 1 + assert set(layer.all_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} + assert set(layer.relevant_topics) == { + "Kantonale_Bauplanung_V1_1.Natur" + } + # Kartoffelfeld from Kantonale_Bauplanung_V1_1.Natur + if layer.alias == "Kartoffelfeld": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + + assert count == 5 project = Project(optimize_strategy=strategy) project.layers = available_layers @@ -1660,7 +1684,7 @@ def _extopt_baustruct_none(self, generator, strategy): QgsProject.instance().clear() - def _extopt_baustruct_group(self, generator, strategy): + def _extopt_baustruct_group(self, generator, strategy, skip_topic_check=False): available_layers = generator.layers() relations, _ = generator.relations(available_layers) @@ -1708,36 +1732,41 @@ def _extopt_baustruct_group(self, generator, strategy): ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - # check relevant topics - count = 0 - for layer in available_layers: - # Strasse from Infrastruktur_V1_1 - if layer.alias == "Strasse": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - # Park from Bauplanung_V1_1 - if layer.alias == "Bauplanung_V1_1.Natur.Park": - count += 1 - assert set(layer.all_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} - assert set(layer.relevant_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} - # Park from Kantonale_Bauplanung_V1_1 - if layer.alias == "Kantonale_Bauplanung_V1_1.Natur.Park": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - # Feld from Bauplanung_V1_1 - if layer.alias == "Feld": - count += 1 - assert set(layer.all_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} - assert set(layer.relevant_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} - # Kartoffelfeld from Kantonale_Bauplanung_V1_1.Natur - if layer.alias == "Kartoffelfeld": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - - assert count == 5 + if not skip_topic_check: + # check relevant topics + count = 0 + for layer in available_layers: + # Strasse from Infrastruktur_V1_1 + if layer.alias == "Strasse": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # Park from Bauplanung_V1_1 + if layer.alias == "Bauplanung_V1_1.Natur.Park": + count += 1 + assert set(layer.all_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} + assert set(layer.relevant_topics) == { + "Kantonale_Bauplanung_V1_1.Natur" + } + # Park from Kantonale_Bauplanung_V1_1 + if layer.alias == "Kantonale_Bauplanung_V1_1.Natur.Park": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # Feld from Bauplanung_V1_1 + if layer.alias == "Feld": + count += 1 + assert set(layer.all_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} + assert set(layer.relevant_topics) == { + "Kantonale_Bauplanung_V1_1.Natur" + } + # Kartoffelfeld from Kantonale_Bauplanung_V1_1.Natur + if layer.alias == "Kartoffelfeld": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + + assert count == 5 project = Project(optimize_strategy=strategy) project.layers = available_layers @@ -1828,7 +1857,7 @@ def _extopt_baustruct_group(self, generator, strategy): QgsProject.instance().clear() - def _extopt_baustruct_hide(self, generator, strategy): + def _extopt_baustruct_hide(self, generator, strategy, skip_topic_check=False): available_layers = generator.layers() relations, _ = generator.relations(available_layers) @@ -1871,36 +1900,41 @@ def _extopt_baustruct_hide(self, generator, strategy): ] assert set(irrelevant_layer_ilinames) == set(expected_irrelevant_layer_ilinames) - # check relevant topics - count = 0 - for layer in available_layers: - # Strasse from Infrastruktur_V1_1 - if layer.alias == "Strasse": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - # Park from Bauplanung_V1_1 - if layer.alias == "Park" and not layer.is_relevant: - count += 1 - assert set(layer.all_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} - assert set(layer.relevant_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} - # Park from Kantonale_Bauplanung_V1_1 - if layer.alias == "Park" and layer.is_relevant: - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - # Feld from Bauplanung_V1_1 - if layer.alias == "Feld": - count += 1 - assert set(layer.all_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} - assert set(layer.relevant_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} - # Kartoffelfeld from Kantonale_Bauplanung_V1_1.Natur - if layer.alias == "Kartoffelfeld": - count += 1 - assert layer.all_topics == [] - assert layer.relevant_topics == [] - - assert count == 5 + if not skip_topic_check: + # check relevant topics + count = 0 + for layer in available_layers: + # Strasse from Infrastruktur_V1_1 + if layer.alias == "Strasse": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # Park from Bauplanung_V1_1 + if layer.alias == "Park" and not layer.is_relevant: + count += 1 + assert set(layer.all_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} + assert set(layer.relevant_topics) == { + "Kantonale_Bauplanung_V1_1.Natur" + } + # Park from Kantonale_Bauplanung_V1_1 + if layer.alias == "Park" and layer.is_relevant: + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + # Feld from Bauplanung_V1_1 + if layer.alias == "Feld": + count += 1 + assert set(layer.all_topics) == {"Kantonale_Bauplanung_V1_1.Natur"} + assert set(layer.relevant_topics) == { + "Kantonale_Bauplanung_V1_1.Natur" + } + # Kartoffelfeld from Kantonale_Bauplanung_V1_1.Natur + if layer.alias == "Kartoffelfeld": + count += 1 + assert layer.all_topics == [] + assert layer.relevant_topics == [] + + assert count == 5 project = Project(optimize_strategy=strategy) project.layers = available_layers From 0a7375b121984979a34046ab5f550f8b03cb37e8 Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 2 Oct 2023 22:28:13 +0200 Subject: [PATCH 34/34] remove unused mssql code and provide proper ignoring a reference to the same layer (because of multi-geometry) --- modelbaker/dataobjects/layers.py | 2 +- modelbaker/dbconnector/mssql_connector.py | 17 ++--------------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/modelbaker/dataobjects/layers.py b/modelbaker/dataobjects/layers.py index ea6557d..bde6a70 100644 --- a/modelbaker/dataobjects/layers.py +++ b/modelbaker/dataobjects/layers.py @@ -282,7 +282,7 @@ def post_generate(self, project): if nm_relation == relation: continue - if nm_relation.referenced_layer == self: + if nm_relation.referenced_layer.ili_name == self.ili_name: continue if ( diff --git a/modelbaker/dbconnector/mssql_connector.py b/modelbaker/dbconnector/mssql_connector.py index f180c0c..48043a6 100644 --- a/modelbaker/dbconnector/mssql_connector.py +++ b/modelbaker/dbconnector/mssql_connector.py @@ -909,26 +909,13 @@ def get_topics_info(self): cur.execute( """ SELECT DISTINCT PARSENAME(cn.iliname,1) as model, - PARSENAME(cn.iliname,2) as topic, - {relevance} + PARSENAME(cn.iliname,2) as topic FROM {schema}.t_ili2db_classname as cn JOIN {schema}.t_ili2db_table_prop as tp ON cn.sqlname = tp.tablename WHERE PARSENAME(cn.iliname,3) != '' and tp.setting != 'ENUM' """.format( - schema=self.schema, - relevance=""" - CASE WHEN (WITH children(childTopic, baseTopic) AS ( - SELECT substring( thisClass, 1, CHARINDEX('.', substring( thisClass, CHARINDEX('.', thisClass)+1))+CHARINDEX('.', thisClass)-1) as childTopic , substring( baseClass, 1, CHARINDEX('.', substring( baseClass, CHARINDEX('.', baseClass)+1))+CHARINDEX('.', baseClass)-1) as baseTopic - FROM {schema}.T_ILI2DB_INHERITANCE - WHERE substring( baseClass, 1, CHARINDEX('.', substring( baseClass, CHARINDEX('.', baseClass)+1))+CHARINDEX('.', baseClass)-1) = substring( CN.IliName, 1, CHARINDEX('.', substring( CN.IliName, CHARINDEX('.', CN.IliName)+1))+CHARINDEX('.', CN.IliName)-1) -- model.topic - UNION - SELECT substring( inheritance.thisClass, 1, CHARINDEX('.', substring( inheritance.thisClass, CHARINDEX('.', inheritance.thisClass)+1))+CHARINDEX('.', inheritance.thisClass)-1) as childTopic , substring( inheritance.baseClass, 1, CHARINDEX('.', substring( inheritance.baseClass, CHARINDEX('.', baseClass)+1))+CHARINDEX('.', inheritance.baseClass)-1) as baseTopic FROM children - JOIN {schema}.T_ILI2DB_INHERITANCE as inheritance ON substring( inheritance.baseClass, 1, CHARINDEX('.', substring( inheritance.baseClass, CHARINDEX('.', baseClass)+1))+CHARINDEX('.', inheritance.baseClass)-1) = children.childTopic - )SELECT count(childTopic) FROM children)>0 THEN 0 ELSE 1 END AS relevance - """.format( - schema=self.schema - ), + schema=self.schema ) ) result = self._get_dict_result(cur)