From 26b1f4fc8072ef1ca55b1cdb64dad89be622250f Mon Sep 17 00:00:00 2001 From: Vincent Bourgeois Date: Wed, 4 Dec 2024 20:01:31 +0100 Subject: [PATCH] test(import): support fieldmapping format v2 --- .../core/gn_synthese/imports/actions.py | 10 +- backend/geonature/core/imports/utils.py | 7 +- .../imports/files/synthese/valid_file.csv | 2 +- .../tests/imports/test_imports_occhab.py | 2 +- .../tests/imports/test_imports_synthese.py | 14 ++- .../geonature/tests/imports/test_mappings.py | 28 +++-- .../gn_module_occhab/imports/actions.py | 10 +- .../aed662bbd88a_add_default_mapping.py | 114 +++++++++--------- 8 files changed, 106 insertions(+), 81 deletions(-) diff --git a/backend/geonature/core/gn_synthese/imports/actions.py b/backend/geonature/core/gn_synthese/imports/actions.py index 7ee33918fd..6b8b7d356f 100644 --- a/backend/geonature/core/gn_synthese/imports/actions.py +++ b/backend/geonature/core/gn_synthese/imports/actions.py @@ -218,7 +218,15 @@ def update_batch_progress(batch, step): do_nomenclatures_mapping( imprt, entity, - {field_name: fields[field_name] for field_name, _ in imprt.fieldmapping.items()}, + { + field_name: fields[field_name] + for field_name, mapping in imprt.fieldmapping.items() + if field_name in fields + and ( + mapping.get("column_src", None) in imprt.columns + or mapping.get("default_value") is not None + ) + }, fill_with_defaults=current_app.config["IMPORT"][ "FILL_MISSING_NOMENCLATURE_WITH_DEFAULT_VALUE" ], diff --git a/backend/geonature/core/imports/utils.py b/backend/geonature/core/imports/utils.py index e50703a7e5..63454d88cd 100644 --- a/backend/geonature/core/imports/utils.py +++ b/backend/geonature/core/imports/utils.py @@ -464,8 +464,11 @@ def get_mapping_data(import_: TImports, entity: Entity): fields = {ef.field.name_field: ef.field for ef in entity.fields} selected_fields = { field_name: fields[field_name] - for field_name, source_field in import_.fieldmapping.items() - if source_field in import_.columns and field_name in fields + for field_name, mapping in import_.fieldmapping.items() + if ( + mapping.get("column_src") in import_.columns or mapping.get("default_value") is not None + ) + and field_name in fields } source_cols = set() for field in selected_fields.values(): diff --git a/backend/geonature/tests/imports/files/synthese/valid_file.csv b/backend/geonature/tests/imports/files/synthese/valid_file.csv index acc1959f6e..1c12d057f1 100644 --- a/backend/geonature/tests/imports/files/synthese/valid_file.csv +++ b/backend/geonature/tests/imports/files/synthese/valid_file.csv @@ -3,5 +3,5 @@ valid;1;1;Relevé n°1;Occurrence n°1;2017-01-01;2017-01-01;12:05:02;12:05:02;6 valid;2;2;Relevé n°2;Occurrence n°2;2017-01-01;2017-01-02;12:05:02;12:05:02;351;351;Rana temporaria Linnaeus, 1758;Grenouille rousse (La);Grenouille rousse;Animalia;Chordés;Amphibiens;Amphibia;Anura;Ranidae;ES;1;1;1500;1565;;;Administrateur test;Théo;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Contact aléatoire tous règnes confondus;4d331cae-65e4-4948-b0b2-a11bc5bb46c2;1;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;10;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Immature;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;830c93c7-288e-40f0-a17f-15fbb50e643a;5b427c76-bd8c-4103-a33c-884c7037aa2b;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497 duplicate id;3;3;Relevé n°3;Occurrence n°3;2017-01-08;;;;67111;67111;Alburnus alburnus (Linnaeus, 1758);Ablette;Ablette;Animalia;Chordés;Poissons;Actinopterygii;Cypriniformes;Leuciscidae;ES;1;1;1600;1600;;;Administrateur test;Donovan;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Contact aléatoire tous règnes confondus;4d331cae-65e4-4948-b0b2-a11bc5bb46c2;1;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;100;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Juvénile;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;2f92f91a-64a2-4684-90e4-140466bb34e3;5937d0f2-c96d-424b-bea4-9e3fdac894ed;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497 duplicate id;3;4;Relevé n°4;Occurrence n°4;2017-01-08;2017-01-08;20:00:00;23:00:00;67111;67111;Alburnus alburnus (Linnaeus, 1758);Ablette;Ablette;Animalia;Chordés;Poissons;Actinopterygii;Cypriniformes;Leuciscidae;ES;1;1;1600;1600;;;Administrateur test;Donovan;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Contact aléatoire tous règnes confondus;4d331cae-65e4-4948-b0b2-a11bc5bb46c2;1;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;100;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Juvénile;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;2f92f91a-64a2-4684-90e4-140466bb34e4;5937d0f2-c96d-424b-bea4-9e3fdac894ed;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497 -count min > count max;5;5;Relevé n°5;Occurrence n°5;2017-01-08;2017/01/08;20:00;23:00:00;67111;67111;Alburnus alburnus (Linnaeus, 1758);Ablette;Ablette;Animalia;Chordés;Poissons;Actinopterygii;Cypriniformes;Leuciscidae;ES;20;;1600;1600;;;Administrateur test;Donovan;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Contact aléatoire tous règnes confondus;4d331cae-65e4-4948-b0b2-a11bc5bb46c2;1;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;100;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Juvénile;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;2f92f91a-64a2-4684-90e4-140466bb34e5;5937d0f2-c96d-424b-bea4-9e3fdac894ed;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497 +count min > count max;5;5;Relevé n°5;Occurrence n°5;2017-01-08;2017/01/08;20:00;23:00:00;67111;67111;Alburnus alburnus (Linnaeus, 1758);Ablette;Ablette;Animalia;Chordés;Poissons;Actinopterygii;Cypriniformes;Leuciscidae;ES;20;5;1600;1600;;;Administrateur test;Donovan;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Contact aléatoire tous règnes confondus;4d331cae-65e4-4948-b0b2-a11bc5bb46c2;1;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;100;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Juvénile;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;2f92f91a-64a2-4684-90e4-140466bb34e5;5937d0f2-c96d-424b-bea4-9e3fdac894ed;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497 valid;6;6;Relevé n°6;Occurrence n°6;2017-01-01;2017-01-01;12:05:02;12:05:02;351;351;Rana temporaria Linnaeus, 1758;Grenouille rousse (La);Grenouille rousse;Animalia;Chordés;Amphibiens;Amphibia;Anura;Ranidae;ES;1;1;1600;1600;;;Administrateur test;Donovan;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Contact aléatoire tous règnes confondus;4d331cae-65e4-4948-b0b2-a11bc5bb46c2;1;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;100;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Juvénile;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;f5515e2a-b30d-11eb-8cc8-af8c2d0867b4;5937d0f2-c96d-424b-bea4-9e3fdac894ed;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497 diff --git a/backend/geonature/tests/imports/test_imports_occhab.py b/backend/geonature/tests/imports/test_imports_occhab.py index b0fe44064f..4865cde719 100644 --- a/backend/geonature/tests/imports/test_imports_occhab.py +++ b/backend/geonature/tests/imports/test_imports_occhab.py @@ -60,7 +60,7 @@ def fieldmapping(occhab_destination): .unique() .all() ) - return {field.name_field: field.name_field for field in fields} + return {field.name_field: {"column_src": field.name_field} for field in fields} @pytest.fixture() diff --git a/backend/geonature/tests/imports/test_imports_synthese.py b/backend/geonature/tests/imports/test_imports_synthese.py index fa076b9fce..b7308bd313 100644 --- a/backend/geonature/tests/imports/test_imports_synthese.py +++ b/backend/geonature/tests/imports/test_imports_synthese.py @@ -208,11 +208,15 @@ def fieldmapping(import_file_name, autogenerate): else: bib_fields = db.session.scalars(sa.select(BibFields).filter_by(display=True)).unique().all() return { - field.name_field: ( - autogenerate - if field.autogenerated - else ([field.name_field, "additional_data2"] if field.multi else field.name_field) - ) + field.name_field: { + "column_src": ( + autogenerate + if field.autogenerated + else ( + [field.name_field, "additional_data2"] if field.multi else field.name_field + ) + ) + } for field in bib_fields } diff --git a/backend/geonature/tests/imports/test_mappings.py b/backend/geonature/tests/imports/test_mappings.py index 4988287198..38dbab98d7 100644 --- a/backend/geonature/tests/imports/test_mappings.py +++ b/backend/geonature/tests/imports/test_mappings.py @@ -37,11 +37,13 @@ def mappings(synthese_destination, users): .all() ) fieldmapping_values = { - field.name_field: ( - True - if field.autogenerated - else ([field.name_field] if field.multi else field.name_field) - ) + field.name_field: { + "column_src": ( + True + if field.autogenerated + else ([field.name_field] if field.multi else field.name_field) + ) + } for field in bib_fields } @@ -266,11 +268,11 @@ def get_mapping(mapping): def test_add_field_mapping(self, users, mappings): fieldmapping = { - "WKT": "geometrie", - "nom_cite": "nomcite", - "cd_nom": "cdnom", - "cd_hab": "cdhab", - "observers": "observateurs", + "WKT": {"column_src": "geometrie"}, + "nom_cite": {"column_src": "nomcite"}, + "cd_nom": {"column_src": "cdnom"}, + "cd_hab": {"column_src": "cdhab"}, + "observers": {"column_src": "observateurs"}, } url = url_for("import.add_mapping", mappingtype="field") @@ -296,7 +298,7 @@ def test_add_field_mapping(self, users, mappings): label=mappings["content_public"].label, ) - r = self.client.post(url, data={"unexisting": "source column"}) + r = self.client.post(url, data={"unexisting": {"column_src": "source column"}}) assert r.status_code == BadRequest.code r = self.client.post(url, data=fieldmapping) @@ -304,7 +306,7 @@ def test_add_field_mapping(self, users, mappings): fieldmapping.update( { - "date_min": "date_debut", + "date_min": {"column_src": "date_debut"}, } ) r = self.client.post(url, data=fieldmapping) @@ -406,7 +408,7 @@ def test_update_field_mapping_values(self, users, mappings): fm = mappings["field_public"] fieldvalues_update = deepcopy(fm.values) - fieldvalues_update["WKT"] = "WKT2" + fieldvalues_update["WKT"] = {"column_src": "WKT2"} fieldvalues_should = deepcopy(fieldvalues_update) del fieldvalues_update["validator"] # should not removed from mapping! r = self.client.post( diff --git a/contrib/gn_module_occhab/backend/gn_module_occhab/imports/actions.py b/contrib/gn_module_occhab/backend/gn_module_occhab/imports/actions.py index 1ecddc529c..a0fd94a8a0 100644 --- a/contrib/gn_module_occhab/backend/gn_module_occhab/imports/actions.py +++ b/contrib/gn_module_occhab/backend/gn_module_occhab/imports/actions.py @@ -474,16 +474,20 @@ def import_data_to_destination(imprt: TImports) -> None: ef.field.name_field: ef.field for ef in entity.fields if ef.field.dest_field != None } insert_fields = {fields["id_station"]} - for field_name, source_field in imprt.fieldmapping.items(): + for field_name, mapping in imprt.fieldmapping.items(): if field_name not in fields: # not a destination field continue field = fields[field_name] + column_src = mapping.get("column_src", None) if field.multi: # TODO@TestImportsOcchab.test_import_valid_file: add testcase - if not set(source_field).isdisjoint(imprt.columns): + if not set(column_src).isdisjoint(imprt.columns): insert_fields |= {field} else: - if source_field in imprt.columns: + if ( + column_src in imprt.columns + or mapping.get("default_value", None) is not None + ): insert_fields |= {field} if entity.code == "station": # unique_dataset_id is replaced with id_dataset diff --git a/contrib/gn_module_occhab/backend/gn_module_occhab/migrations/aed662bbd88a_add_default_mapping.py b/contrib/gn_module_occhab/backend/gn_module_occhab/migrations/aed662bbd88a_add_default_mapping.py index 0914c68b40..43e1339ba6 100644 --- a/contrib/gn_module_occhab/backend/gn_module_occhab/migrations/aed662bbd88a_add_default_mapping.py +++ b/contrib/gn_module_occhab/backend/gn_module_occhab/migrations/aed662bbd88a_add_default_mapping.py @@ -17,7 +17,7 @@ revision = "aed662bbd88a" down_revision = "69494f900cab" branch_labels = None -depends_on = None +depends_on = "e43b01a18850" def get_models(conn): @@ -52,26 +52,28 @@ def upgrade(): sa.insert(FieldMapping).values( id=id_occhab_mapping, values={ - "WKT": "geometry", - "altitude_max": "altitude_max", - "altitude_min": "altitude_min", - "area": "area", - "cd_hab": "cd_hab", - "comment": "comment", - "date_max": "date_fin", - "date_min": "date_debut", - "depth_max": "depth_max", - "depth_min": "depth_min", - "id_nomenclature_area_surface_calculation": "methode_calcul_surface", - "id_nomenclature_exposure": "exposition", - "id_nomenclature_geographic_object": "nature_objet_geo", - "id_station_source": "id_station", - "nom_cite": "nom_cite", - "observers_txt": "observateurs", - "technical_precision": "precision_technique", - "unique_dataset_id": "uuid_jdd", - "unique_id_sinp_habitat": "uuid_habitat", - "unique_id_sinp_station": "uuid_station", + "WKT": {"column_src": "geometry"}, + "altitude_max": {"column_src": "altitude_max"}, + "altitude_min": {"column_src": "altitude_min"}, + "area": {"column_src": "area"}, + "cd_hab": {"column_src": "cd_hab"}, + "comment": {"column_src": "comment"}, + "date_max": {"column_src": "date_fin"}, + "date_min": {"column_src": "date_debut"}, + "depth_max": {"column_src": "depth_max"}, + "depth_min": {"column_src": "depth_min"}, + "id_nomenclature_area_surface_calculation": { + "column_src": "methode_calcul_surface" + }, + "id_nomenclature_exposure": {"column_src": "exposition"}, + "id_nomenclature_geographic_object": {"column_src": "nature_objet_geo"}, + "id_station_source": {"column_src": "id_station"}, + "nom_cite": {"column_src": "nom_cite"}, + "observers_txt": {"column_src": "observateurs"}, + "technical_precision": {"column_src": "precision_technique"}, + "unique_dataset_id": {"column_src": "uuid_jdd"}, + "unique_id_sinp_habitat": {"column_src": "uuid_habitat"}, + "unique_id_sinp_station": {"column_src": "uuid_station"}, }, ) ) @@ -92,39 +94,41 @@ def upgrade(): sa.insert(FieldMapping).values( id=id_mapping_sinp, values={ - "WKT": "WKT", - "altitude_max": "altMax", - "altitude_min": "altMin", - "area": "surf", - "cd_hab": "cdHab", - "comment": "comment", - "date_max": "dateFin", - "date_min": "dateDebut", - "depth_max": "profMax", - "depth_min": "profMin", - "determiner": "persDeterm", - "id_habitat": "idOrigine", - "id_nomenclature_abundance": "abondHab", - "id_nomenclature_area_surface_calculation": "methodeCalculSurface", - "id_nomenclature_collection_technique": "techCollec", - "id_nomenclature_community_interest": "habitatInteretCommunautaire ", - "id_nomenclature_determination_type": "typeDeterm", - "id_nomenclature_exposure": "exposition", - "id_nomenclature_geographic_object": "natObjGeo", - "id_nomenclature_sensitivity": "sensibiliteHab", - "id_station_source": "idOrigEvt", - "is_habitat_complex": "mosaique", - "nom_cite": "nomCite", - "numerization_scale": "echelleNumerisation", - "observers_txt": "observer", - "precision": "precisGeo", - "recovery_percentage": "recouv", - "station_name": "nomStation", - "technical_precision": "precisionTechnique", - "unique_dataset_id": "jddMetaId", - "unique_id_sinp_grp_phyto": "relevePhyto", - "unique_id_sinp_habitat": "idSinpHab", - "unique_id_sinp_station": "permId", + "WKT": {"column_src": "WKT"}, + "altitude_max": {"column_src": "altMax"}, + "altitude_min": {"column_src": "altMin"}, + "area": {"column_src": "surf"}, + "cd_hab": {"column_src": "cdHab"}, + "comment": {"column_src": "comment"}, + "date_max": {"column_src": "dateFin"}, + "date_min": {"column_src": "dateDebut"}, + "depth_max": {"column_src": "profMax"}, + "depth_min": {"column_src": "profMin"}, + "determiner": {"column_src": "persDeterm"}, + "id_habitat": {"column_src": "idOrigine"}, + "id_nomenclature_abundance": {"column_src": "abondHab"}, + "id_nomenclature_area_surface_calculation": {"column_src": "methodeCalculSurface"}, + "id_nomenclature_collection_technique": {"column_src": "techCollec"}, + "id_nomenclature_community_interest": { + "column_src": "habitatInteretCommunautaire " + }, + "id_nomenclature_determination_type": {"column_src": "typeDeterm"}, + "id_nomenclature_exposure": {"column_src": "exposition"}, + "id_nomenclature_geographic_object": {"column_src": "natObjGeo"}, + "id_nomenclature_sensitivity": {"column_src": "sensibiliteHab"}, + "id_station_source": {"column_src": "idOrigEvt"}, + "is_habitat_complex": {"column_src": "mosaique"}, + "nom_cite": {"column_src": "nomCite"}, + "numerization_scale": {"column_src": "echelleNumerisation"}, + "observers_txt": {"column_src": "observer"}, + "precision": {"column_src": "precisGeo"}, + "recovery_percentage": {"column_src": "recouv"}, + "station_name": {"column_src": "nomStation"}, + "technical_precision": {"column_src": "precisionTechnique"}, + "unique_dataset_id": {"column_src": "jddMetaId"}, + "unique_id_sinp_grp_phyto": {"column_src": "relevePhyto"}, + "unique_id_sinp_habitat": {"column_src": "idSinpHab"}, + "unique_id_sinp_station": {"column_src": "permId"}, }, ) ) @@ -138,7 +142,7 @@ def downgrade(): cte = ( sa.select(MappingTemplate.c.id) - .where(MappingTemplate.c.label.in_(["OccHab", "Occurrences d'habitats SINP"])) + .where(MappingTemplate.c.label.in_(["Occhab", "Occurrences d'habitats SINP"])) .cte("mapping_cte") ) op.execute(sa.delete(FieldMapping).where(FieldMapping.c.id == cte.c.id))