Skip to content

Commit

Permalink
test(import): support fieldmapping format v2
Browse files Browse the repository at this point in the history
  • Loading branch information
@20cents
20cents committed Dec 5, 2024
1 parent 3bd6c70 commit 26b1f4f
Show file tree
Hide file tree
Showing 8 changed files with 106 additions and 81 deletions.
10 changes: 9 additions & 1 deletion backend/geonature/core/gn_synthese/imports/actions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,15 @@ def update_batch_progress(batch, step):
do_nomenclatures_mapping(
imprt,
entity,
{field_name: fields[field_name] for field_name, _ in imprt.fieldmapping.items()},
{
field_name: fields[field_name]
for field_name, mapping in imprt.fieldmapping.items()
if field_name in fields
and (
mapping.get("column_src", None) in imprt.columns
or mapping.get("default_value") is not None
)
},
fill_with_defaults=current_app.config["IMPORT"][
"FILL_MISSING_NOMENCLATURE_WITH_DEFAULT_VALUE"
],
Expand Down
7 changes: 5 additions & 2 deletions backend/geonature/core/imports/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -464,8 +464,11 @@ def get_mapping_data(import_: TImports, entity: Entity):
fields = {ef.field.name_field: ef.field for ef in entity.fields}
selected_fields = {
field_name: fields[field_name]
for field_name, source_field in import_.fieldmapping.items()
if source_field in import_.columns and field_name in fields
for field_name, mapping in import_.fieldmapping.items()
if (
mapping.get("column_src") in import_.columns or mapping.get("default_value") is not None
)
and field_name in fields
}
source_cols = set()
for field in selected_fields.values():
Expand Down
2 changes: 1 addition & 1 deletion backend/geonature/tests/imports/files/synthese/valid_file.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@ valid;1;1;Relevé n°1;Occurrence n°1;2017-01-01;2017-01-01;12:05:02;12:05:02;6
valid;2;2;Relevé n°2;Occurrence n°2;2017-01-01;2017-01-02;12:05:02;12:05:02;351;351;Rana temporaria Linnaeus, 1758;Grenouille rousse (La);Grenouille rousse;Animalia;Chordés;Amphibiens;Amphibia;Anura;Ranidae;ES;1;1;1500;1565;;;Administrateur test;Théo;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Contact aléatoire tous règnes confondus;4d331cae-65e4-4948-b0b2-a11bc5bb46c2;1;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;10;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Immature;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;830c93c7-288e-40f0-a17f-15fbb50e643a;5b427c76-bd8c-4103-a33c-884c7037aa2b;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497
duplicate id;3;3;Relevé n°3;Occurrence n°3;2017-01-08;;;;67111;67111;Alburnus alburnus (Linnaeus, 1758);Ablette;Ablette;Animalia;Chordés;Poissons;Actinopterygii;Cypriniformes;Leuciscidae;ES;1;1;1600;1600;;;Administrateur test;Donovan;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Contact aléatoire tous règnes confondus;4d331cae-65e4-4948-b0b2-a11bc5bb46c2;1;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;100;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Juvénile;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;2f92f91a-64a2-4684-90e4-140466bb34e3;5937d0f2-c96d-424b-bea4-9e3fdac894ed;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497
duplicate id;3;4;Relevé n°4;Occurrence n°4;2017-01-08;2017-01-08;20:00:00;23:00:00;67111;67111;Alburnus alburnus (Linnaeus, 1758);Ablette;Ablette;Animalia;Chordés;Poissons;Actinopterygii;Cypriniformes;Leuciscidae;ES;1;1;1600;1600;;;Administrateur test;Donovan;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Contact aléatoire tous règnes confondus;4d331cae-65e4-4948-b0b2-a11bc5bb46c2;1;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;100;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Juvénile;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;2f92f91a-64a2-4684-90e4-140466bb34e4;5937d0f2-c96d-424b-bea4-9e3fdac894ed;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497
count min > count max;5;5;Relevé n°5;Occurrence n°5;2017-01-08;2017/01/08;20:00;23:00:00;67111;67111;Alburnus alburnus (Linnaeus, 1758);Ablette;Ablette;Animalia;Chordés;Poissons;Actinopterygii;Cypriniformes;Leuciscidae;ES;20;;1600;1600;;;Administrateur test;Donovan;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Contact aléatoire tous règnes confondus;4d331cae-65e4-4948-b0b2-a11bc5bb46c2;1;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;100;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Juvénile;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;2f92f91a-64a2-4684-90e4-140466bb34e5;5937d0f2-c96d-424b-bea4-9e3fdac894ed;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497
count min > count max;5;5;Relevé n°5;Occurrence n°5;2017-01-08;2017/01/08;20:00;23:00:00;67111;67111;Alburnus alburnus (Linnaeus, 1758);Ablette;Ablette;Animalia;Chordés;Poissons;Actinopterygii;Cypriniformes;Leuciscidae;ES;20;5;1600;1600;;;Administrateur test;Donovan;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Contact aléatoire tous règnes confondus;4d331cae-65e4-4948-b0b2-a11bc5bb46c2;1;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;100;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Juvénile;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;2f92f91a-64a2-4684-90e4-140466bb34e5;5937d0f2-c96d-424b-bea4-9e3fdac894ed;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497
valid;6;6;Relevé n°6;Occurrence n°6;2017-01-01;2017-01-01;12:05:02;12:05:02;351;351;Rana temporaria Linnaeus, 1758;Grenouille rousse (La);Grenouille rousse;Animalia;Chordés;Amphibiens;Amphibia;Anura;Ranidae;ES;1;1;1600;1600;;;Administrateur test;Donovan;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Contact aléatoire tous règnes confondus;4d331cae-65e4-4948-b0b2-a11bc5bb46c2;1;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;100;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Juvénile;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;f5515e2a-b30d-11eb-8cc8-af8c2d0867b4;5937d0f2-c96d-424b-bea4-9e3fdac894ed;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497
2 changes: 1 addition & 1 deletion backend/geonature/tests/imports/test_imports_occhab.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ def fieldmapping(occhab_destination):
.unique()
.all()
)
return {field.name_field: field.name_field for field in fields}
return {field.name_field: {"column_src": field.name_field} for field in fields}


@pytest.fixture()
Expand Down
14 changes: 9 additions & 5 deletions backend/geonature/tests/imports/test_imports_synthese.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,11 +208,15 @@ def fieldmapping(import_file_name, autogenerate):
else:
bib_fields = db.session.scalars(sa.select(BibFields).filter_by(display=True)).unique().all()
return {
field.name_field: (
autogenerate
if field.autogenerated
else ([field.name_field, "additional_data2"] if field.multi else field.name_field)
)
field.name_field: {
"column_src": (
autogenerate
if field.autogenerated
else (
[field.name_field, "additional_data2"] if field.multi else field.name_field
)
)
}
for field in bib_fields
}

Expand Down
28 changes: 15 additions & 13 deletions backend/geonature/tests/imports/test_mappings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,13 @@ def mappings(synthese_destination, users):
.all()
)
fieldmapping_values = {
field.name_field: (
True
if field.autogenerated
else ([field.name_field] if field.multi else field.name_field)
)
field.name_field: {
"column_src": (
True
if field.autogenerated
else ([field.name_field] if field.multi else field.name_field)
)
}
for field in bib_fields
}

Expand Down Expand Up @@ -266,11 +268,11 @@ def get_mapping(mapping):

def test_add_field_mapping(self, users, mappings):
fieldmapping = {
"WKT": "geometrie",
"nom_cite": "nomcite",
"cd_nom": "cdnom",
"cd_hab": "cdhab",
"observers": "observateurs",
"WKT": {"column_src": "geometrie"},
"nom_cite": {"column_src": "nomcite"},
"cd_nom": {"column_src": "cdnom"},
"cd_hab": {"column_src": "cdhab"},
"observers": {"column_src": "observateurs"},
}
url = url_for("import.add_mapping", mappingtype="field")

Expand All @@ -296,15 +298,15 @@ def test_add_field_mapping(self, users, mappings):
label=mappings["content_public"].label,
)

r = self.client.post(url, data={"unexisting": "source column"})
r = self.client.post(url, data={"unexisting": {"column_src": "source column"}})
assert r.status_code == BadRequest.code

r = self.client.post(url, data=fieldmapping)
assert r.status_code == BadRequest.code # missing date_min

fieldmapping.update(
{
"date_min": "date_debut",
"date_min": {"column_src": "date_debut"},
}
)
r = self.client.post(url, data=fieldmapping)
Expand Down Expand Up @@ -406,7 +408,7 @@ def test_update_field_mapping_values(self, users, mappings):

fm = mappings["field_public"]
fieldvalues_update = deepcopy(fm.values)
fieldvalues_update["WKT"] = "WKT2"
fieldvalues_update["WKT"] = {"column_src": "WKT2"}
fieldvalues_should = deepcopy(fieldvalues_update)
del fieldvalues_update["validator"] # should not removed from mapping!
r = self.client.post(
Expand Down
10 changes: 7 additions & 3 deletions contrib/gn_module_occhab/backend/gn_module_occhab/imports/actions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -474,16 +474,20 @@ def import_data_to_destination(imprt: TImports) -> None:
ef.field.name_field: ef.field for ef in entity.fields if ef.field.dest_field != None
}
insert_fields = {fields["id_station"]}
for field_name, source_field in imprt.fieldmapping.items():
for field_name, mapping in imprt.fieldmapping.items():
if field_name not in fields: # not a destination field
continue
field = fields[field_name]
column_src = mapping.get("column_src", None)
if field.multi:
# [email protected]_import_valid_file: add testcase
if not set(source_field).isdisjoint(imprt.columns):
if not set(column_src).isdisjoint(imprt.columns):

Check warning on line 484 in contrib/gn_module_occhab/backend/gn_module_occhab/imports/actions.py

View check run for this annotation

Codecov / codecov/patch

contrib/gn_module_occhab/backend/gn_module_occhab/imports/actions.py#L484 

Added line #L484 was not covered by tests
insert_fields |= {field}
else:
if source_field in imprt.columns:
if (
column_src in imprt.columns
or mapping.get("default_value", None) is not None
):
insert_fields |= {field}
if entity.code == "station":
# unique_dataset_id is replaced with id_dataset
Expand Down
114 changes: 59 additions & 55 deletions .../gn_module_occhab/backend/gn_module_occhab/migrations/aed662bbd88a_add_default_mapping.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
revision = "aed662bbd88a"
down_revision = "69494f900cab"
branch_labels = None
depends_on = None
depends_on = "e43b01a18850"


def get_models(conn):
Expand Down Expand Up @@ -52,26 +52,28 @@ def upgrade():
sa.insert(FieldMapping).values(
id=id_occhab_mapping,
values={
"WKT": "geometry",
"altitude_max": "altitude_max",
"altitude_min": "altitude_min",
"area": "area",
"cd_hab": "cd_hab",
"comment": "comment",
"date_max": "date_fin",
"date_min": "date_debut",
"depth_max": "depth_max",
"depth_min": "depth_min",
"id_nomenclature_area_surface_calculation": "methode_calcul_surface",
"id_nomenclature_exposure": "exposition",
"id_nomenclature_geographic_object": "nature_objet_geo",
"id_station_source": "id_station",
"nom_cite": "nom_cite",
"observers_txt": "observateurs",
"technical_precision": "precision_technique",
"unique_dataset_id": "uuid_jdd",
"unique_id_sinp_habitat": "uuid_habitat",
"unique_id_sinp_station": "uuid_station",
"WKT": {"column_src": "geometry"},
"altitude_max": {"column_src": "altitude_max"},
"altitude_min": {"column_src": "altitude_min"},
"area": {"column_src": "area"},
"cd_hab": {"column_src": "cd_hab"},
"comment": {"column_src": "comment"},
"date_max": {"column_src": "date_fin"},
"date_min": {"column_src": "date_debut"},
"depth_max": {"column_src": "depth_max"},
"depth_min": {"column_src": "depth_min"},
"id_nomenclature_area_surface_calculation": {
"column_src": "methode_calcul_surface"
},
"id_nomenclature_exposure": {"column_src": "exposition"},
"id_nomenclature_geographic_object": {"column_src": "nature_objet_geo"},
"id_station_source": {"column_src": "id_station"},
"nom_cite": {"column_src": "nom_cite"},
"observers_txt": {"column_src": "observateurs"},
"technical_precision": {"column_src": "precision_technique"},
"unique_dataset_id": {"column_src": "uuid_jdd"},
"unique_id_sinp_habitat": {"column_src": "uuid_habitat"},
"unique_id_sinp_station": {"column_src": "uuid_station"},
},
)
)
Expand All @@ -92,39 +94,41 @@ def upgrade():
sa.insert(FieldMapping).values(
id=id_mapping_sinp,
values={
"WKT": "WKT",
"altitude_max": "altMax",
"altitude_min": "altMin",
"area": "surf",
"cd_hab": "cdHab",
"comment": "comment",
"date_max": "dateFin",
"date_min": "dateDebut",
"depth_max": "profMax",
"depth_min": "profMin",
"determiner": "persDeterm",
"id_habitat": "idOrigine",
"id_nomenclature_abundance": "abondHab",
"id_nomenclature_area_surface_calculation": "methodeCalculSurface",
"id_nomenclature_collection_technique": "techCollec",
"id_nomenclature_community_interest": "habitatInteretCommunautaire ",
"id_nomenclature_determination_type": "typeDeterm",
"id_nomenclature_exposure": "exposition",
"id_nomenclature_geographic_object": "natObjGeo",
"id_nomenclature_sensitivity": "sensibiliteHab",
"id_station_source": "idOrigEvt",
"is_habitat_complex": "mosaique",
"nom_cite": "nomCite",
"numerization_scale": "echelleNumerisation",
"observers_txt": "observer",
"precision": "precisGeo",
"recovery_percentage": "recouv",
"station_name": "nomStation",
"technical_precision": "precisionTechnique",
"unique_dataset_id": "jddMetaId",
"unique_id_sinp_grp_phyto": "relevePhyto",
"unique_id_sinp_habitat": "idSinpHab",
"unique_id_sinp_station": "permId",
"WKT": {"column_src": "WKT"},
"altitude_max": {"column_src": "altMax"},
"altitude_min": {"column_src": "altMin"},
"area": {"column_src": "surf"},
"cd_hab": {"column_src": "cdHab"},
"comment": {"column_src": "comment"},
"date_max": {"column_src": "dateFin"},
"date_min": {"column_src": "dateDebut"},
"depth_max": {"column_src": "profMax"},
"depth_min": {"column_src": "profMin"},
"determiner": {"column_src": "persDeterm"},
"id_habitat": {"column_src": "idOrigine"},
"id_nomenclature_abundance": {"column_src": "abondHab"},
"id_nomenclature_area_surface_calculation": {"column_src": "methodeCalculSurface"},
"id_nomenclature_collection_technique": {"column_src": "techCollec"},
"id_nomenclature_community_interest": {
"column_src": "habitatInteretCommunautaire "
},
"id_nomenclature_determination_type": {"column_src": "typeDeterm"},
"id_nomenclature_exposure": {"column_src": "exposition"},
"id_nomenclature_geographic_object": {"column_src": "natObjGeo"},
"id_nomenclature_sensitivity": {"column_src": "sensibiliteHab"},
"id_station_source": {"column_src": "idOrigEvt"},
"is_habitat_complex": {"column_src": "mosaique"},
"nom_cite": {"column_src": "nomCite"},
"numerization_scale": {"column_src": "echelleNumerisation"},
"observers_txt": {"column_src": "observer"},
"precision": {"column_src": "precisGeo"},
"recovery_percentage": {"column_src": "recouv"},
"station_name": {"column_src": "nomStation"},
"technical_precision": {"column_src": "precisionTechnique"},
"unique_dataset_id": {"column_src": "jddMetaId"},
"unique_id_sinp_grp_phyto": {"column_src": "relevePhyto"},
"unique_id_sinp_habitat": {"column_src": "idSinpHab"},
"unique_id_sinp_station": {"column_src": "permId"},
},
)
)
Expand All @@ -138,7 +142,7 @@ def downgrade():

cte = (
sa.select(MappingTemplate.c.id)
.where(MappingTemplate.c.label.in_(["OccHab", "Occurrences d'habitats SINP"]))
.where(MappingTemplate.c.label.in_(["Occhab", "Occurrences d'habitats SINP"]))
.cte("mapping_cte")
)
op.execute(sa.delete(FieldMapping).where(FieldMapping.c.id == cte.c.id))
Expand Down

0 comments on commit 26b1f4f

Please sign in to comment.