Merge pull request #476 from PnX-SI/develop

Develop > Master / 2.2.1
PnX-SI · Sep 13, 2023 · 9035715 · 9035715
2 parents 87470e5 + 38aa8cf
commit 9035715
Show file tree

Hide file tree

Showing 10 changed files with 96 additions and 80 deletions.
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-2.2.0
+2.2.1
diff --git a/backend/gn_module_import/checks/dataframe/geography.py b/backend/gn_module_import/checks/dataframe/geography.py
@@ -36,7 +36,7 @@ def wkt_to_geometry(value):
 
 def x_y_to_geometry(x, y):
     try:
-        return Point(float(x), float(y))
+        return Point(float(x.replace(",", ".")), float(y.replace(",", ".")))
     except Exception:
         return None
 

diff --git a/backend/gn_module_import/checks/sql/__init__.py b/backend/gn_module_import/checks/sql/__init__.py
@@ -703,10 +703,11 @@ def complete_others_geom_columns(imprt, fields):
 
 def check_is_valid_geography(imprt, fields):
     if "WKT" in fields:
+        # It is useless to check valid WKT when created from X/Y
         where_clause = sa.and_(
             ImportSyntheseData.src_WKT != None,
             ImportSyntheseData.src_WKT != "",
-            sa.not_(ST_IsValid(ST_GeomFromText(ImportSyntheseData.src_WKT))),
+            sa.not_(ST_IsValid(ImportSyntheseData.the_geom_4326)),
         )
         report_erroneous_rows(
             imprt,
@@ -718,58 +719,15 @@ def check_is_valid_geography(imprt, fields):
 
 def check_geography_outside(imprt, fields):
     id_area = current_app.config["IMPORT"]["ID_AREA_RESTRICTION"]
-    where_clause = ()
     if id_area:
         local_srid = db.session.execute(sa.func.Find_SRID("ref_geo", "l_areas", "geom")).scalar()
         area = LAreas.query.filter(LAreas.id_area == id_area).one()
-
-        lat_long_present = sa.and_(
-            ImportSyntheseData.src_longitude != None,
-            ImportSyntheseData.src_longitude != "",
-            ImportSyntheseData.src_latitude != None,
-            ImportSyntheseData.src_latitude != "",
-        )
-        WKT_present = sa.and_(ImportSyntheseData.src_WKT != None, ImportSyntheseData.src_WKT != "")
-
-        if "WKT" in fields:
-            where_clause = sa.and_(
-                WKT_present,
-                sa.not_(lat_long_present),
-                area.geom.ST_Intersects(
-                    ST_Transform(
-                        ST_GeomFromText(ImportSyntheseData.src_WKT, imprt.srid), local_srid
-                    )
-                )
-                == False,
-            )
-            report_erroneous_rows(
-                imprt,
-                error_type="GEOMETRY_OUTSIDE",
-                error_column="WKT",
-                whereclause=where_clause,
-            )
-
-        if "longitude" in fields and "latitude" in fields:
-            where_clause = sa.and_(
-                sa.not_(WKT_present),
-                lat_long_present,
-                area.geom.ST_Intersects(
-                    ST_Transform(
-                        ST_SetSRID(
-                            ST_MakePoint(
-                                ImportSyntheseData.src_longitude.cast(sa.Float),
-                                ImportSyntheseData.src_latitude.cast(sa.Float),
-                            ),
-                            imprt.srid,
-                        ),
-                        local_srid,
-                    ),
-                )
-                == False,
-            )
-            report_erroneous_rows(
-                imprt,
-                error_type="GEOMETRY_OUTSIDE",
-                error_column="longitude",
-                whereclause=where_clause,
-            )
+        report_erroneous_rows(
+            imprt,
+            error_type="GEOMETRY_OUTSIDE",
+            error_column="Champs géométriques",
+            whereclause=sa.and_(
+                ImportSyntheseData.valid == True,
+                ImportSyntheseData.the_geom_local.ST_Disjoint(area.geom),
+            ),
+        )
diff --git a/backend/gn_module_import/migrations/0ff8fc0b4233_remove_archive_schema.py b/backend/gn_module_import/migrations/0ff8fc0b4233_remove_archive_schema.py
@@ -29,7 +29,7 @@ def upgrade():
 
     for archive_table in list(filter(lambda x: x != "cor_import_archives", archive_tables)):
         # Read table with pandas
-        op.drop_column(archive_table, "gn_pk", "gn_import_archives")
+        op.drop_column(archive_table, "gn_pk", schema="gn_import_archives")
         arch_df = pd.read_sql_table(archive_table, con=conn, schema="gn_import_archives")
         update_id = conn.execute(
             imprt.update()

diff --git a/backend/gn_module_import/routes/imports.py b/backend/gn_module_import/routes/imports.py
@@ -530,13 +530,20 @@ def get_import_invalid_rows_as_csv(scope, import_id):
 
     @stream_with_context
     def generate_invalid_rows_csv():
-        inputfile = BytesIO(imprt.source_file)
-        yield inputfile.readline()  # header
+        sourcefile = StringIO(imprt.source_file.decode(imprt.encoding))
+        destfile = StringIO()
+        csvreader = csv.reader(sourcefile, delimiter=imprt.separator)
+        csvwriter = csv.writer(destfile, dialect=csvreader.dialect, lineterminator="\n")
         line_no = 0
-        for row in inputfile:
+        for row in csvreader:
+            # line_no == 0 → csv header
+            if line_no == 0 or line_no in imprt.erroneous_rows:
+                csvwriter.writerow(row)
+                destfile.seek(0)
+                yield destfile.read().encode(imprt.encoding)
+                destfile.seek(0)
+                destfile.truncate()
             line_no += 1
-            if line_no in imprt.erroneous_rows:
-                yield row
 
     response = current_app.response_class(
         generate_invalid_rows_csv(),

diff --git a/backend/gn_module_import/tasks.py b/backend/gn_module_import/tasks.py
@@ -104,8 +104,8 @@ def do_import_checks(self, import_id):
         check_depths,
         check_digital_proof_urls,
         check_mandatory_fields,
-        check_geography_outside,
         check_is_valid_geography,
+        check_geography_outside,
     ]
     with start_sentry_child(op="check.sql", description="run all checks"):
         for i, check in enumerate(sql_checks):

diff --git a/backend/gn_module_import/tests/files/geom_file.csv b/backend/gn_module_import/tests/files/geom_file.csv
@@ -1,16 +1,19 @@
 date_min;cd_nom;nom_cite;observateurs;WKT;latitude;longitude;codecommune;codedepartement;codemaille;erreur attendue
-2017-01-01;67111;Ablette;Toto;;;;05101;;;Valide (codecommune)
+2017-01-01;67111;Ablette;Toto;;;;13088;;;Valide (codecommune)
 2017-01-01;67111;Ablette;Toto;;;;code com invalide;;;INVALID_ATTACHMENT_CODE (codecommune)
-2017-01-01;67111;Ablette;Toto;;;;;05;;Valide (codedépartement)
+2017-01-01;67111;Ablette;Toto;;;;;13;;Valide (codedépartement)  # FIXME invalide altitude_min (bord de mer)
 2017-01-01;67111;Ablette;Toto;;;;;code dep invalide;;INVALID_ATTACHMENT_CODE (codedepartement)
-2017-01-01;67111;Ablette;Toto;;;;;;10kmL93E097N642;Valide (codemaille)
+2017-01-01;67111;Ablette;Toto;;;;;;10kmL93E091N625;Valide (codemaille)
 2017-01-01;67111;Ablette;Toto;;;;;;code maille invalide;INVALID_ATTACHMENT_CODE (codemaille)
 2017-01-01;67111;Ablette;Toto;;;;05101;05;10kmL93E097N642;MULTIPLE_CODE_ATTACHMENT
-2017-01-01;67111;Ablette;Toto;POINT(6.5 44.85);;;05101;05;10kmL93E097N642;Valide (WKT) (FIXME ?)
-2017-01-01;67111;Ablette;Toto;;44.85;6.5;05101;05;10kmL93E097N642;Valide (X/Y) (FIXME ?)
-2017-01-01;67111;Ablette;Toto;POINT(6.5 44.85);44.85;6.5;05101;05;10kmL93E097N642;MULTIPLE_ATTACHMENT_TYPE_CODE
-2017-01-01;67111;Ablette;Toto;POINT(6.5 44.85);;;;;;Valide (WKT)
-2017-01-01;67111;Ablette;Toto;;44.85;6.5;;;;Valide (X/Y)
+2017-01-01;67111;Ablette;Toto;POINT(5.4877 43.3056);;;05101;05;10kmL93E097N642;Valide (WKT)
+2017-01-01;67111;Ablette;Toto;;43.3056;5.4877;05101;05;10kmL93E097N642;Valide (X/Y)
+2017-01-01;67111;Ablette;Toto;POINT(5.4877 43.3056);44.85;6.5;05101;05;10kmL93E097N642;MULTIPLE_ATTACHMENT_TYPE_CODE
+2017-01-01;67111;Ablette;Toto;POINT(5.4877 43.3056);;;;;;Valide (WKT)
+2017-01-01;67111;Ablette;Toto;;43.3056;5.4877;;;;Valide (X/Y)
+2017-01-01;67111;Ablette;Toto;;43,3056;5,4877;;;;Valide (X/Y)
 2017-01-01;67111;Ablette;Toto;POINT(6.5 44.85);44.85;6.5;;;;MULTIPLE_ATTACHMENT_TYPE_CODE
 2017-01-01;67111;Ablette;Toto;;;;;;;NO-GEOM
 2017-01-01;67111;Ablette;Toto;POLYGON((0 0, 1 1, 1 2, 1 1, 0 0));;;;;;INVALID_GEOMETRY
+2017-01-01;67111;Ablette;Toto;POINT(6.5 44.85);;;;;;GEOMETRY_OUTSIDE
+2017-01-01;67111;Ablette;Toto;;44.85;6.5;;;;GEOMETRY_OUTSIDE
diff --git a/backend/gn_module_import/tests/files/multiline_comment_file.csv b/backend/gn_module_import/tests/files/multiline_comment_file.csv
@@ -0,0 +1,6 @@
+date_min;nom_cite;observateurs;WKT;cd_nom;comment_description
+2017-01-01;Ablette;Toto;POINT(6.5 44.85);;"Ligne erronée : MISSING_VALUE (cd_nom)"
+2017-01-01;Ablette;Toto;POINT(6.5 44.85);123456789;"Ligne erronée :
+CD_NOM_NOT_FOUND"
+2017-01-01;Ablette;Toto;POINT(6.5 44.85);67111;"Cette ligne
+est valide"
diff --git a/backend/gn_module_import/tests/test_imports.py b/backend/gn_module_import/tests/test_imports.py
@@ -4,6 +4,7 @@
 from operator import or_
 from functools import reduce
 from unittest.mock import patch
+import csv
 
 import pytest
 from flask import g, url_for, current_app
@@ -140,6 +141,11 @@ def import_file_name():
     return "valid_file.csv"
 
 
+@pytest.fixture()
+def autogenerate():
+    return True
+
+
 @pytest.fixture()
 def import_dataset(datasets, import_file_name):
     ds = datasets["own_dataset"]
@@ -198,12 +204,12 @@ def decoded_import(client, uploaded_import):
 
 
 @pytest.fixture()
-def fieldmapping(import_file_name):
+def fieldmapping(import_file_name, autogenerate):
     if import_file_name == "valid_file.csv":
         return FieldMapping.query.filter_by(label="Synthese GeoNature").one().values
     else:
         return {
-            f.name_field: True
+            f.name_field: autogenerate
             if f.autogenerated
             else ([f.name_field, "cd_nom"] if f.multi else f.name_field)
             for f in BibFields.query.filter_by(display=True)
@@ -937,6 +943,7 @@ def test_import_valid_file(self, users, datasets):
         assert r.status_code == 200, r.data
 
     @pytest.mark.parametrize("import_file_name", ["geom_file.csv"])
+    @pytest.mark.parametrize("autogenerate", [False])
     def test_import_geometry_file(self, area_restriction, prepared_import):
         assert_import_errors(
             prepared_import,
@@ -948,12 +955,11 @@ def test_import_geometry_file(self, area_restriction, prepared_import):
                 (
                     "MULTIPLE_ATTACHMENT_TYPE_CODE",
                     "Champs géométriques",
-                    frozenset([10, 13]),
+                    frozenset([10, 14]),
                 ),
-                ("GEOMETRY_OUTSIDE", "WKT", frozenset([8, 11, 15])),
-                ("GEOMETRY_OUTSIDE", "longitude", frozenset([9, 12])),
-                ("NO-GEOM", "Champs géométriques", frozenset([14])),
-                ("INVALID_GEOMETRY", "WKT", frozenset([15])),
+                ("NO-GEOM", "Champs géométriques", frozenset([15])),
+                ("INVALID_GEOMETRY", "WKT", frozenset([16])),
+                ("GEOMETRY_OUTSIDE", "Champs géométriques", frozenset([17, 18])),
             },
         )
 
@@ -1137,6 +1143,29 @@ def test_import_empty_nomenclatures_file_no_default(
         # et ne pas l’écraser avec la valeur par défaut
         assert obs3.nomenclature_life_stage.label_default == "Alevin"
 
+    @pytest.mark.parametrize("import_file_name", ["multiline_comment_file.csv"])
+    def test_import_multiline_comment_file(self, users, imported_import):
+        assert_import_errors(
+            imported_import,
+            {
+                ("MISSING_VALUE", "cd_nom", frozenset({1})),
+                ("CD_NOM_NOT_FOUND", "cd_nom", frozenset({2})),
+            },
+        )
+
+        obs = Synthese.query.filter_by(source=imported_import.source).first()
+        assert obs.comment_description == "Cette ligne\nest valide"
+
+        set_logged_user_cookie(self.client, users["user"])
+        r = self.client.get(
+            url_for("import.get_import_invalid_rows_as_csv", import_id=imported_import.id_import)
+        )
+        assert r.status_code == 200, r.data
+        csvfile = StringIO(r.data.decode("utf-8"))
+        csvreader = csv.DictReader(csvfile, delimiter=";")
+        rows = list(csvreader)
+        assert rows[1]["comment_description"] == "Ligne erronée :\nCD_NOM_NOT_FOUND"
+
     def test_export_pdf(self, users, imports):
         user = users["user"]
         imprt = imports["own_import"]

diff --git a/docs/CHANGELOG.rst b/docs/CHANGELOG.rst
@@ -2,17 +2,30 @@
 CHANGELOG
 =========
 
+2.2.1 (2023-09-13)
+------------------
+
+**🚀 Nouveautés**
+
+* Prise en charge des virgules comme séparateur des décimales dans les coordonnées des champs X et Y (#473, par @bouttier)
+
+**🐛 Corrections**
+
+* Correction du rapport d'erreur quand des lignes contiennent des sauts de ligne (#464, par @cen-cgeier et @bouttier)
+* Correction de la migration de suppression du schéma ``gn_import_archives`` (#471, par @joelclems)
+
+
 2.2.0 (2023-08-23)
 ------------------
 
 Nécessite la version 2.13.0 (ou plus) de GeoNature.
 
 **🚀 Nouveautés**
 
-* Compatibilité avec GeoNature 2.13.0 et la refonte des permissions, en définissant les permissions disponibles du module (#232)
+* Compatibilité avec GeoNature 2.13.0 et la refonte des permissions, en définissant les permissions disponibles du module (#451)
 * Vérification des permissions sur le nouvel objet "IMPORT" et non plus sur le module lui-même
-* Réimplémentation de la possibilité d'alimenter le champs ``additionnal_data`` de la synthèse avec un ou plusieurs champs du fichier source, ou depuis un champs JSON agrégeant déjà plusieurs informations (#165)
-* Réimplémentation du contrôle de validité des géométries des observations à importer (#453)
+* Ré-implémentation de la possibilité d'alimenter le champs ``additionnal_data`` de la synthèse avec un ou plusieurs champs du fichier source, ou depuis un champs JSON agrégeant déjà plusieurs informations (#165)
+* Ré-implémentation du contrôle de validité des géométries des observations à importer (#453)
 * Amélioration du contrôle des altitudes et des profondeurs (#445)
 * Amélioration du rapport d'import en n'affichant le bloc "Données importées" seulement si l'import a un statut terminé (#457)
 * Amélioration du tri dans la liste des imports en permettant de trier sur les colonnes avec une clé étrangère comme celle des JDD et des auteurs (#460)