Skip to content

Commit

Permalink
Fix(import) fix column validity
Browse files Browse the repository at this point in the history
  • Loading branch information
Pierre-Narcisi committed Nov 27, 2024
1 parent d591f31 commit 836623c
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 8 deletions.
22 changes: 14 additions & 8 deletions backend/geonature/core/imports/checks/sql/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,14 @@
Entity,
EntityField,
BibFields,
TImports,
)


__all__ = ["init_rows_validity", "check_orphan_rows"]


def init_rows_validity(imprt):
def init_rows_validity(imprt: TImports, dataset_name_field: str = "id_dataset"):
"""
Validity columns are three-states:
- None: the row does not contains data for the given entity
Expand Down Expand Up @@ -46,16 +47,21 @@ def init_rows_validity(imprt):
.where(BibFields.name_field.in_(selected_fields_names))
.where(BibFields.entities.any(EntityField.entity == entity))
.where(~BibFields.entities.any(EntityField.entity != entity))
.where(BibFields.name_field != dataset_name_field)
.all()
)
db.session.execute(
sa.update(transient_table)
.where(transient_table.c.id_import == imprt.id_import)
.where(
sa.or_(*[transient_table.c[field.source_column].isnot(None) for field in fields])

if fields:
db.session.execute(
sa.update(transient_table)
.where(transient_table.c.id_import == imprt.id_import)
.where(
sa.or_(
*[transient_table.c[field.source_column].isnot(None) for field in fields]
)
)
.values({entity.validity_column: True})
)
.values({entity.validity_column: True})
)


def check_orphan_rows(imprt):
Expand Down
2 changes: 2 additions & 0 deletions backend/geonature/core/imports/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,8 @@ def update_transient_data_from_dataframe(
updated_cols = ["id_import", "line_no"] + list(updated_cols)
dataframe.replace({np.nan: None}, inplace=True)
records = dataframe[updated_cols].to_dict(orient="records")
if not records:
return
insert_stmt = pg_insert(transient_table)
insert_stmt = insert_stmt.values(records).on_conflict_do_update(
index_elements=updated_cols[:2],
Expand Down

0 comments on commit 836623c

Please sign in to comment.