From 0a32b325e2591f63150d86c0d001822157bbdd02 Mon Sep 17 00:00:00 2001 From: Paul Schilling Date: Wed, 4 Dec 2024 09:19:58 +0100 Subject: [PATCH] [#2903] Support import of Zaaktype configs without library --- src/open_inwoner/openzaak/admin.py | 83 ++++++++++++++++++- src/open_inwoner/openzaak/import_export.py | 2 +- src/open_inwoner/openzaak/tests/test_admin.py | 12 +-- .../openzaak/tests/test_import_export.py | 36 ++++---- .../admin/catalogusconfig_change_list.html | 2 +- .../admin/zaaktypeconfig_change_list.html | 12 +++ 6 files changed, 115 insertions(+), 32 deletions(-) create mode 100644 src/open_inwoner/templates/admin/zaaktypeconfig_change_list.html diff --git a/src/open_inwoner/openzaak/admin.py b/src/open_inwoner/openzaak/admin.py index 4dcd59bde7..10bec8e2e7 100644 --- a/src/open_inwoner/openzaak/admin.py +++ b/src/open_inwoner/openzaak/admin.py @@ -17,7 +17,7 @@ from solo.admin import SingletonModelAdmin from open_inwoner.ckeditor5.widgets import CKEditorWidget -from open_inwoner.openzaak.import_export import CatalogusConfigImport, ZGWConfigExport +from open_inwoner.openzaak.import_export import ZGWConfigExport, ZGWConfigImport from open_inwoner.utils.forms import LimitedUploadFileField from .models import ( @@ -133,7 +133,7 @@ def get_urls(self): path( "import-catalogus-dump/", self.admin_site.admin_view(self.process_file_view), - name="upload_zgw_import_file", + name="upload_catalogus_import_file", ), ] return custom_urls + urls @@ -163,7 +163,7 @@ def process_file_view(self, request): try: import_result = ( - CatalogusConfigImport.import_from_jsonl_file_in_django_storage( + ZGWConfigImport.import_from_jsonl_file_in_django_storage( target_file_name, storage, ) @@ -371,6 +371,7 @@ def has_delete_permission(self, request, obj=None): @admin.register(ZaakTypeConfig) class ZaakTypeConfigAdmin(admin.ModelAdmin): + change_list_template = "admin/zaaktypeconfig_change_list.html" inlines = [ ZaakTypeInformatieObjectTypeConfigInline, ZaakTypeStatusTypeConfigInline, @@ -434,6 +435,17 @@ class ZaakTypeConfigAdmin(admin.ModelAdmin): ] ordering = ("identificatie", "catalogus__domein") + def get_urls(self): + urls = super().get_urls() + custom_urls = [ + path( + "import-zaaktype-dump/", + self.admin_site.admin_view(self.process_file_view), + name="upload_zaaktype_import_file", + ), + ] + return custom_urls + urls + @admin.action(description=_("Export to file")) def export_zaaktype_configs(modeladmin, request, queryset): export = ZGWConfigExport.from_zaaktype_configs(queryset) @@ -446,6 +458,71 @@ def export_zaaktype_configs(modeladmin, request, queryset): ] = 'attachment; filename="zgw-zaaktype-export.json"' return response + def process_file_view(self, request): + form = ImportZGWExportFileForm() + + if request.method == "POST": + form = ImportZGWExportFileForm(request.POST, request.FILES) + if form.is_valid(): + storage = PrivateMediaFileSystemStorage() + timestamp = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + target_file_name = f"zgw_import_dump_{timestamp}.json" + storage.save(target_file_name, request.FILES["zgw_export_file"]) + + try: + import_result = ( + ZGWConfigImport.import_from_jsonl_file_in_django_storage( + target_file_name, + storage, + ) + ) + self.message_user( + request, + _( + "%(num_rows)d item(s) processed in total, with %(error_rows)d failing row(s)." + % { + "num_rows": import_result.total_rows_processed, + "error_rows": len(import_result.import_errors), + } + ), + messages.SUCCESS + if not import_result.import_errors + else messages.WARNING, + ) + if errors := import_result.import_errors: + msgs_deduped = set(error.__str__() for error in errors) + error_msg_iterator = ([msg] for msg in msgs_deduped) + + error_msg_html = format_html_join( + "\n", "

- {}

", error_msg_iterator + ) + error_msg_html = format_html( + _("It was not possible to import the following items:") + + f"
{error_msg_html}
" + ) + self.message_user(request, error_msg_html, messages.ERROR) + + return HttpResponseRedirect( + reverse( + "admin:openzaak_zaaktypeconfig_changelist", + ) + ) + except Exception: + logger.exception("Unable to process ZGW import") + self.message_user( + request, + _( + "We were unable to process your upload. Please regenerate the file and try again." + ), + messages.ERROR, + ) + finally: + storage.delete(target_file_name) + + return TemplateResponse( + request, "admin/import_zgw_export_form.html", {"form": form} + ) + def has_add_permission(self, request): return False diff --git a/src/open_inwoner/openzaak/import_export.py b/src/open_inwoner/openzaak/import_export.py index 06a71d01d5..5b668a0f86 100644 --- a/src/open_inwoner/openzaak/import_export.py +++ b/src/open_inwoner/openzaak/import_export.py @@ -277,7 +277,7 @@ def from_zaaktype_configs(cls, zaaktype_configs: QuerySet) -> Self: @dataclasses.dataclass(frozen=True) -class CatalogusConfigImport: +class ZGWConfigImport: """Import CatalogusConfig(s) and all associated relations.""" total_rows_processed: int = 0 diff --git a/src/open_inwoner/openzaak/tests/test_admin.py b/src/open_inwoner/openzaak/tests/test_admin.py index 7078c62065..d7b79c7166 100644 --- a/src/open_inwoner/openzaak/tests/test_admin.py +++ b/src/open_inwoner/openzaak/tests/test_admin.py @@ -192,7 +192,7 @@ def test_import_flow_reports_success(self) -> None: form = self.app.get( reverse( - "admin:upload_zgw_import_file", + "admin:upload_catalogus_import_file", ), user=self.user, ).form @@ -219,13 +219,13 @@ def test_import_flow_reports_success(self) -> None: ) @mock.patch( - "open_inwoner.openzaak.import_export.CatalogusConfigImport.import_from_jsonl_file_in_django_storage" + "open_inwoner.openzaak.import_export.ZGWConfigImport.import_from_jsonl_file_in_django_storage" ) def test_import_flow_errors_reports_failure_to_user(self, m) -> None: m.side_effect = Exception("something went wrong") form = self.app.get( reverse( - "admin:upload_zgw_import_file", + "admin:upload_catalogus_import_file", ), user=self.user, ).form @@ -255,7 +255,7 @@ def test_import_flow_errors_reports_failure_to_user(self, m) -> None: self.assertEqual( response.request.path, reverse( - "admin:upload_zgw_import_file", + "admin:upload_catalogus_import_file", ), ) @@ -273,7 +273,7 @@ def test_import_flow_reports_errors(self) -> None: form = self.app.get( reverse( - "admin:upload_zgw_import_file", + "admin:upload_catalogus_import_file", ), user=self.user, ).form @@ -334,7 +334,7 @@ def test_import_flow_reports_partial_errors(self) -> None: form = self.app.get( reverse( - "admin:upload_zgw_import_file", + "admin:upload_catalogus_import_file", ), user=self.user, ).form diff --git a/src/open_inwoner/openzaak/tests/test_import_export.py b/src/open_inwoner/openzaak/tests/test_import_export.py index 804f1f12b7..bce6f9d780 100644 --- a/src/open_inwoner/openzaak/tests/test_import_export.py +++ b/src/open_inwoner/openzaak/tests/test_import_export.py @@ -5,7 +5,7 @@ from django.core.files.storage.memory import InMemoryStorage from django.test import TestCase -from open_inwoner.openzaak.import_export import CatalogusConfigImport, ZGWConfigExport +from open_inwoner.openzaak.import_export import ZGWConfigExport, ZGWConfigImport from open_inwoner.openzaak.models import ( CatalogusConfig, ZaakTypeConfig, @@ -352,14 +352,14 @@ def test_import_jsonl_update_success(self): mocks = ZGWExportImportMockData() self.storage.save("import.jsonl", io.StringIO(self.jsonl)) - import_result = CatalogusConfigImport.import_from_jsonl_file_in_django_storage( + import_result = ZGWConfigImport.import_from_jsonl_file_in_django_storage( "import.jsonl", self.storage ) # check import self.assertEqual( import_result, - CatalogusConfigImport( + ZGWConfigImport( total_rows_processed=5, catalogus_configs_imported=1, zaaktype_configs_imported=1, @@ -439,7 +439,7 @@ def test_import_jsonl_missing_statustype_config(self): # we use `asdict` and replace the Exceptions with string representations # because for Exceptions raised from within dataclasses, equality ==/is identity import_result = dataclasses.asdict( - CatalogusConfigImport.import_from_jsonl_file_in_django_storage( + ZGWConfigImport.import_from_jsonl_file_in_django_storage( "import.jsonl", self.storage ) ) @@ -448,7 +448,7 @@ def test_import_jsonl_missing_statustype_config(self): "ZaakTypeConfig identificatie = 'ztc-id-a-0'" ) import_expected = dataclasses.asdict( - CatalogusConfigImport( + ZGWConfigImport( total_rows_processed=6, catalogus_configs_imported=1, zaaktype_configs_imported=1, @@ -485,7 +485,7 @@ def test_import_jsonl_update_statustype_config_missing_zt_config(self): # we use `asdict` and replace the Exceptions with string representations # because for Exceptions raised from within dataclasses, equality ==/is identity import_result = dataclasses.asdict( - CatalogusConfigImport.import_from_jsonl_file_in_django_storage( + ZGWConfigImport.import_from_jsonl_file_in_django_storage( "import.jsonl", self.storage ) ) @@ -494,7 +494,7 @@ def test_import_jsonl_update_statustype_config_missing_zt_config(self): "ZaakTypeConfig identificatie = 'bogus'" ) import_expected = dataclasses.asdict( - CatalogusConfigImport( + ZGWConfigImport( total_rows_processed=6, catalogus_configs_imported=1, zaaktype_configs_imported=1, @@ -531,7 +531,7 @@ def test_import_jsonl_update_reports_duplicate_db_records(self): # we use `asdict` and replace the Exceptions with string representations # because for Exceptions raised from within dataclasses, equality ==/is identity import_result = dataclasses.asdict( - CatalogusConfigImport.import_from_jsonl_file_in_django_storage( + ZGWConfigImport.import_from_jsonl_file_in_django_storage( "import.jsonl", self.storage ) ) @@ -540,7 +540,7 @@ def test_import_jsonl_update_reports_duplicate_db_records(self): "ZaakTypeConfig identificatie = 'ztc-id-a-0'" ) import_expected = dataclasses.asdict( - CatalogusConfigImport( + ZGWConfigImport( total_rows_processed=5, catalogus_configs_imported=1, zaaktype_configs_imported=1, @@ -606,9 +606,7 @@ def test_import_jsonl_fails_with_catalogus_domein_rsin_mismatch(self): with self.assertLogs( logger="open_inwoner.openzaak.import_export", level="ERROR" ) as cm: - import_result = CatalogusConfigImport.from_jsonl_stream_or_string( - import_line - ) + import_result = ZGWConfigImport.from_jsonl_stream_or_string(import_line) self.assertEqual( cm.output, [ @@ -635,7 +633,7 @@ def test_import_jsonl_fails_with_catalogus_domein_rsin_mismatch(self): def test_bad_import_types(self): for bad_type in (set(), list(), b""): with self.assertRaises(ValueError): - CatalogusConfigImport.from_jsonl_stream_or_string(bad_type) + ZGWConfigImport.from_jsonl_stream_or_string(bad_type) def test_valid_input_types_are_accepted(self): ZGWExportImportMockData() @@ -646,10 +644,10 @@ def test_valid_input_types_are_accepted(self): self.jsonl, ): with self.subTest(f"Input type {type(input)}"): - import_result = CatalogusConfigImport.from_jsonl_stream_or_string(input) + import_result = ZGWConfigImport.from_jsonl_stream_or_string(input) self.assertEqual( import_result, - CatalogusConfigImport( + ZGWConfigImport( total_rows_processed=5, catalogus_configs_imported=1, zaaktype_configs_imported=1, @@ -665,9 +663,7 @@ def test_import_is_atomic(self): bad_jsonl = self.jsonl + "\n" + bad_line with self.assertRaises(KeyError): - CatalogusConfigImport.from_jsonl_stream_or_string( - stream_or_string=bad_jsonl - ) + ZGWConfigImport.from_jsonl_stream_or_string(stream_or_string=bad_jsonl) counts = ( CatalogusConfig.objects.count(), @@ -691,8 +687,6 @@ def setUp(self): def test_exports_can_be_imported(self): export = ZGWConfigExport.from_catalogus_configs(CatalogusConfig.objects.all()) - import_result = CatalogusConfigImport.from_jsonl_stream_or_string( - export.as_jsonl() - ) + import_result = ZGWConfigImport.from_jsonl_stream_or_string(export.as_jsonl()) self.assertEqual(import_result.total_rows_processed, 5) diff --git a/src/open_inwoner/templates/admin/catalogusconfig_change_list.html b/src/open_inwoner/templates/admin/catalogusconfig_change_list.html index 41eb9eac60..38a2f2e19e 100644 --- a/src/open_inwoner/templates/admin/catalogusconfig_change_list.html +++ b/src/open_inwoner/templates/admin/catalogusconfig_change_list.html @@ -4,7 +4,7 @@ {% block object-tools-items %} {{ block.super }}
  • - + {% trans "Import from file" %}
    • diff --git a/src/open_inwoner/templates/admin/zaaktypeconfig_change_list.html b/src/open_inwoner/templates/admin/zaaktypeconfig_change_list.html new file mode 100644 index 0000000000..8ea4080c0b --- /dev/null +++ b/src/open_inwoner/templates/admin/zaaktypeconfig_change_list.html @@ -0,0 +1,12 @@ +{% extends "admin/change_list.html" %} +{% load i18n admin_urls %} + +{% block object-tools-items %} +{{ block.super }} +
  • + + {% trans "Import from file" %} + +
    • +{% endblock %} +