[Fixes #12657] Align supported_file_extension_config with the new cli…

…ent configuration (#12673) * [Fixes #12657] Rename ACTIONS into TASKS * [Fixes #12657] Refactor supported types * [Fixes #12657] Refactor supported types * [Fixes #12657] Refactor supported type, fix data retriever and refactor handlers configuration * [Fixes #12657] Refactor supported type, fix data retriever and refactor handlers configuration * [Fixes #12657] Refactor supported type, fix data retriever and refactor handlers configuration * [Fixes #12657] Refactor supported type, fix data retriever and refactor handlers configuration * [Fixes #12657] Refactor supported type, fix data retriever and refactor handlers configuration * [Fixes #12657] Refactor supported type, fix data retriever and refactor handlers configuration
GeoNode · Oct 25, 2024 · 883f933 · 883f933
1 parent 8d58004
commit 883f933
Show file tree

Hide file tree

Showing 55 changed files with 443 additions and 513 deletions.
diff --git a/geonode/geoserver/management/commands/importlayers.py b/geonode/geoserver/management/commands/importlayers.py
@@ -149,6 +149,7 @@ def execute(self):
                             params[name] = os.path.basename(value.name)
 
                     params["non_interactive"] = 'true'
+                    params["action"] = 'upload'
                     response = client.post(
                         urljoin(self.host, "/api/v2/uploads/upload/"),
                         auth=HTTPBasicAuth(self.username, self.password),

diff --git a/geonode/resource/api/tests.py b/geonode/resource/api/tests.py
@@ -157,9 +157,7 @@ def test_endpoint_should_raise_error_if_pk_is_not_passed(self):
 
     def test_endpoint_should_return_the_source(self):
         # creating dummy execution request
-        obj = ExecutionRequest.objects.create(
-            user=self.superuser, func_name="import_new_resource", action="import", source="upload_workflow"
-        )
+        obj = ExecutionRequest.objects.create(user=self.superuser, func_name="import_new_resource", action="upload")
         self.client.force_login(self.superuser)
 
         _url = f"{reverse('executionrequest-list')}/{obj.exec_id}"

diff --git a/geonode/resource/enumerator.py b/geonode/resource/enumerator.py
@@ -22,6 +22,7 @@
 
 class ExecutionRequestAction(enum.Enum):
     IMPORT = _("import")
+    UPLOAD = _("upload")
     CREATE = _("create")
     COPY = _("copy")
     DELETE = _("delete")

diff --git a/geonode/resource/migrations/0009_remove_executionrequest_source_and_more.py b/geonode/resource/migrations/0009_remove_executionrequest_source_and_more.py
@@ -0,0 +1,37 @@
+# Generated by Django 4.2.9 on 2024-10-18 10:41
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("resource", "0008_executionrequest_source"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="executionrequest",
+            name="source",
+        ),
+        migrations.AlterField(
+            model_name="executionrequest",
+            name="action",
+            field=models.CharField(
+                choices=[
+                    ("import", "import"),
+                    ("upload", "upload"),
+                    ("create", "create"),
+                    ("copy", "copy"),
+                    ("delete", "delete"),
+                    ("permissions", "permissions"),
+                    ("update", "update"),
+                    ("ingest", "ingest"),
+                    ("unknown", "unknown"),
+                ],
+                default="unknown",
+                max_length=50,
+                null=True,
+            ),
+        ),
+    ]
diff --git a/geonode/resource/models.py b/geonode/resource/models.py
@@ -60,5 +60,3 @@ class ExecutionRequest(models.Model):
     action = models.CharField(
         max_length=50, choices=ACTION_CHOICES, default=ExecutionRequestAction.UNKNOWN.value, null=True
     )
-
-    source = models.CharField(max_length=250, null=True, default=None)
diff --git a/geonode/security/tests.py b/geonode/security/tests.py
@@ -752,9 +752,7 @@ def test_dataset_permissions(self):
         bobby = get_user_model().objects.get(username="bobby")
 
         self.client.force_login(get_user_model().objects.get(username="admin"))
-        payload = {
-            "base_file": open(f"{project_dir}/tests/fixture/valid.geojson", "rb"),
-        }
+        payload = {"base_file": open(f"{project_dir}/tests/fixture/valid.geojson", "rb"), "action": "upload"}
         response = self.client.post(reverse("importer_upload"), data=payload)
         layer = ResourceHandlerInfo.objects.filter(execution_request=response.json()["execution_id"]).first().resource
         if layer is None:

diff --git a/geonode/settings.py b/geonode/settings.py
@@ -2240,56 +2240,6 @@ def get_geonode_catalogue_service():
     "document_upload",
 )
 
-SUPPORTED_DATASET_FILE_TYPES = [
-    {
-        "id": "shp",
-        "label": "ESRI Shapefile",
-        "format": "vector",
-        "ext": ["shp"],
-        "requires": ["shp", "prj", "dbf", "shx"],
-        "optional": ["xml", "sld"],
-    },
-    {
-        "id": "tiff",
-        "label": "GeoTIFF",
-        "format": "raster",
-        "ext": ["tiff", "tif", "geotiff", "geotif"],
-        "mimeType": ["image/tiff"],
-        "optional": ["xml", "sld"],
-    },
-    {
-        "id": "csv",
-        "label": "Comma Separated Value (CSV)",
-        "format": "vector",
-        "ext": ["csv"],
-        "mimeType": ["text/csv"],
-        "optional": ["xml", "sld"],
-    },
-    {
-        "id": "zip",
-        "label": "Zip Archive",
-        "format": "archive",
-        "ext": ["zip"],
-        "mimeType": ["application/zip"],
-        "optional": ["xml", "sld"],
-    },
-    {
-        "id": "xml",
-        "label": "XML Metadata File",
-        "format": "metadata",
-        "ext": ["xml"],
-        "mimeType": ["application/json"],
-        "needsFiles": ["shp", "prj", "dbf", "shx", "csv", "tiff", "zip", "sld"],
-    },
-    {
-        "id": "sld",
-        "label": "Styled Layer Descriptor (SLD)",
-        "format": "metadata",
-        "ext": ["sld"],
-        "mimeType": ["application/json"],
-        "needsFiles": ["shp", "prj", "dbf", "shx", "csv", "tiff", "zip", "xml"],
-    },
-]
 INSTALLED_APPS += (
     "dynamic_models",
     # "importer",

diff --git a/geonode/storage/tests.py b/geonode/storage/tests.py
@@ -573,23 +573,6 @@ def test_zip_file_should_correctly_index_file_extensions(self):
         # extensions found more than once get indexed
         self.assertIsNotNone(_files.get("csv_file_1"))
 
-    @override_settings(
-        SUPPORTED_DATASET_FILE_TYPES=[
-            {"id": "kmz", "label": "kmz", "format": "vector", "ext": ["kmz"]},
-            {"id": "kml", "label": "kml", "format": "vector", "ext": ["kml"]},
-        ]
-    )
-    def test_zip_file_should_correctly_recognize_main_extension_with_kmz(self):
-        # reinitiate the storage manager with the zip file
-        storage_manager = self.sut(
-            remote_files={"base_file": os.path.join(f"{self.project_root}", "tests/data/Italy.kmz")}
-        )
-        storage_manager.clone_remote_files()
-
-        self.assertIsNotNone(storage_manager.data_retriever.temporary_folder)
-        _files = storage_manager.get_retrieved_paths()
-        self.assertTrue("doc.kml" in _files.get("base_file"), msg=f"files available: {_files}")
-
     def test_zip_file_should_correctly_recognize_main_extension_with_shp(self):
         # zipping files
         storage_manager = self.sut(remote_files=self.local_files_paths)

diff --git a/geonode/tests/test_utils.py b/geonode/tests/test_utils.py
@@ -18,7 +18,6 @@
 #########################################################################
 import copy
 from unittest import TestCase
-from django.test import override_settings
 
 from unittest.mock import patch
 from datetime import datetime, timedelta
@@ -32,8 +31,7 @@
 from geonode.geoserver.helpers import set_attributes
 from geonode.tests.base import GeoNodeBaseTestSupport
 from geonode.br.management.commands.utils.utils import ignore_time
-from geonode.utils import copy_tree, get_supported_datasets_file_types, bbox_to_wkt
-from geonode import settings
+from geonode.utils import copy_tree, bbox_to_wkt
 
 
 class TestCopyTree(GeoNodeBaseTestSupport):
@@ -205,39 +203,6 @@ def setUp(self):
             },
         ]
 
-    @override_settings(
-        ADDITIONAL_DATASET_FILE_TYPES=[
-            {"id": "dummy_type", "label": "Dummy Type", "format": "dummy", "ext": ["dummy"]},
-        ]
-    )
-    def test_should_append_additional_type_if_config_is_provided(self):
-        prev_count = len(settings.SUPPORTED_DATASET_FILE_TYPES)
-        supported_types = get_supported_datasets_file_types()
-        supported_keys = [t.get("id") for t in supported_types]
-        self.assertIn("dummy_type", supported_keys)
-        self.assertEqual(len(supported_keys), prev_count + 1)
-
-    @override_settings(
-        ADDITIONAL_DATASET_FILE_TYPES=[
-            {
-                "id": "shp",
-                "label": "Replaced type",
-                "format": "vector",
-                "ext": ["shp"],
-                "requires": ["shp", "prj", "dbf", "shx"],
-                "optional": ["xml", "sld"],
-            },
-        ]
-    )
-    def test_should_replace_the_type_id_if_already_exists(self):
-        prev_count = len(settings.SUPPORTED_DATASET_FILE_TYPES)
-        supported_types = get_supported_datasets_file_types()
-        supported_keys = [t.get("id") for t in supported_types]
-        self.assertIn("shp", supported_keys)
-        self.assertEqual(len(supported_keys), prev_count)
-        shp_type = [t for t in supported_types if t["id"] == "shp"][0]
-        self.assertEqual(shp_type["label"], "Replaced type")
-
 
 class TestRegionsCrossingDateLine(TestCase):
     def setUp(self):

diff --git a/geonode/upload/api/serializer.py b/geonode/upload/api/serializer.py
@@ -34,15 +34,15 @@ class Meta:
             "sld_file",
             "store_spatial_files",
             "skip_existing_layers",
-            "source",
+            "action",
         )
 
     base_file = serializers.FileField()
     xml_file = serializers.FileField(required=False)
     sld_file = serializers.FileField(required=False)
     store_spatial_files = serializers.BooleanField(required=False, default=True)
     skip_existing_layers = serializers.BooleanField(required=False, default=False)
-    source = serializers.CharField(required=False, default="upload")
+    action = serializers.CharField(required=True)
 
 
 class OverwriteImporterSerializer(ImporterSerializer):

diff --git a/geonode/upload/api/tests.py b/geonode/upload/api/tests.py
@@ -62,9 +62,7 @@ def test_upload_method_not_allowed(self):
     def test_raise_exception_if_file_is_not_a_handled(self):
 
         self.client.force_login(get_user_model().objects.get(username="admin"))
-        payload = {
-            "base_file": SimpleUploadedFile(name="file.invalid", content=b"abc"),
-        }
+        payload = {"base_file": SimpleUploadedFile(name="file.invalid", content=b"abc"), "action": "upload"}
         response = self.client.post(self.url, data=payload)
         self.assertEqual(500, response.status_code)
 
@@ -76,6 +74,7 @@ def test_gpkg_raise_error_with_invalid_payload(self):
                 content=b'{"type": "FeatureCollection", "content": "some-content"}',
             ),
             "store_spatial_files": "invalid",
+            "action": "upload",
         }
         expected = {
             "success": False,
@@ -99,6 +98,7 @@ def test_gpkg_task_is_called(self, patch_upload):
                 content=b'{"type": "FeatureCollection", "content": "some-content"}',
             ),
             "store_spatial_files": True,
+            "action": "upload",
         }
 
         response = self.client.post(self.url, data=payload)
@@ -116,6 +116,7 @@ def test_geojson_task_is_called(self, patch_upload):
                 content=b'{"type": "FeatureCollection", "content": "some-content"}',
             ),
             "store_spatial_files": True,
+            "action": "upload",
         }
 
         response = self.client.post(self.url, data=payload)
@@ -133,6 +134,7 @@ def test_zip_file_is_unzip_and_the_handler_is_found(self, patch_upload):
             "base_file": open(f"{project_dir}/tests/fixture/valid.zip", "rb"),
             "zip_file": open(f"{project_dir}/tests/fixture/valid.zip", "rb"),
             "store_spatial_files": True,
+            "action": "upload",
         }
 
         response = self.client.post(self.url, data=payload)
@@ -191,6 +193,7 @@ def test_asset_is_created_before_the_import_start(self, patch_upload):
                 content=b'{"type": "FeatureCollection", "content": "some-content"}',
             ),
             "store_spatial_files": True,
+            "action": "upload",
         }
 
         response = self.client.post(self.url, data=payload)
@@ -221,6 +224,7 @@ def test_asset_should_be_deleted_if_created_during_with_exception(
                 content=b'{"type": "FeatureCollection", "content": "some-content"}',
             ),
             "store_spatial_files": True,
+            "action": "upload",
         }
 
         response = self.client.post(self.url, data=payload)

diff --git a/geonode/upload/api/views.py b/geonode/upload/api/views.py
@@ -176,7 +176,6 @@ def create(self, request, *args, **kwargs):
             )
 
         handler = orchestrator.get_handler(_data)
-
         # not file but handler means that is a remote resource
         if handler:
             asset = None
@@ -191,8 +190,6 @@ def create(self, request, *args, **kwargs):
 
                     self.validate_upload(request, storage_manager)
 
-                action = ExecutionRequestAction.IMPORT.value
-
                 input_params = {
                     **{"files": files, "handler_module_path": str(handler)},
                     **extracted_params,
@@ -205,15 +202,14 @@ def create(self, request, *args, **kwargs):
                             "asset_module_path": f"{asset.__module__}.{asset.__class__.__name__}",
                         }
                     )
-
+                action = input_params.get("action")
                 execution_id = orchestrator.create_execution_request(
                     user=request.user,
                     func_name=next(iter(handler.get_task_list(action=action))),
                     step=_(next(iter(handler.get_task_list(action=action)))),
                     input_params=input_params,
                     action=action,
                     name=_file.name if _file else extracted_params.get("title", None),
-                    source=extracted_params.get("source"),
                 )
 
                 sig = import_orchestrator.s(files, str(execution_id), handler=str(handler), action=action)
@@ -234,7 +230,7 @@ def create(self, request, *args, **kwargs):
                 logger.exception(e)
                 raise ImportException(detail=e.args[0] if len(e.args) > 0 else e)
 
-        raise ImportException(detail="No handlers found for this dataset type")
+        raise ImportException(detail="No handlers found for this dataset type/action")
 
     def _handle_asset(self, request, asset_dir, storage_manager, _data, handler):
         if storage_manager is None:
@@ -328,7 +324,6 @@ def copy(self, request, *args, **kwargs):
                         **{"handler_module_path": handler_module_path},
                         **extracted_params,
                     },
-                    source="importer_copy",
                 )
 
                 sig = import_orchestrator.s(

diff --git a/geonode/upload/celery_tasks.py b/geonode/upload/celery_tasks.py
@@ -90,7 +90,7 @@ def import_orchestrator(
     step="start_import",
     layer_name=None,
     alternate=None,
-    action=exa.IMPORT.value,
+    action=exa.UPLOAD.value,
     **kwargs,
 ):
     """
@@ -179,7 +179,7 @@ def import_resource(self, execution_id, /, handler_module_path, action, **kwargs
         call_rollback_function(
             execution_id,
             handlers_module_path=handler_module_path,
-            prev_action=exa.IMPORT.value,
+            prev_action=exa.UPLOAD.value,
             layer=None,
             alternate=None,
             error=e,
@@ -309,7 +309,7 @@ def create_geonode_resource(
     layer_name: Optional[str] = None,
     alternate: Optional[str] = None,
     handler_module_path: str = None,
-    action: str = exa.IMPORT.value,
+    action: str = exa.UPLOAD.value,
     **kwargs,
 ):
     """

diff --git a/geonode/upload/handlers/README.md b/geonode/upload/handlers/README.md
@@ -32,7 +32,7 @@ class BaseVectorFileHandler(BaseHandler):
     It must provide the task_lists required to comple the upload
     """
 
-    ACTIONS = {
+    TASKS = {
         exa.IMPORT.value: (),  # define the list of the step (celery task) needed to execute the action for the resource
         exa.COPY.value: (),
         exa.DELETE.value: (),
@@ -242,7 +242,7 @@ class NewVectorFileHandler(BaseVectorFileHandler):
     It must provide the task_lists required to comple the upload
     """
 
-    ACTIONS = {
+    TASKS = {
         exa.IMPORT.value: (
             "start_import",
             "geonode.upload.import_resource",