Possible fix for bulk upload

ecds · Jul 12, 2024 · ccdc73b · ccdc73b
1 parent 36999ce
commit ccdc73b
Show file tree

Hide file tree

Showing 3 changed files with 60 additions and 51 deletions.
diff --git a/readux_ingest_ecds/models.py b/readux_ingest_ecds/models.py
@@ -4,6 +4,8 @@
 from zipfile import ZipFile
 from django.db import models
 from django.conf import settings
+from django.core.files.uploadedfile import InMemoryUploadedFile
+from django.core.files.storage import FileSystemStorage
 from .services.file_services import (
     is_image,
     is_ocr,
@@ -284,14 +286,21 @@ def upload_files(self, files):
         :param files: _description_
         :type files: _type_
         """
-        for uploaded_file in files:
-            with open(
-                os.path.join(
-                    settings.INGEST_TMP_DIR, bulk_path(self, uploaded_file.name)
-                ),
-                "wb",
-            ) as out_file:
-                out_file.write(uploaded_file.read())
+        print(files)
+        print(str(files))
+        if isinstance(files, InMemoryUploadedFile):
+            FileSystemStorage(
+                location=os.path.join(settings.INGEST_TMP_DIR, str(self.id))
+            ).save(files.name, files)
+        else:
+            for uploaded_file in files:
+                with open(
+                    os.path.join(
+                        settings.INGEST_TMP_DIR, bulk_path(self, uploaded_file.name)
+                    ),
+                    "wb",
+                ) as out_file:
+                    out_file.write(uploaded_file.read())
 
     class Meta:
         """Model Meta"""

diff --git a/test_app/tests/test_bulk_admin.py b/test_app/tests/test_bulk_admin.py
@@ -29,9 +29,9 @@ def setUp(self):
         self.user = UserFactory.create(is_superuser=True)
         self.bulk = BulkFactory.create()
 
-    # def teardown_class():
-    #     """Clean up files when done"""
-    #     rmtree(settings.INGEST_TMP_DIR, ignore_errors=True)
+    def teardown_class():
+        """Clean up files when done"""
+        rmtree(settings.INGEST_TMP_DIR, ignore_errors=True)
 
     def test_bulk_admin_save(self):
         """Uploaded multiple files should put files in correct places."""
@@ -76,41 +76,41 @@ def test_bulk_admin_save(self):
             os.path.join(settings.INGEST_PROCESSING_DIR, "pid6_00000008.jpg")
         )
 
-    # def test_bulk_admin_save_multiple(self):
-    #     """It should add three Local objects to this Bulk object"""
-    #     bulk = BulkFactory.create()
-
-    #     assert Local.objects.all().count() == 0
-
-    #     # Add 3 files to POST request
-    #     data = {}
-    #     metadata_file_path = os.path.join(self.fixture_path, "metadata.csv")
-    #     with open(metadata_file_path, "rb") as f:
-    #         metadata_content = files.base.ContentFile(f.read())
-    #     metadata_file = files.File(metadata_content.file, "metadata.csv")
-
-    #     bundle_file_one_path = os.path.join(
-    #         os.path.join(self.fixture_path, "volume2.zip")
-    #     )
-    #     with open(bundle_file_one_path, "rb") as f:
-    #         bundle_file_one_content = files.base.ContentFile(f.read())
-    #     bundle_file_one = files.File(bundle_file_one_content.file, "volume2.zip")
-
-    #     bundle_file_two_path = os.path.join(
-    #         os.path.join(self.fixture_path, "volume3.zip")
-    #     )
-    #     with open(bundle_file_two_path, "rb") as f:
-    #         bundle_file_two_content = files.base.ContentFile(f.read())
-    #     bundle_file_two = files.File(bundle_file_two_content.file, "volume3.zip")
-    #     data["volume_files"] = [metadata_file, bundle_file_one, bundle_file_two]
-
-    #     request_factory = RequestFactory()
-    #     req = request_factory.post("/admin/ingest/bulk/add/", data=data)
-    #     req.user = self.user
-
-    #     bulk_model_admin = BulkAdmin(model=Bulk, admin_site=AdminSite())
-    #     mock_form = BulkVolumeUploadForm()
-    #     bulk_model_admin.save_model(obj=bulk, request=req, form=mock_form, change=None)
-
-    #     bulk.refresh_from_db()
-    #     assert len(bulk.local_uploads.all()) == 3
+    def test_bulk_admin_save_multiple(self):
+        """It should add three Local objects to this Bulk object"""
+        bulk = BulkFactory.create()
+
+        assert Local.objects.all().count() == 0
+
+        # Add 3 files to POST request
+        data = {}
+        metadata_file_path = os.path.join(self.fixture_path, "metadata.csv")
+        with open(metadata_file_path, "rb") as f:
+            metadata_content = files.base.ContentFile(f.read())
+        metadata_file = files.File(metadata_content.file, "metadata.csv")
+
+        bundle_file_one_path = os.path.join(
+            os.path.join(self.fixture_path, "volume2.zip")
+        )
+        with open(bundle_file_one_path, "rb") as f:
+            bundle_file_one_content = files.base.ContentFile(f.read())
+        bundle_file_one = files.File(bundle_file_one_content.file, "volume2.zip")
+
+        bundle_file_two_path = os.path.join(
+            os.path.join(self.fixture_path, "volume3.zip")
+        )
+        with open(bundle_file_two_path, "rb") as f:
+            bundle_file_two_content = files.base.ContentFile(f.read())
+        bundle_file_two = files.File(bundle_file_two_content.file, "volume3.zip")
+        data["volume_files"] = [metadata_file, bundle_file_one, bundle_file_two]
+
+        request_factory = RequestFactory()
+        req = request_factory.post("/admin/ingest/bulk/add/", data=data)
+        req.user = self.user
+
+        bulk_model_admin = BulkAdmin(model=Bulk, admin_site=AdminSite())
+        mock_form = BulkVolumeUploadForm()
+        bulk_model_admin.save_model(obj=bulk, request=req, form=mock_form, change=None)
+
+        bulk.refresh_from_db()
+        assert Local.objects.all().count() == 2
diff --git a/test_app/tests/test_local.py b/test_app/tests/test_local.py
@@ -37,8 +37,8 @@ def setUp(self):
         conn = boto3.resource("s3", region_name="us-east-1")
         conn.create_bucket(Bucket=settings.INGEST_TRIGGER_BUCKET)
 
-    # def teardown_class():
-    #     rmtree(settings.INGEST_TMP_DIR, ignore_errors=True)
+    def teardown_class():
+        rmtree(settings.INGEST_TMP_DIR, ignore_errors=True)
 
     def mock_local(self, bundle, with_manifest=False, metadata={}, from_bulk=False):
         # Note, I tried to use the factory here, but could not get it to override the file for bundle.