From ccdc73b7c4fc819793ad1adba760c6fc9ab75239 Mon Sep 17 00:00:00 2001 From: Jay Varner Date: Fri, 12 Jul 2024 11:03:40 -0400 Subject: [PATCH] Possible fix for bulk upload --- readux_ingest_ecds/models.py | 25 +++++++--- test_app/tests/test_bulk_admin.py | 82 +++++++++++++++---------------- test_app/tests/test_local.py | 4 +- 3 files changed, 60 insertions(+), 51 deletions(-) diff --git a/readux_ingest_ecds/models.py b/readux_ingest_ecds/models.py index 035e0b9..502b02f 100644 --- a/readux_ingest_ecds/models.py +++ b/readux_ingest_ecds/models.py @@ -4,6 +4,8 @@ from zipfile import ZipFile from django.db import models from django.conf import settings +from django.core.files.uploadedfile import InMemoryUploadedFile +from django.core.files.storage import FileSystemStorage from .services.file_services import ( is_image, is_ocr, @@ -284,14 +286,21 @@ def upload_files(self, files): :param files: _description_ :type files: _type_ """ - for uploaded_file in files: - with open( - os.path.join( - settings.INGEST_TMP_DIR, bulk_path(self, uploaded_file.name) - ), - "wb", - ) as out_file: - out_file.write(uploaded_file.read()) + print(files) + print(str(files)) + if isinstance(files, InMemoryUploadedFile): + FileSystemStorage( + location=os.path.join(settings.INGEST_TMP_DIR, str(self.id)) + ).save(files.name, files) + else: + for uploaded_file in files: + with open( + os.path.join( + settings.INGEST_TMP_DIR, bulk_path(self, uploaded_file.name) + ), + "wb", + ) as out_file: + out_file.write(uploaded_file.read()) class Meta: """Model Meta""" diff --git a/test_app/tests/test_bulk_admin.py b/test_app/tests/test_bulk_admin.py index 4113595..c8bde60 100644 --- a/test_app/tests/test_bulk_admin.py +++ b/test_app/tests/test_bulk_admin.py @@ -29,9 +29,9 @@ def setUp(self): self.user = UserFactory.create(is_superuser=True) self.bulk = BulkFactory.create() - # def teardown_class(): - # """Clean up files when done""" - # rmtree(settings.INGEST_TMP_DIR, ignore_errors=True) + def teardown_class(): + """Clean up files when done""" + rmtree(settings.INGEST_TMP_DIR, ignore_errors=True) def test_bulk_admin_save(self): """Uploaded multiple files should put files in correct places.""" @@ -76,41 +76,41 @@ def test_bulk_admin_save(self): os.path.join(settings.INGEST_PROCESSING_DIR, "pid6_00000008.jpg") ) - # def test_bulk_admin_save_multiple(self): - # """It should add three Local objects to this Bulk object""" - # bulk = BulkFactory.create() - - # assert Local.objects.all().count() == 0 - - # # Add 3 files to POST request - # data = {} - # metadata_file_path = os.path.join(self.fixture_path, "metadata.csv") - # with open(metadata_file_path, "rb") as f: - # metadata_content = files.base.ContentFile(f.read()) - # metadata_file = files.File(metadata_content.file, "metadata.csv") - - # bundle_file_one_path = os.path.join( - # os.path.join(self.fixture_path, "volume2.zip") - # ) - # with open(bundle_file_one_path, "rb") as f: - # bundle_file_one_content = files.base.ContentFile(f.read()) - # bundle_file_one = files.File(bundle_file_one_content.file, "volume2.zip") - - # bundle_file_two_path = os.path.join( - # os.path.join(self.fixture_path, "volume3.zip") - # ) - # with open(bundle_file_two_path, "rb") as f: - # bundle_file_two_content = files.base.ContentFile(f.read()) - # bundle_file_two = files.File(bundle_file_two_content.file, "volume3.zip") - # data["volume_files"] = [metadata_file, bundle_file_one, bundle_file_two] - - # request_factory = RequestFactory() - # req = request_factory.post("/admin/ingest/bulk/add/", data=data) - # req.user = self.user - - # bulk_model_admin = BulkAdmin(model=Bulk, admin_site=AdminSite()) - # mock_form = BulkVolumeUploadForm() - # bulk_model_admin.save_model(obj=bulk, request=req, form=mock_form, change=None) - - # bulk.refresh_from_db() - # assert len(bulk.local_uploads.all()) == 3 + def test_bulk_admin_save_multiple(self): + """It should add three Local objects to this Bulk object""" + bulk = BulkFactory.create() + + assert Local.objects.all().count() == 0 + + # Add 3 files to POST request + data = {} + metadata_file_path = os.path.join(self.fixture_path, "metadata.csv") + with open(metadata_file_path, "rb") as f: + metadata_content = files.base.ContentFile(f.read()) + metadata_file = files.File(metadata_content.file, "metadata.csv") + + bundle_file_one_path = os.path.join( + os.path.join(self.fixture_path, "volume2.zip") + ) + with open(bundle_file_one_path, "rb") as f: + bundle_file_one_content = files.base.ContentFile(f.read()) + bundle_file_one = files.File(bundle_file_one_content.file, "volume2.zip") + + bundle_file_two_path = os.path.join( + os.path.join(self.fixture_path, "volume3.zip") + ) + with open(bundle_file_two_path, "rb") as f: + bundle_file_two_content = files.base.ContentFile(f.read()) + bundle_file_two = files.File(bundle_file_two_content.file, "volume3.zip") + data["volume_files"] = [metadata_file, bundle_file_one, bundle_file_two] + + request_factory = RequestFactory() + req = request_factory.post("/admin/ingest/bulk/add/", data=data) + req.user = self.user + + bulk_model_admin = BulkAdmin(model=Bulk, admin_site=AdminSite()) + mock_form = BulkVolumeUploadForm() + bulk_model_admin.save_model(obj=bulk, request=req, form=mock_form, change=None) + + bulk.refresh_from_db() + assert Local.objects.all().count() == 2 diff --git a/test_app/tests/test_local.py b/test_app/tests/test_local.py index f2a7129..d8c365d 100644 --- a/test_app/tests/test_local.py +++ b/test_app/tests/test_local.py @@ -37,8 +37,8 @@ def setUp(self): conn = boto3.resource("s3", region_name="us-east-1") conn.create_bucket(Bucket=settings.INGEST_TRIGGER_BUCKET) - # def teardown_class(): - # rmtree(settings.INGEST_TMP_DIR, ignore_errors=True) + def teardown_class(): + rmtree(settings.INGEST_TMP_DIR, ignore_errors=True) def mock_local(self, bundle, with_manifest=False, metadata={}, from_bulk=False): # Note, I tried to use the factory here, but could not get it to override the file for bundle.