From e40d648a9e4f9ab99f05b61668b073bbeed967c6 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 18 Jul 2024 08:45:56 -0700 Subject: [PATCH 1/4] S3 test --- tests/unit/test_zarrio.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/test_zarrio.py b/tests/unit/test_zarrio.py index 081706a8..5ac9a79d 100644 --- a/tests/unit/test_zarrio.py +++ b/tests/unit/test_zarrio.py @@ -182,3 +182,7 @@ def test_force_open_without_consolidated_fails(self): except ValueError as e: self.fail("ZarrIO.__open_file_consolidated raised an unexpected ValueError: {}".format(e)) + def test_s3_open_with_consolidated_(self): + s3_path = "https://dandiarchive.s3.amazonaws.com/zarr/ccefbc9f-30e7-4a4c-b044-5b59d300040b/" + with ZarrIO(s3_path, mode='r') as read_io: + self.assertIsInstance(read_io.file.store, zarr.storage.ConsolidatedMetadataStore) From 35d8bbf9d9f59ce950b24e896977f2dd32492fe7 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 18 Jul 2024 09:58:32 -0700 Subject: [PATCH 2/4] clean up --- tests/unit/test_fsspec_streaming.py | 9 +++++++++ tests/unit/test_zarrio.py | 5 ----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_fsspec_streaming.py b/tests/unit/test_fsspec_streaming.py index 6fe71e0a..30b9e7b4 100644 --- a/tests/unit/test_fsspec_streaming.py +++ b/tests/unit/test_fsspec_streaming.py @@ -2,6 +2,8 @@ from hdmf_zarr import NWBZarrIO from .utils import check_s3fs_ffspec_installed +import zarr + HAVE_FSSPEC = check_s3fs_ffspec_installed() @@ -24,3 +26,10 @@ def test_fsspec_streaming(self): self.assertEqual(len(nwbfile.electrode_groups), 2) self.assertEqual(len(nwbfile.electrodes), 1152) self.assertEqual(nwbfile.institution, "AIND") + + @unittest.skipIf(not HAVE_FSSPEC, "fsspec not installed") + def test_s3_open_with_consolidated_(self): + s3_path = "https://dandiarchive.s3.amazonaws.com/zarr/ccefbc9f-30e7-4a4c-b044-5b59d300040b/" + with NWBZarrIO(s3_path, mode='r') as read_io: + read_io.open() + self.assertIsInstance(read_io.file.store, zarr.storage.ConsolidatedMetadataStore) diff --git a/tests/unit/test_zarrio.py b/tests/unit/test_zarrio.py index 5ac9a79d..4de48108 100644 --- a/tests/unit/test_zarrio.py +++ b/tests/unit/test_zarrio.py @@ -181,8 +181,3 @@ def test_force_open_without_consolidated_fails(self): read_io._ZarrIO__open_file_consolidated(store=self.store, mode='r') except ValueError as e: self.fail("ZarrIO.__open_file_consolidated raised an unexpected ValueError: {}".format(e)) - - def test_s3_open_with_consolidated_(self): - s3_path = "https://dandiarchive.s3.amazonaws.com/zarr/ccefbc9f-30e7-4a4c-b044-5b59d300040b/" - with ZarrIO(s3_path, mode='r') as read_io: - self.assertIsInstance(read_io.file.store, zarr.storage.ConsolidatedMetadataStore) From 4146317c37b793ac32378b62f7f0d7b66cf80871 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Fri, 19 Jul 2024 09:52:16 -0700 Subject: [PATCH 3/4] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d9745d5..ddc9a05e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 0.9.0 (Upcoming) ### Enhancements * NWBZarrIO load_namespaces=True by default. @mavaylon1 [#204](https://github.com/hdmf-dev/hdmf-zarr/pull/204) +* Added test for opening file with consolidated metadata from DANDI. @mavaylon1 [#206](https://github.com/hdmf-dev/hdmf-zarr/pull/206) ## 0.8.0 (June 4, 2024) ### Bug Fixes From 3c9e6ec36fdbc4b632e12245775cd5e655a2cc4a Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Tue, 23 Jul 2024 15:36:45 -0700 Subject: [PATCH 4/4] review --- tests/unit/test_fsspec_streaming.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit/test_fsspec_streaming.py b/tests/unit/test_fsspec_streaming.py index 30b9e7b4..cbb0bf06 100644 --- a/tests/unit/test_fsspec_streaming.py +++ b/tests/unit/test_fsspec_streaming.py @@ -29,7 +29,13 @@ def test_fsspec_streaming(self): @unittest.skipIf(not HAVE_FSSPEC, "fsspec not installed") def test_s3_open_with_consolidated_(self): + """ + The file is a Zarr file with consolidated metadata. + """ s3_path = "https://dandiarchive.s3.amazonaws.com/zarr/ccefbc9f-30e7-4a4c-b044-5b59d300040b/" with NWBZarrIO(s3_path, mode='r') as read_io: read_io.open() self.assertIsInstance(read_io.file.store, zarr.storage.ConsolidatedMetadataStore) + with NWBZarrIO(s3_path, mode='-r') as read_io: + read_io.open() + self.assertIsInstance(read_io.file.store, zarr.storage.FSStore)