From d8f9464a8388b71b2b9ccf2cbfb3318b181b4cae Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Sat, 9 Nov 2024 12:52:34 -0600 Subject: [PATCH] add s3 test --- src/pynwb/__init__.py | 18 ++++++++++++++---- tests/integration/hdf5/test_io.py | 10 ++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/pynwb/__init__.py b/src/pynwb/__init__.py index e8294057f..2eef7bbe9 100644 --- a/src/pynwb/__init__.py +++ b/src/pynwb/__init__.py @@ -514,11 +514,21 @@ def read_nwb(**kwargs): path = popargs('path', kwargs) file = popargs('file', kwargs) - # open the file with NWBZarrIO and rad the file - io = NWBHDF5IO(path=path, file=file, mode="r", load_namespaces=True) - nwbfile = io.read() + # Streaming case + if path is not None and str(path).startswith("s3://") or str(path).startswith("http"): + import fsspec + print(path) + path = "https://dandiarchive.s3.amazonaws.com/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991" + fsspec_file_system = fsspec.filesystem("http") + ffspec_file = fsspec_file_system.open(str(path), "rb") + + open_file = h5py.File(ffspec_file, "r") + io = NWBHDF5IO(file=open_file) + nwbfile = io.read() + else: + io = NWBHDF5IO(path=path, file=file, mode="r", load_namespaces=True) + nwbfile = io.read() - # return the NWBFile object return nwbfile from . import io as __io # noqa: F401,E402 diff --git a/tests/integration/hdf5/test_io.py b/tests/integration/hdf5/test_io.py index d54201b7b..76921d290 100644 --- a/tests/integration/hdf5/test_io.py +++ b/tests/integration/hdf5/test_io.py @@ -4,6 +4,7 @@ from h5py import File from pathlib import Path import tempfile +import pytest from pynwb import NWBFile, TimeSeries, get_manager, NWBHDF5IO, validate @@ -596,3 +597,12 @@ def test_read_nwb_method_file(self): self.assertContainerEqual(read_nwbfile, self.nwbfile) read_nwbfile.get_read_io().close() + + @pytest.mark.skipif(not pytest.importorskip("fsspec"), reason="fsspec library not available") + def test_read_nwb_method_s3_path(self): + + s3_test_path = "https://dandiarchive.s3.amazonaws.com/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991" + read_nwbfile = NWBHDF5IO.read_nwb(path=s3_test_path) + assert read_nwbfile.identifier == "3f77c586-6139-4777-a05d-f603e90b1330" + + assert read_nwbfile.subject.subject_id == "1" \ No newline at end of file