Skip to content

Commit

Permalink
add s3 test
Browse files Browse the repository at this point in the history
  • Loading branch information
h-mayorquin committed Nov 9, 2024
1 parent 77983c9 commit d8f9464
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 4 deletions.
18 changes: 14 additions & 4 deletions src/pynwb/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -514,11 +514,21 @@ def read_nwb(**kwargs):
path = popargs('path', kwargs)
file = popargs('file', kwargs)

# open the file with NWBZarrIO and rad the file
io = NWBHDF5IO(path=path, file=file, mode="r", load_namespaces=True)
nwbfile = io.read()
# Streaming case
if path is not None and str(path).startswith("s3://") or str(path).startswith("http"):
import fsspec

Check warning on line 519 in src/pynwb/__init__.py

View check run for this annotation

Codecov / codecov/patch

src/pynwb/__init__.py#L518-L519

Added lines #L518 - L519 were not covered by tests
print(path)
path = "https://dandiarchive.s3.amazonaws.com/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991"
fsspec_file_system = fsspec.filesystem("http")
ffspec_file = fsspec_file_system.open(str(path), "rb")

open_file = h5py.File(ffspec_file, "r")
io = NWBHDF5IO(file=open_file)
nwbfile = io.read()

Check warning on line 527 in src/pynwb/__init__.py

View check run for this annotation

Codecov / codecov/patch

src/pynwb/__init__.py#L523-L527

Added lines #L523 - L527 were not covered by tests
else:
io = NWBHDF5IO(path=path, file=file, mode="r", load_namespaces=True)
nwbfile = io.read()

Check warning on line 531 in src/pynwb/__init__.py

View check run for this annotation

Codecov / codecov/patch

src/pynwb/__init__.py#L529-L531

Added lines #L529 - L531 were not covered by tests
# return the NWBFile object
return nwbfile

from . import io as __io # noqa: F401,E402
Expand Down
10 changes: 10 additions & 0 deletions tests/integration/hdf5/test_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from h5py import File
from pathlib import Path
import tempfile
import pytest

from pynwb import NWBFile, TimeSeries, get_manager, NWBHDF5IO, validate

Expand Down Expand Up @@ -596,3 +597,12 @@ def test_read_nwb_method_file(self):
self.assertContainerEqual(read_nwbfile, self.nwbfile)

read_nwbfile.get_read_io().close()

@pytest.mark.skipif(not pytest.importorskip("fsspec"), reason="fsspec library not available")
def test_read_nwb_method_s3_path(self):

s3_test_path = "https://dandiarchive.s3.amazonaws.com/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991"
read_nwbfile = NWBHDF5IO.read_nwb(path=s3_test_path)
assert read_nwbfile.identifier == "3f77c586-6139-4777-a05d-f603e90b1330"

assert read_nwbfile.subject.subject_id == "1"

0 comments on commit d8f9464

Please sign in to comment.