From 19a292c5e6c673a248c170b5aa8696b03eb50fdd Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Tue, 5 Dec 2023 17:25:36 -0800 Subject: [PATCH 1/3] Update run_dandi_read_tests.yml --- .github/workflows/run_dandi_read_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_dandi_read_tests.yml b/.github/workflows/run_dandi_read_tests.yml index 857b32c9a..cb4cdbfd3 100644 --- a/.github/workflows/run_dandi_read_tests.yml +++ b/.github/workflows/run_dandi_read_tests.yml @@ -1,7 +1,7 @@ name: Run DANDI read tests on: - schedule: - - cron: '0 6 * * *' # once per day at 1am ET + # schedule: + # - cron: '0 6 * * *' # once per day at 1am ET workflow_dispatch: jobs: From d5249f7de5844fea1dec9dedff225333c7562606 Mon Sep 17 00:00:00 2001 From: rly Date: Wed, 13 Dec 2023 23:01:23 -0800 Subject: [PATCH 2/3] Update test and add comment to workflow --- .github/workflows/run_dandi_read_tests.yml | 21 +++++++------------ ..._read_dandi.py => read_first_nwb_asset.py} | 19 ++++++++++++----- 2 files changed, 22 insertions(+), 18 deletions(-) rename tests/read_dandi/{test_read_dandi.py => read_first_nwb_asset.py} (79%) diff --git a/.github/workflows/run_dandi_read_tests.yml b/.github/workflows/run_dandi_read_tests.yml index cb4cdbfd3..7148d209e 100644 --- a/.github/workflows/run_dandi_read_tests.yml +++ b/.github/workflows/run_dandi_read_tests.yml @@ -1,5 +1,8 @@ name: Run DANDI read tests on: + # NOTE this is disabled until we can run this systematically instead of randomly + # so we don't get constant error notifications and waste compute cycles + # See https://github.com/NeurodataWithoutBorders/pynwb/issues/1804 # schedule: # - cron: '0 6 * * *' # once per day at 1am ET workflow_dispatch: @@ -7,9 +10,6 @@ on: jobs: run-tests: runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} # necessary for conda steps: - name: Cancel non-latest runs uses: styfle/cancel-workflow-action@0.11.0 @@ -22,19 +22,14 @@ jobs: submodules: 'recursive' fetch-depth: 0 # tags are required for versioneer to determine the version - - name: Set up Conda - uses: conda-incubator/setup-miniconda@v2 + - name: Set up Python + uses: actions/setup-python@v4 with: - auto-update-conda: true - activate-environment: ros3 - environment-file: environment-ros3.yml - python-version: "3.11" - channels: conda-forge - auto-activate-base: false + python-version: '3.11' - name: Install run dependencies run: | - python -m pip install dandi pytest + python -m pip install dandi fsspec requests aiohttp pytest python -m pip uninstall -y pynwb # uninstall pynwb python -m pip install -e . python -m pip list @@ -47,4 +42,4 @@ jobs: - name: Run DANDI read tests run: | - python tests/read_dandi/test_read_dandi.py + python tests/read_dandi/read_dandi.py diff --git a/tests/read_dandi/test_read_dandi.py b/tests/read_dandi/read_first_nwb_asset.py similarity index 79% rename from tests/read_dandi/test_read_dandi.py rename to tests/read_dandi/read_first_nwb_asset.py index f9dafd938..895dbb1c2 100644 --- a/tests/read_dandi/test_read_dandi.py +++ b/tests/read_dandi/read_first_nwb_asset.py @@ -1,5 +1,7 @@ -"""Test reading NWB files from the DANDI Archive using ROS3.""" +"""Test reading NWB files from the DANDI Archive using fsspec.""" from dandi.dandiapi import DandiAPIClient +import fsspec +import h5py import random import sys import traceback @@ -10,9 +12,12 @@ # NOTE: do not name the function with "test_" prefix, otherwise pytest # will try to run it as a test +# TODO read dandisets systematically, not randomly +# see https://github.com/NeurodataWithoutBorders/pynwb/issues/1804 + def read_first_nwb_asset(): - """Test reading the first NWB asset from a random selection of 50 dandisets that uses NWB.""" - num_dandisets_to_read = 50 + """Test reading the first NWB asset from a random selection of 2 dandisets that uses NWB.""" + num_dandisets_to_read = 2 client = DandiAPIClient() dandisets = list(client.get_dandisets()) random.shuffle(dandisets) @@ -20,6 +25,8 @@ def read_first_nwb_asset(): print("Reading NWB files from the following dandisets:") print([d.get_raw_metadata()["identifier"] for d in dandisets_to_read]) + fs = fsspec.filesystem("http") + failed_reads = dict() for i, dandiset in enumerate(dandisets_to_read): dandiset_metadata = dandiset.get_raw_metadata() @@ -47,8 +54,10 @@ def read_first_nwb_asset(): s3_url = first_asset.get_content_url(follow_redirects=1, strip_query=True) try: - with NWBHDF5IO(path=s3_url, driver="ros3") as io: - io.read() + with fs.open(s3_url, "rb") as f: + with h5py.File(f) as file: + with NWBHDF5IO(file=file) as io: + io.read() except Exception as e: print(traceback.format_exc()) failed_reads[dandiset] = e From 3987dcb5ff96d2c2e457583e599b087bfc1b6f39 Mon Sep 17 00:00:00 2001 From: rly Date: Wed, 13 Dec 2023 23:03:08 -0800 Subject: [PATCH 3/3] FIx flake8 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index cccacf048..b216932f9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ per-file-ignores = tests/integration/__init__.py:F401 src/pynwb/testing/__init__.py:F401 src/pynwb/validate.py:T201 - tests/read_dandi/test_read_dandi.py:T201 + tests/read_dandi/read_first_nwb_asset.py:T201 setup.py:T201 test.py:T201 scripts/*:T201