From 15290289f6304a3a83c6fcf8bf4110c15dc1683c Mon Sep 17 00:00:00 2001
From: Ryan Ly <rly@lbl.gov>
Date: Thu, 11 Jan 2024 16:38:29 -0800
Subject: [PATCH] Disable nightly dandi read workflow for now (#1794)

---
 .github/workflows/run_dandi_read_tests.yml    | 25 ++++++++-----------
 setup.cfg                                     |  2 +-
 ..._read_dandi.py => read_first_nwb_asset.py} | 19 ++++++++++----
 3 files changed, 25 insertions(+), 21 deletions(-)
 rename tests/read_dandi/{test_read_dandi.py => read_first_nwb_asset.py} (79%)

diff --git a/.github/workflows/run_dandi_read_tests.yml b/.github/workflows/run_dandi_read_tests.yml
index 857b32c9a..7148d209e 100644
--- a/.github/workflows/run_dandi_read_tests.yml
+++ b/.github/workflows/run_dandi_read_tests.yml
@@ -1,15 +1,15 @@
 name: Run DANDI read tests
 on:
-  schedule:
-    - cron: '0 6 * * *'  # once per day at 1am ET
+  # NOTE this is disabled until we can run this systematically instead of randomly
+  # so we don't get constant error notifications and waste compute cycles
+  # See https://github.com/NeurodataWithoutBorders/pynwb/issues/1804
+  # schedule:
+  #   - cron: '0 6 * * *'  # once per day at 1am ET
   workflow_dispatch:
 
 jobs:
   run-tests:
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash -l {0}  # necessary for conda
     steps:
       - name: Cancel non-latest runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -22,19 +22,14 @@ jobs:
           submodules: 'recursive'
           fetch-depth: 0  # tags are required for versioneer to determine the version
 
-      - name: Set up Conda
-        uses: conda-incubator/setup-miniconda@v2
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
-          auto-update-conda: true
-          activate-environment: ros3
-          environment-file: environment-ros3.yml
-          python-version: "3.11"
-          channels: conda-forge
-          auto-activate-base: false
+          python-version: '3.11'
 
       - name: Install run dependencies
         run: |
-          python -m pip install dandi pytest
+          python -m pip install dandi fsspec requests aiohttp pytest
           python -m pip uninstall -y pynwb  # uninstall pynwb
           python -m pip install -e .
           python -m pip list
@@ -47,4 +42,4 @@ jobs:
 
       - name: Run DANDI read tests
         run: |
-          python tests/read_dandi/test_read_dandi.py
+          python tests/read_dandi/read_dandi.py
diff --git a/setup.cfg b/setup.cfg
index 0b7c5ee50..d44fcc2b1 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -29,7 +29,7 @@ per-file-ignores =
   tests/integration/__init__.py:F401
   src/pynwb/testing/__init__.py:F401
   src/pynwb/validate.py:T201
-  tests/read_dandi/test_read_dandi.py:T201
+  tests/read_dandi/read_first_nwb_asset.py:T201
   setup.py:T201
   test.py:T201
   scripts/*:T201
diff --git a/tests/read_dandi/test_read_dandi.py b/tests/read_dandi/read_first_nwb_asset.py
similarity index 79%
rename from tests/read_dandi/test_read_dandi.py
rename to tests/read_dandi/read_first_nwb_asset.py
index f9dafd938..895dbb1c2 100644
--- a/tests/read_dandi/test_read_dandi.py
+++ b/tests/read_dandi/read_first_nwb_asset.py
@@ -1,5 +1,7 @@
-"""Test reading NWB files from the DANDI Archive using ROS3."""
+"""Test reading NWB files from the DANDI Archive using fsspec."""
 from dandi.dandiapi import DandiAPIClient
+import fsspec
+import h5py
 import random
 import sys
 import traceback
@@ -10,9 +12,12 @@
 # NOTE: do not name the function with "test_" prefix, otherwise pytest
 # will try to run it as a test
 
+# TODO read dandisets systematically, not randomly
+# see https://github.com/NeurodataWithoutBorders/pynwb/issues/1804
+
 def read_first_nwb_asset():
-    """Test reading the first NWB asset from a random selection of 50 dandisets that uses NWB."""
-    num_dandisets_to_read = 50
+    """Test reading the first NWB asset from a random selection of 2 dandisets that uses NWB."""
+    num_dandisets_to_read = 2
     client = DandiAPIClient()
     dandisets = list(client.get_dandisets())
     random.shuffle(dandisets)
@@ -20,6 +25,8 @@ def read_first_nwb_asset():
     print("Reading NWB files from the following dandisets:")
     print([d.get_raw_metadata()["identifier"] for d in dandisets_to_read])
 
+    fs = fsspec.filesystem("http")
+
     failed_reads = dict()
     for i, dandiset in enumerate(dandisets_to_read):
         dandiset_metadata = dandiset.get_raw_metadata()
@@ -47,8 +54,10 @@ def read_first_nwb_asset():
         s3_url = first_asset.get_content_url(follow_redirects=1, strip_query=True)
 
         try:
-            with NWBHDF5IO(path=s3_url, driver="ros3") as io:
-                io.read()
+            with fs.open(s3_url, "rb") as f:
+                with h5py.File(f) as file:
+                    with NWBHDF5IO(file=file) as io:
+                        io.read()
         except Exception as e:
             print(traceback.format_exc())
             failed_reads[dandiset] = e