Merge branch 'dev' into find_all_of_class

NeurodataWithoutBorders · Jan 24, 2024 · 46e8878 · 46e8878
2 parents a441a6d + 7c6868b
commit 46e8878
Show file tree

Hide file tree

Showing 69 changed files with 1,573 additions and 971 deletions.
diff --git a/.codespellrc b/.codespellrc
@@ -1,7 +1,8 @@
 [codespell]
 # in principle .ipynb can be corrected -- a good number of typos there
 # nwb-schema -- excluding since submodule, should have its own fixes/checks
-skip = .git,*.pdf,*.svg,venvs,env,*.ipynb,nwb-schema
+skip = .git,*.pdf,*.svg,venvs,env,nwb-schema
+ignore-regex = ^\s*"image/\S+": ".*
 # it is optin in a url
 # potatos - demanded to be left alone, autogenerated
 ignore-words-list = optin,potatos
diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml
@@ -16,4 +16,4 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
       - name: Codespell
-        uses: codespell-project/actions-codespell@v1
+        uses: codespell-project/actions-codespell@v2
diff --git a/.github/workflows/run_all_tests.yml b/.github/workflows/run_all_tests.yml
@@ -108,6 +108,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: 'recursive'
+          fetch-depth: 0  # tags are required for versioneer to determine the version
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -151,6 +152,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: 'recursive'
+          fetch-depth: 0  # tags are required for versioneer to determine the version
 
       - name: Set up Conda
         uses: conda-incubator/setup-miniconda@v2
@@ -194,9 +196,9 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: linux-python3.11-ros3  , python-ver: "3.11", os: ubuntu-latest }
-          - { name: windows-python3.11-ros3, python-ver: "3.11", os: windows-latest }
-          - { name: macos-python3.11-ros3  , python-ver: "3.11", os: macos-latest }
+          - { name: conda-linux-python3.11-ros3  , python-ver: "3.11", os: ubuntu-latest }
+          - { name: conda-windows-python3.11-ros3, python-ver: "3.11", os: windows-latest }
+          - { name: conda-macos-python3.11-ros3  , python-ver: "3.11", os: macos-latest }
     steps:
       - name: Cancel non-latest runs
         uses: styfle/[email protected]
@@ -207,6 +209,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: 'recursive'
+          fetch-depth: 0  # tags are required for versioneer to determine the version
 
       - name: Set up Conda
         uses: conda-incubator/setup-miniconda@v2
@@ -240,9 +243,9 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: linux-gallery-python3.11-ros3  , python-ver: "3.11", os: ubuntu-latest }
-          - { name: windows-gallery-python3.11-ros3, python-ver: "3.11", os: windows-latest }
-          - { name: macos-gallery-python3.11-ros3  , python-ver: "3.11", os: macos-latest }
+          - { name: conda-linux-gallery-python3.11-ros3  , python-ver: "3.11", os: ubuntu-latest }
+          - { name: conda-windows-gallery-python3.11-ros3, python-ver: "3.11", os: windows-latest }
+          - { name: conda-macos-gallery-python3.11-ros3  , python-ver: "3.11", os: macos-latest }
     steps:
       - name: Cancel non-latest runs
         uses: styfle/[email protected]
@@ -253,6 +256,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: 'recursive'
+          fetch-depth: 0  # tags are required for versioneer to determine the version
 
       - name: Set up Conda
         uses: conda-incubator/setup-miniconda@v2
@@ -278,4 +282,4 @@ jobs:
 
       - name: Run gallery ros3 tests
         run: |
-          python test.py --example-ros3
+          python test.py --example-ros3
diff --git a/.github/workflows/run_coverage.yml b/.github/workflows/run_coverage.yml
@@ -71,7 +71,7 @@ jobs:
 
       - name: Run integration tests and generate coverage report
         run: |
-          python -m coverage run -p test.py --integration --backwards
+          python -m coverage run -p test.py --integration --validation-module --backwards
           # validation CLI tests generate separate .coverage files that need to be merged
           python -m coverage combine
           python -m coverage xml  # codecov uploader requires xml format

diff --git a/.github/workflows/run_dandi_read_tests.yml b/.github/workflows/run_dandi_read_tests.yml
@@ -1,15 +1,15 @@
 name: Run DANDI read tests
 on:
-  schedule:
-    - cron: '0 6 * * *'  # once per day at 1am ET
+  # NOTE this is disabled until we can run this systematically instead of randomly
+  # so we don't get constant error notifications and waste compute cycles
+  # See https://github.com/NeurodataWithoutBorders/pynwb/issues/1804
+  # schedule:
+  #   - cron: '0 6 * * *'  # once per day at 1am ET
   workflow_dispatch:
 
 jobs:
   run-tests:
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash -l {0}  # necessary for conda
     steps:
       - name: Cancel non-latest runs
         uses: styfle/[email protected]
@@ -22,19 +22,14 @@ jobs:
           submodules: 'recursive'
           fetch-depth: 0  # tags are required for versioneer to determine the version
 
-      - name: Set up Conda
-        uses: conda-incubator/setup-miniconda@v2
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
-          auto-update-conda: true
-          activate-environment: ros3
-          environment-file: environment-ros3.yml
-          python-version: "3.11"
-          channels: conda-forge
-          auto-activate-base: false
+          python-version: '3.11'
 
       - name: Install run dependencies
         run: |
-          python -m pip install dandi pytest
+          python -m pip install dandi fsspec requests aiohttp pytest
           python -m pip uninstall -y pynwb  # uninstall pynwb
           python -m pip install -e .
           python -m pip list
@@ -47,4 +42,4 @@ jobs:
 
       - name: Run DANDI read tests
         run: |
-          pytest -rP tests/read_dandi/
+          python tests/read_dandi/read_dandi.py
diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
@@ -92,6 +92,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: 'recursive'
+          fetch-depth: 0  # tags are required for versioneer to determine the version
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -173,7 +174,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: linux-python3.11-ros3  , python-ver: "3.11", os: ubuntu-latest }
+          - { name: conda-linux-python3.11-ros3  , python-ver: "3.11", os: ubuntu-latest }
     steps:
       - name: Cancel non-latest runs
         uses: styfle/[email protected]
@@ -184,6 +185,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: 'recursive'
+          fetch-depth: 0  # tags are required for versioneer to determine the version
 
       - name: Set up Conda
         uses: conda-incubator/setup-miniconda@v2
@@ -217,7 +219,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: linux-gallery-python3.11-ros3  , python-ver: "3.11", os: ubuntu-latest }
+          - { name: conda-linux-gallery-python3.11-ros3  , python-ver: "3.11", os: ubuntu-latest }
     steps:
       - name: Cancel non-latest runs
         uses: styfle/[email protected]
@@ -228,6 +230,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: 'recursive'
+          fetch-depth: 0  # tags are required for versioneer to determine the version
 
       - name: Set up Conda
         uses: conda-incubator/setup-miniconda@v2
@@ -271,6 +274,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: 'recursive'
+          fetch-depth: 0  # tags are required for versioneer to determine the version
 
       - name: Set up Python
         uses: actions/setup-python@v4

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,39 @@
 # PyNWB Changelog
 
+## PyNWB 2.6.0 (Upcoming)
+
+### Enhancements and minor changes
+- For `NWBHDF5IO()`, change the default of arg `load_namespaces` from `False` to `True`. @bendichter [#1748](https://github.com/NeurodataWithoutBorders/pynwb/pull/1748)
+- Add `NWBHDF5IO.can_read()`. @bendichter [#1703](https://github.com/NeurodataWithoutBorders/pynwb/pull/1703)
+- Add `pynwb.get_nwbfile_version()`. @bendichter [#1703](https://github.com/NeurodataWithoutBorders/pynwb/pull/1703)
+- Updated timeseries data checks to warn instead of error when reading invalid files. @stephprince [#1793](https://github.com/NeurodataWithoutBorders/pynwb/pull/1793) and [#1809](https://github.com/NeurodataWithoutBorders/pynwb/pull/1809)
+- Expose the offset, conversion and channel conversion parameters in `mock_ElectricalSeries`. @h-mayorquin [#1796](https://github.com/NeurodataWithoutBorders/pynwb/pull/1796)
+- Expose `starting_time` in `mock_ElectricalSeries`. @h-mayorquin [#1805](https://github.com/NeurodataWithoutBorders/pynwb/pull/1805)
+- Enhance `get_data_in_units()` to work with objects that have a `channel_conversion` attribute like the `ElectricalSeries`. @h-mayorquin [#1806](https://github.com/NeurodataWithoutBorders/pynwb/pull/1806)
+- Refactor validation CLI tests to use `{sys.executable} -m coverage` to use the same Python version and run correctly on Debian systems. @yarikoptic [#1811](https://github.com/NeurodataWithoutBorders/pynwb/pull/1811)
+
+### Bug fixes
+- Fix bug where namespaces were loaded in "w-" mode. @h-mayorquin [#1795](https://github.com/NeurodataWithoutBorders/pynwb/pull/1795)
+- Fix bug where pynwb version was reported as "unknown" to readthedocs @stephprince [#1810](https://github.com/NeurodataWithoutBorders/pynwb/pull/1810)
+
+### Documentation and tutorial enhancements
+- Add RemFile to streaming tutorial. @bendichter [#1761](https://github.com/NeurodataWithoutBorders/pynwb/pull/1761)
+- Fix typos and improve clarify throughout tutorials. @zm711 [#1825](https://github.com/NeurodataWithoutBorders/pynwb/pull/1825)
+
+## PyNWB 2.5.0 (August 18, 2023)
+
+### Enhancements and minor changes
+- Add `TimeSeries.get_timestamps()`. @bendichter [#1741](https://github.com/NeurodataWithoutBorders/pynwb/pull/1741)
+- Add `TimeSeries.get_data_in_units()`. @bendichter [#1745](https://github.com/NeurodataWithoutBorders/pynwb/pull/1745)
+- Updated `ExternalResources` name change to `HERD`, along with HDMF 3.9.0 being the new minimum. @mavaylon1 [#1754](https://github.com/NeurodataWithoutBorders/pynwb/pull/1754)
+
+### Documentation and tutorial enhancements
+- Updated streaming tutorial to ensure code is run on tests and clarify text. @bendichter [#1760](https://github.com/NeurodataWithoutBorders/pynwb/pull/1760) @oruebel [#1762](https://github.com/NeurodataWithoutBorders/pynwb/pull/1762)
+- Fixed minor documentation build warnings and broken links to `basic_trials` tutorial  @oruebel [#1762](https://github.com/NeurodataWithoutBorders/pynwb/pull/1762)
+
+## PyNWB 2.4.1 (July 26, 2023)
+- Stop running validation tests as part of integration tests. They cause issues in CI and can be run separately. @rly [#1740](https://github.com/NeurodataWithoutBorders/pynwb/pull/1740)
+
 ## PyNWB 2.4.0 (July 23, 2023)
 
 ### Enhancements and minor changes

diff --git a/docs/gallery/advanced_io/linking_data.py b/docs/gallery/advanced_io/linking_data.py
@@ -6,57 +6,50 @@
 
 PyNWB supports linking between files using external links.
 
-"""
+Example Use Case: Integrating data from multiple files
+---------------------------------------------------------
 
-####################
-# Example Use Case: Integrating data from multiple files
-# ---------------------------------------------------------
-#
-# NBWContainer classes (e.g., :py:class:`~pynwb.base.TimeSeries`) support the integration of data stored in external
-# HDF5 files with NWB data files via external links. To make things more concrete, let's look at the following use
-# case. We want to simultaneously record multiple data streams during data acquisition. Using the concept of external
-# links allows us to save each data stream to an external HDF5 files during data acquisition and to
-# afterwards link the data into a single NWB:N file. In this case, each recording becomes represented by a
-# separate file-system object that can be set as read-only once the experiment is done.  In the following
-# we are using :py:meth:`~pynwb.base.TimeSeries` as an example, but the same approach works for other
-# NWBContainers as well.
-#
+NBWContainer classes (e.g., :py:class:`~pynwb.base.TimeSeries`) support the integration of data stored in external
+HDF5 files with NWB data files via external links. To make things more concrete, let's look at the following use
+case. We want to simultaneously record multiple data streams during data acquisition. Using the concept of external
+links allows us to save each data stream to an external HDF5 files during data acquisition and to
+afterwards link the data into a single NWB file. In this case, each recording becomes represented by a
+separate file-system object that can be set as read-only once the experiment is done.  In the following
+we are using :py:meth:`~pynwb.base.TimeSeries` as an example, but the same approach works for other
+NWBContainers as well.
 
-####################
-# .. tip::
-#
-#    The same strategies we use here for creating External Links also apply to Soft Links.
-#    The main difference between soft and external links is that soft links point to other
-#    objects within the same file while external links point to objects in external files.
-#
+.. tip::
 
-####################
-# .. tip::
-#
-#    In the case of :py:meth:`~pynwb.base.TimeSeries`, the uncorrected timestamps generated by the acquisition
-#    system can be stored (or linked) in the *sync* group. In the NWB:N format, hardware-recorded time data
-#    must then be corrected to a common time base (e.g., timestamps from all hardware sources aligned) before
-#    it can be included in the *timestamps* of the *TimeSeries*. This means, in the case
-#    of :py:meth:`~pynwb.base.TimeSeries` we need to be careful that we are not including data with incompatible
-#    timestamps in the same file when using external links.
-#
+    The same strategies we use here for creating External Links also apply to Soft Links.
+    The main difference between soft and external links is that soft links point to other
+    objects within the same file while external links point to objects in external files.
 
-####################
-# .. warning::
-#
-#    External links can become stale/break. Since external links are pointing to data in other files
-#    external links may become invalid any time files are modified on the file system, e.g., renamed,
-#    moved or access permissions are changed.
-#
+ .. tip::
 
-####################
-# Creating test data
-# ---------------------------
-#
-# In the following we are creating two :py:meth:`~pynwb.base.TimeSeries` each written to a separate file.
-# We then show how we can integrate these files into a single NWBFile.
+    In the case of :py:meth:`~pynwb.base.TimeSeries`, the uncorrected timestamps generated by the acquisition
+    system can be stored (or linked) in the *sync* group. In the NWB format, hardware-recorded time data
+    must then be corrected to a common time base (e.g., timestamps from all hardware sources aligned) before
+    it can be included in the *timestamps* of the *TimeSeries*. This means, in the case
+    of :py:meth:`~pynwb.base.TimeSeries` we need to be careful that we are not including data with incompatible
+    timestamps in the same file when using external links.
+
+
+.. warning::
+
+    External links can become stale/break. Since external links are pointing to data in other files
+    external links may become invalid any time files are modified on the file system, e.g., renamed,
+    moved or access permissions are changed.
+
+
+Creating test data
+---------------------------
+
+In the following we are creating two :py:meth:`~pynwb.base.TimeSeries` each written to a separate file.
+We then show how we can integrate these files into a single NWBFile.
+"""
 
 # sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_linking_data.png'
+
 from datetime import datetime
 from uuid import uuid4
 
@@ -228,7 +221,7 @@
 # Step 2: Add the container to another NWBFile
 # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 # To integrate both :py:meth:`~pynwb.base.TimeSeries` into a single file we simply create a new
-# :py:meth:`~pynwb.file.NWBFile` and our existing :py:meth:`~pynwb.base.TimeSeries` to it. PyNWB's
+# :py:meth:`~pynwb.file.NWBFile` and add our existing :py:meth:`~pynwb.base.TimeSeries` to it. PyNWB's
 # :py:class:`~pynwb.NWBHDF5IO` backend then automatically detects that the TimeSeries have already
 # been written to another file and will create external links for us.
 #