Merge branch 'dev' into spatialseries_bounds

.github/workflows/check_sphinx_links.yml

@@ -28,7 +28,7 @@ jobs:
       - name: Install Sphinx dependencies and package
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -r requirements-doc.txt
+          python -m pip install -r requirements-doc.txt -r requirements-opt.txt
           python -m pip install .
 
       - name: Check Sphinx internal and external links

.github/workflows/run_all_tests.yml

@@ -26,20 +26,23 @@ jobs:
           - { name: linux-python3.9              , test-tox-env: py39            , build-tox-env: build-py39            , python-ver: "3.9" , os: ubuntu-latest }
           - { name: linux-python3.10             , test-tox-env: py310           , build-tox-env: build-py310           , python-ver: "3.10", os: ubuntu-latest }
           - { name: linux-python3.11             , test-tox-env: py311           , build-tox-env: build-py311           , python-ver: "3.11", os: ubuntu-latest }
+          - { name: linux-python3.11-opt         , test-tox-env: py311-optional  , build-tox-env: build-py311           , python-ver: "3.11", os: ubuntu-latest }
           - { name: linux-python3.12             , test-tox-env: py312           , build-tox-env: build-py312           , python-ver: "3.12", os: ubuntu-latest }
           - { name: linux-python3.12-upgraded    , test-tox-env: py312-upgraded  , build-tox-env: build-py312-upgraded  , python-ver: "3.12", os: ubuntu-latest }
           - { name: linux-python3.12-prerelease  , test-tox-env: py312-prerelease, build-tox-env: build-py312-prerelease, python-ver: "3.12", os: ubuntu-latest }
           - { name: windows-python3.8-minimum    , test-tox-env: py38-minimum    , build-tox-env: build-py38-minimum    , python-ver: "3.8" , os: windows-latest }
           - { name: windows-python3.9            , test-tox-env: py39            , build-tox-env: build-py39            , python-ver: "3.9" , os: windows-latest }
           - { name: windows-python3.10           , test-tox-env: py310           , build-tox-env: build-py310           , python-ver: "3.10", os: windows-latest }
           - { name: windows-python3.11           , test-tox-env: py311           , build-tox-env: build-py311           , python-ver: "3.11", os: windows-latest }
+          - { name: windows-python3.11-opt       , test-tox-env: py311-optional  , build-tox-env: build-py311           , python-ver: "3.11", os: windows-latest }
           - { name: windows-python3.12           , test-tox-env: py312           , build-tox-env: build-py312           , python-ver: "3.12", os: windows-latest }
           - { name: windows-python3.12-upgraded  , test-tox-env: py312-upgraded  , build-tox-env: build-py312-upgraded  , python-ver: "3.12", os: windows-latest }
           - { name: windows-python3.12-prerelease, test-tox-env: py312-prerelease, build-tox-env: build-py312-prerelease, python-ver: "3.11", os: windows-latest }
           - { name: macos-python3.8-minimum      , test-tox-env: py38-minimum    , build-tox-env: build-py38-minimum    , python-ver: "3.8" , os: macos-13 }
           - { name: macos-python3.9              , test-tox-env: py39            , build-tox-env: build-py39            , python-ver: "3.9" , os: macos-13 }
           - { name: macos-python3.10             , test-tox-env: py310           , build-tox-env: build-py310           , python-ver: "3.10", os: macos-latest }
           - { name: macos-python3.11             , test-tox-env: py311           , build-tox-env: build-py311           , python-ver: "3.11", os: macos-latest }
+          - { name: macos-python3.11-opt         , test-tox-env: py311-optional  , build-tox-env: build-py311           , python-ver: "3.11", os: macos-latest }
           - { name: macos-python3.12             , test-tox-env: py312           , build-tox-env: build-py312           , python-ver: "3.12", os: macos-latest }
           - { name: macos-python3.12-upgraded    , test-tox-env: py312-upgraded  , build-tox-env: build-py312-upgraded  , python-ver: "3.12", os: macos-latest }
           - { name: macos-python3.12-prerelease  , test-tox-env: py312-prerelease, build-tox-env: build-py312-prerelease, python-ver: "3.12", os: macos-latest }
@@ -198,7 +201,7 @@ jobs:
         include:
           - { name: conda-linux-python3.12-ros3  , python-ver: "3.12", os: ubuntu-latest }
           - { name: conda-windows-python3.12-ros3, python-ver: "3.12", os: windows-latest }
-          - { name: conda-macos-python3.12-ros3  , python-ver: "3.12", os: macos-13 } # This is due to DANDI not supporting osx-arm64. Will support macos-latest when this changes. 
+          - { name: conda-macos-python3.12-ros3  , python-ver: "3.12", os: macos-13 } # This is due to DANDI not supporting osx-arm64. Will support macos-latest when this changes.
     steps:
       - name: Cancel non-latest runs
         uses: styfle/[email protected]
@@ -245,7 +248,7 @@ jobs:
         include:
           - { name: conda-linux-gallery-python3.12-ros3  , python-ver: "3.12", os: ubuntu-latest }
           - { name: conda-windows-gallery-python3.12-ros3, python-ver: "3.12", os: windows-latest }
-          - { name: conda-macos-gallery-python3.12-ros3  , python-ver: "3.12", os: macos-13 } # This is due to DANDI not supporting osx-arm64. Will support macos-latest when this changes. 
+          - { name: conda-macos-gallery-python3.12-ros3  , python-ver: "3.12", os: macos-13 } # This is due to DANDI not supporting osx-arm64. Will support macos-latest when this changes.
     steps:
       - name: Cancel non-latest runs
         uses: styfle/[email protected]

.github/workflows/run_coverage.yml

@@ -11,7 +11,7 @@ on:
 
 jobs:
   run-coverage:
-    name: ${{ matrix.os }}
+    name: ${{ matrix.os }}, opt reqs ${{ matrix.opt_req }}
     runs-on: ${{ matrix.os }}
     # TODO handle forks
     # run pipeline on either a push event or a PR event on a fork
@@ -21,7 +21,11 @@ jobs:
         shell: bash
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        include:
+          - { os: ubuntu-latest , opt_req: true }
+          - { os: ubuntu-latest , opt_req: false }
+          - { os: windows-latest, opt_req: false }
+          - { os: macos-latest  , opt_req: false }
     env:
       OS: ${{ matrix.os }}
       PYTHON: '3.12'
@@ -47,9 +51,9 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install -r requirements-dev.txt -r requirements.txt
 
-      # - name: Install optional dependencies
-      #   if: ${{ matrix.opt_req }}
-      #   run: python -m pip install -r requirements-opt.txt
+      - name: Install optional dependencies
+        if: ${{ matrix.opt_req }}
+        run: python -m pip install -r requirements-opt.txt
 
       - name: Install package
         run: |

.readthedocs.yaml

@@ -25,6 +25,7 @@ formats: all
 python:
   install:
     - requirements: requirements-doc.txt
+    - requirements: requirements-opt.txt
     - requirements: requirements.txt
 
 # Optionally include all submodules

CHANGELOG.md

@@ -1,9 +1,24 @@
 # PyNWB Changelog
 
-## PyNWB 2.8.0 (Upcoming)
+## PyNWB 2.8.1 (Upcoming)
+
+### Documentation and tutorial enhancements
+- Simplified the introduction to NWB tutorial. @rly [#1914](https://github.com/NeurodataWithoutBorders/pynwb/pull/1914)
+- Simplified the ecephys and ophys tutorials. [#1915](https://github.com/NeurodataWithoutBorders/pynwb/pull/1915)
+
+### Bug fixes
+- Fixed use of `channel_conversion` in `TimeSeries` `get_data_in_units`. @rohanshah [1923](https://github.com/NeurodataWithoutBorders/pynwb/pull/1923)
+
+
+## PyNWB 2.8.0 (May 28, 2024)
 
 ### Enhancements and minor changes
 - Set rate default value inside `mock_ElectricalSeries` to avoid having to set `rate=None` explicitly when passing timestamps. @h-mayorquin [#1894](https://github.com/NeurodataWithoutBorders/pynwb/pull/1894)
+- Integrate validation through the `TypeConfigurator`. @mavaylon1 [#1829](https://github.com/NeurodataWithoutBorders/pynwb/pull/1829)
+- Exposed `aws_region` to `NWBHDF5IO`. @rly [#1903](https://github.com/NeurodataWithoutBorders/pynwb/pull/1903)
+
+### Bug fixes
+- Revert changes in PyNWB 2.7.0 that allow datetimes without a timezone and without a time while issues with DANDI upload of NWB files missing timezone are resolved. @rly [#1908](https://github.com/NeurodataWithoutBorders/pynwb/pull/1908)
 
 ## PyNWB 2.7.0 (May 2, 2024)
 
@@ -325,7 +340,7 @@
 
 ### Tutorial enhancements:
 - Added new tutorial for intracellular electrophysiology to describe the use of the new metadata tables
-  and declared the previous tutoral using ``SweepTable`` as deprecated.  @oruebel (#1349)
+  and declared the previous tutorial using ``SweepTable`` as deprecated.  @oruebel (#1349)
 - Added new tutorial for querying intracellular electrophysiology metadata
   (``docs/gallery/domain/plot_icephys_pandas.py``). @oruebel (#1349, #1383)
 - Added thumbnails for tutorials to improve presentation of online docs.  @oruebel (#1349)

docs/gallery/advanced_io/h5dataio.py

@@ -19,16 +19,20 @@
 #
 
 from datetime import datetime
+
+from dateutil.tz import tzlocal
+
 from pynwb import NWBFile
 
-start_time = datetime(2017, 4, 3, hour=11, minute=0)
+start_time = datetime(2017, 4, 3, 11, tzinfo=tzlocal())
 
 nwbfile = NWBFile(
     session_description="demonstrate advanced HDF5 I/O features",
     identifier="NWB123",
     session_start_time=start_time,
 )
 
+
 ####################
 # Normally if we create a :py:class:`~pynwb.base.TimeSeries` we would do
 

docs/gallery/advanced_io/linking_data.py

@@ -51,12 +51,15 @@
 # sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_linking_data.png'
 
 from datetime import datetime
+from uuid import uuid4
+
 import numpy as np
+from dateutil.tz import tzlocal
+
 from pynwb import NWBHDF5IO, NWBFile, TimeSeries
-from uuid import uuid4
 
 # Create the base data
-start_time = datetime(2017, 4, 3, hour=11, minute=0)
+start_time = datetime(2017, 4, 3, 11, tzinfo=tzlocal())
 data = np.arange(1000).reshape((100, 10))
 timestamps = np.arange(100)
 filename1 = "external1_example.nwb"

docs/gallery/advanced_io/parallelio.py

@@ -32,7 +32,7 @@
 #     from datetime import datetime
 #     from hdmf.backends.hdf5.h5_utils import H5DataIO
 #
-#     start_time = datetime(2018, 4, 25, hour=2, minute=30, second=3)
+#     start_time = datetime(2018, 4, 25, 2, 30, 3, tzinfo=tz.gettz("US/Pacific"))
 #     fname = "test_parallel_pynwb.nwb"
 #     rank = MPI.COMM_WORLD.rank  # The process ID (integer 0-3 for 4-process run)
 #

docs/gallery/advanced_io/plot_iterative_write.py

@@ -110,9 +110,13 @@
 
 # sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_iterative_write.png'
 from datetime import datetime
-from pynwb import NWBHDF5IO, NWBFile, TimeSeries
 from uuid import uuid4
 
+from dateutil.tz import tzlocal
+
+from pynwb import NWBHDF5IO, NWBFile, TimeSeries
+
+
 def write_test_file(filename, data, close_io=True):
     """
 
@@ -125,7 +129,7 @@ def write_test_file(filename, data, close_io=True):
     """
 
     # Create a test NWBfile
-    start_time = datetime(2017, 4, 3, hour=11, minute=30)
+    start_time = datetime(2017, 4, 3, 11, tzinfo=tzlocal())
     nwbfile = NWBFile(
         session_description="demonstrate iterative write",
         identifier=str(uuid4()),

docs/gallery/domain/ecephys.py

@@ -16,7 +16,7 @@
 It is recommended to cover :ref:`basics` before this tutorial.
 
 .. note:: It is recommended to check if your source data is supported by
-  `NeuroConv Extracellular Electrophysiology Gallery <https://neuroconv.readthedocs.io/en/main/conversion_examples_gallery/conversion_example_gallery.html#extracellular-electrophysiology>`_.
+  `NeuroConv Extracellular Electrophysiology Gallery <https://neuroconv.readthedocs.io/en/main/conversion_examples_gallery/#extracellular-electrophysiology>`_.
   If it is supported, it is recommended to use NeuroConv to convert your data.
 
 The following examples will reference variables that may not be defined within the block they are used in. For
@@ -39,7 +39,6 @@
 #
 # When creating a NWB file, the first step is to create the :py:class:`~pynwb.file.NWBFile`.
 
-
 nwbfile = NWBFile(
     session_description="my first synthetic recording",
     identifier=str(uuid4()),
@@ -50,7 +49,8 @@
     lab="Bag End Laboratory",
     institution="University of Middle Earth at the Shire",
     experiment_description="I went on an adventure to reclaim vast treasures.",
-    session_id="LONELYMTN001",
+    keywords=["ecephys", "exploration", "wanderlust"],
+    related_publications="doi:10.1016/j.neuron.2016.12.011",
 )
 
 #######################
@@ -93,7 +93,6 @@
 # additional user-specified metadata as custom columns of the table. We will be adding a ``"label"`` column to the
 # table. Use the following code to add electrodes for an array with 4 shanks and 3 channels per shank.
 
-
 nwbfile.add_electrode_column(name="label", description="label of electrode")
 
 nshanks = 4
@@ -118,10 +117,9 @@
         electrode_counter += 1
 
 #######################
-# Similarly to the ``trials`` table, we can view the ``electrodes`` table in tabular form
+# Similarly to other tables in PyNWB, we can view the ``electrodes`` table in tabular form
 # by converting it to a pandas :py:class:`~pandas.DataFrame`.
 
-
 nwbfile.electrodes.to_dataframe()
 
 #######################
@@ -145,7 +143,6 @@
 # convenience function that creates a :py:class:`~hdmf.common.table.DynamicTableRegion` which references the
 # ``"electrodes"`` table.
 
-
 all_table_region = nwbfile.create_electrode_table_region(
     region=list(range(electrode_counter)),  # reference row indices 0 to N-1
     description="all electrodes",
@@ -156,7 +153,7 @@
 # ^^^^^^^^^^^^^^^^^
 #
 # Now create an :py:class:`~pynwb.ecephys.ElectricalSeries` object to store raw data collected
-# during the experiment, passing in this ``"all_table_region"`` :py:class:`~hdmf.common.table.DynamicTableRegion`
+# during the experiment, passing in this ``all_table_region`` :py:class:`~hdmf.common.table.DynamicTableRegion`
 # reference to all rows of the electrodes table.
 #
 # .. only:: html
@@ -174,10 +171,10 @@
 #     :align: center
 #
 
-
 raw_data = np.random.randn(50, 12)
 raw_electrical_series = ElectricalSeries(
     name="ElectricalSeries",
+    description="Raw acquisition traces",
     data=raw_data,
     electrodes=all_table_region,
     starting_time=0.0,  # timestamp of the first sample in seconds relative to the session start time
@@ -188,7 +185,6 @@
 # Since this :py:class:`~pynwb.ecephys.ElectricalSeries` represents raw data from the data acquisition system,
 # add it to the acquisition group of the :py:class:`~pynwb.file.NWBFile`.
 
-
 nwbfile.add_acquisition(raw_electrical_series)
 
 ####################
@@ -199,10 +195,10 @@
 # again passing in the :py:class:`~hdmf.common.table.DynamicTableRegion` reference to all rows of the ``"electrodes"``
 # table.
 
-
 lfp_data = np.random.randn(50, 12)
 lfp_electrical_series = ElectricalSeries(
     name="ElectricalSeries",
+    description="LFP data",
     data=lfp_data,
     electrodes=all_table_region,
     starting_time=0.0,
@@ -240,7 +236,6 @@
 # This is analogous to how we can store the :py:class:`~pynwb.behavior.Position` object in a processing module
 # created with the method :py:meth:`.NWBFile.create_processing_module`.
 
-
 ecephys_module = nwbfile.create_processing_module(
     name="ecephys", description="processed extracellular electrophysiology data"
 )
@@ -254,13 +249,16 @@
 #
 # Spike times are stored in the :py:class:`~pynwb.misc.Units` table, which is a subclass of
 # :py:class:`~hdmf.common.table.DynamicTable`. Adding columns to the :py:class:`~pynwb.misc.Units` table is analogous
-# to how we can add columns to the ``"electrodes"`` and ``"trials"`` tables.
-#
-# Generate some random spike data and populate the :py:class:`~pynwb.misc.Units` table using the
-# method :py:meth:`.NWBFile.add_unit`.
+# to how we can add columns to the ``"electrodes"`` and ``"trials"`` tables. Use the convenience method
+# :py:meth:`.NWBFile.add_unit_column` to add a new column on the :py:class:`~pynwb.misc.Units` table for the
+# sorting quality of the units.
 
 nwbfile.add_unit_column(name="quality", description="sorting quality")
 
+####################
+# Generate some random spike data and populate the :py:class:`~pynwb.misc.Units` table using the
+# method :py:meth:`.NWBFile.add_unit`.
+
 firing_rate = 20
 n_units = 10
 res = 1000
@@ -272,7 +270,6 @@
 #######################
 # The :py:class:`~pynwb.misc.Units` table can also be converted to a pandas :py:class:`~pandas.DataFrame`.
 
-
 nwbfile.units.to_dataframe()
 
 #######################
@@ -315,7 +312,6 @@
 # Once you have finished adding all of your data to the :py:class:`~pynwb.file.NWBFile`,
 # write the file with :py:class:`~pynwb.NWBHDF5IO`.
 
-
 with NWBHDF5IO("ecephys_tutorial.nwb", "w") as io:
     io.write(nwbfile)