Merge branch 'nwb_schema_2.8.0' into provenance-tracking

CHANGELOG.md

@@ -1,10 +1,31 @@
 # PyNWB Changelog
 
+## PyNWB 3.0.0 (Upcoming)
+
+### Enhancements and minor changes
+- Added support for NWB schema 2.8.0:
+  - Removed `SpatialSeries.bounds` field that was not functional. This will be fixed in a future release. @rly [#1907](https://github.com/NeurodataWithoutBorders/pynwb/pull/1907), 
+
 ## PyNWB 2.8.3 (Upcoming)
 
+### Enhancements and minor changes
+* Added `NWBHDF5IO.read_nwb` convenience method to simplify reading an NWB file. @h-mayorquin [#1979](https://github.com/NeurodataWithoutBorders/pynwb/pull/1979)
+
+### Documentation and tutorial enhancements
+- Added documentation example for `SpikeEventSeries`. @stephprince [#1983](https://github.com/NeurodataWithoutBorders/pynwb/pull/1983)
+- Added documentation example for `AnnotationSeries`. @stephprince [#1989](https://github.com/NeurodataWithoutBorders/pynwb/pull/1989)
+
 ### Performance
 - Cache global type map to speed import 3X. @sneakers-the-rat [#1931](https://github.com/NeurodataWithoutBorders/pynwb/pull/1931)
 
+### Bug fixes
+- Fixed bug in how `ElectrodeGroup.__init__` validates its `position` argument. @oruebel [#1770](https://github.com/NeurodataWithoutBorders/pynwb/pull/1770)
+- Changed `SpatialSeries.reference_frame` from required to optional as specified in the schema. @rly [#1986](https://github.com/NeurodataWithoutBorders/pynwb/pull/1986)
+
+### Enhancements and minor changes
+- Made gain an optional argument for PatchClampSeries to match the schema. @stephprince [#1975](https://github.com/NeurodataWithoutBorders/pynwb/pull/1975)
+- Added warning when writing files with `NWBHDF5IO` without the `.nwb` extension. @stephprince [#1978](https://github.com/NeurodataWithoutBorders/pynwb/pull/1978)
+
 ## PyNWB 2.8.2 (September 9, 2024)
 
 ### Enhancements and minor changes
@@ -43,7 +64,7 @@
 ## PyNWB 2.7.0 (May 2, 2024)
 
 ### Enhancements and minor changes
-- Added `bounds` field to `SpatialSeries` to set optional boundary range (min, max) for each dimension of data. @mavaylon1 [#1869](https://github.com/NeurodataWithoutBorders/pynwb/pull/1869/files)
+- Added `bounds` field to `SpatialSeries` to set optional boundary range (min, max) for each dimension of data. @mavaylon1 [#1869](https://github.com/NeurodataWithoutBorders/pynwb/pull/1869)
 - Added support for NWB schema 2.7.0. See [2.7.0 release notes](https://nwb-schema.readthedocs.io/en/latest/format_release_notes.html) for details
 - Deprecated `ImagingRetinotopy` neurodata type. @rly [#1813](https://github.com/NeurodataWithoutBorders/pynwb/pull/1813)
 - Modified `OptogeneticSeries` to allow 2D data, primarily in extensions of `OptogeneticSeries`. @rly [#1812](https://github.com/NeurodataWithoutBorders/pynwb/pull/1812)

docs/gallery/advanced_io/plot_editing.py

@@ -24,6 +24,9 @@
 
 First, let's create an NWB file with data:
 """
+
+# sphinx_gallery_thumbnail_path = "figures/gallery_thumbnails_editing.png"
+
 from pynwb import NWBHDF5IO, NWBFile, TimeSeries
 from datetime import datetime
 from dateutil.tz import tzlocal

docs/gallery/domain/ecephys.py

@@ -31,7 +31,7 @@
 from dateutil.tz import tzlocal
 
 from pynwb import NWBHDF5IO, NWBFile
-from pynwb.ecephys import LFP, ElectricalSeries
+from pynwb.ecephys import LFP, ElectricalSeries, SpikeEventSeries
 
 #######################
 # Creating and Writing NWB files
@@ -244,8 +244,8 @@
 ####################
 # .. _units_electrode:
 #
-# Spike Times
-# ^^^^^^^^^^^
+# Sorted spike times
+# ^^^^^^^^^^^^^^^^^^
 #
 # Spike times are stored in the :py:class:`~pynwb.misc.Units` table, which is a subclass of
 # :py:class:`~hdmf.common.table.DynamicTable`. Adding columns to the :py:class:`~pynwb.misc.Units` table is analogous
@@ -272,6 +272,29 @@
 
 nwbfile.units.to_dataframe()
 
+####################
+# Unsorted spike times
+# ^^^^^^^^^^^^^^^^^^^^
+#
+# While the :py:class:`~pynwb.misc.Units` table is used to store spike times and waveform data for
+# spike-sorted, single-unit activity, you may also want to store spike times and waveform snippets of
+# unsorted spiking activity (e.g., multi-unit activity detected via threshold crossings during data acquisition).
+# This information can be stored using :py:class:`~pynwb.ecephys.SpikeEventSeries` objects. 
+
+spike_snippets = np.random.rand(20, 3, 40)  # 20 events, 3 channels, 40 samples per event
+shank0 = nwbfile.create_electrode_table_region(
+    region=[0, 1, 2],
+    description="shank0",
+)
+
+
+spike_events = SpikeEventSeries(name='SpikeEvents_Shank0',
+                                description="events detected with 100uV threshold",
+                                data=spike_snippets,
+                                timestamps=np.arange(20),
+                                electrodes=shank0)
+nwbfile.add_acquisition(spike_events)
+
 #######################
 # Designating electrophysiology data
 # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -283,17 +306,17 @@
 # :py:mod:`API documentation <pynwb.ecephys>` and :ref:`basics` for more details on
 # using these objects.
 #
-# For storing spike data, there are two options. Which one you choose depends on what data you have available.
-# If you need to store the complete, continuous raw voltage traces, you should store the traces with
+# For storing unsorted spiking data, there are two options. Which one you choose depends on what data you 
+# have available. If you need to store the complete, continuous raw voltage traces, you should store the traces with
 # :py:class:`~pynwb.ecephys.ElectricalSeries` objects as :ref:`acquisition <basic_timeseries>` data, and use
 # the :py:class:`~pynwb.ecephys.EventDetection` class for identifying the spike events in your raw traces.
 # If you do not want to store the raw voltage traces and only the waveform 'snippets' surrounding spike events,
-# you should use the :py:class:`~pynwb.ecephys.EventWaveform` class, which can store one or more
-# :py:class:`~pynwb.ecephys.SpikeEventSeries` objects.
+# you should use :py:class:`~pynwb.ecephys.SpikeEventSeries` objects.
 #
 # The results of spike sorting (or clustering) should be stored in the top-level :py:class:`~pynwb.misc.Units` table.
-# Note that it is not required to store spike waveforms in order to store spike events or mean waveforms--if you only
-# want to store the spike times of clustered units you can use only the Units table.
+# The :py:class:`~pynwb.misc.Units` table can contain simply the spike times of sorted units, or you can also include 
+# individual and mean waveform information in some of the optional, predefined :py:class:`~pynwb.misc.Units` table 
+# columns: ``waveform_mean``, ``waveform_sd``, or ``waveforms``.
 #
 # For local field potential data, there are two options. Again, which one you choose depends on what data you
 # have available. With both options, you should store your traces with :py:class:`~pynwb.ecephys.ElectricalSeries`

docs/gallery/domain/plot_behavior.py

@@ -105,17 +105,13 @@
 #
 # For position data ``reference_frame`` indicates the zero-position, e.g.
 # the 0,0 point might be the bottom-left corner of an enclosure, as viewed from the tracking camera.
-# In :py:class:`~pynwb.behavior.SpatialSeries`, the ``bounds`` field allows the user to set
-# the boundary range, i.e.,  (min, max), for each dimension of ``data``. The units are the same as in ``data``.
-# This field does not enforce a boundary on the dataset itself.
 
 timestamps = np.linspace(0, 50) / 200
 
 position_spatial_series = SpatialSeries(
     name="SpatialSeries",
     description="Position (x, y) in an open field.",
     data=position_data,
-    bounds=[(0,50), (0,50)],
     timestamps=timestamps,
     reference_frame="(0,0) is bottom left corner",
 )

docs/gallery/general/plot_configurator.py

@@ -8,18 +8,18 @@
 Introduction
 -------------
 Users will create a configuration YAML file that outlines the fields (within a neurodata type)
-they want to be validated against a set of allowed terms. 
+they want to be validated against a set of allowed terms.
 After creating the configuration file, users will need to load the
 configuration file with the :py:func:`~pynwb.load_type_config` method.
 With the configuration loaded, every instance of the neurodata
 types defined in the configuration file will have the respective fields wrapped with a
 :py:class:`~hdmf.term_set.TermSetWrapper`.
 This automatic wrapping is what provides the term validation for the field value.
 For greater control on which datasets and attributes are validated
-against which sets of allowed terms, use the 
+against which sets of allowed terms, use the
 :py:class:`~hdmf.term_set.TermSetWrapper` on individual datasets and attributes instead.
-You can follow the 
-`TermSet tutorial in the HDMF documentation 
+You can follow the
+`TermSet tutorial in the HDMF documentation
 <https://hdmf.readthedocs.io/en/stable/tutorials/plot_term_set.html#sphx-glr-tutorials-plot-term-set-py>`_
 for more information.
 
@@ -42,6 +42,8 @@
 3. Each data type will have a list of fields associated with a :py:class:`~hdmf.term_set.TermSet`.
    The user can use the same or unique TermSet instances for each field.
 """
+# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_configurator.png'
+
 try:
     import linkml_runtime  # noqa: F401
 except ImportError as e:

docs/gallery/general/plot_file.py

@@ -131,6 +131,7 @@
 from pynwb import NWBHDF5IO, NWBFile, TimeSeries
 from pynwb.behavior import Position, SpatialSeries
 from pynwb.file import Subject
+from pynwb.misc import AnnotationSeries
 
 ####################
 # .. _basics_nwbfile:
@@ -285,6 +286,27 @@
 # or using the method :py:meth:`.NWBFile.get_acquisition`:
 nwbfile.get_acquisition("test_timeseries")
 
+####################
+# Other Types of Time Series
+# ^^^^^^^^^^^^^^^^^^^^^^^^^^
+#
+# As mentioned previously, there are many subtypes of :py:class:`~pynwb.base.TimeSeries` that are used to store
+# different kinds of data. One example is :py:class:`~pynwb.misc.AnnotationSeries`, a subclass of 
+# :py:class:`~pynwb.base.TimeSeries` that stores text-based records about the experiment. Similarly to our
+# :py:class:`~pynwb.base.TimeSeries` example above, we can create an :py:class:`~pynwb.misc.AnnotationSeries` 
+# object with text information about a stimulus and add it to the stimulus group in 
+# the :py:class:`~pynwb.file.NWBFile`. 
+
+annotations = AnnotationSeries(name='airpuffs',
+                               data=['Left Airpuff', 'Right Airpuff', 'Right Airpuff'],
+                               description='Airpuff events delivered to the animal',
+                               timestamps=[1.0, 3.0, 8.0])
+
+nwbfile.add_stimulus(annotations)
+
+####################
+# This approach of creating a :py:class:`~pynwb.base.TimeSeries` object and adding it to the appropriate 
+# :py:class:`~pynwb.file.NWBFile` group can be used for all subtypes of :py:class:`~pynwb.base.TimeSeries` data.
 
 ####################
 # .. _basic_spatialseries:

docs/source/conf.py

@@ -244,7 +244,6 @@ def __call__(self, filename):
 # html_theme = 'default'
 # html_theme = "sphinxdoc"
 html_theme = "sphinx_rtd_theme"
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -260,9 +259,6 @@ def __call__(self, filename):
     'css/custom.css',
 ]
 
-# Add any paths that contain custom themes here, relative to this directory.
-# html_theme_path = []
-
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
 # html_title = None

docs/source/export.rst

@@ -53,12 +53,12 @@ on the :py:class:`~pynwb.file.NWBFile` before exporting.
 
 How do I create a copy of an NWB file with different data layouts (e.g., applying compression)?
 ---------------------------------------------------------------------------------------------------------
-Use the `h5repack <https://docs.hdfgroup.org/hdf5/v1_14/_view_tools_edit.html>`_ command line tool from the HDF5 Group.
+Use the `h5repack <https://support.hdfgroup.org/documentation/hdf5/latest/_view_tools_edit.html#secViewToolsEditChange>`_ command line tool from the HDF5 Group.
 
 
 How do I create a copy of an NWB file with different controls over how links are treated and whether copies are deep or shallow?
 ---------------------------------------------------------------------------------------------------------------------------------
-Use the `h5copy <https://docs.hdfgroup.org/hdf5/v1_14/_view_tools_edit.html>`_ command line tool from the HDF5 Group.
+Use the `h5copy <https://support.hdfgroup.org/documentation/hdf5/latest/_view_tools_edit.html#secViewToolsEditCopy>`_ command line tool from the HDF5 Group.
 
 
 How do I generate new object IDs for a newly exported NWB file?
@@ -99,7 +99,7 @@ For example:
            export_io.export(src_io=read_io, nwbfile=nwbfile, write_args={'link_data': False})  # copy linked datasets
            # the written file will contain no links to external datasets
 
-You can also the `h5copy <https://docs.hdfgroup.org/hdf5/v1_14/_view_tools_edit.html>`_ command line tool \
+You can also the `h5copy <https://support.hdfgroup.org/documentation/hdf5/latest/_view_tools_edit.html#secViewToolsEditCopy>`_ command line tool \
 from the HDF5 Group.
 
 

pyproject.toml

@@ -34,7 +34,7 @@ classifiers = [
 ]
 dependencies = [
     "h5py>=2.10",
-    "hdmf>=3.14.3",
+    "hdmf>=3.14.5",
     "numpy>=1.18",
     "pandas>=1.1.5",
     "python-dateutil>=2.7.3",

requirements-min.txt

@@ -1,6 +1,6 @@
 # minimum versions of package dependencies for installing PyNWB
 h5py==2.10  # support for selection of datasets with list of indices added in 2.10
-hdmf==3.14.3
+hdmf==3.14.5
 numpy==1.18
 pandas==1.1.5
 python-dateutil==2.7.3

requirements-opt.txt

@@ -1,3 +1,8 @@
 linkml-runtime==1.7.4; python_version >= "3.9"
 schemasheets==0.2.1; python_version >= "3.9"
 oaklib==0.5.32; python_version >= "3.9"
+
+# for streaming tests
+fsspec==2024.10.0
+requests==2.32.3
+aiohttp==3.10.10

requirements.txt

@@ -1,6 +1,6 @@
 # pinned dependencies to reproduce an entire development environment to use PyNWB
 h5py==3.11.0
-hdmf==3.14.3
+hdmf==3.14.5
 numpy==2.1.1; python_version > "3.9"  # numpy 2.1+ is not compatible with py3.9
 numpy==2.0.2; python_version == "3.9"
 pandas==2.2.2

src/pynwb/__init__.py

@@ -397,6 +397,10 @@ def __init__(self, **kwargs):
         if mode in io_modes_that_create_file or manager is not None or extensions is not None:
             load_namespaces = False
 
+        if mode in io_modes_that_create_file and not str(path).endswith('.nwb'):
+            warn(f"The file path provided: {path} does not end in '.nwb'. "
+                 "It is recommended that NWB files using the HDF5 backend use the '.nwb' extension.", UserWarning)
+
         if load_namespaces:
             tm = get_type_map()
             super().load_namespaces(tm, path, file=file_obj, driver=driver, aws_region=aws_region)
@@ -502,6 +506,34 @@ def export(self, **kwargs):
         kwargs['container'] = nwbfile
         super().export(**kwargs)
 
+    @staticmethod
+    @docval({'name': 'path', 'type': (str, Path), 'doc': 'the path to the HDF5 file', 'default': None},
+            {'name': 'file', 'type': [h5py.File, 'S3File'], 'doc': 'a pre-existing h5py.File object', 'default': None},
+            is_method=False)
+    def read_nwb(**kwargs):
+        """
+        Helper factory method for reading an NWB file and return the NWBFile object
+        """
+        # Retrieve the filepath
+        path = popargs('path', kwargs)
+        file = popargs('file', kwargs)
+
+        path = str(path) if path is not None else None
+
+        # Streaming case
+        if path is not None and (path.startswith("s3://") or path.startswith("http")):
+            import fsspec
+            fsspec_file_system = fsspec.filesystem("http")
+            ffspec_file = fsspec_file_system.open(path, "rb")
+
+            open_file = h5py.File(ffspec_file, "r")
+            io = NWBHDF5IO(file=open_file)
+            nwbfile = io.read()
+        else:
+            io = NWBHDF5IO(path=path, file=file, mode="r", load_namespaces=True)
+            nwbfile = io.read()
+
+        return nwbfile
 
 from . import io as __io  # noqa: F401,E402
 from .core import NWBContainer, NWBData  # noqa: F401,E402

src/pynwb/behavior.py

@@ -26,10 +26,8 @@ class SpatialSeries(TimeSeries):
             {'name': 'data', 'type': ('array_data', 'data', TimeSeries), 'shape': ((None, ), (None, None)), # required
              'doc': ('The data values. Can be 1D or 2D. The first dimension must be time. If 2D, there can be 1, 2, '
                      'or 3 columns, which represent x, y, and z.')},
-            {'name': 'bounds', 'type': list, 'shape': ((1, 2), (2, 2), (3, 2)), 'default': None,
-             'doc': 'The boundary range (min, max) for each dimension of data.'},
-            {'name': 'reference_frame', 'type': str,   # required
-             'doc': 'description defining what the zero-position is'},
+            {'name': 'reference_frame', 'type': str,
+             'doc': 'description defining what the zero-position is', 'default': None},
             {'name': 'unit', 'type': str, 'doc': 'The base unit of measurement (should be SI unit)',
              'default': 'meters'},
             *get_docval(TimeSeries.__init__, 'conversion', 'resolution', 'timestamps', 'starting_time', 'rate',
@@ -38,7 +36,9 @@ def __init__(self, **kwargs):
         """
         Create a SpatialSeries TimeSeries dataset
         """
-        name, data, bounds, reference_frame, unit = popargs('name', 'data', 'bounds', 'reference_frame', 'unit', kwargs)
+        name, data, reference_frame, unit = popargs(
+            'name', 'data', 'reference_frame', 'unit', kwargs
+        )
         super().__init__(name, data, unit, **kwargs)
 
         # NWB 2.5 restricts length of second dimension to be <= 3
@@ -49,7 +49,6 @@ def __init__(self, **kwargs):
                           "The second dimension should have length <= 3 to represent at most x, y, z." %
                           (name, str(data_shape)))
 
-        self.bounds = bounds
         self.reference_frame = reference_frame
 
     @staticmethod