diff --git a/.github/workflows/check_external_links.yml b/.github/workflows/check_sphinx_links.yml similarity index 82% rename from .github/workflows/check_external_links.yml rename to .github/workflows/check_sphinx_links.yml index 1c709ba79..e1eddb97a 100644 --- a/.github/workflows/check_external_links.yml +++ b/.github/workflows/check_sphinx_links.yml @@ -1,4 +1,4 @@ -name: Check Sphinx external links +name: Check Sphinx links on: pull_request: schedule: @@ -6,7 +6,7 @@ on: workflow_dispatch: jobs: - check-external-links: + check-sphinx-links: runs-on: ubuntu-latest steps: - name: Cancel non-latest runs @@ -31,5 +31,5 @@ jobs: python -m pip install -r requirements-doc.txt python -m pip install . - - name: Check Sphinx external links - run: sphinx-build -b linkcheck ./docs/source ./test_build + - name: Check Sphinx internal and external links + run: sphinx-build -W -b linkcheck ./docs/source ./test_build diff --git a/.github/workflows/run_coverage.yml b/.github/workflows/run_coverage.yml index acbc3bd05..18dc00903 100644 --- a/.github/workflows/run_coverage.yml +++ b/.github/workflows/run_coverage.yml @@ -78,8 +78,10 @@ jobs: python -m coverage report -m - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: flags: integration files: coverage.xml fail_ci_if_error: true + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 5befd21e7..a06d0280a 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -8,7 +8,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: '3.8' + python: '3.11' # Build documentation in the docs/ directory with Sphinx sphinx: diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a9b86dd3..c05fddfe8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,32 +1,43 @@ # PyNWB Changelog -## PyNWB 2.6.0 (Upcoming) +## PyNWB 2.7.0 (Upcoming) + +### Bug fixes +- Fixed bug in how `ElectrodeGroup.__init__` validates its `position` argument. @oruebel [#1770](https://github.com/NeurodataWithoutBorders/pynwb/pull/1770) + + +## PyNWB 2.6.0 (February 21, 2024) ### Enhancements and minor changes - For `NWBHDF5IO()`, change the default of arg `load_namespaces` from `False` to `True`. @bendichter [#1748](https://github.com/NeurodataWithoutBorders/pynwb/pull/1748) - Add `NWBHDF5IO.can_read()`. @bendichter [#1703](https://github.com/NeurodataWithoutBorders/pynwb/pull/1703) - Add `pynwb.get_nwbfile_version()`. @bendichter [#1703](https://github.com/NeurodataWithoutBorders/pynwb/pull/1703) +- Fix usage of the `validate` function in the `pynwb.testing.testh5io` classes and cache the spec by default in those classes. @rly [#1782](https://github.com/NeurodataWithoutBorders/pynwb/pull/1782) - Updated timeseries data checks to warn instead of error when reading invalid files. @stephprince [#1793](https://github.com/NeurodataWithoutBorders/pynwb/pull/1793) and [#1809](https://github.com/NeurodataWithoutBorders/pynwb/pull/1809) - Expose the offset, conversion and channel conversion parameters in `mock_ElectricalSeries`. @h-mayorquin [#1796](https://github.com/NeurodataWithoutBorders/pynwb/pull/1796) - Expose `starting_time` in `mock_ElectricalSeries`. @h-mayorquin [#1805](https://github.com/NeurodataWithoutBorders/pynwb/pull/1805) - Enhance `get_data_in_units()` to work with objects that have a `channel_conversion` attribute like the `ElectricalSeries`. @h-mayorquin [#1806](https://github.com/NeurodataWithoutBorders/pynwb/pull/1806) - Refactor validation CLI tests to use `{sys.executable} -m coverage` to use the same Python version and run correctly on Debian systems. @yarikoptic [#1811](https://github.com/NeurodataWithoutBorders/pynwb/pull/1811) +- Fixed tests to address newly caught validation errors. @rly [#1839](https://github.com/NeurodataWithoutBorders/pynwb/pull/1839) ### Bug fixes - Fix bug where namespaces were loaded in "w-" mode. @h-mayorquin [#1795](https://github.com/NeurodataWithoutBorders/pynwb/pull/1795) - Fix bug where pynwb version was reported as "unknown" to readthedocs @stephprince [#1810](https://github.com/NeurodataWithoutBorders/pynwb/pull/1810) -- Fixed bug in how `ElectrodGroup.__init__` validates its `position` argument. @oruebel [#1770](https://github.com/NeurodataWithoutBorders/pynwb/pull/1770) +- Fixed bug to allow linking of `TimeSeries.data` by setting the `data` constructor argument to another `TimeSeries`. @oruebel [#1766](https://github.com/NeurodataWithoutBorders/pynwb/pull/1766) +- Fix recursion error in html representation generation in jupyter notebooks. @stephprince [#1831](https://github.com/NeurodataWithoutBorders/pynwb/pull/1831) ### Documentation and tutorial enhancements - Add RemFile to streaming tutorial. @bendichter [#1761](https://github.com/NeurodataWithoutBorders/pynwb/pull/1761) - Fix typos and improve clarify throughout tutorials. @zm711 [#1825](https://github.com/NeurodataWithoutBorders/pynwb/pull/1825) +- Fix internal links in docstrings and tutorials. @stephprince [#1827](https://github.com/NeurodataWithoutBorders/pynwb/pull/1827) +- Add Zarr IO tutorial @bendichter [#1834](https://github.com/NeurodataWithoutBorders/pynwb/pull/1834) ## PyNWB 2.5.0 (August 18, 2023) ### Enhancements and minor changes -- Add `TimeSeries.get_timestamps()`. @bendichter [#1741](https://github.com/NeurodataWithoutBorders/pynwb/pull/1741) -- Add `TimeSeries.get_data_in_units()`. @bendichter [#1745](https://github.com/NeurodataWithoutBorders/pynwb/pull/1745) +- Added `TimeSeries.get_timestamps()`. @bendichter [#1741](https://github.com/NeurodataWithoutBorders/pynwb/pull/1741) +- Added `TimeSeries.get_data_in_units()`. @bendichter [#1745](https://github.com/NeurodataWithoutBorders/pynwb/pull/1745) - Updated `ExternalResources` name change to `HERD`, along with HDMF 3.9.0 being the new minimum. @mavaylon1 [#1754](https://github.com/NeurodataWithoutBorders/pynwb/pull/1754) ### Documentation and tutorial enhancements @@ -39,15 +50,15 @@ ## PyNWB 2.4.0 (July 23, 2023) ### Enhancements and minor changes -- Add support for `ExternalResources`. @mavaylon1 [#1684](https://github.com/NeurodataWithoutBorders/pynwb/pull/1684) -- Update links for making a release. @mavaylon1 [#1720](https://github.com/NeurodataWithoutBorders/pynwb/pull/1720) +- Added support for `ExternalResources`. @mavaylon1 [#1684](https://github.com/NeurodataWithoutBorders/pynwb/pull/1684) +- Updated links for making a release. @mavaylon1 [#1720](https://github.com/NeurodataWithoutBorders/pynwb/pull/1720) ### Bug fixes - Fixed sphinx-gallery setting to correctly display index in the docs with sphinx-gallery>=0.11. @oruebel [#1733](https://github.com/NeurodataWithoutBorders/pynwb/pull/1733) ### Documentation and tutorial enhancements - Added thumbnail for Optogentics tutorial. @oruebel [#1729](https://github.com/NeurodataWithoutBorders/pynwb/pull/1729) -- Update and fix errors in tutorials. @bendichter @oruebel +- Updated and fixed errors in tutorials. @bendichter @oruebel ## PyNWB 2.3.3 (June 26, 2023) diff --git a/Legal.txt b/Legal.txt index 08061bfbe..eb5bcac88 100644 --- a/Legal.txt +++ b/Legal.txt @@ -1,4 +1,4 @@ -“pynwb” Copyright (c) 2017-2023, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. +“pynwb” Copyright (c) 2017-2024, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. If you have questions about your rights to use or distribute this software, please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. diff --git a/README.rst b/README.rst index ef37476e7..14eb32fce 100644 --- a/README.rst +++ b/README.rst @@ -108,7 +108,7 @@ Citing NWB LICENSE ======= -"pynwb" Copyright (c) 2017-2023, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. +"pynwb" Copyright (c) 2017-2024, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: (1) Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. @@ -124,7 +124,7 @@ You are under no obligation whatsoever to provide any bug fixes, patches, or upg COPYRIGHT ========= -"pynwb" Copyright (c) 2017-2023, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. +"pynwb" Copyright (c) 2017-2024, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. If you have questions about your rights to use or distribute this software, please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. NOTICE. This Software was developed under funding from the U.S. Department of Energy and the U.S. Government consequently retains certain rights. As such, the U.S. Government has been granted for itself and others acting on its behalf a paid-up, nonexclusive, irrevocable, worldwide license in the Software to reproduce, distribute copies to the public, prepare derivative works, and perform publicly and display publicly, and to permit other to do so. diff --git a/docs/Makefile b/docs/Makefile index 80492bbf2..26960f3e7 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -149,7 +149,7 @@ changes: @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + $(SPHINXBUILD) -W -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." diff --git a/docs/gallery/advanced_io/h5dataio.py b/docs/gallery/advanced_io/h5dataio.py index fd3548e4b..3b4391655 100644 --- a/docs/gallery/advanced_io/h5dataio.py +++ b/docs/gallery/advanced_io/h5dataio.py @@ -34,7 +34,7 @@ #################### -# Normally if we create a :py:class:`~pynwb.file.TimeSeries` we would do +# Normally if we create a :py:class:`~pynwb.base.TimeSeries` we would do import numpy as np diff --git a/docs/gallery/advanced_io/plot_editing.py b/docs/gallery/advanced_io/plot_editing.py new file mode 100644 index 000000000..a371dc588 --- /dev/null +++ b/docs/gallery/advanced_io/plot_editing.py @@ -0,0 +1,161 @@ +""" +.. _editing: + +Editing NWB files +================= + +This tutorial demonstrates how to edit NWB files in-place to make small changes to +existing containers. To add or remove containers from an NWB file, see +:ref:`modifying_data`. How and whether it is possible to edit an NWB file depends on the +storage backend and the type of edit. + +.. warning:: + + Manually editing an existing NWB file can make the file invalid if you are not + careful. We highly recommend making a copy before editing and running a validation + check on the file after editing it. See :ref:`validating`. + + +Editing datasets +---------------- +When reading an HDF5 NWB file, PyNWB exposes :py:class:`h5py.Dataset` objects, which can +be edited in place. For this to work, you must open the file in read/write mode +(``"r+"`` or ``"a"``). + +First, let's create an NWB file with data: +""" +from pynwb import NWBHDF5IO, NWBFile, TimeSeries +from datetime import datetime +from dateutil.tz import tzlocal +import numpy as np + +nwbfile = NWBFile( + session_description="my first synthetic recording", + identifier="EXAMPLE_ID", + session_start_time=datetime.now(tzlocal()), + session_id="LONELYMTN", +) + +nwbfile.add_acquisition( + TimeSeries( + name="synthetic_timeseries", + description="Random values", + data=np.random.randn(100, 100), + unit="m", + rate=10e3, + ) +) + +with NWBHDF5IO("test_edit.nwb", "w") as io: + io.write(nwbfile) + +############################################## +# Now, let's edit the values of the dataset + +with NWBHDF5IO("test_edit.nwb", "r+") as io: + nwbfile = io.read() + nwbfile.acquisition["synthetic_timeseries"].data[:10] = 0.0 + + +############################################## +# You can edit the attributes of that dataset through the ``attrs`` attribute: + +with NWBHDF5IO("test_edit.nwb", "r+") as io: + nwbfile = io.read() + nwbfile.acquisition["synthetic_timeseries"].data.attrs["unit"] = "volts" + +############################################## +# Changing the shape of dataset +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Whether it is possible to change the shape of a dataset depends on how the dataset was +# created. If the dataset was created with a flexible shape, then it is possible to +# change in-place. Creating a dataset with a flexible shape is done by specifying the +# ``maxshape`` argument of the :py:class:`~hdmf.backends.hdf5.h5_utils.H5DataIO` class +# constructor. Using a ``None`` value for a component of the ``maxshape`` tuple allows +# the size of the corresponding dimension to grow, such that is can be be reset arbitrarily long +# in that dimension. Chunking is required for datasets with flexible shapes. Setting ``maxshape``, +# hence, automatically sets chunking to ``True``, if not specified. +# +# First, let's create an NWB file with a dataset with a flexible shape: + +from hdmf.backends.hdf5.h5_utils import H5DataIO + +nwbfile = NWBFile( + session_description="my first synthetic recording", + identifier="EXAMPLE_ID", + session_start_time=datetime.now(tzlocal()), + session_id="LONELYMTN", +) + +data_io = H5DataIO(data=np.random.randn(100, 100), maxshape=(None, 100)) + +nwbfile.add_acquisition( + TimeSeries( + name="synthetic_timeseries", + description="Random values", + data=data_io, + unit="m", + rate=10e3, + ) +) + +with NWBHDF5IO("test_edit2.nwb", "w") as io: + io.write(nwbfile) + +############################################## +# The ``None``value in the first component of ``maxshape`` means that the +# the first dimension of the dataset is unlimited. By setting the second dimension +# of ``maxshape`` to ``100``, that dimension is fixed to be no larger than ``100``. +# If you do not specify a``maxshape``, then the shape of the dataset will be fixed +# to the shape that the dataset was created with. Here, you can change the shape of +# the first dimension of this dataset. + + +with NWBHDF5IO("test_edit2.nwb", "r+") as io: + nwbfile = io.read() + nwbfile.acquisition["synthetic_timeseries"].data.resize((200, 100)) + +############################################## +# This will change the shape of the dataset in-place. If you try to change the shape of +# a dataset with a fixed shape, you will get an error. +# +# .. note:: +# There are several types of dataset edits that cannot be done in-place: changing the +# shape of a dataset with a fixed shape, or changing the datatype, compression, +# chunking, max-shape, or fill-value of a dataset. For any of these, we recommend using +# the :py:class:`pynwb.NWBHDF5IO.export` method to export the data to a new file. See +# :ref:`modifying_data` for more information. +# +# Editing groups +# -------------- +# Editing of groups is not yet supported in PyNWB. +# To edit the attributes of a group, open the file and edit it using ``h5py``: + +import h5py + +with h5py.File("test_edit.nwb", "r+") as f: + f["acquisition"]["synthetic_timeseries"].attrs["description"] = "Random values in volts" + +############################################## +# .. warning:: +# Be careful not to edit values that will bring the file out of compliance with the +# NWB specification. +# +# Renaming groups and datasets +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Rename groups and datasets in-place using the :py:meth:`~h5py.Group.move` method. For example, to rename +# the ``"synthetic_timeseries"`` group: + +with h5py.File("test_edit.nwb", "r+") as f: + f["acquisition"].move("synthetic_timeseries", "synthetic_timeseries_renamed") + +############################################## +# You can use this same technique to move a group or dataset to a different location in +# the file. For example, to move the ``"synthetic_timeseries_renamed"`` group to the +# ``"analysis"`` group: + +with h5py.File("test_edit.nwb", "r+") as f: + f["acquisition"].move( + "synthetic_timeseries_renamed", + "/analysis/synthetic_timeseries_renamed", + ) diff --git a/docs/gallery/advanced_io/plot_zarr_io.py b/docs/gallery/advanced_io/plot_zarr_io.py new file mode 100644 index 000000000..b61fe4a03 --- /dev/null +++ b/docs/gallery/advanced_io/plot_zarr_io.py @@ -0,0 +1,98 @@ +""" +Zarr IO +======= + +Zarr is an alternative backend option for NWB files. It is a Python package that +provides an implementation of chunked, compressed, N-dimensional arrays. Zarr is a good +option for large datasets because, like HDF5, it is designed to store data on disk and +only load the data into memory when needed. Zarr is also a good option for parallel +computing because it supports concurrent reads and writes. + +Note that the Zarr native storage formats are optimized for storage in cloud storage +(e.g., S3). For very large files, Zarr will create many files which can lead to +issues for traditional file system (that are not cloud object stores) due to limitations +on the number of files per directory (this affects local disk, GDrive, Dropbox etc.). + +Zarr read and write is provided by the :hdmf-zarr:`hdmf-zarr<>` package. First, create an +an NWBFile using PyNWB. +""" + +# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnail_plot_nwbzarrio.png' + + +from datetime import datetime +from dateutil.tz import tzlocal + +import numpy as np +from pynwb import NWBFile, TimeSeries + +# Create the NWBFile. Substitute your NWBFile generation here. +nwbfile = NWBFile( + session_description="my first synthetic recording", + identifier="EXAMPLE_ID", + session_start_time=datetime.now(tzlocal()), + session_id="LONELYMTN", +) + +####################################################################################### +# Dataset Configuration +# --------------------- +# Like HDF5, Zarr provides options to chunk and compress datasets. To leverage these +# features, replace all :py:class:`~hdmf.backends.hdf5.h5_utils.H5DataIO` with the analogous +# :py:class:`~hdmf_zarr.utils.ZarrDataIO`, which takes compressors specified by the +# :py:mod:`numcodecs` library. For example, here is an example :py:class:`.TimeSeries` +# where the ``data`` Dataset is compressed with a Blosc-zstd compressor: + +from numcodecs import Blosc +from hdmf_zarr import ZarrDataIO + +data_with_zarr_data_io = ZarrDataIO( + data=np.random.randn(100, 100), + chunks=(10, 10), + fillvalue=0, + compressor=Blosc(cname='zstd', clevel=3, shuffle=Blosc.SHUFFLE) +) + +####################################################################################### +# Now add it to the :py:class:`.NWBFile`. + +nwbfile.add_acquisition( + TimeSeries( + name="synthetic_timeseries", + data=data_with_zarr_data_io, + unit="m", + rate=10e3, + ) +) + +####################################################################################### +# Writing to Zarr +# --------------- +# To write NWB files to Zarr, replace the :py:class:`~pynwb.NWBHDF5IO` with +# :py:class:`hdmf_zarr.nwb.NWBZarrIO`. + +from hdmf_zarr.nwb import NWBZarrIO +import os + +path = "zarr_tutorial.nwb.zarr" +absolute_path = os.path.abspath(path) +with NWBZarrIO(path=path, mode="w") as io: + io.write(nwbfile) + +####################################################################################### +# .. note:: +# The main reason for using the ``absolute_path`` here is for testing purposes to +# ensure links and references work as expected. Otherwise, using the relative path +# here instead is fine. +# +# Reading from Zarr +# ----------------- +# To read NWB files from Zarr, replace the :py:class:`~pynwb.NWBHDF5IO` with the analogous +# :py:class:`hdmf_zarr.nwb.NWBZarrIO`. + +with NWBZarrIO(path=absolute_path, mode="r") as io: + read_nwbfile = io.read() + +####################################################################################### +# .. note:: +# For more information, see the :hdmf-zarr:`hdmf-zarr documentation<>`. diff --git a/docs/gallery/advanced_io/streaming.py b/docs/gallery/advanced_io/streaming.py index 760e2da71..4bdc992b8 100644 --- a/docs/gallery/advanced_io/streaming.py +++ b/docs/gallery/advanced_io/streaming.py @@ -23,6 +23,11 @@ Now you can get the url of a particular NWB file using the dandiset ID and the path of that file within the dandiset. +.. note:: + + To learn more about the dandi API see the + `DANDI Python API docs `_ + """ # sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_streaming.png' diff --git a/docs/gallery/domain/images.py b/docs/gallery/domain/images.py index 8b59b75f0..58e9a8e8b 100644 --- a/docs/gallery/domain/images.py +++ b/docs/gallery/domain/images.py @@ -190,7 +190,7 @@ # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # :py:class:`~pynwb.image.RGBAImage` is for storing data of color image with transparency. -# :py:attr:`~pynwb.image.RGBAImage.data` must be 3D where the first and second dimensions +# ``RGBAImage.data`` must be 3D where the first and second dimensions # represent x and y. The third dimension has length 4 and represents the RGBA value. # @@ -208,7 +208,7 @@ # ^^^^^^^^^^^^^^^^^^^^^^^^^^ # # :py:class:`~pynwb.image.RGBImage` is for storing data of RGB color image. -# :py:attr:`~pynwb.image.RGBImage.data` must be 3D where the first and second dimensions +# ``RGBImage.data`` must be 3D where the first and second dimensions # represent x and y. The third dimension has length 3 and represents the RGB value. # @@ -224,8 +224,7 @@ # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # :py:class:`~pynwb.image.GrayscaleImage` is for storing grayscale image data. -# :py:attr:`~pynwb.image.GrayscaleImage.data` must be 2D where the first and second dimensions -# represent x and y. +# ``GrayscaleImage.data`` must be 2D where the first and second dimensions represent x and y. # gs_logo = GrayscaleImage( @@ -300,7 +299,7 @@ #################### # Here `data` contains the (0-indexed) index of the displayed image as they are ordered -# in the :py:class:`~pynwb.base.ImageReference`. +# in the :py:class:`~pynwb.base.ImageReferences`. # # Writing the images to an NWB File # --------------------------------------- diff --git a/docs/gallery/domain/ophys.py b/docs/gallery/domain/ophys.py index b8ddb1ae5..277e408db 100644 --- a/docs/gallery/domain/ophys.py +++ b/docs/gallery/domain/ophys.py @@ -540,7 +540,7 @@ # Data arrays are read passively from the file. # Calling the data attribute on a :py:class:`~pynwb.base.TimeSeries` # such as a :py:class:`~pynwb.ophys.RoiResponseSeries` does not read the data -# values, but presents an :py:class:`~h5py` object that can be indexed to read data. +# values, but presents an ``h5py`` object that can be indexed to read data. # You can use the ``[:]`` operator to read the entire data array into memory. # Load and print all the data values of the :py:class:`~pynwb.ophys.RoiResponseSeries` # object representing the fluorescence data. @@ -558,7 +558,7 @@ # # It is often preferable to read only a portion of the data. To do this, index # or slice into the data attribute just like if you were indexing or slicing a -# :py:class:`~numpy` array. +# :py:mod:`numpy` array. # # The following code prints elements ``0:10`` in the first dimension (time) # and ``0:3`` (ROIs) in the second dimension from the fluorescence data we have written. diff --git a/docs/gallery/domain/plot_icephys.py b/docs/gallery/domain/plot_icephys.py index 109ec5fcc..410375909 100644 --- a/docs/gallery/domain/plot_icephys.py +++ b/docs/gallery/domain/plot_icephys.py @@ -350,7 +350,7 @@ ##################################################################### # .. note:: Since :py:meth:`~pynwb.file.NWBFile.add_intracellular_recording` can automatically add # the objects to the NWBFile we do not need to separately call -# :py:meth:`~pynwb.file.NWBFile.add_stimulus` and :py:meth:`~pynwb.file.NWBFile.add_acquistion` +# :py:meth:`~pynwb.file.NWBFile.add_stimulus` and :py:meth:`~pynwb.file.NWBFile.add_acquisition` # to add our stimulus and response, but it is still fine to do so. # # .. note:: The ``id`` parameter in the call is optional and if the ``id`` is omitted then PyNWB will @@ -495,8 +495,7 @@ # .. note:: The same process applies to all our other tables as well. We can use the # corresponding :py:meth:`~pynwb.file.NWBFile.get_intracellular_recordings`, # :py:meth:`~pynwb.file.NWBFile.get_icephys_sequential_recordings`, -# :py:meth:`~pynwb.file.NWBFile.get_icephys_repetitions`, and -# :py:meth:`~pynwb.file.NWBFile.get_icephys_conditions` functions instead. +# :py:meth:`~pynwb.file.NWBFile.get_icephys_repetitions` functions instead. # In general, we can always use the get functions instead of accessing the property # of the file. # @@ -507,7 +506,7 @@ # # Add a single simultaneous recording consisting of a set of intracellular recordings. # Again, setting the id for a simultaneous recording is optional. The recordings -# argument of the :py:meth:`~pynwb.file.NWBFile.add_simultaneous_recording` function +# argument of the :py:meth:`~pynwb.file.NWBFile.add_icephys_simultaneous_recording` function # here is simply a list of ints with the indices of the corresponding rows in # the :py:class:`~pynwb.icephys.IntracellularRecordingsTable` # @@ -564,7 +563,7 @@ # Add a single sequential recording consisting of a set of simultaneous recordings. # Again, setting the id for a sequential recording is optional. Also this table is # optional and will be created automatically by NWBFile. The ``simultaneous_recordings`` -# argument of the :py:meth:`~pynwb.file.NWBFile.add_sequential_recording` function +# argument of the :py:meth:`~pynwb.file.NWBFile.add_icephys_sequential_recording` function # here is simply a list of ints with the indices of the corresponding rows in # the :py:class:`~pynwb.icephys.SimultaneousRecordingsTable`. @@ -579,7 +578,7 @@ # Add a single repetition consisting of a set of sequential recordings. Again, setting # the id for a repetition is optional. Also this table is optional and will be created # automatically by NWBFile. The ``sequential_recordings argument`` of the -# :py:meth:`~pynwb.file.NWBFile.add_sequential_recording` function here is simply +# :py:meth:`~pynwb.file.NWBFile.add_icephys_repetition` function here is simply # a list of ints with the indices of the corresponding rows in # the :py:class:`~pynwb.icephys.SequentialRecordingsTable`. @@ -592,7 +591,7 @@ # Add a single experimental condition consisting of a set of repetitions. Again, # setting the id for a condition is optional. Also this table is optional and # will be created automatically by NWBFile. The ``repetitions`` argument of -# the :py:meth:`~pynwb.file.NWBFile.add_icephys_condition` function again is +# the :py:meth:`~pynwb.file.NWBFile.add_icephys_experimental_condition` function again is # simply a list of ints with the indices of the correspondingto rows in the # :py:class:`~pynwb.icephys.RepetitionsTable`. diff --git a/docs/gallery/general/add_remove_containers.py b/docs/gallery/general/add_remove_containers.py index 80c5cb032..86aa373b2 100644 --- a/docs/gallery/general/add_remove_containers.py +++ b/docs/gallery/general/add_remove_containers.py @@ -70,33 +70,15 @@ # file path, and it is not possible to remove objects from an NWB file. You can use the # :py:meth:`NWBHDF5IO.export ` method, detailed below, to modify an NWB file in these ways. # -# .. warning:: -# -# NWB datasets that have been written to disk are read as :py:class:`h5py.Dataset ` objects. -# Directly modifying the data in these :py:class:`h5py.Dataset ` objects immediately -# modifies the data on disk -# (the :py:meth:`NWBHDF5IO.write ` method does not need to be called and the -# :py:class:`~pynwb.NWBHDF5IO` instance does not need to be closed). Directly modifying datasets in this way -# can lead to files that do not validate or cannot be opened, so exercise caution when using this method. -# Note: only chunked datasets or datasets with ``maxshape`` set can be resized. -# See the `h5py chunked storage documentation `_ -# for more details. - -############################################################################### -# .. note:: -# -# It is not possible to modify the attributes (fields) of an NWB container in memory. - -############################################################################### # Exporting a written NWB file to a new file path -# --------------------------------------------------- +# ----------------------------------------------- # Use the :py:meth:`NWBHDF5IO.export ` method to read data from an existing NWB file, # modify the data, and write the modified data to a new file path. Modifications to the data can be additions or # removals of objects, such as :py:class:`~pynwb.base.TimeSeries` objects. This is especially useful if you -# have raw data and processed data in the same NWB file and you want to create a new NWB file with all of the -# contents of the original file except for the raw data for sharing with collaborators. +# have raw data and processed data in the same NWB file and you want to create a new NWB file with all the contents of +# the original file except for the raw data for sharing with collaborators. # -# To remove existing containers, use the :py:class:`~hdmf.utils.LabelledDict.pop` method on any +# To remove existing containers, use the :py:meth:`~hdmf.utils.LabelledDict.pop` method on any # :py:class:`~hdmf.utils.LabelledDict` object, such as ``NWBFile.acquisition``, ``NWBFile.processing``, # ``NWBFile.analysis``, ``NWBFile.processing``, ``NWBFile.scratch``, ``NWBFile.devices``, ``NWBFile.stimulus``, # ``NWBFile.stimulus_template``, ``NWBFile.electrode_groups``, ``NWBFile.imaging_planes``, @@ -200,7 +182,7 @@ export_io.export(src_io=read_io, nwbfile=read_nwbfile) ############################################################################### -# More information about export -# --------------------------------- # For more information about the export functionality, see :ref:`export` # and the PyNWB documentation for :py:meth:`NWBHDF5IO.export `. +# +# For more information about editing a file in place, see :ref:`editing`. diff --git a/docs/gallery/general/object_id.py b/docs/gallery/general/object_id.py index 206142715..a4de45625 100644 --- a/docs/gallery/general/object_id.py +++ b/docs/gallery/general/object_id.py @@ -10,7 +10,7 @@ unique and used widely across computing platforms as if they are unique. The object ID of an NWB container object can be accessed using the -:py:meth:`~hdmf.container.AbstractContainer.object_id` method. +:py:attr:`~hdmf.container.AbstractContainer.object_id` method. .. _UUID: https://en.wikipedia.org/wiki/Universally_unique_identifier diff --git a/docs/gallery/general/plot_file.py b/docs/gallery/general/plot_file.py index beead22f6..154991877 100644 --- a/docs/gallery/general/plot_file.py +++ b/docs/gallery/general/plot_file.py @@ -24,7 +24,7 @@ * :ref:`modules_overview`, i.e., objects for storing and grouping analyses, and * experiment metadata and other metadata related to data provenance. -The following sections describe the :py:class:`~pynwb.base.TimeSeries` and :py:class:`~pynwb.base.ProcessingModules` +The following sections describe the :py:class:`~pynwb.base.TimeSeries` and :py:class:`~pynwb.base.ProcessingModule` classes in further detail. .. _timeseries_overview: @@ -569,7 +569,7 @@ #################### # :py:class:`~hdmf.common.table.DynamicTable` and its subclasses can be converted to a pandas -# :py:class:`~pandas.DataFrame` for convenient analysis using :py:meth:`.DynamicTable.to_dataframe`. +# :py:class:`~pandas.DataFrame` for convenient analysis using :py:meth:`~hdmf.common.table.DynamicTable.to_dataframe`. nwbfile.trials.to_dataframe() diff --git a/docs/gallery/general/plot_timeintervals.py b/docs/gallery/general/plot_timeintervals.py index a04a400c5..4069fd4a4 100644 --- a/docs/gallery/general/plot_timeintervals.py +++ b/docs/gallery/general/plot_timeintervals.py @@ -9,16 +9,14 @@ :py:class:`~pynwb.epoch.TimeIntervals` type. The :py:class:`~pynwb.epoch.TimeIntervals` type is a :py:class:`~hdmf.common.table.DynamicTable` with the following columns: -1. :py:meth:`~pynwb.epoch.TimeIntervals.start_time` and :py:meth:`~pynwb.epoch.TimeIntervals.stop_time` - describe the start and stop times of intervals as floating point offsets in seconds relative to the - :py:meth:`~pynwb.file.NWBFile.timestamps_reference_time` of the file. In addition, -2. :py:class:`~pynwb.epoch.TimeIntervals.tags` is an optional, indexed column used to associate user-defined string - tags with intervals (0 or more tags per time interval) -3. :py:class:`~pynwb.epoch.TimeIntervals.timeseries` is an optional, indexed - :py:class:`~pynwb.base.TimeSeriesReferenceVectorData` column to map intervals directly to ranges in select, - relevant :py:class:`~pynwb.base.TimeSeries` (0 or more per time interval) +1. ``start_time`` and ``stop_time`` describe the start and stop times of intervals as floating point offsets in seconds + relative to the :py:meth:`~pynwb.file.NWBFile.timestamps_reference_time` of the file. In addition, +2. ``tags`` is an optional, indexed column used to associate user-defined string tags with intervals (0 or more tags per + time interval) +3. ``timeseries`` is an optional, indexed :py:class:`~pynwb.base.TimeSeriesReferenceVectorData` column to map intervals + directly to ranges in select, relevant :py:class:`~pynwb.base.TimeSeries` (0 or more per time interval) 4. as a :py:class:`~hdmf.common.table.DynamicTable` user may add additional columns to - :py:meth:`~pynwb.epoch.TimeIntervals` via :py:class:`~hdmf.common.table.DynamicTable.add_column` + :py:meth:`~pynwb.epoch.TimeIntervals` via :py:meth:`~hdmf.common.table.DynamicTable.add_column` .. hint:: :py:meth:`~pynwb.epoch.TimeIntervals` is intended for storing general annotations of time ranges. @@ -84,12 +82,10 @@ # ^^^^^^ # # Trials can be added to an NWB file using the methods :py:meth:`~pynwb.file.NWBFile.add_trial` -# By default, NWBFile only requires trial :py:meth:`~pynwb.file.NWBFile.add_trial.start_time` -# and :py:meth:`~pynwb.file.NWBFile.add_trial.end_time`. The :py:meth:`~pynwb.file.NWBFile.add_trial.tags` -# and :py:meth:`~pynwb.file.NWBFile.add_trial.timeseries` are optional. For -# :py:meth:`~pynwb.file.NWBFile.add_trial.timeseries` we only need to supply the :py:class:`~pynwb.base.TimeSeries`. +# By default, NWBFile only requires trial ``start_time`` and ``stop_time``. The ``tags`` and ``timeseries`` are +# optional. For ``timeseries`` we only need to supply the :py:class:`~pynwb.base.TimeSeries`. # PyNWB automatically calculates the corresponding index range (described by ``idx_start`` and ``count``) for -# the supplied :py:class:`~pynwb.base.TimeSeries based on the given ``start_time`` and ``stop_time`` and +# the supplied :py:class:`~pynwb.base.TimeSeries` based on the given ``start_time`` and ``stop_time`` and # the :py:meth:`~pynwb.base.TimeSeries.timestamps` (or :py:class:`~pynwb.base.TimeSeries.starting_time` # and :py:meth:`~pynwb.base.TimeSeries.rate`) of the given :py:class:`~pynwb.base.TimeSeries`. # @@ -199,7 +195,7 @@ # # To define custom, experiment-specific :py:class:`~pynwb.epoch.TimeIntervals` we can add them # either: 1) when creating the :py:class:`~pynwb.file.NWBFile` by defining the -# :py:meth:`~pynwb.file.NWBFile.__init__.intervals` constructor argument or 2) via the +# ``intervals`` constructor argument or 2) via the # :py:meth:`~pynwb.file.NWBFile.add_time_intervals` or :py:meth:`~pynwb.file.NWBFile.create_time_intervals` # after the :py:class:`~pynwb.file.NWBFile` has been created. # @@ -286,9 +282,9 @@ # Adding TimeSeries references to other tables # -------------------------------------------- # -# Since :py:class:`~pynwb.base.TimeSeriesReferenceVectorData` is a regular :py:class:`~hdmf.common.table.VectoData` +# Since :py:class:`~pynwb.base.TimeSeriesReferenceVectorData` is a regular :py:class:`~hdmf.common.table.VectorData` # type, we can use it to add references to intervals in :py:class:`~pynwb.base.TimeSeries` to any -# :py:class:`~hdmf.common.table.DynamicTable`. In the :py:class:`~pynwb.icephys.IntracellularRecordingTable`, e.g., +# :py:class:`~hdmf.common.table.DynamicTable`. In the :py:class:`~pynwb.icephys.IntracellularRecordingsTable`, e.g., # it is used to reference the recording of the stimulus and response associated with a particular intracellular # electrophysiology recording. # diff --git a/docs/make.bat b/docs/make.bat index dcafe003d..0db0fd778 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -183,7 +183,7 @@ if "%1" == "changes" ( ) if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + %SPHINXBUILD% -W -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck if errorlevel 1 exit /b 1 echo. echo.Link check complete; look for any errors in the above output ^ diff --git a/docs/source/conf.py b/docs/source/conf.py index 143d9d2c6..6b101c23c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -148,6 +148,9 @@ def __call__(self, filename): 'fsspec': ("https://filesystem-spec.readthedocs.io/en/latest/", None), 'nwbwidgets': ("https://nwb-widgets.readthedocs.io/en/latest/", None), 'nwb-overview': ("https://nwb-overview.readthedocs.io/en/latest/", None), + 'zarr': ("https://zarr.readthedocs.io/en/stable/", None), + 'hdmf-zarr': ("https://hdmf-zarr.readthedocs.io/en/latest/", None), + 'numcodecs': ("https://numcodecs.readthedocs.io/en/latest/", None), } extlinks = { @@ -159,8 +162,13 @@ def __call__(self, filename): 'hdmf-docs': ('https://hdmf.readthedocs.io/en/stable/%s', '%s'), 'dandi': ('https://www.dandiarchive.org/%s', '%s'), "nwbinspector": ("https://nwbinspector.readthedocs.io/en/dev/%s", "%s"), + 'hdmf-zarr': ('https://hdmf-zarr.readthedocs.io/en/latest/%s', '%s'), } +nitpicky = True +nitpick_ignore = [('py:class', 'Intracomm'), + ('py:class', 'BaseStorageSpec')] + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -176,7 +184,7 @@ def __call__(self, filename): # General information about the project. project = u'PyNWB' -copyright = u'2017-2023, Neurodata Without Borders' +copyright = u'2017-2024, Neurodata Without Borders' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/docs/source/figures/gallery_thumbnail_plot_nwbzarrio.png b/docs/source/figures/gallery_thumbnail_plot_nwbzarrio.png new file mode 100644 index 000000000..8926a47ff Binary files /dev/null and b/docs/source/figures/gallery_thumbnail_plot_nwbzarrio.png differ diff --git a/docs/source/overview_citing.rst b/docs/source/overview_citing.rst index bc72e017c..8fda20363 100644 --- a/docs/source/overview_citing.rst +++ b/docs/source/overview_citing.rst @@ -35,7 +35,7 @@ If you use PyNWB in your research, please use the following citation: Using RRID ---------- -* ResourceID: `SCR_017452 `_ +* ResourceID: `SCR_017452 `_ * Proper Citation: **(PyNWB, RRID:SCR_017452)** diff --git a/environment-ros3.yml b/environment-ros3.yml index c84b4c090..07838258e 100644 --- a/environment-ros3.yml +++ b/environment-ros3.yml @@ -6,7 +6,7 @@ channels: dependencies: - python==3.11 - h5py==3.8.0 - - hdmf==3.5.4 + - hdmf==3.12.2 - matplotlib==3.7.1 - numpy==1.24.2 - pandas==2.0.0 diff --git a/license.txt b/license.txt index 7e54d7dbd..07d29b4fe 100644 --- a/license.txt +++ b/license.txt @@ -1,4 +1,4 @@ -“pynwb” Copyright (c) 2017-2023, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. +“pynwb” Copyright (c) 2017-2024, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/requirements-doc.txt b/requirements-doc.txt index 2050f4439..c37aee646 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -12,3 +12,4 @@ dataframe_image # used to render large dataframe as image in the sphinx galler lxml # used by dataframe_image when using the matplotlib backend hdf5plugin dandi>=0.46.6 +hdmf-zarr diff --git a/requirements-min.txt b/requirements-min.txt index 8f52348f1..0e8bde429 100644 --- a/requirements-min.txt +++ b/requirements-min.txt @@ -1,6 +1,6 @@ # minimum versions of package dependencies for installing PyNWB h5py==2.10 # support for selection of datasets with list of indices added in 2.10 -hdmf==3.9.0 +hdmf==3.12.2 numpy==1.18 pandas==1.1.5 python-dateutil==2.7.3 diff --git a/requirements.txt b/requirements.txt index 2ad7b813e..ad5d748bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # pinned dependencies to reproduce an entire development environment to use PyNWB -h5py==3.8.0 -hdmf==3.9.0 -numpy==1.24.2 -pandas==2.0.0 +h5py==3.10.0 +hdmf==3.12.2 +numpy==1.26.1 +pandas==2.1.2 python-dateutil==2.8.2 -setuptools==65.5.1 +setuptools==65.5.1 \ No newline at end of file diff --git a/setup.py b/setup.py index 90aebf55f..2a9ecb19e 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ reqs = [ 'h5py>=2.10', - 'hdmf>=3.9.0', + 'hdmf>=3.12.2', 'numpy>=1.16', 'pandas>=1.1.5', 'python-dateutil>=2.7.3', diff --git a/src/pynwb/__init__.py b/src/pynwb/__init__.py index 6e3b3104f..5e29caede 100644 --- a/src/pynwb/__init__.py +++ b/src/pynwb/__init__.py @@ -147,15 +147,19 @@ def _dec(cls): _dec(container_cls) -def get_nwbfile_version(h5py_file: h5py.File): +@docval({'name': 'h5py_file', 'type': h5py.File, 'doc': 'An NWB file'}, rtype=tuple, + is_method=False,) +def get_nwbfile_version(**kwargs): """ Get the NWB version of the file if it is an NWB file. - :returns: Tuple consisting of: 1) the original version string as stored in the file and - 2) a tuple with the parsed components of the version string, consisting of integers - and strings, e.g., (2, 5, 1, beta). (None, None) will be returned if the file is not a valid NWB file - or the nwb_version is missing, e.g., in the case when no data has been written to the file yet. + + :Returns: Tuple consisting of: 1) the + original version string as stored in the file and 2) a tuple with the parsed components of the version string, + consisting of integers and strings, e.g., (2, 5, 1, beta). (None, None) will be returned if the file is not a + valid NWB file or the nwb_version is missing, e.g., in the case when no data has been written to the file yet. """ # Get the version string for the NWB file + h5py_file = getargs('h5py_file', kwargs) try: nwb_version_string = h5py_file.attrs['nwb_version'] # KeyError occurs when the file is empty (e.g., when creating a new file nothing has been written) @@ -251,7 +255,7 @@ def can_read(path: str): 'doc': 'a path to a namespace, a TypeMap, or a list consisting paths to namespaces and TypeMaps', 'default': None}, {'name': 'file', 'type': [h5py.File, 'S3File'], 'doc': 'a pre-existing h5py.File object', 'default': None}, - {'name': 'comm', 'type': "Intracomm", 'doc': 'the MPI communicator to use for parallel I/O', + {'name': 'comm', 'type': 'Intracomm', 'doc': 'the MPI communicator to use for parallel I/O', 'default': None}, {'name': 'driver', 'type': str, 'doc': 'driver for h5py to use when opening HDF5 file', 'default': None}, {'name': 'herd_path', 'type': str, 'doc': 'The path to the HERD', @@ -327,7 +331,8 @@ def read(self, **kwargs): {'name': 'nwbfile', 'type': 'NWBFile', 'doc': 'the NWBFile object to export. If None, then the entire contents of src_io will be exported', 'default': None}, - {'name': 'write_args', 'type': dict, 'doc': 'arguments to pass to :py:meth:`write_builder`', + {'name': 'write_args', 'type': dict, + 'doc': 'arguments to pass to :py:meth:`~hdmf.backends.io.HDMFIO.write_builder`', 'default': None}) def export(self, **kwargs): """ diff --git a/src/pynwb/base.py b/src/pynwb/base.py index 42f7b7ff3..e9093db52 100644 --- a/src/pynwb/base.py +++ b/src/pynwb/base.py @@ -287,6 +287,42 @@ def __get_links(self, links): def __add_link(self, links_key, link): self.fields.setdefault(links_key, list()).append(link) + def _generate_field_html(self, key, value, level, access_code): + def find_location_in_memory_nwbfile(current_location: str, neurodata_object) -> str: + """ + Method for determining the location of a neurodata object within an in-memory NWBFile object. Adapted from + neuroconv package. + """ + parent = neurodata_object.parent + if parent is None: + return neurodata_object.name + "/" + current_location + elif parent.name == 'root': + # Items in defined top-level places like acquisition, intervals, etc. do not act as 'containers' + # in that they do not set the `.parent` attribute; ask if object is in their in-memory dictionaries + # instead + for parent_field_name, parent_field_value in parent.fields.items(): + if isinstance(parent_field_value, dict) and neurodata_object.name in parent_field_value: + return parent_field_name + "/" + neurodata_object.name + "/" + current_location + return neurodata_object.name + "/" + current_location + return find_location_in_memory_nwbfile( + current_location=neurodata_object.name + "/" + current_location, neurodata_object=parent + ) + + # reassign value if linked timestamp or linked data to avoid recursion error + if key in ['timestamps', 'data'] and isinstance(value, TimeSeries): + path_to_linked_object = find_location_in_memory_nwbfile(key, value) + if key == 'timestamps': + value = value.timestamps + elif key == 'data': + value = value.data + key = f'{key} (link to {path_to_linked_object})' + + if key in ['timestamp_link', 'data_link']: + linked_key = 'timestamps' if key == 'timestamp_link' else 'data' + value = [find_location_in_memory_nwbfile(linked_key, v) for v in value] + + return super()._generate_field_html(key, value, level, access_code) + @property def time_unit(self): return self.__time_unit @@ -514,7 +550,7 @@ class TimeSeriesReferenceVectorData(VectorData): then this indicates an invalid link (in practice both ``idx_start`` and ``count`` must always either both be positive or both be negative). When selecting data via the :py:meth:`~pynwb.base.TimeSeriesReferenceVectorData.get` or - :py:meth:`~pynwb.base. TimeSeriesReferenceVectorData.__getitem__` + :py:meth:`~object.__getitem__` functions, ``(-1, -1, TimeSeries)`` values are replaced by the corresponding :py:class:`~pynwb.base.TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_NONE_TYPE` tuple to avoid exposing NWB storage internals to the user and simplifying the use of and checking @@ -527,11 +563,11 @@ class TimeSeriesReferenceVectorData(VectorData): TIME_SERIES_REFERENCE_TUPLE = TimeSeriesReference """Return type when calling :py:meth:`~pynwb.base.TimeSeriesReferenceVectorData.get` or - :py:meth:`~pynwb.base. TimeSeriesReferenceVectorData.__getitem__`.""" + :py:meth:`~object.__getitem__`""" TIME_SERIES_REFERENCE_NONE_TYPE = TIME_SERIES_REFERENCE_TUPLE(None, None, None) """Tuple used to represent None values when calling :py:meth:`~pynwb.base.TimeSeriesReferenceVectorData.get` or - :py:meth:`~pynwb.base. TimeSeriesReferenceVectorData.__getitem__`. See also + :py:meth:`~object.__getitem__`. See also :py:class:`~pynwb.base.TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE`""" @docval({'name': 'name', 'type': str, 'doc': 'the name of this VectorData', 'default': 'timeseries'}, diff --git a/src/pynwb/file.py b/src/pynwb/file.py index b473e571a..8bd7ff447 100644 --- a/src/pynwb/file.py +++ b/src/pynwb/file.py @@ -612,7 +612,7 @@ def __check_epochs(self): def add_epoch_column(self, **kwargs): """ Add a column to the epoch table. - See :py:meth:`~pynwb.core.TimeIntervals.add_column` for more details + See :py:meth:`~hdmf.common.table.DynamicTable.add_column` for more details """ self.__check_epochs() self.epoch_tags.update(kwargs.pop('tags', list())) @@ -783,7 +783,7 @@ def add_trial_column(self, **kwargs): def add_trial(self, **kwargs): """ Add a trial to the trial table. - See :py:meth:`~hdmf.common.table.DynamicTable.add_interval` for more details. + See :py:meth:`~pynwb.epoch.TimeIntervals.add_interval` for more details. Required fields are *start_time*, *stop_time*, and any columns that have been added (through calls to `add_trial_columns`). diff --git a/src/pynwb/icephys.py b/src/pynwb/icephys.py index 04382abbc..2de21d571 100644 --- a/src/pynwb/icephys.py +++ b/src/pynwb/icephys.py @@ -48,7 +48,7 @@ class IntracellularElectrode(NWBContainer): {'name': 'seal', 'type': str, 'doc': 'Information about seal used for recording.', 'default': None}, {'name': 'location', 'type': str, 'doc': 'Area, layer, comments on estimation, stereotaxis coordinates (if in vivo, etc).', 'default': None}, - {'name': 'resistance', 'type': str, 'doc': 'Electrode resistance, unit: Ohm.', 'default': None}, + {'name': 'resistance', 'type': str, 'doc': 'Electrode resistance, unit - Ohm.', 'default': None}, {'name': 'filtering', 'type': str, 'doc': 'Electrode specific filtering.', 'default': None}, {'name': 'initial_access_resistance', 'type': str, 'doc': 'Initial access resistance.', 'default': None}, {'name': 'cell_id', 'type': str, 'doc': 'Unique ID of cell.', 'default': None} @@ -164,11 +164,11 @@ class CurrentClampSeries(PatchClampSeries): 'capacitance_compensation') @docval(*get_docval(PatchClampSeries.__init__, 'name', 'data', 'electrode'), # required - {'name': 'gain', 'type': float, 'doc': 'Units: Volt/Volt'}, + {'name': 'gain', 'type': float, 'doc': 'Units - Volt/Volt'}, *get_docval(PatchClampSeries.__init__, 'stimulus_description'), - {'name': 'bias_current', 'type': float, 'doc': 'Unit: Amp', 'default': None}, - {'name': 'bridge_balance', 'type': float, 'doc': 'Unit: Ohm', 'default': None}, - {'name': 'capacitance_compensation', 'type': float, 'doc': 'Unit: Farad', 'default': None}, + {'name': 'bias_current', 'type': float, 'doc': 'Unit - Amp', 'default': None}, + {'name': 'bridge_balance', 'type': float, 'doc': 'Unit - Ohm', 'default': None}, + {'name': 'capacitance_compensation', 'type': float, 'doc': 'Unit - Farad', 'default': None}, *get_docval(PatchClampSeries.__init__, 'resolution', 'conversion', 'timestamps', 'starting_time', 'rate', 'comments', 'description', 'control', 'control_description', 'sweep_number', 'offset'), {'name': 'unit', 'type': str, 'doc': "The base unit of measurement (must be 'volts')", @@ -267,15 +267,15 @@ class VoltageClampSeries(PatchClampSeries): 'whole_cell_series_resistance_comp') @docval(*get_docval(PatchClampSeries.__init__, 'name', 'data', 'electrode'), # required - {'name': 'gain', 'type': float, 'doc': 'Units: Volt/Amp'}, # required + {'name': 'gain', 'type': float, 'doc': 'Units - Volt/Amp'}, # required *get_docval(PatchClampSeries.__init__, 'stimulus_description'), - {'name': 'capacitance_fast', 'type': float, 'doc': 'Unit: Farad', 'default': None}, - {'name': 'capacitance_slow', 'type': float, 'doc': 'Unit: Farad', 'default': None}, - {'name': 'resistance_comp_bandwidth', 'type': float, 'doc': 'Unit: Hz', 'default': None}, - {'name': 'resistance_comp_correction', 'type': float, 'doc': 'Unit: percent', 'default': None}, - {'name': 'resistance_comp_prediction', 'type': float, 'doc': 'Unit: percent', 'default': None}, - {'name': 'whole_cell_capacitance_comp', 'type': float, 'doc': 'Unit: Farad', 'default': None}, - {'name': 'whole_cell_series_resistance_comp', 'type': float, 'doc': 'Unit: Ohm', 'default': None}, + {'name': 'capacitance_fast', 'type': float, 'doc': 'Unit - Farad', 'default': None}, + {'name': 'capacitance_slow', 'type': float, 'doc': 'Unit - Farad', 'default': None}, + {'name': 'resistance_comp_bandwidth', 'type': float, 'doc': 'Unit - Hz', 'default': None}, + {'name': 'resistance_comp_correction', 'type': float, 'doc': 'Unit - percent', 'default': None}, + {'name': 'resistance_comp_prediction', 'type': float, 'doc': 'Unit - percent', 'default': None}, + {'name': 'whole_cell_capacitance_comp', 'type': float, 'doc': 'Unit - Farad', 'default': None}, + {'name': 'whole_cell_series_resistance_comp', 'type': float, 'doc': 'Unit - Ohm', 'default': None}, *get_docval(PatchClampSeries.__init__, 'resolution', 'conversion', 'timestamps', 'starting_time', 'rate', 'comments', 'description', 'control', 'control_description', 'sweep_number', 'offset'), {'name': 'unit', 'type': str, 'doc': "The base unit of measurement (must be 'amperes')", diff --git a/src/pynwb/image.py b/src/pynwb/image.py index 0295183b8..518ec8a8c 100644 --- a/src/pynwb/image.py +++ b/src/pynwb/image.py @@ -46,7 +46,7 @@ class ImageSeries(TimeSeries): 'is specified. If unit (and data) are not specified, then unit will be set to "unknown".'), 'default': None}, {'name': 'format', 'type': str, - 'doc': 'Format of image. Three types: 1) Image format; tiff, png, jpg, etc. 2) external 3) raw.', + 'doc': 'Format of image. Three types - 1) Image format; tiff, png, jpg, etc. 2) external 3) raw.', 'default': None}, {'name': 'external_file', 'type': ('array_data', 'data'), 'doc': 'Path or URL to one or more external file(s). Field only present if format=external. ' diff --git a/src/pynwb/io/base.py b/src/pynwb/io/base.py index 5b5aac48b..db9c259ef 100644 --- a/src/pynwb/io/base.py +++ b/src/pynwb/io/base.py @@ -73,6 +73,19 @@ def timestamps_carg(self, builder, manager): else: return tstamps_builder.data + @NWBContainerMapper.object_attr("data") + def data_attr(self, container, manager): + ret = container.fields.get('data') + if isinstance(ret, TimeSeries): + owner = ret + curr = owner.fields.get('data') + while isinstance(curr, TimeSeries): + owner = curr + curr = owner.fields.get('data') + data_builder = manager.build(owner) + ret = LinkBuilder(data_builder['data'], 'data') + return ret + @NWBContainerMapper.constructor_arg("data") def data_carg(self, builder, manager): # handle case where a TimeSeries is read and missing data @@ -105,7 +118,10 @@ def unit_carg(self, builder, manager): data_builder = manager.construct(target.parent) else: data_builder = target - unit_value = data_builder.attributes.get('unit') + if isinstance(data_builder, TimeSeries): # Data linked in another timeseries + unit_value = data_builder.unit + else: # DatasetBuilder owned by this timeseries + unit_value = data_builder.attributes.get('unit') if unit_value is None: return timeseries_cls.DEFAULT_UNIT return unit_value diff --git a/src/pynwb/io/utils.py b/src/pynwb/io/utils.py index 23b41b6b4..e607b9589 100644 --- a/src/pynwb/io/utils.py +++ b/src/pynwb/io/utils.py @@ -17,7 +17,7 @@ def get_nwb_version(builder: Builder, include_prerelease=False) -> Tuple[int, .. if include_prerelease=True, (2, 0, 0, "b") is returned; else, (2, 0, 0) is returned. :param builder: Any builder within an NWB file. - :type builder: Builder + :type builder: :py:class:`~hdmf.build.builders.Builder` :param include_prerelease: Whether to include prerelease information in the returned tuple. :type include_prerelease: bool :return: The version of the NWB file, as a tuple. diff --git a/src/pynwb/misc.py b/src/pynwb/misc.py index 4d977b4f2..14c2e08d1 100644 --- a/src/pynwb/misc.py +++ b/src/pynwb/misc.py @@ -265,7 +265,7 @@ class DecompositionSeries(TimeSeries): 'shape': (None, None, None)}, *get_docval(TimeSeries.__init__, 'description'), {'name': 'metric', 'type': str, # required - 'doc': "metric of analysis. recommended: 'phase', 'amplitude', 'power'"}, + 'doc': "metric of analysis. recommended - 'phase', 'amplitude', 'power'"}, {'name': 'unit', 'type': str, 'doc': 'SI unit of measurement', 'default': 'no unit'}, {'name': 'bands', 'type': DynamicTable, 'doc': 'a table for describing the frequency bands that the signal was decomposed into', 'default': None}, diff --git a/src/pynwb/ophys.py b/src/pynwb/ophys.py index e3d9f8f6d..bdff47592 100644 --- a/src/pynwb/ophys.py +++ b/src/pynwb/ophys.py @@ -57,15 +57,15 @@ class ImagingPlane(NWBContainer): 'doc': 'Rate images are acquired, in Hz. If the corresponding TimeSeries is present, the rate should be ' 'stored there instead.', 'default': None}, {'name': 'manifold', 'type': 'array_data', - 'doc': ('DEPRECATED: Physical position of each pixel. size=("height", "width", "xyz"). ' + 'doc': ('DEPRECATED - Physical position of each pixel. size=("height", "width", "xyz"). ' 'Deprecated in favor of origin_coords and grid_spacing.'), 'default': None}, {'name': 'conversion', 'type': float, - 'doc': ('DEPRECATED: Multiplier to get from stored values to specified unit (e.g., 1e-3 for millimeters) ' + 'doc': ('DEPRECATED - Multiplier to get from stored values to specified unit (e.g., 1e-3 for millimeters) ' 'Deprecated in favor of origin_coords and grid_spacing.'), 'default': 1.0}, {'name': 'unit', 'type': str, - 'doc': 'DEPRECATED: Base unit that coordinates are stored in (e.g., Meters). ' + 'doc': 'DEPRECATED - Base unit that coordinates are stored in (e.g., Meters). ' 'Deprecated in favor of origin_coords_unit and grid_spacing_unit.', 'default': 'meters'}, {'name': 'reference_frame', 'type': str, diff --git a/src/pynwb/retinotopy.py b/src/pynwb/retinotopy.py index a345177c0..fd32037e8 100644 --- a/src/pynwb/retinotopy.py +++ b/src/pynwb/retinotopy.py @@ -123,9 +123,9 @@ class ImagingRetinotopy(NWBDataInterface): 'Description should be something like ["altitude", "azimuth"] or ["radius", "theta"].'}, {'name': 'focal_depth_image', 'type': FocalDepthImage, 'doc': 'Gray-scale image taken with same settings/parameters (e.g., focal depth, wavelength) ' - 'as data collection. Array format: [rows][columns].'}, + 'as data collection. Array format - [rows][columns].'}, {'name': 'vasculature_image', 'type': RetinotopyImage, - 'doc': 'Gray-scale anatomical image of cortical surface. Array structure: [rows][columns].'}, + 'doc': 'Gray-scale anatomical image of cortical surface. Array structure - [rows][columns].'}, {'name': 'name', 'type': str, 'doc': 'the name of this container', 'default': 'ImagingRetinotopy'}) def __init__(self, **kwargs): axis_1_phase_map, axis_1_power_map, axis_2_phase_map, axis_2_power_map, axis_descriptions, \ diff --git a/src/pynwb/spec.py b/src/pynwb/spec.py index 94271118a..fe97b6eae 100644 --- a/src/pynwb/spec.py +++ b/src/pynwb/spec.py @@ -62,7 +62,7 @@ def neurodata_type_inc(self): class BaseStorageOverride: ''' This class is used for the purpose of overriding - BaseStorageSpec classmethods, without creating diamond + :py:class:`~hdmf.spec.spec.BaseStorageSpec` classmethods, without creating diamond inheritance hierarchies. ''' diff --git a/src/pynwb/testing/icephys_testutils.py b/src/pynwb/testing/icephys_testutils.py index 732f312e6..3de4619d4 100644 --- a/src/pynwb/testing/icephys_testutils.py +++ b/src/pynwb/testing/icephys_testutils.py @@ -60,10 +60,10 @@ def create_icephys_testfile(filename=None, add_custom_columns=True, randomize_da :param randomize_data: Randomize data values in the stimulus and response :type randomize_data: bool - :returns: ICEphysFile NWBFile object created for writing. NOTE: If filename is provided then + :returns: NWBFile object with icephys data created for writing. NOTE: If filename is provided then the file is written to disk, but the function does not read the file back. If you want to use the file from disk then you will need to read it with NWBHDF5IO. - :rtype: ICEphysFile + :rtype: NWBFile """ nwbfile = NWBFile( session_description='my first synthetic recording', diff --git a/src/pynwb/testing/mock/base.py b/src/pynwb/testing/mock/base.py index 45b95fc08..39617fe11 100644 --- a/src/pynwb/testing/mock/base.py +++ b/src/pynwb/testing/mock/base.py @@ -15,7 +15,7 @@ def mock_TimeSeries( conversion: float = 1.0, timestamps=None, starting_time: Optional[float] = None, - rate: Optional[float] = 10.0, + rate: Optional[float] = None, comments: str = "no comments", description: str = "no description", control=None, @@ -24,6 +24,8 @@ def mock_TimeSeries( nwbfile: Optional[NWBFile] = None, offset=0., ) -> TimeSeries: + if timestamps is None and rate is None: + rate = 10.0 # Hz time_series = TimeSeries( name=name or name_generator("TimeSeries"), data=data if data is not None else np.array([1, 2, 3, 4]), diff --git a/src/pynwb/testing/testh5io.py b/src/pynwb/testing/testh5io.py index b45407bfb..c7b3bfdcc 100644 --- a/src/pynwb/testing/testh5io.py +++ b/src/pynwb/testing/testh5io.py @@ -79,7 +79,7 @@ def test_roundtrip_export(self): self.assertIs(self.read_exported_nwbfile.objects[self.container.object_id], self.read_container) self.assertContainerEqual(self.read_container, self.container, ignore_hdmf_attrs=True) - def roundtripContainer(self, cache_spec=False): + def roundtripContainer(self, cache_spec=True): """Add the Container to an NWBFile, write it to file, read the file, and return the Container from the file. """ session_description = 'a file to test writing and reading a %s' % self.container_type @@ -116,7 +116,7 @@ def roundtripContainer(self, cache_spec=False): self.reader = None raise e - def roundtripExportContainer(self, cache_spec=False): + def roundtripExportContainer(self, cache_spec=True): """ Add the test Container to an NWBFile, write it to file, read the file, export the read NWBFile to another file, and return the test Container from the file @@ -163,18 +163,14 @@ def getContainer(self, nwbfile): def validate(self): """ Validate the created files """ if os.path.exists(self.filename): - with NWBHDF5IO(self.filename, mode='r') as io: - errors = pynwb_validate(io) - if errors: - for err in errors: - raise Exception(err) + errors, _ = pynwb_validate(paths=[self.filename]) + if errors: + raise Exception("\n".join(errors)) if os.path.exists(self.export_filename): - with NWBHDF5IO(self.filename, mode='r') as io: - errors = pynwb_validate(io) - if errors: - for err in errors: - raise Exception(err) + errors, _ = pynwb_validate(paths=[self.export_filename]) + if errors: + raise Exception("\n".join(errors)) class AcquisitionH5IOMixin(NWBH5IOMixin): @@ -294,7 +290,7 @@ def test_roundtrip_export(self): self.assertIs(self.read_exported_nwbfile.objects[self.container.object_id], self.read_container) self.assertContainerEqual(self.read_container, self.container, ignore_hdmf_attrs=True) - def roundtripContainer(self, cache_spec=False): + def roundtripContainer(self, cache_spec=True): """Write the file, validate the file, read the file, and return the Container from the file. """ @@ -325,7 +321,7 @@ def roundtripContainer(self, cache_spec=False): self.reader = None raise e - def roundtripExportContainer(self, cache_spec=False): + def roundtripExportContainer(self, cache_spec=True): """ Roundtrip the container, then export the read NWBFile to a new file, validate the files, and return the test Container from the file. @@ -366,13 +362,11 @@ def roundtripExportContainer(self, cache_spec=False): def validate(self): """Validate the created files.""" if os.path.exists(self.filename): - with NWBHDF5IO(self.filename, mode='r') as io: - errors = pynwb_validate(io) - if errors: - raise Exception("\n".join(errors)) + errors, _ = pynwb_validate(paths=[self.filename]) + if errors: + raise Exception("\n".join(errors)) if os.path.exists(self.export_filename): - with NWBHDF5IO(self.filename, mode='r') as io: - errors = pynwb_validate(io) - if errors: - raise Exception("\n".join(errors)) + errors, _ = pynwb_validate(paths=[self.export_filename]) + if errors: + raise Exception("\n".join(errors)) diff --git a/src/pynwb/validate.py b/src/pynwb/validate.py index 62aa41426..827249cbb 100644 --- a/src/pynwb/validate.py +++ b/src/pynwb/validate.py @@ -120,7 +120,11 @@ def _get_cached_namespaces_to_validate( is_method=False, ) def validate(**kwargs): - """Validate NWB file(s) against a namespace or its cached namespaces.""" + """Validate NWB file(s) against a namespace or its cached namespaces. + + NOTE: If an io object is provided and no namespace name is specified, then the file will be validated + against the core namespace, even if use_cached_namespaces is True. + """ from . import NWBHDF5IO # TODO: modularize to avoid circular import io, paths, use_cached_namespaces, namespace, verbose, driver = getargs( diff --git a/tests/back_compat/test_read.py b/tests/back_compat/test_read.py index 792d26e7a..16a119690 100644 --- a/tests/back_compat/test_read.py +++ b/tests/back_compat/test_read.py @@ -29,6 +29,12 @@ class TestReadOldVersions(TestCase): "- expected an array of shape '[None]', got non-array data 'one publication'")], '1.0.3_str_pub.nwb': [("root/general/related_publications (general/related_publications): incorrect shape " "- expected an array of shape '[None]', got non-array data 'one publication'")], + '1.5.1_timeseries_no_data.nwb': [("TimeSeries/data/data (acquisition/test_timeseries/data): argument missing")], + '1.5.1_timeseries_no_unit.nwb': [("TimeSeries/data/unit (acquisition/test_timeseries/data): argument missing")], + '1.5.1_imageseries_no_data.nwb': [("ImageSeries/data/data (acquisition/test_imageseries/data): " + "argument missing")], + '1.5.1_imageseries_no_unit.nwb': [("ImageSeries/data/unit (acquisition/test_imageseries/data): " + "argument missing")], } def get_io(self, path): diff --git a/tests/integration/hdf5/test_base.py b/tests/integration/hdf5/test_base.py index 1e855f3ce..60f8510ff 100644 --- a/tests/integration/hdf5/test_base.py +++ b/tests/integration/hdf5/test_base.py @@ -46,6 +46,27 @@ def test_timestamps_linking(self): tsb = nwbfile.acquisition['b'] self.assertIs(tsa.timestamps, tsb.timestamps) + def test_data_linking(self): + ''' Test that data get linked to in TimeSeries ''' + tsa = TimeSeries(name='a', data=np.linspace(0, 1, 1000), timestamps=np.arange(1000.), unit='m') + tsb = TimeSeries(name='b', data=tsa, timestamps=np.arange(1000.), unit='m') + tsc = TimeSeries(name='c', data=tsb, timestamps=np.arange(1000.), unit='m') + nwbfile = NWBFile(identifier='foo', + session_start_time=datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal()), + session_description='bar') + nwbfile.add_acquisition(tsa) + nwbfile.add_acquisition(tsb) + nwbfile.add_acquisition(tsc) + with NWBHDF5IO(self.path, 'w') as io: + io.write(nwbfile) + with NWBHDF5IO(self.path, 'r') as io: + nwbfile = io.read() + tsa = nwbfile.acquisition['a'] + tsb = nwbfile.acquisition['b'] + tsc = nwbfile.acquisition['c'] + self.assertIs(tsa.data, tsb.data) + self.assertIs(tsa.data, tsc.data) + class TestImagesIO(AcquisitionH5IOMixin, TestCase): diff --git a/tests/integration/hdf5/test_misc.py b/tests/integration/hdf5/test_misc.py index 6afd7971e..cd9ab1706 100644 --- a/tests/integration/hdf5/test_misc.py +++ b/tests/integration/hdf5/test_misc.py @@ -109,20 +109,36 @@ class TestDecompositionSeriesIO(NWBH5IOMixin, TestCase): def setUpContainer(self): """ Return the test DecompositionSeries to read/write """ - self.timeseries = TimeSeries(name='dummy timeseries', description='desc', - data=np.ones((3, 3)), unit='flibs', - timestamps=np.ones((3,))) - bands = DynamicTable(name='bands', description='band info for LFPSpectralAnalysis', columns=[ - VectorData(name='band_name', description='name of bands', data=['alpha', 'beta', 'gamma']), - VectorData(name='band_limits', description='low and high cutoffs in Hz', data=np.ones((3, 2))) - ]) - spec_anal = DecompositionSeries(name='LFPSpectralAnalysis', - description='my description', - data=np.ones((3, 3, 3)), - timestamps=np.ones((3,)), - source_timeseries=self.timeseries, - metric='amplitude', - bands=bands) + self.timeseries = TimeSeries( + name='dummy timeseries', + description='desc', + data=np.ones((3, 3)), + unit='flibs', + timestamps=np.ones((3,)), + ) + bands = DynamicTable( + name='bands', + description='band info for LFPSpectralAnalysis', + columns=[ + VectorData(name='band_name', description='name of bands', data=['alpha', 'beta', 'gamma']), + VectorData(name='band_limits', description='low and high cutoffs in Hz', data=np.ones((3, 2))), + VectorData(name='band_mean', description='mean gaussian filters in Hz', data=np.ones((3,))), + VectorData( + name='band_stdev', + description='standard deviation of gaussian filters in Hz', + data=np.ones((3,)) + ), + ], + ) + spec_anal = DecompositionSeries( + name='LFPSpectralAnalysis', + description='my description', + data=np.ones((3, 3, 3)), + timestamps=np.ones((3,)), + source_timeseries=self.timeseries, + metric='amplitude', + bands=bands, + ) return spec_anal @@ -144,27 +160,48 @@ def make_electrode_table(self): """ Make an electrode table, electrode group, and device """ self.table = get_electrode_table() self.dev1 = Device(name='dev1') - self.group = ElectrodeGroup(name='tetrode1', - description='tetrode description', - location='tetrode location', - device=self.dev1) - for i in range(4): + self.group = ElectrodeGroup( + name='tetrode1', + description='tetrode description', + location='tetrode location', + device=self.dev1 + ) + for _ in range(4): self.table.add_row(location='CA1', group=self.group, group_name='tetrode1') def setUpContainer(self): """ Return the test ElectricalSeries to read/write """ self.make_electrode_table(self) - region = DynamicTableRegion(name='source_channels', - data=[0, 2], - description='the first and third electrodes', - table=self.table) + region = DynamicTableRegion( + name='source_channels', + data=[0, 2], + description='the first and third electrodes', + table=self.table + ) data = np.random.randn(100, 2, 30) timestamps = np.arange(100)/100 - ds = DecompositionSeries(name='test_DS', - data=data, - source_channels=region, - timestamps=timestamps, - metric='amplitude') + bands = DynamicTable( + name='bands', + description='band info for LFPSpectralAnalysis', + columns=[ + VectorData(name='band_name', description='name of bands', data=['alpha', 'beta', 'gamma']), + VectorData(name='band_limits', description='low and high cutoffs in Hz', data=np.ones((3, 2))), + VectorData(name='band_mean', description='mean gaussian filters in Hz', data=np.ones((3,))), + VectorData( + name='band_stdev', + description='standard deviation of gaussian filters in Hz', + data=np.ones((3,)) + ), + ], + ) + ds = DecompositionSeries( + name='test_DS', + data=data, + source_channels=region, + timestamps=timestamps, + metric='amplitude', + bands=bands, + ) return ds def addContainer(self, nwbfile): diff --git a/tests/integration/hdf5/test_modular_storage.py b/tests/integration/hdf5/test_modular_storage.py index 6c86fc615..fba5d02db 100644 --- a/tests/integration/hdf5/test_modular_storage.py +++ b/tests/integration/hdf5/test_modular_storage.py @@ -14,29 +14,35 @@ class TestTimeSeriesModular(TestCase): def setUp(self): - self.start_time = datetime(1971, 1, 1, 12, tzinfo=tzutc()) + # File paths + self.data_filename = os.path.join(os.getcwd(), 'test_time_series_modular_data.nwb') + self.link_filename = os.path.join(os.getcwd(), 'test_time_series_modular_link.nwb') + # Make the data container file write + self.start_time = datetime(1971, 1, 1, 12, tzinfo=tzutc()) self.data = np.arange(2000).reshape((1000, 2)) self.timestamps = np.linspace(0, 1, 1000) - + # The container before roundtrip self.container = TimeSeries( name='data_ts', unit='V', data=self.data, timestamps=self.timestamps ) + self.data_read_io = None # HDF5IO used for reading the main data file + self.read_data_nwbfile = None # The NWBFile read after roundtrip + self.read_data_container = None # self.container after roundtrip - self.data_filename = os.path.join(os.getcwd(), 'test_time_series_modular_data.nwb') - self.link_filename = os.path.join(os.getcwd(), 'test_time_series_modular_link.nwb') - - self.read_container = None - self.link_read_io = None - self.data_read_io = None + # Variables for the second file which links the main data file + self.link_container = None # The container with the links before write + self.read_link_container = None # The container with the links after roundtrip + self.read_link_nwbfile = None # The NWBFile container containing link_container after roundtrip + self.link_read_io = None # HDF5IO use for reading the read_link_nwbfile def tearDown(self): - if self.read_container: - self.read_container.data.file.close() - self.read_container.timestamps.file.close() + if self.read_link_container: + self.read_link_container.data.file.close() + self.read_link_container.timestamps.file.close() if self.link_read_io: self.link_read_io.close() if self.data_read_io: @@ -64,49 +70,83 @@ def roundtripContainer(self): data_write_io.write(data_file) # read data file - with HDF5IO(self.data_filename, 'r', manager=get_manager()) as self.data_read_io: - data_file_obt = self.data_read_io.read() - - # write "link file" with timeseries.data that is an external link to the timeseries in "data file" - # also link timeseries.timestamps.data to the timeseries.timestamps in "data file" - with HDF5IO(self.link_filename, 'w', manager=get_manager()) as link_write_io: - link_file = NWBFile( - session_description='a test file', - identifier='link_file', - session_start_time=self.start_time - ) - self.link_container = TimeSeries( - name='test_mod_ts', - unit='V', - data=data_file_obt.get_acquisition('data_ts'), # test direct link - timestamps=H5DataIO( - data=data_file_obt.get_acquisition('data_ts').timestamps, - link_data=True # test with setting link data - ) - ) - link_file.add_acquisition(self.link_container) - link_write_io.write(link_file) + self.data_read_io = HDF5IO(self.data_filename, 'r', manager=get_manager()) + self.read_data_nwbfile = self.data_read_io.read() + self.read_data_container = self.read_data_nwbfile.get_acquisition('data_ts') - # note that self.link_container contains a link to a dataset that is now closed + # write "link file" with timeseries.data that is an external link to the timeseries in "data file" + # also link timeseries.timestamps.data to the timeseries.timestamps in "data file" + with HDF5IO(self.link_filename, 'w', manager=get_manager()) as link_write_io: + link_file = NWBFile( + session_description='a test file', + identifier='link_file', + session_start_time=self.start_time + ) + self.link_container = TimeSeries( + name='test_mod_ts', + unit='V', + data=H5DataIO( + data=self.read_data_container.data, + link_data=True # test with setting link data + ), + timestamps=H5DataIO( + data=self.read_data_container.timestamps, + link_data=True # test with setting link data + ) + ) + link_file.add_acquisition(self.link_container) + link_write_io.write(link_file) # read the link file self.link_read_io = HDF5IO(self.link_filename, 'r', manager=get_manager()) - self.read_nwbfile = self.link_read_io.read() - return self.getContainer(self.read_nwbfile) + self.read_link_nwbfile = self.link_read_io.read() + return self.getContainer(self.read_link_nwbfile) def test_roundtrip(self): - self.read_container = self.roundtripContainer() - - # make sure we get a completely new object - self.assertIsNotNone(str(self.container)) # added as a test to make sure printing works + # Roundtrip the container + self.read_link_container = self.roundtripContainer() + + # 1. Make sure our containers are set correctly for the test + # 1.1: Make sure the container we read is not identical to the container we used for writing + self.assertNotEqual(id(self.link_container), id(self.read_link_container)) + self.assertNotEqual(id(self.container), id(self.read_data_container)) + # 1.2: Make sure the container we read is indeed the correct container we should use for testing + self.assertIs(self.read_link_nwbfile.objects[self.link_container.object_id], self.read_link_container) + self.assertIs(self.read_data_nwbfile.objects[self.container.object_id], self.read_data_container) + # 1.3: Make sure the object_ids of the container we wrote and read are the same + self.assertEqual(self.read_link_container.object_id, self.link_container.object_id) + self.assertEqual(self.read_data_container.object_id, self.container.object_id) + # 1.4: Make sure the object_ids between the source data and link data container are different + self.assertNotEqual(self.read_link_container.object_id, self.read_data_container.object_id) + + # Test that printing works for the source data and linked data container + self.assertIsNotNone(str(self.container)) + self.assertIsNotNone(str(self.read_data_container)) self.assertIsNotNone(str(self.link_container)) - self.assertIsNotNone(str(self.read_container)) - self.assertFalse(self.link_container.timestamps.valid) - self.assertTrue(self.read_container.timestamps.id.valid) - self.assertNotEqual(id(self.link_container), id(self.read_container)) - self.assertIs(self.read_nwbfile.objects[self.link_container.object_id], self.read_container) - self.assertContainerEqual(self.read_container, self.container, ignore_name=True, ignore_hdmf_attrs=True) - self.assertEqual(self.read_container.object_id, self.link_container.object_id) + self.assertIsNotNone(str(self.read_link_container)) + + # Test that timestamps and data are valid after write + self.assertTrue(self.read_link_container.timestamps.id.valid) + self.assertTrue(self.read_link_container.data.id.valid) + self.assertTrue(self.read_data_container.timestamps.id.valid) + self.assertTrue(self.read_data_container.data.id.valid) + + # Make sure the data in the read data container and linked data container match the original container + self.assertContainerEqual(self.read_link_container, self.container, ignore_name=True, ignore_hdmf_attrs=True) + self.assertContainerEqual(self.read_data_container, self.container, ignore_name=True, ignore_hdmf_attrs=True) + + # Make sure the timestamps and data are linked correctly. I.e., the filename of the h5py dataset should + # match between the data file and the file with links even-though they have been read from different files + self.assertEqual( + self.read_data_container.data.file.filename, # Path where the source data is stored + self.read_link_container.data.file.filename # Path where the linked h5py dataset points to + ) + self.assertEqual( + self.read_data_container.timestamps.file.filename, # Path where the source data is stored + self.read_link_container.timestamps.file.filename # Path where the linked h5py dataset points to + ) + + # validate both the source data and linked data file via the pynwb validator self.validate() def test_link_root(self): @@ -159,3 +199,55 @@ def validate(self): def getContainer(self, nwbfile): return nwbfile.get_acquisition('test_mod_ts') + + +class TestTimeSeriesModularLinkViaTimeSeries(TestTimeSeriesModular): + """ + Same as TestTimeSeriesModular but creating links by setting TimeSeries.data + and TimeSeries.timestamps to the other TimeSeries on construction, rather than + using H5DataIO. + """ + def setUp(self): + super().setUp() + self.skipTest("This behavior is currently broken. See issue .") + + def roundtripContainer(self): + # create and write data file + data_file = NWBFile( + session_description='a test file', + identifier='data_file', + session_start_time=self.start_time + ) + data_file.add_acquisition(self.container) + + with HDF5IO(self.data_filename, 'w', manager=get_manager()) as data_write_io: + data_write_io.write(data_file) + + # read data file + self.data_read_io = HDF5IO(self.data_filename, 'r', manager=get_manager()) + self.read_data_nwbfile = self.data_read_io.read() + self.read_data_container = self.read_data_nwbfile.get_acquisition('data_ts') + + # write "link file" with timeseries.data that is an external link to the timeseries in "data file" + # also link timeseries.timestamps.data to the timeseries.timestamps in "data file" + with HDF5IO(self.link_filename, 'w', manager=get_manager()) as link_write_io: + link_file = NWBFile( + session_description='a test file', + identifier='link_file', + session_start_time=self.start_time + ) + self.link_container = TimeSeries( + name='test_mod_ts', + unit='V', + data=self.read_data_container, # <--- This is the main difference to TestTimeSeriesModular + timestamps=self.read_data_container # <--- This is the main difference to TestTimeSeriesModular + ) + link_file.add_acquisition(self.link_container) + link_write_io.write(link_file) + + # note that self.link_container contains a link to a dataset that is now closed + + # read the link file + self.link_read_io = HDF5IO(self.link_filename, 'r', manager=get_manager()) + self.read_link_nwbfile = self.link_read_io.read() + return self.getContainer(self.read_link_nwbfile) diff --git a/tests/integration/ros3/test_ros3.py b/tests/integration/ros3/test_ros3.py index c2f7b562d..95a891760 100644 --- a/tests/integration/ros3/test_ros3.py +++ b/tests/integration/ros3/test_ros3.py @@ -4,6 +4,7 @@ from pynwb.testing import TestCase import urllib.request import h5py +import warnings class TestRos3Streaming(TestCase): @@ -28,16 +29,28 @@ def setUp(self): def test_read(self): s3_path = 'https://dandiarchive.s3.amazonaws.com/ros3test.nwb' - with NWBHDF5IO(s3_path, mode='r', driver='ros3') as io: - nwbfile = io.read() - test_data = nwbfile.acquisition['ts_name'].data[:] - self.assertEqual(len(test_data), 3) + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=r"Ignoring cached namespace .*", + category=UserWarning, + ) + with NWBHDF5IO(s3_path, mode='r', driver='ros3') as io: + nwbfile = io.read() + test_data = nwbfile.acquisition['ts_name'].data[:] + self.assertEqual(len(test_data), 3) def test_dandi_read(self): - with NWBHDF5IO(path=self.s3_test_path, mode='r', driver='ros3') as io: - nwbfile = io.read() - test_data = nwbfile.acquisition['TestData'].data[:] - self.assertEqual(len(test_data), 3) + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=r"Ignoring cached namespace .*", + category=UserWarning, + ) + with NWBHDF5IO(path=self.s3_test_path, mode='r', driver='ros3') as io: + nwbfile = io.read() + test_data = nwbfile.acquisition['TestData'].data[:] + self.assertEqual(len(test_data), 3) def test_dandi_get_cached_namespaces(self): expected_namespaces = ["core"] diff --git a/tests/unit/test_base.py b/tests/unit/test_base.py index ad4ce6739..f8c08f68f 100644 --- a/tests/unit/test_base.py +++ b/tests/unit/test_base.py @@ -464,6 +464,20 @@ def test_file_with_starting_time_and_timestamps_in_construct_mode(self): timestamps=[1, 2, 3, 4, 5] ) + def test_repr_html(self): + """ Test that html representation of linked timestamp data will occur as expected and will not cause a + RecursionError + """ + data1 = [0, 1, 2, 3] + data2 = [4, 5, 6, 7] + timestamps = [0.0, 0.1, 0.2, 0.3] + ts1 = TimeSeries(name="test_ts1", data=data1, unit="grams", timestamps=timestamps) + ts2 = TimeSeries(name="test_ts2", data=data2, unit="grams", timestamps=ts1) + pm = ProcessingModule(name="processing", description="a test processing module") + pm.add(ts1) + pm.add(ts2) + self.assertIn('(link to processing/test_ts1/timestamps)', pm._repr_html_()) + class TestImage(TestCase): def test_init(self): diff --git a/tests/unit/test_misc.py b/tests/unit/test_misc.py index 99e0d6f87..9350d1d2e 100644 --- a/tests/unit/test_misc.py +++ b/tests/unit/test_misc.py @@ -33,7 +33,13 @@ def test_init(self): timestamps=[1., 2., 3.]) bands = DynamicTable(name='bands', description='band info for LFPSpectralAnalysis', columns=[ VectorData(name='band_name', description='name of bands', data=['alpha', 'beta', 'gamma']), - VectorData(name='band_limits', description='low and high cutoffs in Hz', data=np.ones((3, 2))) + VectorData(name='band_limits', description='low and high cutoffs in Hz', data=np.ones((3, 2))), + VectorData(name='band_mean', description='mean gaussian filters in Hz', data=np.ones((3,))), + VectorData( + name='band_stdev', + description='standard deviation of gaussian filters in Hz', + data=np.ones((3,)) + ), ]) spec_anal = DecompositionSeries(name='LFPSpectralAnalysis', description='my description', @@ -49,6 +55,8 @@ def test_init(self): np.testing.assert_equal(spec_anal.timestamps, [1., 2., 3.]) self.assertEqual(spec_anal.bands['band_name'].data, ['alpha', 'beta', 'gamma']) np.testing.assert_equal(spec_anal.bands['band_limits'].data, np.ones((3, 2))) + np.testing.assert_equal(spec_anal.bands['band_mean'].data, np.ones((3,))) + np.testing.assert_equal(spec_anal.bands['band_stdev'].data, np.ones((3,))) self.assertEqual(spec_anal.source_timeseries, timeseries) self.assertEqual(spec_anal.metric, 'amplitude') diff --git a/tests/unit/test_mock.py b/tests/unit/test_mock.py index d24e47551..72174f018 100644 --- a/tests/unit/test_mock.py +++ b/tests/unit/test_mock.py @@ -90,6 +90,17 @@ def test_mock(mock_function): mock_function() +def test_mock_TimeSeries_w_timestamps(): + ts = mock_TimeSeries(timestamps=[0, 1, 2, 3]) + assert ts.timestamps is not None + assert len(ts.timestamps) == 4 + + +def test_mock_TimeSeries_w_no_time(): + ts = mock_TimeSeries() + assert ts.rate == 10.0 + + @pytest.mark.parametrize("mock_function", mock_functions) def test_mock_write(mock_function, tmp_path): if mock_function is mock_NWBFile: