diff --git a/CHANGELOG.md b/CHANGELOG.md index 06066042..798d984f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,17 +3,29 @@ ## 0.1.7 (Upcoming) ### Bugs -* Use path relative to the current Zarr file in the definition of links and references to avoid breaking - links when moving Zarr files @oruebel [#46](https://github.com/hdmf-dev/hdmf-zarr/pull/46) -* Fix bugs in requirements defined in setup.py @oruebel [#46](https://github.com/hdmf-dev/hdmf-zarr/pull/46) -* Update dateset used in conversion tutorial, which caused warnings @oruebel [#56](https://github.com/hdmf-dev/hdmf-zarr/pull/56) +* Updated the storage of links/references to use paths relative to the current Zarr file to avoid breaking + links/reference when moving Zarr files @oruebel [#46](https://github.com/hdmf-dev/hdmf-zarr/pull/46) +* Fixed bugs in requirements defined in setup.py @oruebel [#46](https://github.com/hdmf-dev/hdmf-zarr/pull/46) +* Fixed bug regarding Sphinx external links @mavaylon1 [#53](https://github.com/hdmf-dev/hdmf-zarr/pull/53) +* Updated gallery tests to use test_gallery.py and necessary package dependcies + @mavaylon1 [#53](https://github.com/hdmf-dev/hdmf-zarr/pull/53) +* Updated dateset used in conversion tutorial, which caused warnings + @oruebel [#56](https://github.com/hdmf-dev/hdmf-zarr/pull/56) ### Docs -* Add tutorial illustrating how to create a new NWB file with NWBZarrIO @oruebel [#46](https://github.com/hdmf-dev/hdmf-zarr/pull/46) -* Add docs for describing the mapping of HDMF schema to Zarr storage @oruebel [#48](https://github.com/hdmf-dev/hdmf-zarr/pull/48) +* Added tutorial illustrating how to create a new NWB file with NWBZarrIO + @oruebel [#46](https://github.com/hdmf-dev/hdmf-zarr/pull/46) +* Added docs for describing the mapping of HDMF schema to Zarr storage + @oruebel [#48](https://github.com/hdmf-dev/hdmf-zarr/pull/48) +* Added ``docs/gallery/resources`` for storing local files used by the tutorial galleries + @oruebel [#61](https://github.com/hdmf-dev/hdmf-zarr/pull/61) +* Removed dependency on ``dandi`` library for data download in the conversion tutorial by storing the NWB files as + local resources @oruebel [#61](https://github.com/hdmf-dev/hdmf-zarr/pull/61) ## 0.1.0 ### New features -- Created new optional Zarr-based I/O backend for writing files using Zarr's `zarr.store.DirectoryStore` backend, including support for iterative write, chunking, compression, simple and compound data types, links, object references, namespace and spec I/O. +* Created new optional Zarr-based I/O backend for writing files using Zarr's `zarr.store.DirectoryStore` backend, + including support for iterative write, chunking, compression, simple and compound data types, links, object + references, namespace and spec I/O. diff --git a/docs/gallery/plot_convert_nwb_hdf5.py b/docs/gallery/plot_convert_nwb_hdf5.py index 4bfbc9bd..c26006f1 100644 --- a/docs/gallery/plot_convert_nwb_hdf5.py +++ b/docs/gallery/plot_convert_nwb_hdf5.py @@ -5,7 +5,8 @@ This tutorial illustrates how to convert data between HDF5 and Zarr using a Neurodata Without Borders (NWB) file from the DANDI data archive as an example. In this tutorial we will convert our example file from HDF5 to Zarr and then -back again to HDF5. +back again to HDF5. The NWB standard is defined using :hdmf-docs:`HDMF <>` and uses the +:py:class:`~ hdmf.backends.hdf5.h5tools.HDF5IO` HDF5 backend from HDMF for storage. """ @@ -13,29 +14,36 @@ # Setup # ----- # -# We first **download a small NWB file** from the DANDI neurophysiology data archive as an example. -# The NWB standard is defined using HDMF and uses the :py:class:`~ hdmf.backends.hdf5.h5tools.HDF5IO` -# HDF5 backend from HDMF for storage. +# Here we use a small NWB file from the DANDI neurophysiology data archive from +# `DANDIset 000009 <https://dandiarchive.org/dandiset/000009/0.220126.1903>`_ as an example. +# To download the file directly from DANDI we can use: +# +# .. code-block:: python +# :linenos: +# +# from dandi.dandiapi import DandiAPIClient +# dandiset_id = "000009" +# filepath = "sub-anm00239123/sub-anm00239123_ses-20170627T093549_ecephys+ogen.nwb" # ~0.5MB file +# with DandiAPIClient() as client: +# asset = client.get_dandiset(dandiset_id, 'draft').get_asset_by_path(filepath) +# s3_path = asset.get_content_url(follow_redirects=1, strip_query=True) +# filename = os.path.basename(asset.path) +# asset.download(filename) +# +# We here use a local copy of a small file from this DANDIset as an example: +# # sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnail_plot_convert_nwb.png' import os import shutil -from dandi.dandiapi import DandiAPIClient - -dandiset_id = "000009" -filepath = "sub-anm00239123/sub-anm00239123_ses-20170627T093549_ecephys+ogen.nwb" # ~0.5MB file -with DandiAPIClient() as client: - asset = client.get_dandiset(dandiset_id, 'draft').get_asset_by_path(filepath) - s3_path = asset.get_content_url(follow_redirects=1, strip_query=True) - filename = os.path.basename(asset.path) -asset.download(filename) - -############################################################################### -# Next we define the names of the files to generate as part of this tutorial and clean up any -# data from previous executions of this tutorial. -zarr_filename = "test_zarr_" + filename + ".zarr" -hdf_filename = "test_hdf5_" + filename +# Input file to convert +basedir = "resources" +filename = os.path.join(basedir, "sub_anm00239123_ses_20170627T093549_ecephys_and_ogen.nwb") +# Zarr file to generate for converting from HDF5 to Zarr +zarr_filename = "test_zarr_" + os.path.basename(filename) + ".zarr" +# HDF5 file to generate for converting from Zarr to HDF5 +hdf_filename = "test_hdf5_" + os.path.basename(filename) # Delete our converted HDF5 and Zarr file from previous runs of this notebook for fname in [zarr_filename, hdf_filename]: @@ -91,6 +99,7 @@ # :pynwb-docs:`Trials <tutorials/general/plot_timeintervals.html>` table. zf.trials.to_dataframe()[['start_time', 'stop_time', 'type', 'photo_stim_type']] +zr.close() ############################################################################### # Convert the Zarr file back to HDF5 diff --git a/docs/gallery/plot_nwb_zarrio.py b/docs/gallery/plot_nwb_zarrio.py index e86a00a7..8065b262 100644 --- a/docs/gallery/plot_nwb_zarrio.py +++ b/docs/gallery/plot_nwb_zarrio.py @@ -135,24 +135,12 @@ with NWBZarrIO(path=path, mode="w") as io: io.write(nwbfile) -############################################################################### -# Test opening the file -# --------------------- -with NWBZarrIO(path=path, mode="r") as io: - infile = io.read() - ############################################################################### # Test opening with the absolute path instead # ------------------------------------------- -with NWBZarrIO(path=absolute_path, mode="r") as io: - infile = io.read() - -############################################################################### -# Test changing the current directory -# ------------------------------------ - -import os -os.chdir(os.path.abspath(os.path.join(os.getcwd(), "../"))) - +# +# The main reason for using the ``absolute_path`` here is for testing purposes +# to ensure links and references work as expected. Otherwise, using the +# relative ``path`` here instead is fine. with NWBZarrIO(path=absolute_path, mode="r") as io: infile = io.read() diff --git a/docs/gallery/plot_zarr_dataset_io.py b/docs/gallery/plot_zarr_dataset_io.py index 7f13434b..753ac2f5 100644 --- a/docs/gallery/plot_zarr_dataset_io.py +++ b/docs/gallery/plot_zarr_dataset_io.py @@ -110,3 +110,7 @@ (c.name, str(c.data.chunks), str(c.data.compressor))) + +############################################################################### +# +zarr_io.close() diff --git a/docs/gallery/plot_zarr_io.py b/docs/gallery/plot_zarr_io.py index 89a9dd01..0d4965fe 100644 --- a/docs/gallery/plot_zarr_io.py +++ b/docs/gallery/plot_zarr_io.py @@ -86,6 +86,9 @@ intable = zarr_io.read() intable.to_dataframe() +############################################################################### +# +zarr_io.close() ############################################################################### # Converting to/from HDF5 using ``export`` diff --git a/docs/gallery/resources/README.rst b/docs/gallery/resources/README.rst new file mode 100644 index 00000000..1a762d3a --- /dev/null +++ b/docs/gallery/resources/README.rst @@ -0,0 +1,24 @@ +Resources +========= + +sub_anm00239123_ses_20170627T093549_ecephys_and_ogen.nwb +-------------------------------------------------------- + +This NWB file was downloaded from `DANDIset 000009 <https://dandiarchive.org/dandiset/000009/0.220126.1903>`_ +The file was modified to replace ``:`` characters used in the name of the ``ElectrodeGroup`` called ``ADunit: 32`` in +``'general/extracellular_ephys/`` to ``'ADunit_32'``. The dataset ``general/extracellular_ephys/electrodes/group_name`` +as part of the electrodes table was updated accordingly to list the appropriate group name. This is to avoid issues +on Windows file systems that do not support ``:`` as part of folder names. The asses can be downloaded from DANDI via: + +.. code-block:: python + :linenos: + + from dandi.dandiapi import DandiAPIClient + dandiset_id = "000009" + filepath = "sub-anm00239123/sub-anm00239123_ses-20170627T093549_ecephys+ogen.nwb" # ~0.5MB file + with DandiAPIClient() as client: + asset = client.get_dandiset(dandiset_id, 'draft').get_asset_by_path(filepath) + s3_path = asset.get_content_url(follow_redirects=1, strip_query=True) + filename = os.path.basename(asset.path) + asset.download(filename) + diff --git a/docs/gallery/resources/sub_anm00239123_ses_20170627T093549_ecephys_and_ogen.nwb b/docs/gallery/resources/sub_anm00239123_ses_20170627T093549_ecephys_and_ogen.nwb new file mode 100644 index 00000000..a7b4c87b Binary files /dev/null and b/docs/gallery/resources/sub_anm00239123_ses_20170627T093549_ecephys_and_ogen.nwb differ diff --git a/docs/source/conf.py b/docs/source/conf.py index 2dfedaef..de6ff8cc 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -77,8 +77,8 @@ intersphinx_mapping = { 'python': ('https://docs.python.org/3.10', None), 'numpy': ('https://numpy.org/doc/stable/', None), - 'scipy': ('https://docs.scipy.org/doc/scipy/reference', None), - 'matplotlib': ('https://matplotlib.org', None), + 'scipy': ('https://docs.scipy.org/doc/scipy/', None), + 'matplotlib': ('https://matplotlib.org/stable/', None), 'h5py': ('https://docs.h5py.org/en/latest/', None), 'pandas': ('https://pandas.pydata.org/pandas-docs/stable/', None), 'hdmf': ('https://hdmf.readthedocs.io/en/stable/', None), diff --git a/docs/source/overview.rst b/docs/source/overview.rst index eb58bdc0..d8ca65d7 100644 --- a/docs/source/overview.rst +++ b/docs/source/overview.rst @@ -30,3 +30,4 @@ Known Limitations - Currently the :py:class:`~hdmf_zarr.backend.ZarrIO` backend uses Zarr's :py:class:`~zarr.storage.DirectoryStore` only. Other `Zarr stores <https://zarr.readthedocs.io/en/stable/api/storage.html>`_ could be added but will require proper treatment of links and references for those backends as links are not supported in Zarr (see `zarr-python issues #389 <https://github.com/zarr-developers/zarr-python/issues/389>`_. - Exporting of HDF5 files with external links is not yet fully implemented/tested. (see `hdmf-zarr issue #49 <https://github.com/hdmf-dev/hdmf-zarr/issues/49>`_. - Object references are currently always resolved on read (as are links) rather than being loaded lazily (see `hdmf-zarr issue #50 <https://github.com/hdmf-dev/hdmf-zarr/issues/50>`_. +- Special characters (e.g., ``:``, ``<``, ``>``, ``"``, ``/``, ``\``, ``|``, ``?``, or ``*``) may not be supported by all file systems (e.g., on Windows) and as such should not be used as part of the names of Datasets or Groups as Zarr needs to create folders on the filesystem for these objects. diff --git a/requirements-doc.txt b/requirements-doc.txt index d307b741..32a790cf 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -1,6 +1,6 @@ # dependencies to generate the documentation for HDMF -sphinx -sphinx_rtd_theme -sphinx-gallery -dandi matplotlib +sphinx>=4 # improved support for docutils>=0.17 +sphinx_rtd_theme>=1 # <1 does not work with docutils>=0.17 +sphinx-gallery +sphinx-copybutton diff --git a/setup.py b/setup.py index 13f45c46..dd9f4800 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ 'numcodecs>=0.9.1', 'pynwb>=2.0.0', 'setuptools', - 'numpy>=1.23; python_version >"3.7"' + 'numpy>=1.22, <1.24; python_version>"3.7"' ] print(reqs) diff --git a/test.py b/test.py index 195ae477..a5a55603 100644 --- a/test.py +++ b/test.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # NOTE this script is currently used in CI *only* to test the sphinx gallery examples using python test.py -e - import warnings import re import argparse @@ -67,6 +66,8 @@ def _import_from_file(script): def run_example_tests(): global TOTAL, FAILURES, ERRORS logging.info('running example tests') + + # get list of example scripts examples_scripts = list() for root, dirs, files in os.walk(os.path.join(os.path.dirname(__file__), "docs", "gallery")): for f in files: @@ -74,12 +75,18 @@ def run_example_tests(): examples_scripts.append(os.path.join(root, f)) TOTAL += len(examples_scripts) + curr_dir = os.getcwd() for script in examples_scripts: + os.chdir(curr_dir) # Reset the working directory + script_abs = os.path.abspath(script) # Determine the full path of the script + # Set the working dir to be relative to the script to allow the use of relative file paths in the scripts + os.chdir(os.path.dirname(script_abs)) try: logging.info("Executing %s" % script) ws = list() with warnings.catch_warnings(record=True) as tmp: - _import_from_file(script) + # Import/run the example gallery + _import_from_file(script_abs) for w in tmp: # ignore RunTimeWarnings about importing if isinstance(w.message, RuntimeWarning) and not warning_re.match(str(w.message)): ws.append(w) @@ -89,6 +96,8 @@ def run_example_tests(): print(traceback.format_exc()) FAILURES += 1 ERRORS += 1 + # Make sure to reset the working directory at the end + os.chdir(curr_dir) def main(): diff --git a/test_gallery.py b/test_gallery.py new file mode 100644 index 00000000..731a9f0d --- /dev/null +++ b/test_gallery.py @@ -0,0 +1,151 @@ +"""Test that the Sphinx Gallery files run without warnings or errors. +See tox.ini for usage. +""" + +import importlib.util +import logging +import os +import os.path +import sys +import traceback +import warnings + +TOTAL = 0 +FAILURES = 0 +ERRORS = 0 + + +def _import_from_file(script): + modname = os.path.basename(script) + spec = importlib.util.spec_from_file_location(os.path.basename(script), script) + module = importlib.util.module_from_spec(spec) + sys.modules[modname] = module + spec.loader.exec_module(module) + + +_numpy_warning_re = ( + "numpy.ufunc size changed, may indicate binary incompatibility. Expected 216, got 192" +) + +_distutils_warning_re = ( + "distutils Version classes are deprecated. Use packaging.version instead." +) + +_experimental_warning_re = ( + "The ZarrIO backend is experimental. It is under active development. " + "The ZarrIO backend may change any time and backward compatibility is not guaranteed." +) + +_user_warning_transpose = ( + "ElectricalSeries 'ElectricalSeries': The second dimension of data does not match the " + "length of electrodes. Your data may be transposed." +) + +_deprication_warning_map = ( + 'Classes in map.py should be imported from hdmf.build. Importing from hdmf.build.map will be removed ' + 'in HDMF 3.0.' +) + +_deprication_warning_fmt_docval_args = ( + "fmt_docval_args will be deprecated in a future version of HDMF. Instead of using fmt_docval_args, " + "call the function directly with the kwargs. Please note that fmt_docval_args " + "removes all arguments not accepted by the function's docval, so if you are passing kwargs that " + "includes extra arguments and the function's docval does not allow extra arguments (allow_extra=True " + "is set), then you will need to pop the extra arguments out of kwargs before calling the function." +) + +_deprication_warning_call_docval_func = ( + "call the function directly with the kwargs. Please note that call_docval_func " + "removes all arguments not accepted by the function's docval, so if you are passing kwargs that " + "includes extra arguments and the function's docval does not allow extra arguments (allow_extra=True " + "is set), then you will need to pop the extra arguments out of kwargs before calling the function." +) + + +def run_gallery_tests(): + global TOTAL, FAILURES, ERRORS + logging.info("Testing execution of Sphinx Gallery files") + + # get all python file names in docs/gallery + gallery_file_names = list() + for root, _, files in os.walk( + os.path.join(os.path.dirname(__file__), "docs", "gallery") + ): + for f in files: + if f.endswith(".py"): + gallery_file_names.append(os.path.join(root, f)) + + warnings.simplefilter("error") + + TOTAL += len(gallery_file_names) + curr_dir = os.getcwd() + for script in gallery_file_names: + logging.info("Executing %s" % script) + os.chdir(curr_dir) # Reset the working directory + script_abs = os.path.abspath(script) # Determine the full path of the script + # Set the working dir to be relative to the script to allow the use of relative file paths in the scripts + os.chdir(os.path.dirname(script_abs)) + try: + with warnings.catch_warnings(record=True): + warnings.filterwarnings( + "ignore", message=_deprication_warning_map, category=DeprecationWarning + ) + warnings.filterwarnings( + "ignore", message=_deprication_warning_fmt_docval_args, category=PendingDeprecationWarning + ) + warnings.filterwarnings( + "ignore", message=_deprication_warning_call_docval_func, category=PendingDeprecationWarning + ) + warnings.filterwarnings( + "ignore", message=_experimental_warning_re, category=UserWarning + ) + warnings.filterwarnings( + "ignore", message=_user_warning_transpose, category=UserWarning + ) + warnings.filterwarnings( + # this warning is triggered from pandas when HDMF is installed with the minimum requirements + "ignore", message=_distutils_warning_re, category=DeprecationWarning + ) + warnings.filterwarnings( + # this warning is triggered when some numpy extension code in an upstream package was compiled + # against a different version of numpy than the one installed + "ignore", message=_numpy_warning_re, category=RuntimeWarning + ) + _import_from_file(script_abs) + except Exception: + print(traceback.format_exc()) + FAILURES += 1 + ERRORS += 1 + # Make sure to reset the working directory at the end + os.chdir(curr_dir) + + +def main(): + logging_format = ( + "======================================================================\n" + "%(asctime)s - %(levelname)s - %(message)s" + ) + logging.basicConfig(format=logging_format, level=logging.INFO) + + run_gallery_tests() + + final_message = "Ran %s tests" % TOTAL + exitcode = 0 + if ERRORS > 0 or FAILURES > 0: + exitcode = 1 + _list = list() + if ERRORS > 0: + _list.append("errors=%d" % ERRORS) + if FAILURES > 0: + _list.append("failures=%d" % FAILURES) + final_message = "%s - FAILED (%s)" % (final_message, ",".join(_list)) + else: + final_message = "%s - OK" % final_message + + logging.info(final_message) + + return exitcode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tox.ini b/tox.ini index 36e731b4..f8e7798e 100644 --- a/tox.ini +++ b/tox.ini @@ -133,7 +133,7 @@ deps = -rrequirements-doc.txt commands = - python test.py --example + python test_gallery.py [testenv:gallery-py37] basepython = python3.7