From 40d2ab7895b32da8b4066122fcb612862cb782e5 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Thu, 19 Dec 2024 13:50:39 -0800 Subject: [PATCH] Prepare 0.10.0 release (#251) * Fix badges * Remove references to region refs in docs * Update pyproject.toml python range and dev status * Update changelog date * Fix asset typo and change caps on dandiset * Remove invalid sphinx_rtd_theme option * Update conda workflows to address warning * Incorporate fix from #238 --- .github/workflows/run_all_tests.yml | 3 ++- .github/workflows/run_tests.yml | 3 ++- CHANGELOG.md | 2 +- README.rst | 22 ++++++++-------- docs/gallery/plot_convert_nwb_hdf5.py | 4 +-- docs/gallery/resources/README.rst | 4 +-- docs/source/conf.py | 1 - docs/source/overview.rst | 1 - docs/source/storage.rst | 36 +-------------------------- pyproject.toml | 5 ++-- src/hdmf_zarr/backend.py | 5 +++- 11 files changed, 28 insertions(+), 58 deletions(-) diff --git a/.github/workflows/run_all_tests.yml b/.github/workflows/run_all_tests.yml index 129cdad8..f264ac5a 100644 --- a/.github/workflows/run_all_tests.yml +++ b/.github/workflows/run_all_tests.yml @@ -155,12 +155,13 @@ jobs: auto-update-conda: true python-version: ${{ matrix.python-ver }} channels: conda-forge + conda-remove-defaults: "true" - name: Install build dependencies run: | conda config --set always_yes yes --set changeps1 no conda info - conda install -c conda-forge tox + conda install tox - name: Conda reporting run: | diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 0fd692c5..63dc5a58 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -139,12 +139,13 @@ jobs: auto-update-conda: true python-version: ${{ matrix.python-ver }} channels: conda-forge + conda-remove-defaults: "true" - name: Install build dependencies run: | conda config --set always_yes yes --set changeps1 no conda info - conda install -c conda-forge tox + conda install tox - name: Conda reporting run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 453504a3..cc7b69ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # HDMF-ZARR Changelog -## 0.10.0 (Upcoming) +## 0.10.0 (December 18, 2024) ### Enhancements * Added initial refactor of export, supporting references and internal/external links from Zarr to Zarr. This will introduce breaking changes that could lead to existing exported files to be invalid. This update removes '.' as the object default file source. @mavaylon1 [#194](https://github.com/hdmf-dev/hdmf-zarr/pull/194) diff --git a/README.rst b/README.rst index 5af642ff..40e13295 100644 --- a/README.rst +++ b/README.rst @@ -22,30 +22,30 @@ CI / Health Status .. image:: https://codecov.io/gh/hdmf-dev/hdmf-zarr/branch/dev/graph/badge.svg :target: https://codecov.io/gh/hdmf-dev/hdmf-zarr -.. image:: https://github.com/hdmf-dev/hdmf-zarr/workflows/Run%20code%20coverage/badge.svg +.. image:: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/run_coverage.yml/badge.svg :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/run_coverage.yml -.. image:: https://github.com/hdmf-dev/hdmf-zarr/workflows/Run%20tests/badge.svg +.. image:: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/run_tests.yml/badge.svg :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/run_tests.yml -.. image:: https://github.com/hdmf-dev/hdmf-zarr/workflows/Run%20all%20tests/badge.svg +.. image:: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/run_all_tests.yml/badge.svg :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/run_all_tests.yml -.. image:: https://github.com/hdmf-dev/hdmf-zarr/workflows/Check%20Sphinx%20external%20links/badge.svg +.. image:: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/check_external_links.yml/badge.svg :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/check_external_links.yml -.. image:: https://github.com/hdmf-dev/hdmf-zarr/workflows/Deploy%20release/badge.svg +.. image:: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/deploy_release.yml/badge.svg :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/deploy_release.yml -.. image:: https://github.com/hdmf-dev/hdmf-zarr/workflows/black/badge.svg - :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/black.yml +.. image:: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/ruff.yml/badge.svg + :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/ruff.yml -.. image:: https://github.com/hdmf-dev/hdmf-zarr/workflows/Run%20style%20check/badge.svg - :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/run_flake8.yml +.. image:: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/codespell.yml/badge.svg + :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/codespell.yml +.. image:: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/HDMF_dev.yaml/badge.svg + :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/HDMF_dev.yaml - -Citing hdmf-zarr ---------------- If you use HDMF or hdmf_zarr in your research, please use the following citation: diff --git a/docs/gallery/plot_convert_nwb_hdf5.py b/docs/gallery/plot_convert_nwb_hdf5.py index 6565afe2..e98126ac 100644 --- a/docs/gallery/plot_convert_nwb_hdf5.py +++ b/docs/gallery/plot_convert_nwb_hdf5.py @@ -14,7 +14,7 @@ # ----- # # Here we use a small NWB file from the DANDI neurophysiology data archive from -# `DANDIset 000009 `_ as an example. +# `Dandiset 000009 `_ as an example. # To download the file directly from DANDI we can use: # # .. code-block:: python @@ -29,7 +29,7 @@ # filename = os.path.basename(asset.path) # asset.download(filename) # -# We here use a local copy of a small file from this DANDIset as an example: +# We here use a local copy of a small file from this Dandiset as an example: # sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnail_plot_convert_nwb.png' diff --git a/docs/gallery/resources/README.rst b/docs/gallery/resources/README.rst index 1a762d3a..f7b595de 100644 --- a/docs/gallery/resources/README.rst +++ b/docs/gallery/resources/README.rst @@ -4,11 +4,11 @@ Resources sub_anm00239123_ses_20170627T093549_ecephys_and_ogen.nwb -------------------------------------------------------- -This NWB file was downloaded from `DANDIset 000009 `_ +This NWB file was downloaded from `Dandiset 000009 `_ The file was modified to replace ``:`` characters used in the name of the ``ElectrodeGroup`` called ``ADunit: 32`` in ``'general/extracellular_ephys/`` to ``'ADunit_32'``. The dataset ``general/extracellular_ephys/electrodes/group_name`` as part of the electrodes table was updated accordingly to list the appropriate group name. This is to avoid issues -on Windows file systems that do not support ``:`` as part of folder names. The asses can be downloaded from DANDI via: +on Windows file systems that do not support ``:`` as part of folder names. The asset can be downloaded from DANDI via: .. code-block:: python :linenos: diff --git a/docs/source/conf.py b/docs/source/conf.py index 39669770..d6c4f697 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -119,7 +119,6 @@ html_theme_options = { 'logo_only': False, # Only show the hdmf-zarr logo without the documentation title - 'display_version': True, 'prev_next_buttons_location': 'bottom', # Show previous/next button at the bottom 'style_external_links': True, # Add marker to indicate external links 'vcs_pageview_mode': '', diff --git a/docs/source/overview.rst b/docs/source/overview.rst index 73ad6f65..690bc8f5 100644 --- a/docs/source/overview.rst +++ b/docs/source/overview.rst @@ -29,7 +29,6 @@ Supported features Known Limitations ----------------- -- Support for region references is not yet implemented. See also :ref:`sec-zarr-storage-references-region` for details. - The Zarr backend is currently experimental and may still change. - Attributes are stored as JSON documents in Zarr (using the DirectoryStore). As such, all attributes must be JSON serializable. The :py:class:`~hdmf_zarr.backend.ZarrIO` backend attempts to cast types to JSON serializable types as much as possible. - Currently the :py:class:`~hdmf_zarr.backend.ZarrIO` backend supports Zarr's directory-based stores :py:class:`~zarr.storage.DirectoryStore`, :py:class:`~zarr.storage.NestedDirectoryStore`, and :py:class:`~zarr.storage.TempStore`. Other `Zarr stores `_ could be added but will require proper treatment of links and references for those backends as links are not supported in Zarr (see `zarr-python issues #389 `_. diff --git a/docs/source/storage.rst b/docs/source/storage.rst index 864fe420..e9386003 100644 --- a/docs/source/storage.rst +++ b/docs/source/storage.rst @@ -260,7 +260,7 @@ Storing object references in Attributes Object references are stored in a attributes as dicts with the following keys: * ``zarr_dtype`` : Indicating the data type for the attribute. For object references - ``zarr_dtype`` is set to ``"object"`` (or ``"region"`` for :ref:`sec-zarr-storage-references-region`) + ``zarr_dtype`` is set to ``"object"`` * ``value``: The value of the object references, i.e., here the py:class:`~hdmf_zarr.utils.ZarrReference` dictionary with the ``source``, ``path``, ``object_id``, and ``source_object_id`` keys defining the object reference, with the definition of the keys being the same as @@ -280,36 +280,6 @@ For example in NWB, the attribute ``ElectricalSeries.electrodes.table`` would be "zarr_dtype": "object" } -.. _sec-zarr-storage-references-region: - -Region references ------------------ - -Region references are similar to object references, but instead of references other Datasets or Groups, -region references link to subsets of another Dataset. To identify region references, the reserved attribute -``zarr_dtype`` is set to ``'region'`` (see also :ref:`sec-zarr-storage-attributes-reserved`). In addition -to the ``source`` and ``path``, the py:class:`~hdmf_zarr.utils.ZarrReference` object will also need to -store the definition of the ``region`` that is being referenced, e.g., a slice or list on point indices. - -.. warning:: - - Region references are not yet fully implemented in :py:class:`~hdmf_zarr.backend.ZarrIO`. - To implement region references will require updating: - 1) py:class:`~hdmf_zarr.utils.ZarrReference` to add a ``region`` key to support storing - the selection for the region, - 2) :py:meth:`~hdmf_zarr.backend.ZarrIO._create_ref` to support passing in the region definition to - be added to the py:class:`~hdmf_zarr.utils.ZarrReference`, - 3) :py:meth:`~hdmf_zarr.backend.ZarrIO.write_dataset` already partially implements the required - logic for creating region references by checking for :py:class:`hdmf.build.RegionBuilder` inputs - but will likely need updates as well - 4) :py:meth:`~hdmf_zarr.backend.ZarrIO.__read_dataset` to support reading region references, - which may also require updates to :py:meth:`~hdmf_zarr.backend.ZarrIO.__parse_ref` and - :py:meth:`~hdmf_zarr.backend.ZarrIO.__resolve_ref`, and - 5) and possibly other parts of :py:class:`~hdmf_zarr.backend.ZarrIO`. - 6) The py:class:`~hdmf_zarr.zarr_utils.ContainerZarrRegionDataset` and - py:class:`~hdmf_zarr.zarr_utils.ContainerZarrRegionDataset` classes will also need to be finalized - to support region references. - .. _sec-zarr-storage-dtypes: @@ -357,10 +327,6 @@ The mappings of data types is as follows | * "reference" | dataset. See | | | * "object" | :ref:`sec-zarr-storage-references` | | +--------------------------+------------------------------------+----------------+ - | * region | Reference to a region | | - | | of another dataset. See | | - | | :ref:sec-zarr-storage-references` | | - +--------------------------+------------------------------------+----------------+ | * compound dtype | Compound data type | | +--------------------------+------------------------------------+----------------+ | * "isodatetime" | ASCII ISO8061 datetime string. | variable | diff --git a/pyproject.toml b/pyproject.toml index 65be5d24..bde404eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ name = "hdmf_zarr" authors = [ { name="Oliver Ruebel", email="oruebel@lbl.gov" }, { name="Matthew Avaylon", email="mavaylon@lbl.gov" }, + { name="Ryan Ly", email="rly@lbl.gov" }, ] description = "A package defining a Zarr I/O backend for HDMF" readme = "README.rst" @@ -18,8 +19,9 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "License :: OSI Approved :: BSD License", - "Development Status :: 5 - Production/Stable", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Intended Audience :: Science/Research", "Operating System :: Microsoft :: Windows", @@ -130,7 +132,6 @@ line-length = 120 [tool.ruff.lint.per-file-ignores] "docs/gallery/*" = ["E402", "T201"] -"src/*/__init__.py" = ["F401"] "test_gallery.py" = ["T201"] [tool.ruff.lint.mccabe] diff --git a/src/hdmf_zarr/backend.py b/src/hdmf_zarr/backend.py index 5477e533..b6394488 100644 --- a/src/hdmf_zarr/backend.py +++ b/src/hdmf_zarr/backend.py @@ -803,7 +803,10 @@ def resolve_ref(self, zarr_ref): source_file = str(zarr_ref["source"]) # Resolve the path relative to the current file if not self.is_remote(): - source_file = os.path.abspath(source_file) + if isinstance(self.source, str) and self.source.startswith(("s3://")): + source_file = self.source + else: + source_file = os.path.abspath(source_file) else: # get rid of extra "/" and "./" in the path root and source_file root_path = str(self.path).rstrip("/")