From 6afa3f59867d7e164cc52ef0efa62bf8756dc28c Mon Sep 17 00:00:00 2001 From: rly Date: Wed, 18 Dec 2024 12:40:30 -0800 Subject: [PATCH 1/8] Fix badges --- README.rst | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index 5af642ff..40e13295 100644 --- a/README.rst +++ b/README.rst @@ -22,30 +22,30 @@ CI / Health Status .. image:: https://codecov.io/gh/hdmf-dev/hdmf-zarr/branch/dev/graph/badge.svg :target: https://codecov.io/gh/hdmf-dev/hdmf-zarr -.. image:: https://github.com/hdmf-dev/hdmf-zarr/workflows/Run%20code%20coverage/badge.svg +.. image:: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/run_coverage.yml/badge.svg :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/run_coverage.yml -.. image:: https://github.com/hdmf-dev/hdmf-zarr/workflows/Run%20tests/badge.svg +.. image:: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/run_tests.yml/badge.svg :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/run_tests.yml -.. image:: https://github.com/hdmf-dev/hdmf-zarr/workflows/Run%20all%20tests/badge.svg +.. image:: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/run_all_tests.yml/badge.svg :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/run_all_tests.yml -.. image:: https://github.com/hdmf-dev/hdmf-zarr/workflows/Check%20Sphinx%20external%20links/badge.svg +.. image:: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/check_external_links.yml/badge.svg :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/check_external_links.yml -.. image:: https://github.com/hdmf-dev/hdmf-zarr/workflows/Deploy%20release/badge.svg +.. image:: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/deploy_release.yml/badge.svg :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/deploy_release.yml -.. image:: https://github.com/hdmf-dev/hdmf-zarr/workflows/black/badge.svg - :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/black.yml +.. image:: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/ruff.yml/badge.svg + :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/ruff.yml -.. image:: https://github.com/hdmf-dev/hdmf-zarr/workflows/Run%20style%20check/badge.svg - :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/run_flake8.yml +.. image:: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/codespell.yml/badge.svg + :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/codespell.yml +.. image:: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/HDMF_dev.yaml/badge.svg + :target: https://github.com/hdmf-dev/hdmf-zarr/actions/workflows/HDMF_dev.yaml - -Citing hdmf-zarr ---------------- If you use HDMF or hdmf_zarr in your research, please use the following citation: From 7ab7d6b061da04046686ef40c95220a9d53bfeb6 Mon Sep 17 00:00:00 2001 From: rly Date: Wed, 18 Dec 2024 12:40:41 -0800 Subject: [PATCH 2/8] Remove references to region refs in docs --- docs/source/overview.rst | 1 - docs/source/storage.rst | 36 +----------------------------------- 2 files changed, 1 insertion(+), 36 deletions(-) diff --git a/docs/source/overview.rst b/docs/source/overview.rst index 73ad6f65..690bc8f5 100644 --- a/docs/source/overview.rst +++ b/docs/source/overview.rst @@ -29,7 +29,6 @@ Supported features Known Limitations ----------------- -- Support for region references is not yet implemented. See also :ref:`sec-zarr-storage-references-region` for details. - The Zarr backend is currently experimental and may still change. - Attributes are stored as JSON documents in Zarr (using the DirectoryStore). As such, all attributes must be JSON serializable. The :py:class:`~hdmf_zarr.backend.ZarrIO` backend attempts to cast types to JSON serializable types as much as possible. - Currently the :py:class:`~hdmf_zarr.backend.ZarrIO` backend supports Zarr's directory-based stores :py:class:`~zarr.storage.DirectoryStore`, :py:class:`~zarr.storage.NestedDirectoryStore`, and :py:class:`~zarr.storage.TempStore`. Other `Zarr stores `_ could be added but will require proper treatment of links and references for those backends as links are not supported in Zarr (see `zarr-python issues #389 `_. diff --git a/docs/source/storage.rst b/docs/source/storage.rst index 864fe420..e9386003 100644 --- a/docs/source/storage.rst +++ b/docs/source/storage.rst @@ -260,7 +260,7 @@ Storing object references in Attributes Object references are stored in a attributes as dicts with the following keys: * ``zarr_dtype`` : Indicating the data type for the attribute. For object references - ``zarr_dtype`` is set to ``"object"`` (or ``"region"`` for :ref:`sec-zarr-storage-references-region`) + ``zarr_dtype`` is set to ``"object"`` * ``value``: The value of the object references, i.e., here the py:class:`~hdmf_zarr.utils.ZarrReference` dictionary with the ``source``, ``path``, ``object_id``, and ``source_object_id`` keys defining the object reference, with the definition of the keys being the same as @@ -280,36 +280,6 @@ For example in NWB, the attribute ``ElectricalSeries.electrodes.table`` would be "zarr_dtype": "object" } -.. _sec-zarr-storage-references-region: - -Region references ------------------ - -Region references are similar to object references, but instead of references other Datasets or Groups, -region references link to subsets of another Dataset. To identify region references, the reserved attribute -``zarr_dtype`` is set to ``'region'`` (see also :ref:`sec-zarr-storage-attributes-reserved`). In addition -to the ``source`` and ``path``, the py:class:`~hdmf_zarr.utils.ZarrReference` object will also need to -store the definition of the ``region`` that is being referenced, e.g., a slice or list on point indices. - -.. warning:: - - Region references are not yet fully implemented in :py:class:`~hdmf_zarr.backend.ZarrIO`. - To implement region references will require updating: - 1) py:class:`~hdmf_zarr.utils.ZarrReference` to add a ``region`` key to support storing - the selection for the region, - 2) :py:meth:`~hdmf_zarr.backend.ZarrIO._create_ref` to support passing in the region definition to - be added to the py:class:`~hdmf_zarr.utils.ZarrReference`, - 3) :py:meth:`~hdmf_zarr.backend.ZarrIO.write_dataset` already partially implements the required - logic for creating region references by checking for :py:class:`hdmf.build.RegionBuilder` inputs - but will likely need updates as well - 4) :py:meth:`~hdmf_zarr.backend.ZarrIO.__read_dataset` to support reading region references, - which may also require updates to :py:meth:`~hdmf_zarr.backend.ZarrIO.__parse_ref` and - :py:meth:`~hdmf_zarr.backend.ZarrIO.__resolve_ref`, and - 5) and possibly other parts of :py:class:`~hdmf_zarr.backend.ZarrIO`. - 6) The py:class:`~hdmf_zarr.zarr_utils.ContainerZarrRegionDataset` and - py:class:`~hdmf_zarr.zarr_utils.ContainerZarrRegionDataset` classes will also need to be finalized - to support region references. - .. _sec-zarr-storage-dtypes: @@ -357,10 +327,6 @@ The mappings of data types is as follows | * "reference" | dataset. See | | | * "object" | :ref:`sec-zarr-storage-references` | | +--------------------------+------------------------------------+----------------+ - | * region | Reference to a region | | - | | of another dataset. See | | - | | :ref:sec-zarr-storage-references` | | - +--------------------------+------------------------------------+----------------+ | * compound dtype | Compound data type | | +--------------------------+------------------------------------+----------------+ | * "isodatetime" | ASCII ISO8061 datetime string. | variable | From 4cd8aeb26d0702bc03a3b79c4fb53ea90e1edaee Mon Sep 17 00:00:00 2001 From: rly Date: Wed, 18 Dec 2024 12:40:51 -0800 Subject: [PATCH 3/8] Update pyproject.toml python range and dev status --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 65be5d24..bde404eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ name = "hdmf_zarr" authors = [ { name="Oliver Ruebel", email="oruebel@lbl.gov" }, { name="Matthew Avaylon", email="mavaylon@lbl.gov" }, + { name="Ryan Ly", email="rly@lbl.gov" }, ] description = "A package defining a Zarr I/O backend for HDMF" readme = "README.rst" @@ -18,8 +19,9 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "License :: OSI Approved :: BSD License", - "Development Status :: 5 - Production/Stable", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Intended Audience :: Science/Research", "Operating System :: Microsoft :: Windows", @@ -130,7 +132,6 @@ line-length = 120 [tool.ruff.lint.per-file-ignores] "docs/gallery/*" = ["E402", "T201"] -"src/*/__init__.py" = ["F401"] "test_gallery.py" = ["T201"] [tool.ruff.lint.mccabe] From 97e426caf7f5c0ca103ad9a3050bb826592ec510 Mon Sep 17 00:00:00 2001 From: rly Date: Wed, 18 Dec 2024 12:40:55 -0800 Subject: [PATCH 4/8] Update changelog date --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 453504a3..cc7b69ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # HDMF-ZARR Changelog -## 0.10.0 (Upcoming) +## 0.10.0 (December 18, 2024) ### Enhancements * Added initial refactor of export, supporting references and internal/external links from Zarr to Zarr. This will introduce breaking changes that could lead to existing exported files to be invalid. This update removes '.' as the object default file source. @mavaylon1 [#194](https://github.com/hdmf-dev/hdmf-zarr/pull/194) From 0764bd4a327ec84ab5b216cb0b273cc19e1b7c45 Mon Sep 17 00:00:00 2001 From: rly Date: Wed, 18 Dec 2024 12:43:29 -0800 Subject: [PATCH 5/8] Fix asset typo and change caps on dandiset --- docs/gallery/plot_convert_nwb_hdf5.py | 4 ++-- docs/gallery/resources/README.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/gallery/plot_convert_nwb_hdf5.py b/docs/gallery/plot_convert_nwb_hdf5.py index 6565afe2..e98126ac 100644 --- a/docs/gallery/plot_convert_nwb_hdf5.py +++ b/docs/gallery/plot_convert_nwb_hdf5.py @@ -14,7 +14,7 @@ # ----- # # Here we use a small NWB file from the DANDI neurophysiology data archive from -# `DANDIset 000009 `_ as an example. +# `Dandiset 000009 `_ as an example. # To download the file directly from DANDI we can use: # # .. code-block:: python @@ -29,7 +29,7 @@ # filename = os.path.basename(asset.path) # asset.download(filename) # -# We here use a local copy of a small file from this DANDIset as an example: +# We here use a local copy of a small file from this Dandiset as an example: # sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnail_plot_convert_nwb.png' diff --git a/docs/gallery/resources/README.rst b/docs/gallery/resources/README.rst index 1a762d3a..f7b595de 100644 --- a/docs/gallery/resources/README.rst +++ b/docs/gallery/resources/README.rst @@ -4,11 +4,11 @@ Resources sub_anm00239123_ses_20170627T093549_ecephys_and_ogen.nwb -------------------------------------------------------- -This NWB file was downloaded from `DANDIset 000009 `_ +This NWB file was downloaded from `Dandiset 000009 `_ The file was modified to replace ``:`` characters used in the name of the ``ElectrodeGroup`` called ``ADunit: 32`` in ``'general/extracellular_ephys/`` to ``'ADunit_32'``. The dataset ``general/extracellular_ephys/electrodes/group_name`` as part of the electrodes table was updated accordingly to list the appropriate group name. This is to avoid issues -on Windows file systems that do not support ``:`` as part of folder names. The asses can be downloaded from DANDI via: +on Windows file systems that do not support ``:`` as part of folder names. The asset can be downloaded from DANDI via: .. code-block:: python :linenos: From c9719c1f2dc38cfca5604235d2b9acc5a1a2358b Mon Sep 17 00:00:00 2001 From: rly Date: Wed, 18 Dec 2024 16:33:59 -0800 Subject: [PATCH 6/8] Remove invalid sphinx_rtd_theme option --- docs/source/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 39669770..d6c4f697 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -119,7 +119,6 @@ html_theme_options = { 'logo_only': False, # Only show the hdmf-zarr logo without the documentation title - 'display_version': True, 'prev_next_buttons_location': 'bottom', # Show previous/next button at the bottom 'style_external_links': True, # Add marker to indicate external links 'vcs_pageview_mode': '', From 10e993374a02a3d0f458c7bab1d343e015c9c8e7 Mon Sep 17 00:00:00 2001 From: rly Date: Wed, 18 Dec 2024 16:54:48 -0800 Subject: [PATCH 7/8] Update conda workflows to address warning --- .github/workflows/run_all_tests.yml | 3 ++- .github/workflows/run_tests.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_all_tests.yml b/.github/workflows/run_all_tests.yml index 129cdad8..f264ac5a 100644 --- a/.github/workflows/run_all_tests.yml +++ b/.github/workflows/run_all_tests.yml @@ -155,12 +155,13 @@ jobs: auto-update-conda: true python-version: ${{ matrix.python-ver }} channels: conda-forge + conda-remove-defaults: "true" - name: Install build dependencies run: | conda config --set always_yes yes --set changeps1 no conda info - conda install -c conda-forge tox + conda install tox - name: Conda reporting run: | diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 0fd692c5..63dc5a58 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -139,12 +139,13 @@ jobs: auto-update-conda: true python-version: ${{ matrix.python-ver }} channels: conda-forge + conda-remove-defaults: "true" - name: Install build dependencies run: | conda config --set always_yes yes --set changeps1 no conda info - conda install -c conda-forge tox + conda install tox - name: Conda reporting run: | From 72263e485c5c014f4dcb9ea44f9c601e495bb953 Mon Sep 17 00:00:00 2001 From: rly Date: Wed, 18 Dec 2024 17:06:04 -0800 Subject: [PATCH 8/8] Incorporate fix from #238 --- src/hdmf_zarr/backend.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/hdmf_zarr/backend.py b/src/hdmf_zarr/backend.py index 5477e533..b6394488 100644 --- a/src/hdmf_zarr/backend.py +++ b/src/hdmf_zarr/backend.py @@ -803,7 +803,10 @@ def resolve_ref(self, zarr_ref): source_file = str(zarr_ref["source"]) # Resolve the path relative to the current file if not self.is_remote(): - source_file = os.path.abspath(source_file) + if isinstance(self.source, str) and self.source.startswith(("s3://")): + source_file = self.source + else: + source_file = os.path.abspath(source_file) else: # get rid of extra "/" and "./" in the path root and source_file root_path = str(self.path).rstrip("/")