Merge branch 'dev' into gallery

CHANGELOG.md

@@ -1,11 +1,16 @@
 # HDMF-ZARR Changelog
 
+## 1.0.0 (Upcoming)
+### Enhancements
+* Added support for Pathlib paths. @mavaylon1 [#212](https://github.com/hdmf-dev/hdmf-zarr/pull/212)
+
 ## 0.9.0 (September 16, 2024)
 ### Enhancements
 * Added support for appending a dataset of references. @mavaylon1 [#203](https://github.com/hdmf-dev/hdmf-zarr/pull/203)
 * NWBZarrIO load_namespaces=True by default. @mavaylon1 [#204](https://github.com/hdmf-dev/hdmf-zarr/pull/204)
 * Added test for opening file with consolidated metadata from DANDI. @mavaylon1 [#206](https://github.com/hdmf-dev/hdmf-zarr/pull/206)
 * Add dimension labels compatible with xarray. @mavaylon1 [#207](https://github.com/hdmf-dev/hdmf-zarr/pull/207)
+* Added link_data --> clear_cache relationship to support repacking zarr nwbfiles: [#215](https://github.com/hdmf-dev/hdmf-zarr/pull/215)
 
 ## 0.8.0 (June 4, 2024)
 ### Bug Fixes

src/hdmf_zarr/backend.py

@@ -48,6 +48,9 @@
 from hdmf.query import HDMFDataset
 from hdmf.container import Container
 
+from pathlib import Path
+
+
 # Module variables
 ROOT_NAME = 'root'
 """
@@ -84,7 +87,7 @@ def can_read(path):
             return False
 
     @docval({'name': 'path',
-             'type': (str, *SUPPORTED_ZARR_STORES),
+             'type': (str, Path, *SUPPORTED_ZARR_STORES),
              'doc': 'the path to the Zarr file or a supported Zarr store'},
             {'name': 'manager', 'type': BuildManager, 'doc': 'the BuildManager to use for I/O', 'default': None},
             {'name': 'mode', 'type': str,
@@ -115,6 +118,8 @@ def __init__(self, **kwargs):
         else:
             self.__synchronizer = synchronizer
         self.__mode = mode
+        if isinstance(path, Path):
+            path = str(path)
         self.__path = path
         self.__file = None
         self.__storage_options = storage_options
@@ -195,7 +200,7 @@ def is_remote(self):
              'type': (NamespaceCatalog, TypeMap),
              'doc': 'the NamespaceCatalog or TypeMap to load namespaces into'},
             {'name': 'path',
-             'type': (str, *SUPPORTED_ZARR_STORES),
+             'type': (str, Path, *SUPPORTED_ZARR_STORES),
              'doc': 'the path to the Zarr file or a supported Zarr store'},
             {'name': 'storage_options', 'type': dict,
              'doc': 'Zarr storage options to read remote folders',
@@ -362,6 +367,8 @@ def export(self, **kwargs):
         write_args['export_source'] = src_io.source  # pass export_source=src_io.source to write_builder
         ckwargs = kwargs.copy()
         ckwargs['write_args'] = write_args
+        if not write_args.get('link_data', True):
+            ckwargs['clear_cache'] = True
         super().export(**ckwargs)
         if cache_spec:
             self.__cache_spec()
@@ -1305,6 +1312,13 @@ def __list_fill__(self, parent, name, data, options=None):  # noqa: C901
             except ValueError:
                 for i in range(len(data)):
                     dset[i] = data[i]
+            except TypeError: # If data is an h5py.Dataset with strings, they may need to be decoded
+                for c in np.ndindex(data_shape):
+                    o = data
+                    for i in c:
+                        o = o[i]
+                    # bytes are not JSON serializable
+                    dset[c] = o if not isinstance(o, (bytes, np.bytes_)) else o.decode("utf-8")
         return dset
 
     def __scalar_fill__(self, parent, name, data, options=None):

tests/unit/test_io_convert.py

@@ -949,7 +949,7 @@ def __get_data_array(self, foo_container):
 
     def test_maxshape(self):
         """test when maxshape is set for the dataset"""
-        data = H5DataIO(data=list(range(5)), maxshape=(None,))
+        data = H5DataIO(data=list(range(5)), maxshape=(5,))
         self.__roundtrip_data(data=data)
         self.assertContainerEqual(self.out_container, self.read_container, ignore_hdmf_attrs=True)
 

tests/unit/test_zarrio.py

@@ -24,6 +24,7 @@
 import os
 import shutil
 import warnings
+import pathlib
 
 
 CUR_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -136,6 +137,16 @@ def setUp(self):
         self.store = [NestedDirectoryStore(p) for p in self.store_path]
 
 
+#########################################
+#  Pathlib Tests
+#########################################
+class TestPathlib(BaseTestZarrWriter):
+    """Test writing of builder with Zarr using a custom DirectoryStore"""
+    def setUp(self):
+        super().setUp()
+        self.store = pathlib.Path(self.store_path)
+
+
 #########################################
 #  Consolidate Metadata tests
 #########################################
@@ -195,7 +206,7 @@ class TestDimensionLabels(BuildDatasetShapeMixin):
     Workflow:
     i) We need to define a `get_dataset_inc_spec` to set the dim in the spec (via BuildDatasetShapeMixin)
     ii) Create and write a BarDataHolder with a BarData.
-    iii) Read and check that the _ARRAY_DIMENSIONS attribute is set. 
+    iii) Read and check that the _ARRAY_DIMENSIONS attribute is set.
     """
     def tearDown(self):
         shutil.rmtree(self.store)