[Backend Configuration IIIb] Configure DynamicTable router for set_data_io

CHANGELOG.md

@@ -15,6 +15,7 @@
 * Added a `from_nwbfile` class method constructor to all `BackendConfiguration` models. [PR #673](https://github.com/catalystneuro/neuroconv/pull/673)
 * Added compression to `FicTracDataInterface`. [PR #678](https://github.com/catalystneuro/neuroconv/pull/678)
 * Exposed `block_index` to all OpenEphys interfaces. [PR #695](https://github.com/catalystneuro/neuroconv/pull/695)
+* Added support for `DynamicTable` columns in the `configure_backend` tool function. [PR #700](https://github.com/catalystneuro/neuroconv/pull/700)
 
 
 

requirements-minimal.txt

@@ -3,7 +3,7 @@ jsonschema>=3.2.0
 PyYAML>=5.4
 scipy>=1.4.1
 h5py>=3.9.0
-hdmf>=3.11.0
+hdmf @ git+https://github.com/hdmf-dev/hdmf.git@dev
 hdmf_zarr>=0.4.0
 pynwb>=2.3.2;python_version>='3.8'
 pydantic>=1.10.13,<2.0.0

src/neuroconv/tools/nwb_helpers/__init__.py

@@ -1,4 +1,8 @@
 """Collection of Pydantic models and helper functions for configuring dataset IO parameters for different backends."""
+# Mark these imports as private to avoid polluting the namespace; only used in global BACKEND_NWB_IO mapping
+from hdmf_zarr import NWBZarrIO as _NWBZarrIO
+from pynwb import NWBHDF5IO as _NWBHDF5IO
+
 from ._backend_configuration import get_default_backend_configuration
 from ._configuration_models._base_backend import BackendConfiguration
 from ._configuration_models._base_dataset_io import DatasetInfo, DatasetIOConfiguration
@@ -24,13 +28,17 @@
 
 BACKEND_CONFIGURATIONS = dict(hdf5=HDF5BackendConfiguration, zarr=ZarrBackendConfiguration)
 DATASET_IO_CONFIGURATIONS = dict(hdf5=HDF5DatasetIOConfiguration, zarr=ZarrDatasetIOConfiguration)
+BACKEND_NWB_IO = dict(hdf5=_NWBHDF5IO, zarr=_NWBZarrIO)
 
 __all__ = [
+    "AVAILABLE_HDF5_COMPRESSION_METHODS",
+    "AVAILABLE_ZARR_COMPRESSION_METHODS",
+    "BACKEND_CONFIGURATIONS",
+    "DATASET_IO_CONFIGURATIONS",
+    "BACKEND_NWB_IO",
     "get_default_backend_configuration",
     "get_default_dataset_io_configurations",
     "configure_backend",
-    "AVAILABLE_HDF5_COMPRESSION_METHODS",
-    "AVAILABLE_ZARR_COMPRESSION_METHODS",
     "BackendConfiguration",
     "DatasetIOConfiguration",
     "get_default_dataset_io_configurations",

src/neuroconv/tools/nwb_helpers/_configure_backend.py

@@ -1,7 +1,8 @@
 """Collection of helper functions related to configuration of datasets dependent on backend."""
 from typing import Union
 
-from pynwb import NWBFile
+from hdmf.common import Data
+from pynwb import NWBFile, TimeSeries
 
 from ._configuration_models._hdf5_backend import HDF5BackendConfiguration
 from ._configuration_models._zarr_backend import ZarrBackendConfiguration
@@ -21,4 +22,15 @@ def configure_backend(
 
         # TODO: update buffer shape in iterator, if present
 
-        nwbfile_objects[object_id].set_data_io(dataset_name=dataset_name, data_io_class=data_io_class, **data_io_kwargs)
+       nwbfile_object = nwbfile_objects[object_id]
+        if isinstance(nwbfile_object, Data):
+            nwbfile_object.set_data_io(data_io_class=data_io_class, data_io_kwargs=data_io_kwargs)
+        elif isinstance(nwbfile_object, TimeSeries):
+            nwbfile_object.set_data_io(
+                dataset_name=dataset_name, data_io_class=data_io_class, **data_io_kwargs
+            )
+        else:  # Strictly speaking, it would be odd if a backend_configuration led to this, but might as well be safe
+            raise NotImplementedError(
+                f"Unsupported object type {type(nwbfile_object)} for backend "
+                f"configuration of {nwbfile_object.name}!"
+            )

src/neuroconv/tools/spikeinterface/spikeinterface.py

@@ -2,6 +2,7 @@
 import warnings
 from collections import defaultdict
 from numbers import Real
+from pathlib import Path
 from typing import List, Literal, Optional, Union
 
 import numpy as np

tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_configure_backend_defaults.py

@@ -5,14 +5,14 @@
 import numcodecs
 import numpy as np
 import pytest
+from hdmf.common import DynamicTable, VectorData
 from hdmf.data_utils import DataChunkIterator
-from hdmf_zarr import NWBZarrIO
-from pynwb import NWBHDF5IO
 from pynwb.testing.mock.base import mock_TimeSeries
 from pynwb.testing.mock.file import mock_NWBFile
 
 from neuroconv.tools.hdmf import SliceableDataChunkIterator
 from neuroconv.tools.nwb_helpers import (
+    BACKEND_NWB_IO,
     configure_backend,
     get_default_backend_configuration,
 )
@@ -31,8 +31,6 @@
 def test_simple_time_series(
     tmpdir: Path, case_name: str, iterator: callable, iterator_options: dict, backend: Literal["hdf5", "zarr"]
 ):
-    BACKEND_NWB_IO = dict(hdf5=NWBHDF5IO, zarr=NWBZarrIO)
-
     array = np.zeros(shape=(30_000 * 5, 384), dtype="int16")
     data = iterator(array, **iterator_options)
 
@@ -44,7 +42,7 @@ def test_simple_time_series(
     dataset_configuration = backend_configuration.dataset_configurations["acquisition/TestTimeSeries/data"]
     configure_backend(nwbfile=nwbfile, backend_configuration=backend_configuration)
 
-    nwbfile_path = str(tmpdir / f"test_configure_{backend}_defaults_{case_name}_data.nwb.h5")
+    nwbfile_path = str(tmpdir / f"test_configure_defaults_{case_name}_time_series.nwb.{backend}")
     with BACKEND_NWB_IO[backend](path=nwbfile_path, mode="w") as io:
         io.write(nwbfile)
 
@@ -58,3 +56,34 @@ def test_simple_time_series(
             assert written_data.compression == "gzip"
         elif backend == "zarr":
             assert written_data.compressor == numcodecs.GZip(level=1)
+
+
+@pytest.mark.parametrize("backend", ["hdf5", "zarr"])
+def test_simple_dynamic_table(tmpdir: Path, backend: Literal["hdf5", "zarr"]):
+    data = np.zeros(shape=(30_000 * 5, 384), dtype="int16")
+
+    nwbfile = mock_NWBFile()
+    dynamic_table = DynamicTable(
+        name="TestDynamicTable", description="", columns=[VectorData(name="TestColumn", description="", data=data)]
+    )
+    nwbfile.add_acquisition(dynamic_table)
+
+    backend_configuration = get_default_backend_configuration(nwbfile=nwbfile, backend=backend)
+    dataset_configuration = backend_configuration.dataset_configurations["acquisition/TestDynamicTable/TestColumn/data"]
+    configure_backend(nwbfile=nwbfile, backend_configuration=backend_configuration)
+
+    nwbfile_path = str(tmpdir / f"test_configure_defaults_dynamic_table.nwb.{backend}")
+    NWB_IO = BACKEND_NWB_IO[backend]
+    with NWB_IO(path=nwbfile_path, mode="w") as io:
+        io.write(nwbfile)
+
+    with NWB_IO(path=nwbfile_path, mode="r") as io:
+        written_nwbfile = io.read()
+        written_data = written_nwbfile.acquisition["TestDynamicTable"]["TestColumn"].data
+
+        assert written_data.chunks == dataset_configuration.chunk_shape
+
+        if backend == "hdf5":
+            assert written_data.compression == "gzip"
+        elif backend == "zarr":
+            assert written_data.compressor == numcodecs.GZip(level=1)

tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_configure_backend_overrides.py

@@ -5,14 +5,14 @@
 import numcodecs
 import numpy as np
 import pytest
+from hdmf.common import DynamicTable, VectorData
 from hdmf.data_utils import DataChunkIterator
-from hdmf_zarr import NWBZarrIO
-from pynwb import NWBHDF5IO
 from pynwb.testing.mock.base import mock_TimeSeries
 from pynwb.testing.mock.file import mock_NWBFile
 
 from neuroconv.tools.hdmf import SliceableDataChunkIterator
 from neuroconv.tools.nwb_helpers import (
+    BACKEND_NWB_IO,
     configure_backend,
     get_default_backend_configuration,
 )
@@ -31,8 +31,6 @@
 def test_simple_time_series_override(
     tmpdir: Path, case_name: str, iterator: callable, iterator_options: dict, backend: Literal["hdf5", "zarr"]
 ):
-    BACKEND_NWB_IO = dict(hdf5=NWBHDF5IO, zarr=NWBZarrIO)
-
     array = np.zeros(shape=(30_000 * 5, 384), dtype="int16")
     data = iterator(array, **iterator_options)
 
@@ -59,7 +57,7 @@ def test_simple_time_series_override(
     if case_name != "unwrapped":  # TODO: eventually, even this case will be buffered automatically
         assert nwbfile.acquisition["TestTimeSeries"].data
 
-    nwbfile_path = str(tmpdir / f"test_configure_{backend}_defaults_{case_name}_data.nwb.h5")
+    nwbfile_path = str(tmpdir / f"test_configure_defaults_{case_name}_data.nwb.{backend}")
     with BACKEND_NWB_IO[backend](path=nwbfile_path, mode="w") as io:
         io.write(nwbfile)
 
@@ -74,3 +72,45 @@ def test_simple_time_series_override(
             assert written_data.compression_opts == higher_gzip_level
         elif backend == "zarr":
             assert written_data.compressor == numcodecs.GZip(level=5)
+
+
+@pytest.mark.parametrize("backend", ["hdf5", "zarr"])
+def test_simple_dynamic_table_override(tmpdir: Path, backend: Literal["hdf5", "zarr"]):
+    data = np.zeros(shape=(30_000 * 5, 384), dtype="int16")
+
+    nwbfile = mock_NWBFile()
+    dynamic_table = DynamicTable(
+        name="TestDynamicTable", description="", columns=[VectorData(name="TestColumn", description="", data=data)]
+    )
+    nwbfile.add_acquisition(dynamic_table)
+
+    backend_configuration = get_default_backend_configuration(nwbfile=nwbfile, backend=backend)
+    dataset_configuration = backend_configuration.dataset_configurations["acquisition/TestDynamicTable/TestColumn/data"]
+
+    smaller_chunk_shape = (30_000, 64)
+    dataset_configuration.chunk_shape = smaller_chunk_shape
+
+    higher_gzip_level = 5
+    if backend == "hdf5":
+        dataset_configuration.compression_options = dict(level=higher_gzip_level)
+    elif backend == "zarr":
+        dataset_configuration.compression_options = dict(level=higher_gzip_level)
+
+    configure_backend(nwbfile=nwbfile, backend_configuration=backend_configuration)
+
+    nwbfile_path = str(tmpdir / f"test_configure_defaults_dynamic_table.nwb.{backend}")
+    NWB_IO = BACKEND_NWB_IO[backend]
+    with NWB_IO(path=nwbfile_path, mode="w") as io:
+        io.write(nwbfile)
+
+    with NWB_IO(path=nwbfile_path, mode="r") as io:
+        written_nwbfile = io.read()
+        written_data = written_nwbfile.acquisition["TestDynamicTable"]["TestColumn"].data
+
+        assert written_data.chunks == smaller_chunk_shape
+
+        if backend == "hdf5":
+            assert written_data.compression == "gzip"
+            assert written_data.compression_opts == higher_gzip_level
+        elif backend == "zarr":
+            assert written_data.compressor == numcodecs.GZip(level=5)