Auto detect backend type in converters and interfaces (#840)

Co-authored-by: Ben Dichter <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

CHANGELOG.md

@@ -22,6 +22,9 @@
 
 ### Improvements
 * Fixed docstrings related to backend configurations for various methods. [PR #822](https://github.com/catalystneuro/neuroconv/pull/822)
+* Added automatic `backend` detection when a `backend_configuration` is passed to an interface or converter. [PR #840]
+  (https://github.
+  com/catalystneuro/neuroconv/pull/840)
 
 ### Testing
 * Add general test for metadata in-place modification by interfaces. [PR #815](https://github.com/catalystneuro/neuroconv/pull/815)

src/neuroconv/basedatainterface.py

@@ -86,8 +86,8 @@ def create_nwbfile(self, metadata: Optional[dict] = None, **conversion_options)
         if metadata is None:
             metadata = self.get_metadata()
 
-        nwbfile = make_nwbfile_from_metadata(metadata)
-        self.add_to_nwbfile(nwbfile, metadata=metadata, **conversion_options)
+        nwbfile = make_nwbfile_from_metadata(metadata=metadata)
+        self.add_to_nwbfile(nwbfile=nwbfile, metadata=metadata, **conversion_options)
         return nwbfile
 
     @abstractmethod
@@ -115,7 +115,7 @@ def run_conversion(
         # TODO: when all H5DataIO prewraps are gone, introduce Zarr safely
         # backend: Union[Literal["hdf5", "zarr"]],
         # backend_configuration: Optional[Union[HDF5BackendConfiguration, ZarrBackendConfiguration]] = None,
-        backend: Literal["hdf5"] = "hdf5",
+        backend: Optional[Literal["hdf5"]] = None,
         backend_configuration: Optional[HDF5BackendConfiguration] = None,
         **conversion_options,
     ):
@@ -134,9 +134,10 @@ def run_conversion(
         overwrite : bool, default: False
             Whether to overwrite the NWBFile if one exists at the nwbfile_path.
             The default is False (append mode).
-        backend : "hdf5", default: "hdf5"
+        backend : "hdf5", optional
             The type of backend to use when writing the file.
-            Additional backend types will be added soon.
+            If a `backend_configuration` is not specified, the default type will be "hdf5".
+            If a `backend_configuration` is specified, then the type will be auto-detected.
         backend_configuration : HDF5BackendConfiguration, optional
             The configuration model to use when configuring the datasets for this backend.
             To customize, call the `.get_default_backend_configuration(...)` method, modify the returned
@@ -151,6 +152,21 @@ def run_conversion(
             )
         if backend_configuration is not None and nwbfile is None:
             raise ValueError("When specifying a custom `backend_configuration`, you must also provide an `nwbfile`.")
+        if backend is not None and backend_configuration is not None:
+            if backend == backend_configuration.backend:
+                warnings.warn(
+                    f"Both `backend` and `backend_configuration` were specified as type '{backend}'. "
+                    "To suppress this warning, specify only `backend_configuration`."
+                )
+            else:
+                raise ValueError(
+                    f"Both `backend` and `backend_configuration` were specified and are conflicting."
+                    f"{backend=}, {backend_configuration.backend=}."
+                    "These values must match. To suppress this error, specify only `backend_configuration`."
+                )
+
+        if backend is None:
+            backend = backend_configuration.backend if backend_configuration is not None else "hdf5"
 
         if metadata is None:
             metadata = self.get_metadata()

src/neuroconv/nwbconverter.py

@@ -1,4 +1,5 @@
 import json
+import warnings
 from collections import Counter
 from pathlib import Path
 from typing import Dict, List, Literal, Optional, Tuple, Union
@@ -13,6 +14,7 @@
     configure_backend,
     get_default_backend_configuration,
     get_default_nwbfile_metadata,
+    make_nwbfile_from_metadata,
     make_or_load_nwbfile,
 )
 from .utils import (
@@ -120,6 +122,30 @@ def _validate_source_data(self, source_data: Dict[str, dict], verbose: bool = Tr
         if verbose:
             print("Source data is valid!")
 
+    def create_nwbfile(self, metadata: Optional[dict] = None, conversion_options: Optional[dict] = None) -> NWBFile:
+        """
+        Create and return an in-memory pynwb.NWBFile object with this interface's data added to it.
+
+        Parameters
+        ----------
+        metadata : dict, optional
+            Metadata dictionary with information used to create the NWBFile.
+        conversion_options : dict, optional
+            Similar to source_data, a dictionary containing keywords for each interface for which non-default
+            conversion specification is requested.
+
+        Returns
+        -------
+        nwbfile : pynwb.NWBFile
+            The in-memory object with this interface's data added to it.
+        """
+        if metadata is None:
+            metadata = self.get_metadata()
+
+        nwbfile = make_nwbfile_from_metadata(metadata=metadata)
+        self.add_to_nwbfile(nwbfile=nwbfile, metadata=metadata, conversion_options=conversion_options)
+        return nwbfile
+
     def add_to_nwbfile(self, nwbfile: NWBFile, metadata, conversion_options: Optional[dict] = None) -> None:
         conversion_options = conversion_options or dict()
         for interface_name, data_interface in self.data_interface_objects.items():
@@ -136,7 +162,7 @@ def run_conversion(
         # TODO: when all H5DataIO prewraps are gone, introduce Zarr safely
         # backend: Union[Literal["hdf5", "zarr"]],
         # backend_configuration: Optional[Union[HDF5BackendConfiguration, ZarrBackendConfiguration]] = None,
-        backend: Literal["hdf5"] = "hdf5",
+        backend: Optional[Literal["hdf5"]] = None,
         backend_configuration: Optional[HDF5BackendConfiguration] = None,
         conversion_options: Optional[dict] = None,
     ) -> None:
@@ -155,8 +181,10 @@ def run_conversion(
         overwrite : bool, default: False
             Whether to overwrite the NWBFile if one exists at the nwbfile_path.
             The default is False (append mode).
-        backend : "hdf5" or a HDF5BackendConfiguration, default: "hdf5"
+        backend : "hdf5", optional
             The type of backend to use when writing the file.
+            If a `backend_configuration` is not specified, the default type will be "hdf5".
+            If a `backend_configuration` is specified, then the type will be auto-detected.
         backend_configuration : HDF5BackendConfiguration, optional
             The configuration model to use when configuring the datasets for this backend.
             To customize, call the `.get_default_backend_configuration(...)` method, modify the returned
@@ -166,6 +194,22 @@ def run_conversion(
             Similar to source_data, a dictionary containing keywords for each interface for which non-default
             conversion specification is requested.
         """
+        if backend is not None and backend_configuration is not None:
+            if backend == backend_configuration.backend:
+                warnings.warn(
+                    f"Both `backend` and `backend_configuration` were specified as type '{backend}'. "
+                    "To suppress this warning, specify only `backend_configuration`."
+                )
+            else:
+                raise ValueError(
+                    f"Both `backend` and `backend_configuration` were specified and are conflicting."
+                    f"{backend=}, {backend_configuration.backend=}."
+                    "These values must match. To suppress this error, specify only `backend_configuration`."
+                )
+
+        if backend is None:
+            backend = backend_configuration.backend if backend_configuration is not None else "hdf5"
+
         if metadata is None:
             metadata = self.get_metadata()
 

src/neuroconv/tools/testing/data_interface_mixins.py

@@ -98,16 +98,22 @@ def check_no_metadata_mutation(self):
 
         assert metadata == metadata_in
 
-    def check_run_conversion_default_backend(self, nwbfile_path: str, backend: Literal["hdf5", "zarr"] = "hdf5"):
+    def check_run_conversion_with_backend(self, nwbfile_path: str, backend: Literal["hdf5", "zarr"] = "hdf5"):
         metadata = self.interface.get_metadata()
         if "session_start_time" not in metadata["NWBFile"]:
             metadata["NWBFile"].update(session_start_time=datetime.now().astimezone())
 
         self.interface.run_conversion(
-            nwbfile_path=nwbfile_path, overwrite=True, metadata=metadata, backend=backend, **self.conversion_options
+            nwbfile_path=nwbfile_path,
+            overwrite=True,
+            metadata=metadata,
+            backend=backend,
+            **self.conversion_options,
         )
 
-    def check_run_conversion_custom_backend(self, nwbfile_path: str, backend: Literal["hdf5", "zarr"] = "hdf5"):
+    def check_run_conversion_with_backend_configuration(
+        self, nwbfile_path: str, backend: Literal["hdf5", "zarr"] = "hdf5"
+    ):
         metadata = self.interface.get_metadata()
         if "session_start_time" not in metadata["NWBFile"]:
             metadata["NWBFile"].update(session_start_time=datetime.now().astimezone())
@@ -116,15 +122,13 @@ def check_run_conversion_custom_backend(self, nwbfile_path: str, backend: Litera
         backend_configuration = self.interface.get_default_backend_configuration(nwbfile=nwbfile, backend=backend)
         self.interface.run_conversion(
             nwbfile_path=nwbfile_path,
-            overwrite=True,
             nwbfile=nwbfile,
-            metadata=metadata,
-            backend=backend,
+            overwrite=True,
             backend_configuration=backend_configuration,
             **self.conversion_options,
         )
 
-    def check_run_conversion_default_backend_in_nwbconverter(
+    def check_run_conversion_in_nwbconverter_with_backend(
         self, nwbfile_path: str, backend: Literal["hdf5", "zarr"] = "hdf5"
     ):
         class TestNWBConverter(NWBConverter):
@@ -147,6 +151,33 @@ class TestNWBConverter(NWBConverter):
             conversion_options=conversion_options,
         )
 
+    def check_run_conversion_in_nwbconverter_with_backend_configuration(
+        self, nwbfile_path: str, backend: Union["hdf5", "zarr"] = "hdf5"
+    ):
+        class TestNWBConverter(NWBConverter):
+            data_interface_classes = dict(Test=type(self.interface))
+
+        test_kwargs = self.test_kwargs[0] if isinstance(self.test_kwargs, list) else self.test_kwargs
+        source_data = dict(Test=test_kwargs)
+        converter = TestNWBConverter(source_data=source_data)
+
+        metadata = converter.get_metadata()
+        if "session_start_time" not in metadata["NWBFile"]:
+            metadata["NWBFile"].update(session_start_time=datetime.now().astimezone())
+
+        conversion_options = dict(Test=self.conversion_options)
+
+        nwbfile = converter.create_nwbfile(metadata=metadata, conversion_options=conversion_options)
+        backend_configuration = converter.get_default_backend_configuration(nwbfile=nwbfile, backend=backend)
+        converter.run_conversion(
+            nwbfile_path=nwbfile_path,
+            nwbfile=nwbfile,
+            overwrite=True,
+            metadata=metadata,
+            backend_configuration=backend_configuration,
+            conversion_options=conversion_options,
+        )
+
     @abstractmethod
     def check_read_nwb(self, nwbfile_path: str):
         """Read the produced NWB file and compare it to the interface."""
@@ -182,12 +213,13 @@ def test_all_conversion_checks(self):
 
                 self.check_no_metadata_mutation()
 
-                self.check_run_conversion_default_backend_in_nwbconverter(
+                self.check_run_conversion_in_nwbconverter_with_backend(nwbfile_path=self.nwbfile_path, backend="hdf5")
+                self.check_run_conversion_in_nwbconverter_with_backend_configuration(
                     nwbfile_path=self.nwbfile_path, backend="hdf5"
                 )
 
-                self.check_run_conversion_default_backend(nwbfile_path=self.nwbfile_path, backend="hdf5")
-                self.check_run_conversion_custom_backend(nwbfile_path=self.nwbfile_path, backend="hdf5")
+                self.check_run_conversion_with_backend(nwbfile_path=self.nwbfile_path, backend="hdf5")
+                self.check_run_conversion_with_backend_configuration(nwbfile_path=self.nwbfile_path, backend="hdf5")
 
                 self.check_read_nwb(nwbfile_path=self.nwbfile_path)
 
@@ -595,12 +627,13 @@ def test_all_conversion_checks(self):
 
                 self.check_no_metadata_mutation()
 
-                self.check_run_conversion_default_backend_in_nwbconverter(
+                self.check_run_conversion_in_nwbconverter_with_backend(nwbfile_path=self.nwbfile_path, backend="hdf5")
+                self.check_run_conversion_in_nwbconverter_with_backend_configuration(
                     nwbfile_path=self.nwbfile_path, backend="hdf5"
                 )
 
-                self.check_run_conversion_default_backend(nwbfile_path=self.nwbfile_path, backend="hdf5")
-                self.check_run_conversion_custom_backend(nwbfile_path=self.nwbfile_path, backend="hdf5")
+                self.check_run_conversion_with_backend(nwbfile_path=self.nwbfile_path, backend="hdf5")
+                self.check_run_conversion_with_backend_configuration(nwbfile_path=self.nwbfile_path, backend="hdf5")
 
                 self.check_read_nwb(nwbfile_path=self.nwbfile_path)
 

tests/test_on_data/test_recording_interfaces.py

@@ -176,11 +176,19 @@ def test_interface_alignment(self):
                 self.check_nwbfile_temporal_alignment()
 
     # EDF has simultaneous access issues; can't have multiple interfaces open on the same file at once...
-    def check_run_conversion_default_backend_in_nwbconverter(
+    def check_run_conversion_in_nwbconverter_with_backend(
         self, nwbfile_path: str, backend: Literal["hdf5", "zarr"] = "hdf5"
     ):
         pass
 
+    def check_run_conversion_in_nwbconverter_with_backend_configuration(
+        self, nwbfile_path: str, backend: Literal["hdf5", "zarr"] = "hdf5"
+    ):
+        pass
+
+    def check_run_conversion_with_backend(self, nwbfile_path: str, backend: Literal["hdf5", "zarr"] = "hdf5"):
+        pass
+
 
 class TestIntanRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase):
     data_interface_cls = IntanRecordingInterface