Enable zarr backend testing in data tests [2] (#1083)

catalystneuro · Sep 17, 2024 · 4dad0ef · 4dad0ef
1 parent fa69900
commit 4dad0ef
Show file tree

Hide file tree

Showing 4 changed files with 44 additions and 40 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,7 @@
 ## Features
 * Added chunking/compression for string-only compound objects: [PR #1042](https://github.com/catalystneuro/neuroconv/pull/1042)
 * Added automated EFS volume creation and mounting to the `submit_aws_job` helper function. [PR #1018](https://github.com/catalystneuro/neuroconv/pull/1018)
+* Added a mock for segmentation extractors interfaces in ophys: `MockSegmentationInterface`  [PR #1067](https://github.com/catalystneuro/neuroconv/pull/1067)
 * Added a `MockSortingInterface` for testing purposes. [PR #1065](https://github.com/catalystneuro/neuroconv/pull/1065)
 
 
@@ -21,6 +22,9 @@
 * Add writing to zarr test for to the test on data [PR #1056](https://github.com/catalystneuro/neuroconv/pull/1056)
 * Modified the CI to avoid running doctests twice [PR #1077](https://github.com/catalystneuro/neuroconv/pull/#1077)
 * Consolidated daily workflows into one workflow and added email notifications [PR #1081](https://github.com/catalystneuro/neuroconv/pull/1081)
+* Added zarr tests for the test on data with checking equivalent backends [PR #1083](https://github.com/catalystneuro/neuroconv/pull/1083)
+
+
 
 ## v0.6.3
 

diff --git a/src/neuroconv/basedatainterface.py b/src/neuroconv/basedatainterface.py
@@ -127,11 +127,8 @@ def run_conversion(
         nwbfile: Optional[NWBFile] = None,
         metadata: Optional[dict] = None,
         overwrite: bool = False,
-        # TODO: when all H5DataIO prewraps are gone, introduce Zarr safely
-        # backend: Union[Literal["hdf5", "zarr"]],
-        # backend_configuration: Optional[Union[HDF5BackendConfiguration, ZarrBackendConfiguration]] = None,
-        backend: Optional[Literal["hdf5"]] = None,
-        backend_configuration: Optional[HDF5BackendConfiguration] = None,
+        backend: Optional[Literal["hdf5", "zarr"]] = None,
+        backend_configuration: Optional[Union[HDF5BackendConfiguration, ZarrBackendConfiguration]] = None,
         **conversion_options,
     ):
         """
@@ -148,11 +145,11 @@ def run_conversion(
         overwrite : bool, default: False
             Whether to overwrite the NWBFile if one exists at the nwbfile_path.
             The default is False (append mode).
-        backend : "hdf5", optional
+        backend : {"hdf5", "zarr"}, optional
             The type of backend to use when writing the file.
             If a `backend_configuration` is not specified, the default type will be "hdf5".
             If a `backend_configuration` is specified, then the type will be auto-detected.
-        backend_configuration : HDF5BackendConfiguration, optional
+        backend_configuration : HDF5BackendConfiguration or ZarrBackendConfiguration, optional
             The configuration model to use when configuring the datasets for this backend.
             To customize, call the `.get_default_backend_configuration(...)` method, modify the returned
             BackendConfiguration object, and pass that instead.

diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py
@@ -105,33 +105,8 @@ def test_no_metadata_mutation(self, setup_interface):
         metadata_before_add_method = deepcopy(metadata)
 
         self.interface.add_to_nwbfile(nwbfile=nwbfile, metadata=metadata, **self.conversion_options)
-
         assert metadata == metadata_before_add_method
 
-    def check_run_conversion_with_backend(self, nwbfile_path: str, backend: Literal["hdf5", "zarr"] = "hdf5"):
-        metadata = self.interface.get_metadata()
-        if "session_start_time" not in metadata["NWBFile"]:
-            metadata["NWBFile"].update(session_start_time=datetime.now().astimezone())
-
-        self.interface.run_conversion(
-            nwbfile_path=nwbfile_path,
-            overwrite=True,
-            metadata=metadata,
-            backend=backend,
-            **self.conversion_options,
-        )
-
-    def check_configure_backend_for_equivalent_nwbfiles(self, backend: Literal["hdf5", "zarr"] = "hdf5"):
-        metadata = self.interface.get_metadata()
-        if "session_start_time" not in metadata["NWBFile"]:
-            metadata["NWBFile"].update(session_start_time=datetime.now().astimezone())
-
-        nwbfile_1 = self.interface.create_nwbfile(metadata=metadata, **self.conversion_options)
-        nwbfile_2 = self.interface.create_nwbfile(metadata=metadata, **self.conversion_options)
-
-        backend_configuration = get_default_backend_configuration(nwbfile=nwbfile_1, backend=backend)
-        configure_backend(nwbfile=nwbfile_2, backend_configuration=backend_configuration)
-
     def check_run_conversion_with_backend_configuration(
         self, nwbfile_path: str, backend: Literal["hdf5", "zarr"] = "hdf5"
     ):
@@ -204,11 +179,6 @@ def check_read_nwb(self, nwbfile_path: str):
         """Read the produced NWB file and compare it to the interface."""
         pass
 
-    def check_basic_zarr_read(self, nwbfile_path: str):
-        """Ensure NWBZarrIO can read the file."""
-        with NWBZarrIO(path=nwbfile_path, mode="r") as io:
-            io.read()
-
     def check_extracted_metadata(self, metadata: dict):
         """Override this method to make assertions about specific extracted metadata values."""
         pass
@@ -235,7 +205,20 @@ def test_run_conversion_with_backend(self, setup_interface, tmp_path, backend):
         )
 
         if backend == "zarr":
-            self.check_basic_zarr_read(nwbfile_path)
+            with NWBZarrIO(path=nwbfile_path, mode="r") as io:
+                io.read()
+
+    @pytest.mark.parametrize("backend", ["hdf5", "zarr"])
+    def test_configure_backend_for_equivalent_nwbfiles(self, setup_interface, tmp_path, backend):
+        metadata = self.interface.get_metadata()
+        if "session_start_time" not in metadata["NWBFile"]:
+            metadata["NWBFile"].update(session_start_time=datetime.now().astimezone())
+
+        nwbfile_1 = self.interface.create_nwbfile(metadata=metadata, **self.conversion_options)
+        nwbfile_2 = self.interface.create_nwbfile(metadata=metadata, **self.conversion_options)
+
+        backend_configuration = get_default_backend_configuration(nwbfile=nwbfile_1, backend=backend)
+        configure_backend(nwbfile=nwbfile_2, backend_configuration=backend_configuration)
 
     def test_all_conversion_checks(self, setup_interface, tmp_path):
         interface, test_name = setup_interface
@@ -247,7 +230,6 @@ def test_all_conversion_checks(self, setup_interface, tmp_path):
         # Now run the checks using the setup objects
         self.check_conversion_options_schema_valid()
         self.check_metadata()
-        self.check_configure_backend_for_equivalent_nwbfiles()
 
         self.check_run_conversion_in_nwbconverter_with_backend(nwbfile_path=nwbfile_path, backend="hdf5")
         self.check_run_conversion_in_nwbconverter_with_backend_configuration(nwbfile_path=nwbfile_path, backend="hdf5")
@@ -746,7 +728,6 @@ def test_all_conversion_checks(self, setup_interface, tmp_path):
         # Now run the checks using the setup objects
         self.check_conversion_options_schema_valid()
         self.check_metadata()
-        self.check_configure_backend_for_equivalent_nwbfiles()
 
         self.check_run_conversion_in_nwbconverter_with_backend(nwbfile_path=nwbfile_path, backend="hdf5")
         self.check_run_conversion_in_nwbconverter_with_backend_configuration(nwbfile_path=nwbfile_path, backend="hdf5")
@@ -900,6 +881,9 @@ def test_run_conversion_with_backend(self):
     def test_no_metadata_mutation(self):
         pass
 
+    def test_configure_backend_for_equivalent_nwbfiles(self):
+        pass
+
     def check_metadata_schema_valid(self):
         schema = self.interface.get_metadata_schema()
         Draft7Validator.check_schema(schema=schema)
@@ -1263,6 +1247,9 @@ def test_run_conversion_with_backend(self):
     def test_no_metadata_mutation(self):
         pass
 
+    def test_configure_backend_for_equivalent_nwbfiles(self):
+        pass
+
     def check_metadata_schema_valid(self):
         schema = self.interface.get_metadata_schema()
         Draft7Validator.check_schema(schema=schema)

diff --git a/tests/test_on_data/ecephys/test_recording_interfaces.py b/tests/test_on_data/ecephys/test_recording_interfaces.py
@@ -179,6 +179,19 @@ class TestEDFRecordingInterface(RecordingExtractorInterfaceTestMixin):
     def check_extracted_metadata(self, metadata: dict):
         assert metadata["NWBFile"]["session_start_time"] == datetime(2022, 3, 2, 10, 42, 19)
 
+    def check_run_conversion_with_backend(self, nwbfile_path: str, backend="hdf5"):
+        metadata = self.interface.get_metadata()
+        if "session_start_time" not in metadata["NWBFile"]:
+            metadata["NWBFile"].update(session_start_time=datetime.now().astimezone())
+
+        self.interface.run_conversion(
+            nwbfile_path=nwbfile_path,
+            overwrite=True,
+            metadata=metadata,
+            backend=backend,
+            **self.conversion_options,
+        )
+
     def test_all_conversion_checks(self, setup_interface, tmp_path):
         # Create a unique test name and file path
         nwbfile_path = str(tmp_path / f"{self.__class__.__name__}.nwb")
@@ -205,6 +218,9 @@ def test_run_conversion_with_backend(self):
     def test_interface_alignment(self):
         pass
 
+    def test_configure_backend_for_equivalent_nwbfiles(self):
+        pass
+
 
 class TestIntanRecordingInterfaceRHS(RecordingExtractorInterfaceTestMixin):
     data_interface_cls = IntanRecordingInterface