Merge branch 'main' into make_metadata_encoder_public_again

CHANGELOG.md

@@ -1,11 +1,12 @@
 # Upcoming
 
 ## Deprecations
-* Completely removed compression settings from most places[PR #1126](https://github.com/catalystneuro/neuroconv/pull/1126)
+* Completely removed compression settings from most places [PR #1126](https://github.com/catalystneuro/neuroconv/pull/1126)
 
 ## Bug Fixes
 * datetime objects now can be validated as conversion options [#1139](https://github.com/catalystneuro/neuroconv/pull/1126)
 * Make `NWBMetaDataEncoder` public again [PR #1142](https://github.com/catalystneuro/neuroconv/pull/1142)
+* Fix a bug where data in `DeepLabCutInterface` failed to write when `ndx-pose` was not imported. [#1144](https://github.com/catalystneuro/neuroconv/pull/1144)
 
 ## Features
 * Propagate the `unit_electrode_indices` argument from the spikeinterface tools to `BaseSortingExtractorInterface`. This allows users to map units to the electrode table when adding sorting data [PR #1124](https://github.com/catalystneuro/neuroconv/pull/1124)

src/neuroconv/datainterfaces/behavior/audio/audiointerface.py

@@ -46,6 +46,10 @@ def __init__(self, file_paths: list[FilePath], verbose: bool = False):
 
         verbose : bool, default: False
         """
+        # This import is to assure that ndx_sound is in the global namespace when an pynwb.io object is created.
+        # For more detail, see https://github.com/rly/ndx-pose/issues/36
+        import ndx_sound  # noqa: F401
+
         suffixes = [suffix for file_path in file_paths for suffix in Path(file_path).suffixes]
         format_is_not_supported = [
             suffix for suffix in suffixes if suffix not in [".wav"]
@@ -166,7 +170,6 @@ def add_to_nwbfile(
         stub_frames: int = 1000,
         write_as: Literal["stimulus", "acquisition"] = "stimulus",
         iterator_options: Optional[dict] = None,
-        compression_options: Optional[dict] = None,  # TODO: remove completely after 10/1/2024
         overwrite: bool = False,
         verbose: bool = True,
     ):
@@ -224,7 +227,6 @@ def add_to_nwbfile(
                 write_as=write_as,
                 starting_time=starting_times[file_index],
                 iterator_options=iterator_options,
-                compression_options=compression_options,  # TODO: remove completely after 10/1/2024; still passing for deprecation warning
             )
 
         return nwbfile

src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py

@@ -279,13 +279,14 @@ def _write_pes_to_nwbfile(
         else:
             timestamps_cleaned = timestamps
 
+        timestamps = np.asarray(timestamps_cleaned).astype("float64", copy=False)
         pes = PoseEstimationSeries(
             name=f"{animal}_{keypoint}" if animal else keypoint,
             description=f"Keypoint {keypoint} from individual {animal}.",
             data=data[:, :2],
             unit="pixels",
             reference_frame="(0,0) corresponds to the bottom left corner of the video.",
-            timestamps=timestamps_cleaned,
+            timestamps=timestamps,
             confidence=data[:, 2],
             confidence_definition="Softmax output of the deep neural network.",
         )
@@ -298,6 +299,7 @@ def _write_pes_to_nwbfile(
 
     # TODO, taken from the original implementation, improve it if the video is passed
     dimensions = [list(map(int, image_shape.split(",")))[1::2]]
+    dimensions = np.array(dimensions, dtype="uint32")
     pose_estimation_default_kwargs = dict(
         pose_estimation_series=pose_estimation_series,
         description="2D keypoint coordinates estimated using DeepLabCut.",

src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py

@@ -5,7 +5,6 @@
 from pydantic import FilePath, validate_call
 from pynwb.file import NWBFile
 
-# import ndx_pose
 from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface
 
 
@@ -48,6 +47,9 @@ def __init__(
         verbose: bool, default: True
             Controls verbosity.
         """
+        # This import is to assure that the ndx_pose is in the global namespace when an pynwb.io object is created
+        from ndx_pose import PoseEstimation, PoseEstimationSeries  # noqa: F401
+
         from ._dlc_utils import _read_config
 
         file_path = Path(file_path)

src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py

@@ -80,6 +80,10 @@ def __init__(
         verbose : bool, default: True
             controls verbosity. ``True`` by default.
         """
+        # This import is to assure that the ndx_pose is in the global namespace when an pynwb.io object is created
+        # For more detail, see https://github.com/rly/ndx-pose/issues/36
+        import ndx_pose  # noqa: F401
+
         from neuroconv.datainterfaces.behavior.video.video_utils import (
             VideoCaptureContext,
         )

src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py

@@ -73,6 +73,10 @@ def __init__(
         verbose : bool, optional
             Whether to print verbose output, by default True
         """
+        # This import is to assure that the ndx_events is in the global namespace when an pynwb.io object is created
+        # For more detail, see https://github.com/rly/ndx-pose/issues/36
+        import ndx_events  # noqa: F401
+
         if aligned_timestamp_names is None:
             aligned_timestamp_names = []
         super().__init__(

src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py

@@ -43,6 +43,7 @@ def __init__(self, folder_path: DirectoryPath, verbose: bool = True):
             folder_path=folder_path,
             verbose=verbose,
         )
+        # This module should be here so ndx_fiber_photometry is in the global namespace when an pynwb.io object is created
         import ndx_fiber_photometry  # noqa: F401
 
     def get_metadata(self) -> DeepDict:

src/neuroconv/tools/audio/audio.py

@@ -15,7 +15,6 @@ def add_acoustic_waveform_series(
     starting_time: float = 0.0,
     write_as: Literal["stimulus", "acquisition"] = "stimulus",
     iterator_options: Optional[dict] = None,
-    compression_options: Optional[dict] = None,  # TODO: remove completely after 10/1/2024
 ) -> NWBFile:
     """
 
@@ -53,17 +52,6 @@ def add_acoustic_waveform_series(
         "acquisition",
     ], "Acoustic series can be written either as 'stimulus' or 'acquisition'."
 
-    # TODO: remove completely after 10/1/2024
-    if compression_options is not None:
-        warn(
-            message=(
-                "Specifying compression methods and their options at the level of tool functions has been deprecated. "
-                "Please use the `configure_backend` tool function for this purpose."
-            ),
-            category=DeprecationWarning,
-            stacklevel=2,
-        )
-
     iterator_options = iterator_options or dict()
 
     container = nwbfile.acquisition if write_as == "acquisition" else nwbfile.stimulus

tests/test_on_data/behavior/test_behavior_interfaces.py

@@ -1,3 +1,4 @@
+import sys
 import unittest
 from datetime import datetime, timezone
 from pathlib import Path
@@ -10,7 +11,6 @@
 from natsort import natsorted
 from ndx_miniscope import Miniscope
 from ndx_miniscope.utils import get_timestamps
-from ndx_pose import PoseEstimation, PoseEstimationSeries
 from numpy.testing import assert_array_equal
 from parameterized import param, parameterized
 from pynwb import NWBHDF5IO
@@ -105,6 +105,8 @@ def check_extracted_metadata(self, metadata: dict):
         assert metadata["Behavior"][self.pose_estimation_name] == self.expected_metadata[self.pose_estimation_name]
 
     def check_read_nwb(self, nwbfile_path: str):
+        from ndx_pose import PoseEstimation, PoseEstimationSeries
+
         with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io:
             nwbfile = io.read()
 
@@ -381,6 +383,49 @@ def check_read_nwb(self, nwbfile_path: str):
             assert all(expected_pose_estimation_series_are_in_nwb_file)
 
 
+@pytest.fixture
+def clean_pose_extension_import():
+    modules_to_remove = [m for m in sys.modules if m.startswith("ndx_pose")]
+    for module in modules_to_remove:
+        del sys.modules[module]
+
+
+@pytest.mark.skipif(
+    platform == "darwin" and python_version < version.parse("3.10"),
+    reason="interface not supported on macOS with Python < 3.10",
+)
+def test_deep_lab_cut_import_pose_extension_bug(clean_pose_extension_import, tmp_path):
+    """
+    Test that the DeepLabCutInterface writes correctly without importing the ndx-pose extension.
+    See issues:
+    https://github.com/catalystneuro/neuroconv/issues/1114
+    https://github.com/rly/ndx-pose/issues/36
+
+    """
+
+    interface_kwargs = dict(
+        file_path=str(
+            BEHAVIOR_DATA_PATH
+            / "DLC"
+            / "open_field_without_video"
+            / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5"
+        ),
+        config_file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "open_field_without_video" / "config.yaml"),
+    )
+
+    interface = DeepLabCutInterface(**interface_kwargs)
+    metadata = interface.get_metadata()
+    metadata["NWBFile"]["session_start_time"] = datetime(2023, 7, 24, 9, 30, 55, 440600, tzinfo=timezone.utc)
+
+    nwbfile_path = tmp_path / "test.nwb"
+    interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata, overwrite=True)
+    with NWBHDF5IO(path=nwbfile_path, mode="r") as io:
+        read_nwbfile = io.read()
+        pose_estimation_container = read_nwbfile.processing["behavior"]["PoseEstimation"]
+
+        assert len(pose_estimation_container.fields) > 0
+
+
 @pytest.mark.skipif(
     platform == "darwin" and python_version < version.parse("3.10"),
     reason="interface not supported on macOS with Python < 3.10",

tests/test_on_data/behavior/test_lightningpose_converter.py

@@ -5,7 +5,6 @@
 from warnings import warn
 
 from hdmf.testing import TestCase
-from ndx_pose import PoseEstimation
 from pynwb import NWBHDF5IO
 from pynwb.image import ImageSeries
 
@@ -134,6 +133,8 @@ def test_run_conversion_add_conversion_options(self):
         self.assertNWBFileStructure(nwbfile_path=nwbfile_path, **self.conversion_options)
 
     def assertNWBFileStructure(self, nwbfile_path: str, stub_test: bool = False):
+        from ndx_pose import PoseEstimation
+
         with NWBHDF5IO(path=nwbfile_path) as io:
             nwbfile = io.read()