From dcd248b3faec7f9b6f003784dbd899945b565df8 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 22 Nov 2024 12:55:53 -0600 Subject: [PATCH] Move imports of extensions to the init of the interfaces (#1144) Co-authored-by: Paul Adkisson --- CHANGELOG.md | 3 +- .../behavior/audio/audiointerface.py | 6 ++- .../behavior/deeplabcut/_dlc_utils.py | 4 +- .../deeplabcut/deeplabcutdatainterface.py | 4 +- .../lightningposedatainterface.py | 4 ++ .../behavior/medpc/medpcdatainterface.py | 4 ++ .../tdt_fp/tdtfiberphotometrydatainterface.py | 1 + src/neuroconv/tools/audio/audio.py | 12 ----- .../behavior/test_behavior_interfaces.py | 47 ++++++++++++++++++- .../behavior/test_lightningpose_converter.py | 3 +- 10 files changed, 69 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 002aed660..c0c3bd729 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,11 @@ # Upcoming ## Deprecations -* Completely removed compression settings from most places[PR #1126](https://github.com/catalystneuro/neuroconv/pull/1126) +* Completely removed compression settings from most places [PR #1126](https://github.com/catalystneuro/neuroconv/pull/1126) ## Bug Fixes * datetime objects now can be validated as conversion options [#1139](https://github.com/catalystneuro/neuroconv/pull/1126) +* Fix a bug where data in `DeepLabCutInterface` failed to write when `ndx-pose` was not imported. [#1144](https://github.com/catalystneuro/neuroconv/pull/1144) ## Features * Propagate the `unit_electrode_indices` argument from the spikeinterface tools to `BaseSortingExtractorInterface`. This allows users to map units to the electrode table when adding sorting data [PR #1124](https://github.com/catalystneuro/neuroconv/pull/1124) diff --git a/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py b/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py index fc3f08fb8..6561096b5 100644 --- a/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py +++ b/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py @@ -46,6 +46,10 @@ def __init__(self, file_paths: list[FilePath], verbose: bool = False): verbose : bool, default: False """ + # This import is to assure that ndx_sound is in the global namespace when an pynwb.io object is created. + # For more detail, see https://github.com/rly/ndx-pose/issues/36 + import ndx_sound # noqa: F401 + suffixes = [suffix for file_path in file_paths for suffix in Path(file_path).suffixes] format_is_not_supported = [ suffix for suffix in suffixes if suffix not in [".wav"] @@ -166,7 +170,6 @@ def add_to_nwbfile( stub_frames: int = 1000, write_as: Literal["stimulus", "acquisition"] = "stimulus", iterator_options: Optional[dict] = None, - compression_options: Optional[dict] = None, # TODO: remove completely after 10/1/2024 overwrite: bool = False, verbose: bool = True, ): @@ -224,7 +227,6 @@ def add_to_nwbfile( write_as=write_as, starting_time=starting_times[file_index], iterator_options=iterator_options, - compression_options=compression_options, # TODO: remove completely after 10/1/2024; still passing for deprecation warning ) return nwbfile diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py b/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py index 5d1224e85..14866510d 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py @@ -279,13 +279,14 @@ def _write_pes_to_nwbfile( else: timestamps_cleaned = timestamps + timestamps = np.asarray(timestamps_cleaned).astype("float64", copy=False) pes = PoseEstimationSeries( name=f"{animal}_{keypoint}" if animal else keypoint, description=f"Keypoint {keypoint} from individual {animal}.", data=data[:, :2], unit="pixels", reference_frame="(0,0) corresponds to the bottom left corner of the video.", - timestamps=timestamps_cleaned, + timestamps=timestamps, confidence=data[:, 2], confidence_definition="Softmax output of the deep neural network.", ) @@ -298,6 +299,7 @@ def _write_pes_to_nwbfile( # TODO, taken from the original implementation, improve it if the video is passed dimensions = [list(map(int, image_shape.split(",")))[1::2]] + dimensions = np.array(dimensions, dtype="uint32") pose_estimation_default_kwargs = dict( pose_estimation_series=pose_estimation_series, description="2D keypoint coordinates estimated using DeepLabCut.", diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py b/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py index f45913061..147fcf6ea 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py @@ -5,7 +5,6 @@ from pydantic import FilePath, validate_call from pynwb.file import NWBFile -# import ndx_pose from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface @@ -48,6 +47,9 @@ def __init__( verbose: bool, default: True Controls verbosity. """ + # This import is to assure that the ndx_pose is in the global namespace when an pynwb.io object is created + from ndx_pose import PoseEstimation, PoseEstimationSeries # noqa: F401 + from ._dlc_utils import _read_config file_path = Path(file_path) diff --git a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py index f103b7c9a..dbd425b5b 100644 --- a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py @@ -80,6 +80,10 @@ def __init__( verbose : bool, default: True controls verbosity. ``True`` by default. """ + # This import is to assure that the ndx_pose is in the global namespace when an pynwb.io object is created + # For more detail, see https://github.com/rly/ndx-pose/issues/36 + import ndx_pose # noqa: F401 + from neuroconv.datainterfaces.behavior.video.video_utils import ( VideoCaptureContext, ) diff --git a/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py b/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py index 6a4127663..09f9111d7 100644 --- a/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py @@ -73,6 +73,10 @@ def __init__( verbose : bool, optional Whether to print verbose output, by default True """ + # This import is to assure that the ndx_events is in the global namespace when an pynwb.io object is created + # For more detail, see https://github.com/rly/ndx-pose/issues/36 + import ndx_events # noqa: F401 + if aligned_timestamp_names is None: aligned_timestamp_names = [] super().__init__( diff --git a/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py b/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py index aa58f6ae4..8b092464e 100644 --- a/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py @@ -43,6 +43,7 @@ def __init__(self, folder_path: DirectoryPath, verbose: bool = True): folder_path=folder_path, verbose=verbose, ) + # This module should be here so ndx_fiber_photometry is in the global namespace when an pynwb.io object is created import ndx_fiber_photometry # noqa: F401 def get_metadata(self) -> DeepDict: diff --git a/src/neuroconv/tools/audio/audio.py b/src/neuroconv/tools/audio/audio.py index 44d10de63..064f36155 100644 --- a/src/neuroconv/tools/audio/audio.py +++ b/src/neuroconv/tools/audio/audio.py @@ -15,7 +15,6 @@ def add_acoustic_waveform_series( starting_time: float = 0.0, write_as: Literal["stimulus", "acquisition"] = "stimulus", iterator_options: Optional[dict] = None, - compression_options: Optional[dict] = None, # TODO: remove completely after 10/1/2024 ) -> NWBFile: """ @@ -53,17 +52,6 @@ def add_acoustic_waveform_series( "acquisition", ], "Acoustic series can be written either as 'stimulus' or 'acquisition'." - # TODO: remove completely after 10/1/2024 - if compression_options is not None: - warn( - message=( - "Specifying compression methods and their options at the level of tool functions has been deprecated. " - "Please use the `configure_backend` tool function for this purpose." - ), - category=DeprecationWarning, - stacklevel=2, - ) - iterator_options = iterator_options or dict() container = nwbfile.acquisition if write_as == "acquisition" else nwbfile.stimulus diff --git a/tests/test_on_data/behavior/test_behavior_interfaces.py b/tests/test_on_data/behavior/test_behavior_interfaces.py index b43e65206..0b5c63376 100644 --- a/tests/test_on_data/behavior/test_behavior_interfaces.py +++ b/tests/test_on_data/behavior/test_behavior_interfaces.py @@ -1,3 +1,4 @@ +import sys import unittest from datetime import datetime, timezone from pathlib import Path @@ -10,7 +11,6 @@ from natsort import natsorted from ndx_miniscope import Miniscope from ndx_miniscope.utils import get_timestamps -from ndx_pose import PoseEstimation, PoseEstimationSeries from numpy.testing import assert_array_equal from parameterized import param, parameterized from pynwb import NWBHDF5IO @@ -105,6 +105,8 @@ def check_extracted_metadata(self, metadata: dict): assert metadata["Behavior"][self.pose_estimation_name] == self.expected_metadata[self.pose_estimation_name] def check_read_nwb(self, nwbfile_path: str): + from ndx_pose import PoseEstimation, PoseEstimationSeries + with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: nwbfile = io.read() @@ -381,6 +383,49 @@ def check_read_nwb(self, nwbfile_path: str): assert all(expected_pose_estimation_series_are_in_nwb_file) +@pytest.fixture +def clean_pose_extension_import(): + modules_to_remove = [m for m in sys.modules if m.startswith("ndx_pose")] + for module in modules_to_remove: + del sys.modules[module] + + +@pytest.mark.skipif( + platform == "darwin" and python_version < version.parse("3.10"), + reason="interface not supported on macOS with Python < 3.10", +) +def test_deep_lab_cut_import_pose_extension_bug(clean_pose_extension_import, tmp_path): + """ + Test that the DeepLabCutInterface writes correctly without importing the ndx-pose extension. + See issues: + https://github.com/catalystneuro/neuroconv/issues/1114 + https://github.com/rly/ndx-pose/issues/36 + + """ + + interface_kwargs = dict( + file_path=str( + BEHAVIOR_DATA_PATH + / "DLC" + / "open_field_without_video" + / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5" + ), + config_file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "open_field_without_video" / "config.yaml"), + ) + + interface = DeepLabCutInterface(**interface_kwargs) + metadata = interface.get_metadata() + metadata["NWBFile"]["session_start_time"] = datetime(2023, 7, 24, 9, 30, 55, 440600, tzinfo=timezone.utc) + + nwbfile_path = tmp_path / "test.nwb" + interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata, overwrite=True) + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + read_nwbfile = io.read() + pose_estimation_container = read_nwbfile.processing["behavior"]["PoseEstimation"] + + assert len(pose_estimation_container.fields) > 0 + + @pytest.mark.skipif( platform == "darwin" and python_version < version.parse("3.10"), reason="interface not supported on macOS with Python < 3.10", diff --git a/tests/test_on_data/behavior/test_lightningpose_converter.py b/tests/test_on_data/behavior/test_lightningpose_converter.py index 4d0f8ab89..dd93632a4 100644 --- a/tests/test_on_data/behavior/test_lightningpose_converter.py +++ b/tests/test_on_data/behavior/test_lightningpose_converter.py @@ -5,7 +5,6 @@ from warnings import warn from hdmf.testing import TestCase -from ndx_pose import PoseEstimation from pynwb import NWBHDF5IO from pynwb.image import ImageSeries @@ -134,6 +133,8 @@ def test_run_conversion_add_conversion_options(self): self.assertNWBFileStructure(nwbfile_path=nwbfile_path, **self.conversion_options) def assertNWBFileStructure(self, nwbfile_path: str, stub_test: bool = False): + from ndx_pose import PoseEstimation + with NWBHDF5IO(path=nwbfile_path) as io: nwbfile = io.read()