From baccf8f96c37a9465db64b50a6f8652e4ead7779 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 19 Dec 2023 14:40:50 -0500 Subject: [PATCH] Exposed block_index to OpenEphys interfaces (#695) * exposed block_index to all open ephys and added test * Update CHANGELOG.md * Update tests/test_on_data/test_recording_interfaces.py Co-authored-by: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: Heberto Mayorquin Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGELOG.md | 3 +- .../openephys/openephysbinarydatainterface.py | 28 +++++++++---- .../openephys/openephysdatainterface.py | 16 +++++++- .../openephys/openephyslegacydatainterface.py | 26 +++++++++--- .../test_on_data/test_recording_interfaces.py | 41 ++++++++++++++----- 5 files changed, 86 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da38e1274..fe02a486e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,8 @@ ### Improvements * `nwbinspector` has been removed as a minimal dependency. It becomes an extra (optional) dependency with `neuroconv[dandi]`. [PR #672](https://github.com/catalystneuro/neuroconv/pull/672) * Added a `from_nwbfile` class method constructor to all `BackendConfiguration` models. [PR #673](https://github.com/catalystneuro/neuroconv/pull/673) -* Added compression to `FicTracDataInterface` [PR #678](https://github.com/catalystneuro/neuroconv/pull/678) +* Added compression to `FicTracDataInterface`. [PR #678](https://github.com/catalystneuro/neuroconv/pull/678) +* Exposed `block_index` to all OpenEphys interfaces. [PR #695](https://github.com/catalystneuro/neuroconv/pull/695) diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py b/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py index 5e985c168..e6b4614fb 100644 --- a/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py @@ -20,8 +20,11 @@ def _open_with_pyopenephys(folder_path: FolderPathType): class OpenEphysBinaryRecordingInterface(BaseRecordingExtractorInterface): - """Primary data interface for converting binary OpenEphys data (.dat files). Uses - :py:class:`~spikeinterface.extractors.OpenEphysBinaryRecordingExtractor`.""" + """ + Primary data interface for converting binary OpenEphys data (.dat files). + + Uses :py:class:`~spikeinterface.extractors.OpenEphysBinaryRecordingExtractor`. + """ ExtractorName = "OpenEphysBinaryRecordingExtractor" @@ -47,6 +50,7 @@ def __init__( self, folder_path: FolderPathType, stream_name: Optional[str] = None, + block_index: Optional[int] = None, stub_test: bool = False, verbose: bool = True, es_key: str = "ElectricalSeries", @@ -61,23 +65,25 @@ def __init__( stream_name : str, optional The name of the recording stream to load; only required if there is more than one stream detected. Call `OpenEphysRecordingInterface.get_stream_names(folder_path=...)` to see what streams are available. + block_index : int, optional, default: None + The index of the block to extract from the data. stub_test : bool, default: False verbose : bool, default: True es_key : str, default: "ElectricalSeries" """ - try: _open_with_pyopenephys(folder_path=folder_path) except Exception as error: # Type of error might depend on pyopenephys version and/or platform error_case_1 = ( - type(error) == Exception + type(error) is Exception and str(error) == "Only 'binary' and 'openephys' format are supported by pyopenephys" ) - error_case_2 = type(error) == OSError and "Unique settings file not found in" in str(error) + error_case_2 = type(error) is OSError and "Unique settings file not found in" in str(error) if error_case_1 or error_case_2: # Raise a more informative error instead. raise ValueError( - "Unable to identify the OpenEphys folder structure! Please check that your `folder_path` contains sub-folders of the " + "Unable to identify the OpenEphys folder structure! " + "Please check that your `folder_path` contains sub-folders of the " "following form: 'experiment' -> 'recording' -> 'continuous'." ) else: @@ -86,15 +92,19 @@ def __init__( available_streams = self.get_stream_names(folder_path=folder_path) if len(available_streams) > 1 and stream_name is None: raise ValueError( - "More than one stream is detected! Please specify which stream you wish to load with the `stream_name` argument. " - "To see what streams are available, call `OpenEphysRecordingInterface.get_stream_names(folder_path=...)`." + "More than one stream is detected! " + "Please specify which stream you wish to load with the `stream_name` argument. " + "To see what streams are available, call " + " `OpenEphysRecordingInterface.get_stream_names(folder_path=...)`." ) if stream_name is not None and stream_name not in available_streams: raise ValueError( f"The selected stream '{stream_name}' is not in the available streams '{available_streams}'!" ) - super().__init__(folder_path=folder_path, stream_name=stream_name, verbose=verbose, es_key=es_key) + super().__init__( + folder_path=folder_path, stream_name=stream_name, block_index=block_index, verbose=verbose, es_key=es_key + ) if stub_test: self.subset_channels = [0, 1] diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/openephysdatainterface.py b/src/neuroconv/datainterfaces/ecephys/openephys/openephysdatainterface.py index 8ba26410c..e22a4633a 100644 --- a/src/neuroconv/datainterfaces/ecephys/openephys/openephysdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/openephys/openephysdatainterface.py @@ -16,11 +16,13 @@ def __new__( cls, folder_path: FolderPathType, stream_name: Optional[str] = None, + block_index: Optional[int] = None, verbose: bool = True, es_key: str = "ElectricalSeries", ): """ Abstract class that defines which interface class to use for a given Open Ephys recording. + For "legacy" format (.continuous files) the interface redirects to OpenEphysLegacyRecordingInterface. For "binary" format (.dat files) the interface redirects to OpenEphysBinaryRecordingInterface. @@ -32,6 +34,8 @@ def __new__( The name of the recording stream. When the recording stream is not specified the channel stream is chosen if available. When channel stream is not available the name of the stream must be specified. + block_index : int, optional, default: None + The index of the block to extract from the data. verbose : bool, default: True es_key : str, default: "ElectricalSeries" """ @@ -40,12 +44,20 @@ def __new__( folder_path = Path(folder_path) if any(folder_path.rglob("*.continuous")): return OpenEphysLegacyRecordingInterface( - folder_path=folder_path, stream_name=stream_name, verbose=verbose, es_key=es_key + folder_path=folder_path, + stream_name=stream_name, + block_index=block_index, + verbose=verbose, + es_key=es_key, ) elif any(folder_path.rglob("*.dat")): return OpenEphysBinaryRecordingInterface( - folder_path=folder_path, stream_name=stream_name, verbose=verbose, es_key=es_key + folder_path=folder_path, + stream_name=stream_name, + block_index=block_index, + verbose=verbose, + es_key=es_key, ) else: diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py b/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py index d6eba47d1..99828acef 100644 --- a/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py @@ -7,8 +7,11 @@ class OpenEphysLegacyRecordingInterface(BaseRecordingExtractorInterface): - """Primary data interface for converting legacy Open Ephys data (.continuous files). - Uses :py:class:`~spikeinterface.extractors.OpenEphysLegacyRecordingExtractor`.""" + """ + Primary data interface for converting legacy Open Ephys data (.continuous files). + + Uses :py:class:`~spikeinterface.extractors.OpenEphysLegacyRecordingExtractor`. + """ @classmethod def get_stream_names(cls, folder_path: FolderPathType) -> List[str]: @@ -31,11 +34,13 @@ def __init__( self, folder_path: FolderPathType, stream_name: Optional[str] = None, + block_index: Optional[int] = None, verbose: bool = True, es_key: str = "ElectricalSeries", ): """ Initialize reading of OpenEphys legacy recording (.continuous files). + See :py:class:`~spikeinterface.extractors.OpenEphysLegacyRecordingExtractor` for options. Parameters @@ -44,21 +49,27 @@ def __init__( Path to OpenEphys directory. stream_name : str, optional The name of the recording stream. + block_index : int, optional, default: None + The index of the block to extract from the data. verbose : bool, default: True es_key : str, default: "ElectricalSeries" """ available_streams = self.get_stream_names(folder_path=folder_path) if len(available_streams) > 1 and stream_name is None: raise ValueError( - "More than one stream is detected! Please specify which stream you wish to load with the `stream_name` argument. " - "To see what streams are available, call `OpenEphysRecordingInterface.get_stream_names(folder_path=...)`." + "More than one stream is detected! " + "Please specify which stream you wish to load with the `stream_name` argument. " + "To see what streams are available, call " + "`OpenEphysRecordingInterface.get_stream_names(folder_path=...)`." ) if stream_name is not None and stream_name not in available_streams: raise ValueError( f"The selected stream '{stream_name}' is not in the available streams '{available_streams}'!" ) - super().__init__(folder_path=folder_path, stream_name=stream_name, verbose=verbose, es_key=es_key) + super().__init__( + folder_path=folder_path, stream_name=stream_name, block_index=block_index, verbose=verbose, es_key=es_key + ) def get_metadata(self): metadata = super().get_metadata() @@ -74,7 +85,10 @@ def get_metadata(self): extracted_date, extracted_timestamp = date_created.split(" ") if len(extracted_timestamp) != len("%H%M%S"): warn( - f"The timestamp for starting time from openephys metadata is ambiguous ('{extracted_timestamp}')! Only the date will be auto-populated in metadata. Please update the timestamp manually to record this value with the highest known temporal resolution." + "The timestamp for starting time from openephys metadata is ambiguous " + f"('{extracted_timestamp}')! Only the date will be auto-populated in metadata. " + "Please update the timestamp manually to record this value with the highest known " + "temporal resolution." ) session_start_time = datetime.strptime(extracted_date, "%d-%b-%Y") else: diff --git a/tests/test_on_data/test_recording_interfaces.py b/tests/test_on_data/test_recording_interfaces.py index b383d3f58..57ff2e843 100644 --- a/tests/test_on_data/test_recording_interfaces.py +++ b/tests/test_on_data/test_recording_interfaces.py @@ -211,7 +211,8 @@ def check_extracted_metadata(self, metadata: dict): assert len(metadata["Ecephys"]["Device"]) == 1 assert metadata["Ecephys"]["Device"][0]["name"] == "Neuronexus-32" assert metadata["Ecephys"]["Device"][0]["description"] == "The ecephys device for the MEArec recording." - # assert len(metadata["Ecephys"]["ElectrodeGroup"]) == 1 # do not test this condition because in the test we are setting a mock probe + # assert len(metadata["Ecephys"]["ElectrodeGroup"]) == 1 + # do not test this condition because in the test we are setting a mock probe assert metadata["Ecephys"]["ElectrodeGroup"][0]["device"] == "Neuronexus-32" assert metadata["Ecephys"]["ElectricalSeries"]["description"] == ( '{"angle_tol": 15, "bursting": false, "chunk_duration": 0, "color_noise_floor": 1, ' @@ -341,7 +342,8 @@ def test_folder_structure_assertion(self): with self.assertRaisesWith( exc_type=ValueError, exc_msg=( - "Unable to identify the OpenEphys folder structure! Please check that your `folder_path` contains sub-folders of the " + "Unable to identify the OpenEphys folder structure! " + "Please check that your `folder_path` contains sub-folders of the " "following form: 'experiment' -> 'recording' -> 'continuous'." ), ): @@ -354,8 +356,10 @@ def test_stream_name_missing_assertion(self): with self.assertRaisesWith( exc_type=ValueError, exc_msg=( - "More than one stream is detected! Please specify which stream you wish to load with the `stream_name` argument. " - "To see what streams are available, call `OpenEphysRecordingInterface.get_stream_names(folder_path=...)`." + "More than one stream is detected! " + "Please specify which stream you wish to load with the `stream_name` argument. " + "To see what streams are available, call " + " `OpenEphysRecordingInterface.get_stream_names(folder_path=...)`." ), ): OpenEphysBinaryRecordingInterface( @@ -409,6 +413,23 @@ def check_extracted_metadata(self, metadata: dict): assert metadata["NWBFile"]["session_start_time"] == datetime(2020, 11, 24, 15, 46, 56) +class TestOpenEphysBinaryRecordingInterfaceWithBlocks_version_0_6_block_1_stream_1( + RecordingExtractorInterfaceTestMixin, TestCase +): + """From Issue #695, exposed `block_index` argument and added tests on data that include multiple blocks.""" + + data_interface_cls = OpenEphysBinaryRecordingInterface + interface_kwargs = dict( + folder_path=str(DATA_PATH / "openephysbinary" / "v0.6.x_neuropixels_multiexp_multistream" / "Record Node 101"), + stream_name="Record Node 101#NI-DAQmx-103.PXIe-6341", + block_index=1, + ) + save_directory = OUTPUT_PATH + + def check_extracted_metadata(self, metadata: dict): + assert metadata["NWBFile"]["session_start_time"] == datetime(2022, 5, 3, 10, 52, 24) + + class TestOpenEphysLegacyRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): data_interface_cls = OpenEphysLegacyRecordingInterface interface_kwargs = dict(folder_path=str(DATA_PATH / "openephys" / "OpenEphys_SampleData_1")) @@ -438,12 +459,12 @@ class TestOpenEphysRecordingInterfaceRouter(RecordingExtractorInterfaceTestMixin class TestSpikeGadgetsRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): data_interface_cls = SpikeGadgetsRecordingInterface interface_kwargs = [ - dict(file_path=str(DATA_PATH / "spikegadgets" / f"20210225_em8_minirec2_ac.rec")), - dict(file_path=str(DATA_PATH / "spikegadgets" / f"20210225_em8_minirec2_ac.rec"), gains=[0.195]), - dict(file_path=str(DATA_PATH / "spikegadgets" / f"20210225_em8_minirec2_ac.rec"), gains=[0.385] * 512), - dict(file_path=str(DATA_PATH / "spikegadgets" / f"W122_06_09_2019_1_fromSD.rec")), - dict(file_path=str(DATA_PATH / "spikegadgets" / f"W122_06_09_2019_1_fromSD.rec"), gains=[0.195]), - dict(file_path=str(DATA_PATH / "spikegadgets" / f"W122_06_09_2019_1_fromSD.rec"), gains=[0.385] * 128), + dict(file_path=str(DATA_PATH / "spikegadgets" / "20210225_em8_minirec2_ac.rec")), + dict(file_path=str(DATA_PATH / "spikegadgets" / "20210225_em8_minirec2_ac.rec"), gains=[0.195]), + dict(file_path=str(DATA_PATH / "spikegadgets" / "20210225_em8_minirec2_ac.rec"), gains=[0.385] * 512), + dict(file_path=str(DATA_PATH / "spikegadgets" / "W122_06_09_2019_1_fromSD.rec")), + dict(file_path=str(DATA_PATH / "spikegadgets" / "W122_06_09_2019_1_fromSD.rec"), gains=[0.195]), + dict(file_path=str(DATA_PATH / "spikegadgets" / "W122_06_09_2019_1_fromSD.rec"), gains=[0.385] * 128), ] save_directory = OUTPUT_PATH