From 58120d6d3ca617527ad3e82fcfd50d36c65b80c4 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 3 Dec 2024 09:45:57 -0600 Subject: [PATCH 01/10] accept folder path on spikeinterface --- .../ecephys/spikeglx/spikeglxdatainterface.py | 22 +++++++++++-------- tests/test_on_data/ecephys/test_lfp.py | 4 +--- .../ecephys/test_recording_interfaces.py | 4 +--- .../ecephys/test_spikeglx_converter.py | 14 +++++++----- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py index c15516431..622eb6353 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py @@ -4,7 +4,7 @@ from typing import Optional import numpy as np -from pydantic import FilePath, validate_call +from pydantic import DirectoryPath, FilePath, validate_call from .spikeglx_utils import ( add_recording_extractor_properties, @@ -45,7 +45,6 @@ def get_source_schema(cls) -> dict: def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: extractor_kwargs = source_data.copy() - extractor_kwargs.pop("file_path") extractor_kwargs["folder_path"] = self.folder_path extractor_kwargs["all_annotations"] = True extractor_kwargs["stream_id"] = self.stream_id @@ -54,9 +53,11 @@ def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: @validate_call def __init__( self, - file_path: FilePath, - verbose: bool = True, + folder_path: Optional[DirectoryPath] = None, + stream_id: Optional[str] = None, es_key: Optional[str] = None, + verbose: bool = True, + file_path: Optional[FilePath] = None, ): """ Parameters @@ -68,7 +69,13 @@ def __init__( es_key : str, default: "ElectricalSeries" """ - self.stream_id = fetch_stream_id_for_spikelgx_file(file_path) + if file_path is not None and stream_id is None: + self.stream_id = fetch_stream_id_for_spikelgx_file(file_path) + self.folder_path = Path(file_path).parent + else: + self.stream_id = stream_id + self.folder_path = Path(folder_path) + if es_key is None: if "lf" in self.stream_id: es_key = "ElectricalSeriesLF" @@ -76,15 +83,12 @@ def __init__( es_key = "ElectricalSeriesAP" else: raise ValueError("Cannot automatically determine es_key from path") - file_path = Path(file_path) - self.folder_path = file_path.parent super().__init__( - file_path=file_path, + folder_path=folder_path, verbose=verbose, es_key=es_key, ) - self.source_data["file_path"] = str(file_path) self.meta = self.recording_extractor.neo_reader.signals_info_dict[(0, self.stream_id)]["meta"] # Set electrodes properties diff --git a/tests/test_on_data/ecephys/test_lfp.py b/tests/test_on_data/ecephys/test_lfp.py index c46f8d297..010516d86 100644 --- a/tests/test_on_data/ecephys/test_lfp.py +++ b/tests/test_on_data/ecephys/test_lfp.py @@ -57,9 +57,7 @@ class TestEcephysLFPNwbConversions(unittest.TestCase): param( data_interface=SpikeGLXRecordingInterface, interface_kwargs=dict( - file_path=( - DATA_PATH / "spikeglx" / "Noise4Sam_g0" / "Noise4Sam_g0_imec0" / "Noise4Sam_g0_t0.imec0.lf.bin" - ) + folder_path=DATA_PATH / "spikeglx" / "Noise4Sam_g0" / "Noise4Sam_g0_imec0", stream_id="imec0.lf" ), expected_write_module="raw", ), diff --git a/tests/test_on_data/ecephys/test_recording_interfaces.py b/tests/test_on_data/ecephys/test_recording_interfaces.py index 7677ded22..0520b5b42 100644 --- a/tests/test_on_data/ecephys/test_recording_interfaces.py +++ b/tests/test_on_data/ecephys/test_recording_interfaces.py @@ -641,9 +641,7 @@ def test_extracted_metadata(self, setup_interface): class TestSpikeGLXRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = SpikeGLXRecordingInterface interface_kwargs = dict( - file_path=str( - ECEPHY_DATA_PATH / "spikeglx" / "Noise4Sam_g0" / "Noise4Sam_g0_imec0" / "Noise4Sam_g0_t0.imec0.ap.bin" - ) + folder_path=ECEPHY_DATA_PATH / "spikeglx" / "Noise4Sam_g0" / "Noise4Sam_g0_imec0", stream_id="imec0.ap" ) save_directory = OUTPUT_PATH diff --git a/tests/test_on_data/ecephys/test_spikeglx_converter.py b/tests/test_on_data/ecephys/test_spikeglx_converter.py index af98789c1..40df49311 100644 --- a/tests/test_on_data/ecephys/test_spikeglx_converter.py +++ b/tests/test_on_data/ecephys/test_spikeglx_converter.py @@ -33,7 +33,7 @@ def assertNWBFileStructure(self, nwbfile_path: FilePath, expected_session_start_ with NWBHDF5IO(path=nwbfile_path) as io: nwbfile = io.read() - assert nwbfile.session_start_time == expected_session_start_time + assert nwbfile.session_start_time.replace(tzinfo=None) == expected_session_start_time assert "ElectricalSeriesAPImec0" in nwbfile.acquisition assert "ElectricalSeriesLFImec0" in nwbfile.acquisition @@ -74,7 +74,7 @@ def test_single_probe_spikeglx_converter(self): nwbfile_path = self.tmpdir / "test_single_probe_spikeglx_converter.nwb" converter.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata) - expected_session_start_time = datetime(2020, 11, 3, 10, 35, 10).astimezone() + expected_session_start_time = datetime(2020, 11, 3, 10, 35, 10) self.assertNWBFileStructure(nwbfile_path=nwbfile_path, expected_session_start_time=expected_session_start_time) def test_in_converter_pipe(self): @@ -84,7 +84,7 @@ def test_in_converter_pipe(self): nwbfile_path = self.tmpdir / "test_spikeglx_converter_in_converter_pipe.nwb" converter_pipe.run_conversion(nwbfile_path=nwbfile_path) - expected_session_start_time = datetime(2020, 11, 3, 10, 35, 10).astimezone() + expected_session_start_time = datetime(2020, 11, 3, 10, 35, 10) self.assertNWBFileStructure(nwbfile_path=nwbfile_path, expected_session_start_time=expected_session_start_time) def test_in_nwbconverter(self): @@ -101,7 +101,7 @@ class TestConverter(NWBConverter): nwbfile_path = self.tmpdir / "test_spikeglx_converter_in_nwbconverter.nwb" converter.run_conversion(nwbfile_path=nwbfile_path) - expected_session_start_time = datetime(2020, 11, 3, 10, 35, 10).astimezone() + expected_session_start_time = datetime(2020, 11, 3, 10, 35, 10) self.assertNWBFileStructure(nwbfile_path=nwbfile_path, expected_session_start_time=expected_session_start_time) @@ -118,7 +118,9 @@ def assertNWBFileStructure(self, nwbfile_path: FilePath, expected_session_start_ with NWBHDF5IO(path=nwbfile_path) as io: nwbfile = io.read() - assert nwbfile.session_start_time == expected_session_start_time + # Do the comparison without timezone information to avoid CI timezone issues + # The timezone is set by pynbw automatically + assert nwbfile.session_start_time.replace(tzinfo=None) == expected_session_start_time # TODO: improve name of segments using 'Segment{index}' for clarity assert "ElectricalSeriesAPImec00" in nwbfile.acquisition @@ -167,7 +169,7 @@ def test_multi_probe_spikeglx_converter(self): nwbfile_path = self.tmpdir / "test_multi_probe_spikeglx_converter.nwb" converter.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata) - expected_session_start_time = datetime(2022, 5, 19, 17, 37, 47).astimezone() + expected_session_start_time = datetime(2022, 5, 19, 17, 37, 47) self.assertNWBFileStructure(nwbfile_path=nwbfile_path, expected_session_start_time=expected_session_start_time) From b560f29be85e226139939e50084107a7e536ae50 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 4 Dec 2024 10:35:23 -0600 Subject: [PATCH 02/10] test passing no metadata --- .../ecephys/spikeglx/spikeglxconverter.py | 31 +++++++------------ .../ecephys/spikeglx/spikeglxdatainterface.py | 28 ++++++++++------- .../ecephys/spikeglx/spikeglxnidqinterface.py | 27 +++++++++++----- .../ecephys/test_spikeglx_converter.py | 8 ++--- 4 files changed, 50 insertions(+), 44 deletions(-) diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py index 007c3177c..029955d24 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py @@ -29,8 +29,10 @@ def get_source_schema(cls): @classmethod def get_streams(cls, folder_path: DirectoryPath) -> list[str]: + "Return the stream ids available in the folder." from spikeinterface.extractors import SpikeGLXRecordingExtractor + # The first entry is the stream ids the second is the stream names return SpikeGLXRecordingExtractor.get_streams(folder_path=folder_path)[0] @validate_call @@ -61,28 +63,17 @@ def __init__( """ folder_path = Path(folder_path) - streams = streams or self.get_streams(folder_path=folder_path) + streams_ids = streams or self.get_streams(folder_path=folder_path) data_interfaces = dict() - for stream in streams: - if "ap" in stream: - probe_name = stream[:5] - file_path = ( - folder_path / f"{folder_path.stem}_{probe_name}" / f"{folder_path.stem}_t0.{probe_name}.ap.bin" - ) - es_key = f"ElectricalSeriesAP{probe_name.capitalize()}" - interface = SpikeGLXRecordingInterface(file_path=file_path, es_key=es_key) - elif "lf" in stream: - probe_name = stream[:5] - file_path = ( - folder_path / f"{folder_path.stem}_{probe_name}" / f"{folder_path.stem}_t0.{probe_name}.lf.bin" - ) - es_key = f"ElectricalSeriesLF{probe_name.capitalize()}" - interface = SpikeGLXRecordingInterface(file_path=file_path, es_key=es_key) - elif "nidq" in stream: - file_path = folder_path / f"{folder_path.stem}_t0.nidq.bin" - interface = SpikeGLXNIDQInterface(file_path=file_path) - data_interfaces.update({str(stream): interface}) # Without str() casting, is a numpy string + + nidq_streams = [stream_id for stream_id in streams_ids if stream_id == "nidq"] + electrical_streams = [stream_id for stream_id in streams_ids if stream_id not in nidq_streams] + for stream_id in electrical_streams: + data_interfaces[stream_id] = SpikeGLXRecordingInterface(folder_path=folder_path, stream_id=stream_id) + + for stream_id in nidq_streams: + data_interfaces[stream_id] = SpikeGLXNIDQInterface(folder_path=folder_path) super().__init__(data_interfaces=data_interfaces, verbose=verbose) diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py index 622eb6353..3d00bb00b 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py @@ -62,34 +62,40 @@ def __init__( """ Parameters ---------- + folder_path: DirectoryPath + Folder path containing the binary files of the SpikeGLX recording. + stream_id: str, optional + Stream ID of the SpikeGLX recording. file_path : FilePathType Path to .bin file. Point to .ap.bin for SpikeGLXRecordingInterface and .lf.bin for SpikeGLXLFPInterface. verbose : bool, default: True Whether to output verbose text. - es_key : str, default: "ElectricalSeries" + es_key : str, the key to access the metadata of the ElectricalSeries. """ if file_path is not None and stream_id is None: self.stream_id = fetch_stream_id_for_spikelgx_file(file_path) self.folder_path = Path(file_path).parent + else: self.stream_id = stream_id self.folder_path = Path(folder_path) - if es_key is None: - if "lf" in self.stream_id: - es_key = "ElectricalSeriesLF" - elif "ap" in self.stream_id: - es_key = "ElectricalSeriesAP" - else: - raise ValueError("Cannot automatically determine es_key from path") - super().__init__( folder_path=folder_path, verbose=verbose, es_key=es_key, ) - self.meta = self.recording_extractor.neo_reader.signals_info_dict[(0, self.stream_id)]["meta"] + + signal_info_key = (0, self.stream_id) # Key format is (segment_index, stream_id) + self._signals_info_dict = self.recording_extractor.neo_reader.signals_info_dict[signal_info_key] + self.meta = self._signals_info_dict["meta"] + + if es_key is None: + stream_kind = self._signals_info_dict["stream_kind"] # ap or lf + stream_kind_caps = stream_kind.upper() + device = self._signals_info_dict["device"].capitalize() # imec0, imec1, etc. + self.es_key = f"ElectricalSeries{stream_kind_caps}{device}" # Set electrodes properties add_recording_extractor_properties(self.recording_extractor) @@ -104,7 +110,7 @@ def get_metadata(self) -> dict: device = get_device_metadata(self.meta) # Should follow pattern 'Imec0', 'Imec1', etc. - probe_name = self.stream_id[:5].capitalize() + probe_name = self._signals_info_dict["device"].capitalize() device["name"] = f"Neuropixel{probe_name}" # Add groups metadata diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py index 3cf50080a..7afa48000 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py @@ -1,7 +1,8 @@ from pathlib import Path +from typing import Optional import numpy as np -from pydantic import ConfigDict, FilePath, validate_call +from pydantic import ConfigDict, DirectoryPath, FilePath, validate_call from .spikeglx_utils import get_session_start_time from ..baserecordingextractorinterface import BaseRecordingExtractorInterface @@ -29,7 +30,6 @@ def get_source_schema(cls) -> dict: def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: extractor_kwargs = source_data.copy() - extractor_kwargs.pop("file_path") extractor_kwargs["folder_path"] = self.folder_path extractor_kwargs["stream_id"] = self.stream_id return extractor_kwargs @@ -37,7 +37,8 @@ def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def __init__( self, - file_path: FilePath, + folder_path: Optional[DirectoryPath] = None, + file_path: Optional[FilePath] = None, verbose: bool = True, load_sync_channel: bool = False, es_key: str = "ElectricalSeriesNIDQ", @@ -49,6 +50,8 @@ def __init__( Parameters ---------- + folder_path : DirectoryPath + Path to the folder containing the .nidq.bin file. file_path : FilePathType Path to .nidq.bin file. verbose : bool, default: True @@ -59,10 +62,17 @@ def __init__( es_key : str, default: "ElectricalSeriesNIDQ" """ - self.file_path = Path(file_path) - self.folder_path = self.file_path.parent + if file_path is None and folder_path is None: + raise ValueError("Either 'file_path' or 'folder_path' must be provided.") + + if file_path is not None: + file_path = Path(file_path) + self.folder_path = file_path.parent + + if folder_path is not None: + self.folder_path = Path(folder_path) + super().__init__( - file_path=self.file_path, verbose=verbose, load_sync_channel=load_sync_channel, es_key=es_key, @@ -72,7 +82,10 @@ def __init__( self.recording_extractor.set_property( key="group_name", values=["NIDQChannelGroup"] * self.recording_extractor.get_num_channels() ) - self.meta = self.recording_extractor.neo_reader.signals_info_dict[(0, "nidq")]["meta"] + + signal_info_key = (0, self.stream_id) # Key format is (segment_index, stream_id) + self._signals_info_dict = self.recording_extractor.neo_reader.signals_info_dict[signal_info_key] + self.meta = self._signals_info_dict["meta"] def get_metadata(self) -> dict: metadata = super().get_metadata() diff --git a/tests/test_on_data/ecephys/test_spikeglx_converter.py b/tests/test_on_data/ecephys/test_spikeglx_converter.py index 40df49311..544e7348f 100644 --- a/tests/test_on_data/ecephys/test_spikeglx_converter.py +++ b/tests/test_on_data/ecephys/test_spikeglx_converter.py @@ -131,7 +131,7 @@ def assertNWBFileStructure(self, nwbfile_path: FilePath, expected_session_start_ assert "ElectricalSeriesLFImec01" in nwbfile.acquisition assert "ElectricalSeriesLFImec10" in nwbfile.acquisition assert "ElectricalSeriesLFImec11" in nwbfile.acquisition - assert len(nwbfile.acquisition) == 8 + assert len(nwbfile.acquisition) == 16 assert "NeuropixelImec0" in nwbfile.devices assert "NeuropixelImec1" in nwbfile.devices @@ -143,7 +143,7 @@ def assertNWBFileStructure(self, nwbfile_path: FilePath, expected_session_start_ def test_multi_probe_spikeglx_converter(self): converter = SpikeGLXConverterPipe( - folder_path=SPIKEGLX_PATH / "multi_trigger_multi_gate" / "SpikeGLX" / "5-19-2022-CI0" / "5-19-2022-CI0_g0" + folder_path=SPIKEGLX_PATH / "multi_trigger_multi_gate" / "SpikeGLX" / "5-19-2022-CI0" ) metadata = converter.get_metadata() @@ -162,10 +162,6 @@ def test_multi_probe_spikeglx_converter(self): device_metadata = test_ecephys_metadata.pop("Device") expected_device_metadata = expected_ecephys_metadata.pop("Device") - assert device_metadata == expected_device_metadata - - assert test_ecephys_metadata == expected_ecephys_metadata - nwbfile_path = self.tmpdir / "test_multi_probe_spikeglx_converter.nwb" converter.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata) From ca00c34b3da4f4d2404d0f6b34bd84af5f1f965f Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 4 Dec 2024 10:39:37 -0600 Subject: [PATCH 03/10] restore metadata test --- tests/test_on_data/ecephys/test_spikeglx_converter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_on_data/ecephys/test_spikeglx_converter.py b/tests/test_on_data/ecephys/test_spikeglx_converter.py index 544e7348f..1745a61aa 100644 --- a/tests/test_on_data/ecephys/test_spikeglx_converter.py +++ b/tests/test_on_data/ecephys/test_spikeglx_converter.py @@ -162,6 +162,9 @@ def test_multi_probe_spikeglx_converter(self): device_metadata = test_ecephys_metadata.pop("Device") expected_device_metadata = expected_ecephys_metadata.pop("Device") + assert device_metadata == expected_device_metadata + assert test_ecephys_metadata == expected_ecephys_metadata + nwbfile_path = self.tmpdir / "test_multi_probe_spikeglx_converter.nwb" converter.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata) From 1b350f19ea2f672a4af413c901489ae06fbe1366 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 4 Dec 2024 15:42:57 -0600 Subject: [PATCH 04/10] fix yaml tests --- .../test_yaml/test_yaml_conversion_specification.py | 6 +++--- .../test_yaml/test_yaml_conversion_specification_cli.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_on_data/test_yaml/test_yaml_conversion_specification.py b/tests/test_on_data/test_yaml/test_yaml_conversion_specification.py index 61c71cf86..3d60e5752 100644 --- a/tests/test_on_data/test_yaml/test_yaml_conversion_specification.py +++ b/tests/test_on_data/test_yaml/test_yaml_conversion_specification.py @@ -60,7 +60,7 @@ def test_run_conversion_from_yaml(): assert nwbfile.institution == "My Institution" assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") assert nwbfile.subject.subject_id == "1" - assert "ElectricalSeriesAP" in nwbfile.acquisition + assert "ElectricalSeriesAPImec0" in nwbfile.acquisition nwbfile_path_2 = OUTPUT_PATH / "example_converter_spec_2.nwb" assert nwbfile_path_2.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path_2}'!" @@ -111,7 +111,7 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): assert nwbfile.institution == "My Institution" assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") assert nwbfile.subject.subject_id == "Mouse 1" - assert "ElectricalSeriesAP" in nwbfile.acquisition + assert "ElectricalSeriesAPImec0" in nwbfile.acquisition nwbfile_path = self.test_folder / "sub-Mouse-1_ses-20201109T211909.nwb" assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " @@ -123,7 +123,7 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): assert nwbfile.institution == "My Institution" assert nwbfile.session_start_time == datetime.fromisoformat("2020-11-09T21:19:09+00:00") assert nwbfile.subject.subject_id == "Mouse 1" - assert "ElectricalSeriesAP" in nwbfile.acquisition + assert "ElectricalSeriesAPImec0" in nwbfile.acquisition nwbfile_path = self.test_folder / "example_defined_name.nwb" assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " diff --git a/tests/test_on_data/test_yaml/test_yaml_conversion_specification_cli.py b/tests/test_on_data/test_yaml/test_yaml_conversion_specification_cli.py index cc4391623..2b36d5874 100644 --- a/tests/test_on_data/test_yaml/test_yaml_conversion_specification_cli.py +++ b/tests/test_on_data/test_yaml/test_yaml_conversion_specification_cli.py @@ -34,7 +34,7 @@ def test_run_conversion_from_yaml_cli(self): assert nwbfile.institution == "My Institution" assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") assert nwbfile.subject.subject_id == "1" - assert "ElectricalSeriesAP" in nwbfile.acquisition + assert "ElectricalSeriesAPImec0" in nwbfile.acquisition nwbfile_path = self.test_folder / "example_converter_spec_2.nwb" assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " @@ -81,7 +81,7 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): assert nwbfile.institution == "My Institution" assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") assert nwbfile.subject.subject_id == "Mouse 1" - assert "ElectricalSeriesAP" in nwbfile.acquisition + assert "ElectricalSeriesAPImec0" in nwbfile.acquisition nwbfile_path = self.test_folder / "sub-Mouse-1_ses-20201109T211909.nwb" assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " @@ -93,7 +93,7 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): assert nwbfile.institution == "My Institution" assert nwbfile.session_start_time == datetime.fromisoformat("2020-11-09T21:19:09+00:00") assert nwbfile.subject.subject_id == "Mouse 1" - assert "ElectricalSeriesAP" in nwbfile.acquisition + assert "ElectricalSeriesAPImec0" in nwbfile.acquisition nwbfile_path = self.test_folder / "example_defined_name.nwb" assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " From 683ba3907da09693ff89b372871e8feaa5e2008e Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 4 Dec 2024 19:12:12 -0600 Subject: [PATCH 05/10] Revert "fix yaml tests" This reverts commit 1b350f19ea2f672a4af413c901489ae06fbe1366. --- .../test_yaml/test_yaml_conversion_specification.py | 6 +++--- .../test_yaml/test_yaml_conversion_specification_cli.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_on_data/test_yaml/test_yaml_conversion_specification.py b/tests/test_on_data/test_yaml/test_yaml_conversion_specification.py index 3d60e5752..61c71cf86 100644 --- a/tests/test_on_data/test_yaml/test_yaml_conversion_specification.py +++ b/tests/test_on_data/test_yaml/test_yaml_conversion_specification.py @@ -60,7 +60,7 @@ def test_run_conversion_from_yaml(): assert nwbfile.institution == "My Institution" assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") assert nwbfile.subject.subject_id == "1" - assert "ElectricalSeriesAPImec0" in nwbfile.acquisition + assert "ElectricalSeriesAP" in nwbfile.acquisition nwbfile_path_2 = OUTPUT_PATH / "example_converter_spec_2.nwb" assert nwbfile_path_2.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path_2}'!" @@ -111,7 +111,7 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): assert nwbfile.institution == "My Institution" assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") assert nwbfile.subject.subject_id == "Mouse 1" - assert "ElectricalSeriesAPImec0" in nwbfile.acquisition + assert "ElectricalSeriesAP" in nwbfile.acquisition nwbfile_path = self.test_folder / "sub-Mouse-1_ses-20201109T211909.nwb" assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " @@ -123,7 +123,7 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): assert nwbfile.institution == "My Institution" assert nwbfile.session_start_time == datetime.fromisoformat("2020-11-09T21:19:09+00:00") assert nwbfile.subject.subject_id == "Mouse 1" - assert "ElectricalSeriesAPImec0" in nwbfile.acquisition + assert "ElectricalSeriesAP" in nwbfile.acquisition nwbfile_path = self.test_folder / "example_defined_name.nwb" assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " diff --git a/tests/test_on_data/test_yaml/test_yaml_conversion_specification_cli.py b/tests/test_on_data/test_yaml/test_yaml_conversion_specification_cli.py index 2b36d5874..cc4391623 100644 --- a/tests/test_on_data/test_yaml/test_yaml_conversion_specification_cli.py +++ b/tests/test_on_data/test_yaml/test_yaml_conversion_specification_cli.py @@ -34,7 +34,7 @@ def test_run_conversion_from_yaml_cli(self): assert nwbfile.institution == "My Institution" assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") assert nwbfile.subject.subject_id == "1" - assert "ElectricalSeriesAPImec0" in nwbfile.acquisition + assert "ElectricalSeriesAP" in nwbfile.acquisition nwbfile_path = self.test_folder / "example_converter_spec_2.nwb" assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " @@ -81,7 +81,7 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): assert nwbfile.institution == "My Institution" assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") assert nwbfile.subject.subject_id == "Mouse 1" - assert "ElectricalSeriesAPImec0" in nwbfile.acquisition + assert "ElectricalSeriesAP" in nwbfile.acquisition nwbfile_path = self.test_folder / "sub-Mouse-1_ses-20201109T211909.nwb" assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " @@ -93,7 +93,7 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): assert nwbfile.institution == "My Institution" assert nwbfile.session_start_time == datetime.fromisoformat("2020-11-09T21:19:09+00:00") assert nwbfile.subject.subject_id == "Mouse 1" - assert "ElectricalSeriesAPImec0" in nwbfile.acquisition + assert "ElectricalSeriesAP" in nwbfile.acquisition nwbfile_path = self.test_folder / "example_defined_name.nwb" assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " From 25e6486fcc38f75b92838276345de4493c63ff98 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 4 Dec 2024 19:58:55 -0600 Subject: [PATCH 06/10] convert and interface have consistent metadata --- .../ecephys/spikeglx/spikeglxdatainterface.py | 11 ++++++++++- .../ecephys/spikeglx_single_probe_metadata.json | 12 ++++++------ .../test_on_data/ecephys/test_spikeglx_converter.py | 12 ++++++------ 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py index 3d00bb00b..842ecd8fc 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py @@ -95,7 +95,16 @@ def __init__( stream_kind = self._signals_info_dict["stream_kind"] # ap or lf stream_kind_caps = stream_kind.upper() device = self._signals_info_dict["device"].capitalize() # imec0, imec1, etc. - self.es_key = f"ElectricalSeries{stream_kind_caps}{device}" + + electrical_series_name = f"ElectricalSeries{stream_kind_caps}" + + # Add imec{probe_index} to the electrical series name when there are multiple probes + # or undefined, `typeImEnabled` is present in the meta of all the production probes + self.probes_enabled_in_run = int(self.meta.get("typeImEnabled", 0)) + if self.probes_enabled_in_run != 1: + electrical_series_name += f"{device}" + + self.es_key = electrical_series_name # Set electrodes properties add_recording_extractor_properties(self.recording_extractor) diff --git a/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json b/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json index 637124cd0..20f11742b 100644 --- a/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json +++ b/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json @@ -29,9 +29,9 @@ "device": "NIDQBoard" } ], - "ElectricalSeriesAPImec0": { - "name": "ElectricalSeriesAPImec0", - "description": "Acquisition traces for the ElectricalSeriesAPImec0." + "ElectricalSeriesAP": { + "name": "ElectricalSeriesAP", + "description": "Acquisition traces for the ElectricalSeriesAP." }, "Electrodes": [ { @@ -51,9 +51,9 @@ "name": "ElectricalSeriesNIDQ", "description": "Raw acquisition traces from the NIDQ (.nidq.bin) channels." }, - "ElectricalSeriesLFImec0": { - "name": "ElectricalSeriesLFImec0", - "description": "Acquisition traces for the ElectricalSeriesLFImec0." + "ElectricalSeriesLF": { + "name": "ElectricalSeriesLF", + "description": "Acquisition traces for the ElectricalSeriesLF." } } } diff --git a/tests/test_on_data/ecephys/test_spikeglx_converter.py b/tests/test_on_data/ecephys/test_spikeglx_converter.py index 1745a61aa..970a815af 100644 --- a/tests/test_on_data/ecephys/test_spikeglx_converter.py +++ b/tests/test_on_data/ecephys/test_spikeglx_converter.py @@ -35,8 +35,8 @@ def assertNWBFileStructure(self, nwbfile_path: FilePath, expected_session_start_ assert nwbfile.session_start_time.replace(tzinfo=None) == expected_session_start_time - assert "ElectricalSeriesAPImec0" in nwbfile.acquisition - assert "ElectricalSeriesLFImec0" in nwbfile.acquisition + assert "ElectricalSeriesAP" in nwbfile.acquisition + assert "ElectricalSeriesLF" in nwbfile.acquisition assert "ElectricalSeriesNIDQ" in nwbfile.acquisition assert len(nwbfile.acquisition) == 3 @@ -194,7 +194,7 @@ def test_electrode_table_writing(tmp_path): np.testing.assert_array_equal(saved_channel_names, expected_channel_names_nidq) # Test AP - electrical_series = nwbfile.acquisition["ElectricalSeriesAPImec0"] + electrical_series = nwbfile.acquisition["ElectricalSeriesAP"] ap_electrodes_table_region = electrical_series.electrodes region_indices = ap_electrodes_table_region.data recording_extractor = converter.data_interface_objects["imec0.ap"].recording_extractor @@ -204,7 +204,7 @@ def test_electrode_table_writing(tmp_path): np.testing.assert_array_equal(saved_channel_names, expected_channel_names_ap) # Test LF - electrical_series = nwbfile.acquisition["ElectricalSeriesLFImec0"] + electrical_series = nwbfile.acquisition["ElectricalSeriesLF"] lf_electrodes_table_region = electrical_series.electrodes region_indices = lf_electrodes_table_region.data recording_extractor = converter.data_interface_objects["imec0.lf"].recording_extractor @@ -223,7 +223,7 @@ def test_electrode_table_writing(tmp_path): # Test round trip with spikeinterface recording_extractor_ap = NwbRecordingExtractor( file_path=nwbfile_path, - electrical_series_name="ElectricalSeriesAPImec0", + electrical_series_name="ElectricalSeriesAP", ) channel_ids = recording_extractor_ap.get_channel_ids() @@ -231,7 +231,7 @@ def test_electrode_table_writing(tmp_path): recording_extractor_lf = NwbRecordingExtractor( file_path=nwbfile_path, - electrical_series_name="ElectricalSeriesLFImec0", + electrical_series_name="ElectricalSeriesLF", ) channel_ids = recording_extractor_lf.get_channel_ids() From 78b913271bb905cd721f103ef0dcedf756b5f300 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 5 Dec 2024 08:42:25 -0600 Subject: [PATCH 07/10] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a37093e39..9bc03adcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,11 +13,13 @@ ## Bug Fixes * datetime objects now can be validated as conversion options [#1139](https://github.com/catalystneuro/neuroconv/pull/1126) * Fix a bug where data in `DeepLabCutInterface` failed to write when `ndx-pose` was not imported. [#1144](https://github.com/catalystneuro/neuroconv/pull/1144) +* `SpikeGLXConverterPipe` converter now accepts multi-probe structures with multi-trigger and does not assume a specific folder structure [#1150](https://github.com/catalystneuro/neuroconv/pull/1150) ## Features * Propagate the `unit_electrode_indices` argument from the spikeinterface tools to `BaseSortingExtractorInterface`. This allows users to map units to the electrode table when adding sorting data [PR #1124](https://github.com/catalystneuro/neuroconv/pull/1124) * Imaging interfaces have a new conversion option `always_write_timestamps` that can be used to force writing timestamps even if neuroconv's heuristics indicates regular sampling rate [PR #1125](https://github.com/catalystneuro/neuroconv/pull/1125) * Added .csv support to DeepLabCutInterface [PR #1140](https://github.com/catalystneuro/neuroconv/pull/1140) +* `SpikeGLXRecordingInterface` now also accepts `folder_path` making its behavior requivalent to SpikeInterface [#1150](https://github.com/catalystneuro/neuroconv/pull/1150) ## Improvements * Use mixing tests for ecephy's mocks [PR #1136](https://github.com/catalystneuro/neuroconv/pull/1136) From e702a589dbbf253b092487ab1ba34fa927f8e178 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Mon, 9 Dec 2024 11:17:51 -0500 Subject: [PATCH 08/10] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e86193c0..bcbe82e5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ * Propagate the `unit_electrode_indices` argument from the spikeinterface tools to `BaseSortingExtractorInterface`. This allows users to map units to the electrode table when adding sorting data [PR #1124](https://github.com/catalystneuro/neuroconv/pull/1124) * Imaging interfaces have a new conversion option `always_write_timestamps` that can be used to force writing timestamps even if neuroconv's heuristics indicates regular sampling rate [PR #1125](https://github.com/catalystneuro/neuroconv/pull/1125) * Added .csv support to DeepLabCutInterface [PR #1140](https://github.com/catalystneuro/neuroconv/pull/1140) -* `SpikeGLXRecordingInterface` now also accepts `folder_path` making its behavior requivalent to SpikeInterface [#1150](https://github.com/catalystneuro/neuroconv/pull/1150) +* `SpikeGLXRecordingInterface` now also accepts `folder_path` making its behavior equivalent to SpikeInterface [#1150](https://github.com/catalystneuro/neuroconv/pull/1150) * Added the `rclone_transfer_batch_job` helper function for executing Rclone data transfers in AWS Batch jobs. [PR #1085](https://github.com/catalystneuro/neuroconv/pull/1085) * Added the `deploy_neuroconv_batch_job` helper function for deploying NeuroConv AWS Batch jobs. [PR #1086](https://github.com/catalystneuro/neuroconv/pull/1086) From 71b3299c45aa89166ceb6e21cef1f8adf0e9c03c Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 9 Dec 2024 11:05:17 -0600 Subject: [PATCH 09/10] fix paths --- .../ecephys/spikeglx/spikeglxdatainterface.py | 6 +++--- .../ecephys/spikeglx/spikeglxnidqinterface.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py index 842ecd8fc..95799c55e 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py @@ -53,11 +53,11 @@ def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: @validate_call def __init__( self, + file_path: Optional[FilePath] = None, + verbose: bool = True, + es_key: Optional[str] = None, folder_path: Optional[DirectoryPath] = None, stream_id: Optional[str] = None, - es_key: Optional[str] = None, - verbose: bool = True, - file_path: Optional[FilePath] = None, ): """ Parameters diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py index 7afa48000..1d7079716 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py @@ -37,11 +37,11 @@ def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def __init__( self, - folder_path: Optional[DirectoryPath] = None, file_path: Optional[FilePath] = None, verbose: bool = True, load_sync_channel: bool = False, es_key: str = "ElectricalSeriesNIDQ", + folder_path: Optional[DirectoryPath] = None, ): """ Read channel data from the NIDQ board for the SpikeGLX recording. From 4d21be6f2a8826e0ea090edfdc8597aadd636719 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 10 Dec 2024 07:55:50 -0600 Subject: [PATCH 10/10] small dostring improvement to trigger CI read the docs --- .../datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py index 95799c55e..00419e036 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py @@ -66,6 +66,7 @@ def __init__( Folder path containing the binary files of the SpikeGLX recording. stream_id: str, optional Stream ID of the SpikeGLX recording. + Examples are 'nidq', 'imec0.ap', 'imec0.lf', 'imec1.ap', 'imec1.lf', etc. file_path : FilePathType Path to .bin file. Point to .ap.bin for SpikeGLXRecordingInterface and .lf.bin for SpikeGLXLFPInterface. verbose : bool, default: True