From d2c465f6e57bb36b9650c681fcd552a314df0bdb Mon Sep 17 00:00:00 2001 From: Jeremy Magland Date: Fri, 10 Nov 2023 14:44:42 -0500 Subject: [PATCH 01/17] add set_probe to BaseRecordingExtractorInterface --- CHANGELOG.md | 1 + .../baserecordingextractorinterface.py | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f1d23080..2a07bb408 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Features * Added Pydantic data models of `BackendConfiguration` for both HDF5 and Zarr datasets (container/mapper of all the `DatasetConfiguration`s for a particular file). [PR #568](https://github.com/catalystneuro/neuroconv/pull/568) +* Added set_probe() method to BaseRecordingExtractorInterface diff --git a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py index 7dd4fed2a..6282e1997 100644 --- a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py @@ -225,6 +225,32 @@ def set_aligned_segment_starting_times(self, aligned_segment_starting_times: Lis ] self.set_aligned_segment_timestamps(aligned_segment_timestamps=aligned_segment_timestamps) + def set_probe(self, probe, *, group_mode: Union[Literal["by_shank"], Literal["by_probe"]]): + """ + Set the probe information via a ProbeInterface object. + + Parameters + ---------- + probe : probeinterface.Probe + The probe object. + group_mode : {'by_shank', 'by_probe'} + How to group the channels. If 'by_shank', channels are grouped by the shank_id column. + If 'by_probe', channels are grouped by the probe_id column. + This is a required parameter to avoid the pitfall of using the wrong mode. + """ + # Set the probe to the recording extractor + self.recording_extractor.set_probe( + probe, + in_place=True, + group_mode=group_mode, + ) + # Spike interface sets the "group" property + # Buy neuroconv expects "group_name" to be set + self.recording_extractor.set_property( + "group_name", + self.recording_extractor.get_property("group").astype(str) + ) + def align_by_interpolation( self, unaligned_timestamps: np.ndarray, From 3a27e5254d4453f1ca8cfd607c617434281507f2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 10 Nov 2023 20:25:59 +0000 Subject: [PATCH 02/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../ecephys/baserecordingextractorinterface.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py index 6282e1997..2c249bc77 100644 --- a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py @@ -246,10 +246,7 @@ def set_probe(self, probe, *, group_mode: Union[Literal["by_shank"], Literal["by ) # Spike interface sets the "group" property # Buy neuroconv expects "group_name" to be set - self.recording_extractor.set_property( - "group_name", - self.recording_extractor.get_property("group").astype(str) - ) + self.recording_extractor.set_property("group_name", self.recording_extractor.get_property("group").astype(str)) def align_by_interpolation( self, From 99830992a5e1baa92a2ca4c66bbd89cd201883d4 Mon Sep 17 00:00:00 2001 From: Jeremy Magland Date: Fri, 10 Nov 2023 16:35:50 -0500 Subject: [PATCH 03/17] Adjust comment in baserecordingextractorinterface.py Co-authored-by: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> --- .../datainterfaces/ecephys/baserecordingextractorinterface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py index 2c249bc77..46978404f 100644 --- a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py @@ -245,7 +245,7 @@ def set_probe(self, probe, *, group_mode: Union[Literal["by_shank"], Literal["by group_mode=group_mode, ) # Spike interface sets the "group" property - # Buy neuroconv expects "group_name" to be set + # But neuroconv allows "group_name" property to override spike interface "group" value self.recording_extractor.set_property("group_name", self.recording_extractor.get_property("group").astype(str)) def align_by_interpolation( From 8332459c503a3f1bfe658abc8ea1e7e42f90e197 Mon Sep 17 00:00:00 2001 From: Jeremy Magland Date: Mon, 13 Nov 2023 20:04:48 -0500 Subject: [PATCH 04/17] cover set_probe with tests --- .../tools/testing/data_interface_mixins.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index ed1f1566e..2b4fa855e 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -109,6 +109,8 @@ def test_conversion_as_lone_interface(self): self.case = num self.test_kwargs = kwargs self.interface = self.data_interface_cls(**self.test_kwargs) + if isinstance(self.interface, BaseRecordingExtractorInterface): + self.interface.set_probe(_create_mock_probe_for_recording(self.interface.recording_extractor), group_mode="by_shank") self.check_metadata_schema_valid() self.check_conversion_options_schema_valid() self.check_metadata() @@ -119,6 +121,23 @@ def test_conversion_as_lone_interface(self): # Any extra custom checks to run self.run_custom_checks() +def _create_mock_probe_for_recording(recording): + import spikeinterface as si + import probeinterface as pi + R: si.BaseRecording = recording + n = R.get_num_channels() + positions = np.zeros((n, 2)) + for i in range(n): + x = i // 4 + y = i % 4 + positions[i] = x, y + positions *= 20 + probe = pi.Probe(ndim=2, si_units='um') + probe.set_contacts(positions=positions, shapes='circle', shape_params={'radius': 5}) + probe.set_device_channel_indices(np.arange(n)) + # set shank ids as 0, 1, 2, 0, 1, 2, ... + probe.set_shank_ids(np.arange(n) % 3) + return probe class TemporalAlignmentMixin: """ @@ -300,6 +319,9 @@ def check_read_nwb(self, nwbfile_path: str): # are specified, which occurs during check_recordings_equal when there is only one channel if self.nwb_recording.get_channel_ids()[0] != self.nwb_recording.get_channel_ids()[-1]: check_recordings_equal(RX1=recording, RX2=self.nwb_recording, return_scaled=False) + for property_name in ['x_rel', 'y_rel', 'z_rel', 'group']: + if property_name in recording.get_property_keys() or property_name in self.nwb_recording.get_property_keys(): + assert_array_equal(recording.get_property(property_name), self.nwb_recording.get_property(property_name)) if recording.has_scaled_traces() and self.nwb_recording.has_scaled_traces(): check_recordings_equal(RX1=recording, RX2=self.nwb_recording, return_scaled=True) From d3f6e2004b0fe31e70ce9e26db6e628e0b963333 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 01:05:45 +0000 Subject: [PATCH 05/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../tools/testing/data_interface_mixins.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index 2b4fa855e..c6803fa66 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -110,7 +110,9 @@ def test_conversion_as_lone_interface(self): self.test_kwargs = kwargs self.interface = self.data_interface_cls(**self.test_kwargs) if isinstance(self.interface, BaseRecordingExtractorInterface): - self.interface.set_probe(_create_mock_probe_for_recording(self.interface.recording_extractor), group_mode="by_shank") + self.interface.set_probe( + _create_mock_probe_for_recording(self.interface.recording_extractor), group_mode="by_shank" + ) self.check_metadata_schema_valid() self.check_conversion_options_schema_valid() self.check_metadata() @@ -121,9 +123,11 @@ def test_conversion_as_lone_interface(self): # Any extra custom checks to run self.run_custom_checks() + def _create_mock_probe_for_recording(recording): - import spikeinterface as si import probeinterface as pi + import spikeinterface as si + R: si.BaseRecording = recording n = R.get_num_channels() positions = np.zeros((n, 2)) @@ -132,13 +136,14 @@ def _create_mock_probe_for_recording(recording): y = i % 4 positions[i] = x, y positions *= 20 - probe = pi.Probe(ndim=2, si_units='um') - probe.set_contacts(positions=positions, shapes='circle', shape_params={'radius': 5}) + probe = pi.Probe(ndim=2, si_units="um") + probe.set_contacts(positions=positions, shapes="circle", shape_params={"radius": 5}) probe.set_device_channel_indices(np.arange(n)) # set shank ids as 0, 1, 2, 0, 1, 2, ... probe.set_shank_ids(np.arange(n) % 3) return probe + class TemporalAlignmentMixin: """ Generic class for testing temporal alignment methods. @@ -319,9 +324,14 @@ def check_read_nwb(self, nwbfile_path: str): # are specified, which occurs during check_recordings_equal when there is only one channel if self.nwb_recording.get_channel_ids()[0] != self.nwb_recording.get_channel_ids()[-1]: check_recordings_equal(RX1=recording, RX2=self.nwb_recording, return_scaled=False) - for property_name in ['x_rel', 'y_rel', 'z_rel', 'group']: - if property_name in recording.get_property_keys() or property_name in self.nwb_recording.get_property_keys(): - assert_array_equal(recording.get_property(property_name), self.nwb_recording.get_property(property_name)) + for property_name in ["x_rel", "y_rel", "z_rel", "group"]: + if ( + property_name in recording.get_property_keys() + or property_name in self.nwb_recording.get_property_keys() + ): + assert_array_equal( + recording.get_property(property_name), self.nwb_recording.get_property(property_name) + ) if recording.has_scaled_traces() and self.nwb_recording.has_scaled_traces(): check_recordings_equal(RX1=recording, RX2=self.nwb_recording, return_scaled=True) From 1b189ceb67a5c53695170827cae214566bb0c789 Mon Sep 17 00:00:00 2001 From: Jeremy Magland Date: Mon, 13 Nov 2023 21:00:05 -0500 Subject: [PATCH 06/17] only test probe for select interfaces --- .../tools/testing/data_interface_mixins.py | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index c6803fa66..5e83d5b76 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -19,6 +19,7 @@ from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import ( BaseSortingExtractorInterface, ) +from neuroconv.datainterfaces.ecephys.intan.intandatainterface import IntanRecordingInterface from neuroconv.datainterfaces.ophys.baseimagingextractorinterface import ( BaseImagingExtractorInterface, ) @@ -28,6 +29,8 @@ from neuroconv.utils import NWBMetaDataEncoder +interfaces_for_testing_probe = [IntanRecordingInterface] + class DataInterfaceTestMixin: """ Generic class for testing DataInterfaces. @@ -109,7 +112,9 @@ def test_conversion_as_lone_interface(self): self.case = num self.test_kwargs = kwargs self.interface = self.data_interface_cls(**self.test_kwargs) - if isinstance(self.interface, BaseRecordingExtractorInterface): + do_set_probe = self.data_interface_cls in interfaces_for_testing_probe + if do_set_probe: + print('---------------------------------------------------------------------------------- IIII') self.interface.set_probe( _create_mock_probe_for_recording(self.interface.recording_extractor), group_mode="by_shank" ) @@ -324,14 +329,16 @@ def check_read_nwb(self, nwbfile_path: str): # are specified, which occurs during check_recordings_equal when there is only one channel if self.nwb_recording.get_channel_ids()[0] != self.nwb_recording.get_channel_ids()[-1]: check_recordings_equal(RX1=recording, RX2=self.nwb_recording, return_scaled=False) - for property_name in ["x_rel", "y_rel", "z_rel", "group"]: - if ( - property_name in recording.get_property_keys() - or property_name in self.nwb_recording.get_property_keys() - ): - assert_array_equal( - recording.get_property(property_name), self.nwb_recording.get_property(property_name) - ) + check_probe = self.data_interface_cls in interfaces_for_testing_probe + if check_probe: + for property_name in ["rel_x", "rel_y", "rel_z", "group"]: + if ( + property_name in recording.get_property_keys() + or property_name in self.nwb_recording.get_property_keys() + ): + assert_array_equal( + recording.get_property(property_name), self.nwb_recording.get_property(property_name) + ) if recording.has_scaled_traces() and self.nwb_recording.has_scaled_traces(): check_recordings_equal(RX1=recording, RX2=self.nwb_recording, return_scaled=True) From f2259dfbcf62ac69bc62df3bfbbde5f37eee5176 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 02:00:21 +0000 Subject: [PATCH 07/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/neuroconv/tools/testing/data_interface_mixins.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index 5e83d5b76..1e175bc48 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -19,7 +19,9 @@ from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import ( BaseSortingExtractorInterface, ) -from neuroconv.datainterfaces.ecephys.intan.intandatainterface import IntanRecordingInterface +from neuroconv.datainterfaces.ecephys.intan.intandatainterface import ( + IntanRecordingInterface, +) from neuroconv.datainterfaces.ophys.baseimagingextractorinterface import ( BaseImagingExtractorInterface, ) @@ -28,9 +30,9 @@ ) from neuroconv.utils import NWBMetaDataEncoder - interfaces_for_testing_probe = [IntanRecordingInterface] + class DataInterfaceTestMixin: """ Generic class for testing DataInterfaces. @@ -114,7 +116,7 @@ def test_conversion_as_lone_interface(self): self.interface = self.data_interface_cls(**self.test_kwargs) do_set_probe = self.data_interface_cls in interfaces_for_testing_probe if do_set_probe: - print('---------------------------------------------------------------------------------- IIII') + print("---------------------------------------------------------------------------------- IIII") self.interface.set_probe( _create_mock_probe_for_recording(self.interface.recording_extractor), group_mode="by_shank" ) From 18776a7be75eb8b6817659fa651a466bd4ee0a5a Mon Sep 17 00:00:00 2001 From: Jeremy Magland Date: Mon, 13 Nov 2023 21:01:34 -0500 Subject: [PATCH 08/17] remove debug line --- src/neuroconv/tools/testing/data_interface_mixins.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index 5e83d5b76..88e045c1b 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -114,7 +114,6 @@ def test_conversion_as_lone_interface(self): self.interface = self.data_interface_cls(**self.test_kwargs) do_set_probe = self.data_interface_cls in interfaces_for_testing_probe if do_set_probe: - print('---------------------------------------------------------------------------------- IIII') self.interface.set_probe( _create_mock_probe_for_recording(self.interface.recording_extractor), group_mode="by_shank" ) From 893ec147e1d8d3168a032bb7864b3b49d85aa29f Mon Sep 17 00:00:00 2001 From: Jeremy Magland Date: Mon, 20 Nov 2023 13:26:36 -0500 Subject: [PATCH 09/17] Update src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py Co-authored-by: Ben Dichter --- .../datainterfaces/ecephys/baserecordingextractorinterface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py index 46978404f..423f7c8b3 100644 --- a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py @@ -225,7 +225,7 @@ def set_aligned_segment_starting_times(self, aligned_segment_starting_times: Lis ] self.set_aligned_segment_timestamps(aligned_segment_timestamps=aligned_segment_timestamps) - def set_probe(self, probe, *, group_mode: Union[Literal["by_shank"], Literal["by_probe"]]): + def set_probe(self, probe, *, group_mode: Literal["by_shank", "by_probe"]): """ Set the probe information via a ProbeInterface object. From b946c64c606fe3c38246b71416946334fe9d5487 Mon Sep 17 00:00:00 2001 From: Jeremy Magland Date: Mon, 20 Nov 2023 14:09:48 -0500 Subject: [PATCH 10/17] adjust _create_mock_probe() --- .../tools/testing/data_interface_mixins.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index fe41e5b50..28309f22f 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -117,7 +117,7 @@ def test_conversion_as_lone_interface(self): do_set_probe = self.data_interface_cls in interfaces_for_testing_probe if do_set_probe: self.interface.set_probe( - _create_mock_probe_for_recording(self.interface.recording_extractor), group_mode="by_shank" + _create_mock_probe(num_channels=self.interface.recording_extractor.get_num_channels()), group_mode="by_shank" ) self.check_metadata_schema_valid() self.check_conversion_options_schema_valid() @@ -130,23 +130,20 @@ def test_conversion_as_lone_interface(self): self.run_custom_checks() -def _create_mock_probe_for_recording(recording): +def _create_mock_probe(*, num_channels: int): import probeinterface as pi - import spikeinterface as si - R: si.BaseRecording = recording - n = R.get_num_channels() - positions = np.zeros((n, 2)) - for i in range(n): + positions = np.zeros((num_channels, 2)) + for i in range(num_channels): x = i // 4 y = i % 4 positions[i] = x, y positions *= 20 probe = pi.Probe(ndim=2, si_units="um") probe.set_contacts(positions=positions, shapes="circle", shape_params={"radius": 5}) - probe.set_device_channel_indices(np.arange(n)) + probe.set_device_channel_indices(np.arange(num_channels)) # set shank ids as 0, 1, 2, 0, 1, 2, ... - probe.set_shank_ids(np.arange(n) % 3) + probe.set_shank_ids(np.arange(num_channels) % 3) return probe From 645c2c0f5b97ed044b024673e6c74b210014b274 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 20 Nov 2023 19:10:11 +0000 Subject: [PATCH 11/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/neuroconv/tools/testing/data_interface_mixins.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index 28309f22f..712279f49 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -117,7 +117,8 @@ def test_conversion_as_lone_interface(self): do_set_probe = self.data_interface_cls in interfaces_for_testing_probe if do_set_probe: self.interface.set_probe( - _create_mock_probe(num_channels=self.interface.recording_extractor.get_num_channels()), group_mode="by_shank" + _create_mock_probe(num_channels=self.interface.recording_extractor.get_num_channels()), + group_mode="by_shank", ) self.check_metadata_schema_valid() self.check_conversion_options_schema_valid() From 8cd1aa84dce6d0947c30e741e6bd993b6b885064 Mon Sep 17 00:00:00 2001 From: Jeremy Magland Date: Mon, 20 Nov 2023 14:18:52 -0500 Subject: [PATCH 12/17] adjust _create_mock_probe() to be more realistic --- .../tools/testing/data_interface_mixins.py | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index 28309f22f..0b844a3e6 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -133,17 +133,26 @@ def test_conversion_as_lone_interface(self): def _create_mock_probe(*, num_channels: int): import probeinterface as pi + # The shank ids will be 0, 0, 0, ..., 1, 1, 1, ..., 2, 2, 2, ... + shank_ids: List[int] = [] positions = np.zeros((num_channels, 2)) - for i in range(num_channels): - x = i // 4 - y = i % 4 - positions[i] = x, y - positions *= 20 + num_shanks = 3 + channels_per_shank = num_channels // 3 + for i in range(num_shanks): + # x0, y0 is the position of the first electrode in the shank + x0 = 0 + y0 = i * 200 + for j in range(channels_per_shank): + if len(shank_ids) == num_channels: + break + shank_ids.append(i) + x = x0 + j * 10 + y = y0 + (j % 2) * 10 + positions[len(shank_ids) - 1] = x, y probe = pi.Probe(ndim=2, si_units="um") probe.set_contacts(positions=positions, shapes="circle", shape_params={"radius": 5}) probe.set_device_channel_indices(np.arange(num_channels)) - # set shank ids as 0, 1, 2, 0, 1, 2, ... - probe.set_shank_ids(np.arange(num_channels) % 3) + probe.set_shank_ids(shank_ids) return probe From 75aa8e120909f91631c14372262903361dd99ec4 Mon Sep 17 00:00:00 2001 From: Jeremy Magland Date: Mon, 20 Nov 2023 15:11:19 -0500 Subject: [PATCH 13/17] fix _create_mock_probe() --- src/neuroconv/tools/testing/data_interface_mixins.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index 4d3447003..6be342e2a 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -138,7 +138,8 @@ def _create_mock_probe(*, num_channels: int): shank_ids: List[int] = [] positions = np.zeros((num_channels, 2)) num_shanks = 3 - channels_per_shank = num_channels // 3 + # ceil division + channels_per_shank = (num_channels + num_shanks - 1) // num_shanks for i in range(num_shanks): # x0, y0 is the position of the first electrode in the shank x0 = 0 From 46f592dd6924228ca3051d0ba07e30a4c7a4d2b0 Mon Sep 17 00:00:00 2001 From: Jeremy Magland Date: Tue, 28 Nov 2023 09:10:57 -0500 Subject: [PATCH 14/17] adjust set_probe tests --- .../baserecordingextractorinterface.py | 2 +- .../tools/testing/data_interface_mixins.py | 37 ++----------------- src/neuroconv/tools/testing/mock_probes.py | 28 ++++++++++++++ .../test_on_data/test_recording_interfaces.py | 2 +- 4 files changed, 34 insertions(+), 35 deletions(-) create mode 100644 src/neuroconv/tools/testing/mock_probes.py diff --git a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py index 423f7c8b3..bfb4e9991 100644 --- a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py @@ -225,7 +225,7 @@ def set_aligned_segment_starting_times(self, aligned_segment_starting_times: Lis ] self.set_aligned_segment_timestamps(aligned_segment_timestamps=aligned_segment_timestamps) - def set_probe(self, probe, *, group_mode: Literal["by_shank", "by_probe"]): + def set_probe(self, probe, group_mode: Literal["by_shank", "by_probe"]): """ Set the probe information via a ProbeInterface object. diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index 6be342e2a..1fb4f1970 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -19,9 +19,6 @@ from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import ( BaseSortingExtractorInterface, ) -from neuroconv.datainterfaces.ecephys.intan.intandatainterface import ( - IntanRecordingInterface, -) from neuroconv.datainterfaces.ophys.baseimagingextractorinterface import ( BaseImagingExtractorInterface, ) @@ -30,7 +27,7 @@ ) from neuroconv.utils import NWBMetaDataEncoder -interfaces_for_testing_probe = [IntanRecordingInterface] +from .mock_probes import generate_mock_probe class DataInterfaceTestMixin: @@ -114,10 +111,11 @@ def test_conversion_as_lone_interface(self): self.case = num self.test_kwargs = kwargs self.interface = self.data_interface_cls(**self.test_kwargs) - do_set_probe = self.data_interface_cls in interfaces_for_testing_probe + do_set_probe = isinstance(self.interface, BaseRecordingExtractorInterface) if do_set_probe: + assert isinstance(self.interface, BaseRecordingExtractorInterface) self.interface.set_probe( - _create_mock_probe(num_channels=self.interface.recording_extractor.get_num_channels()), + generate_mock_probe(num_channels=self.interface.recording_extractor.get_num_channels()), group_mode="by_shank", ) self.check_metadata_schema_valid() @@ -131,33 +129,6 @@ def test_conversion_as_lone_interface(self): self.run_custom_checks() -def _create_mock_probe(*, num_channels: int): - import probeinterface as pi - - # The shank ids will be 0, 0, 0, ..., 1, 1, 1, ..., 2, 2, 2, ... - shank_ids: List[int] = [] - positions = np.zeros((num_channels, 2)) - num_shanks = 3 - # ceil division - channels_per_shank = (num_channels + num_shanks - 1) // num_shanks - for i in range(num_shanks): - # x0, y0 is the position of the first electrode in the shank - x0 = 0 - y0 = i * 200 - for j in range(channels_per_shank): - if len(shank_ids) == num_channels: - break - shank_ids.append(i) - x = x0 + j * 10 - y = y0 + (j % 2) * 10 - positions[len(shank_ids) - 1] = x, y - probe = pi.Probe(ndim=2, si_units="um") - probe.set_contacts(positions=positions, shapes="circle", shape_params={"radius": 5}) - probe.set_device_channel_indices(np.arange(num_channels)) - probe.set_shank_ids(shank_ids) - return probe - - class TemporalAlignmentMixin: """ Generic class for testing temporal alignment methods. diff --git a/src/neuroconv/tools/testing/mock_probes.py b/src/neuroconv/tools/testing/mock_probes.py new file mode 100644 index 000000000..0624d2c7d --- /dev/null +++ b/src/neuroconv/tools/testing/mock_probes.py @@ -0,0 +1,28 @@ +from typing import List +import numpy as np + + +def generate_mock_probe(num_channels: int, num_shanks: int = 3): + import probeinterface as pi + + # The shank ids will be 0, 0, 0, ..., 1, 1, 1, ..., 2, 2, 2, ... + shank_ids: List[int] = [] + positions = np.zeros((num_channels, 2)) + # ceil division + channels_per_shank = (num_channels + num_shanks - 1) // num_shanks + for i in range(num_shanks): + # x0, y0 is the position of the first electrode in the shank + x0 = 0 + y0 = i * 200 + for j in range(channels_per_shank): + if len(shank_ids) == num_channels: + break + shank_ids.append(i) + x = x0 + j * 10 + y = y0 + (j % 2) * 10 + positions[len(shank_ids) - 1] = x, y + probe = pi.Probe(ndim=2, si_units="um") + probe.set_contacts(positions=positions, shapes="circle", shape_params={"radius": 5}) + probe.set_device_channel_indices(np.arange(num_channels)) + probe.set_shank_ids(shank_ids) + return probe diff --git a/tests/test_on_data/test_recording_interfaces.py b/tests/test_on_data/test_recording_interfaces.py index 325a1e984..b383d3f58 100644 --- a/tests/test_on_data/test_recording_interfaces.py +++ b/tests/test_on_data/test_recording_interfaces.py @@ -211,7 +211,7 @@ def check_extracted_metadata(self, metadata: dict): assert len(metadata["Ecephys"]["Device"]) == 1 assert metadata["Ecephys"]["Device"][0]["name"] == "Neuronexus-32" assert metadata["Ecephys"]["Device"][0]["description"] == "The ecephys device for the MEArec recording." - assert len(metadata["Ecephys"]["ElectrodeGroup"]) == 1 + # assert len(metadata["Ecephys"]["ElectrodeGroup"]) == 1 # do not test this condition because in the test we are setting a mock probe assert metadata["Ecephys"]["ElectrodeGroup"][0]["device"] == "Neuronexus-32" assert metadata["Ecephys"]["ElectricalSeries"]["description"] == ( '{"angle_tol": 15, "bursting": false, "chunk_duration": 0, "color_noise_floor": 1, ' From d781a46cfea1e30774ce17d771854dc040c87ed6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 14:12:11 +0000 Subject: [PATCH 15/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/neuroconv/tools/testing/mock_probes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/neuroconv/tools/testing/mock_probes.py b/src/neuroconv/tools/testing/mock_probes.py index 0624d2c7d..f5a3cea88 100644 --- a/src/neuroconv/tools/testing/mock_probes.py +++ b/src/neuroconv/tools/testing/mock_probes.py @@ -1,4 +1,5 @@ from typing import List + import numpy as np From 3f8cab599baa361ab0a5b7140c0900a2b145a923 Mon Sep 17 00:00:00 2001 From: Jeremy Magland Date: Tue, 28 Nov 2023 09:19:11 -0500 Subject: [PATCH 16/17] RecordingExtractorInterfaceTestMixin: remove check for testing probe --- .../tools/testing/data_interface_mixins.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index 1fb4f1970..09018c8cc 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -309,16 +309,14 @@ def check_read_nwb(self, nwbfile_path: str): # are specified, which occurs during check_recordings_equal when there is only one channel if self.nwb_recording.get_channel_ids()[0] != self.nwb_recording.get_channel_ids()[-1]: check_recordings_equal(RX1=recording, RX2=self.nwb_recording, return_scaled=False) - check_probe = self.data_interface_cls in interfaces_for_testing_probe - if check_probe: - for property_name in ["rel_x", "rel_y", "rel_z", "group"]: - if ( - property_name in recording.get_property_keys() - or property_name in self.nwb_recording.get_property_keys() - ): - assert_array_equal( - recording.get_property(property_name), self.nwb_recording.get_property(property_name) - ) + for property_name in ["rel_x", "rel_y", "rel_z", "group"]: + if ( + property_name in recording.get_property_keys() + or property_name in self.nwb_recording.get_property_keys() + ): + assert_array_equal( + recording.get_property(property_name), self.nwb_recording.get_property(property_name) + ) if recording.has_scaled_traces() and self.nwb_recording.has_scaled_traces(): check_recordings_equal(RX1=recording, RX2=self.nwb_recording, return_scaled=True) From 69e5d9405b4a21aef2d9a80711c88fad00babff1 Mon Sep 17 00:00:00 2001 From: Jeremy Magland Date: Tue, 28 Nov 2023 15:25:26 -0500 Subject: [PATCH 17/17] recording extractor interface tests --- .../baserecordingextractorinterface.py | 11 +++++++ .../tools/testing/data_interface_mixins.py | 32 +++++++++++++++---- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py index bfb4e9991..c9f8ac2e5 100644 --- a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py @@ -248,6 +248,17 @@ def set_probe(self, probe, group_mode: Literal["by_shank", "by_probe"]): # But neuroconv allows "group_name" property to override spike interface "group" value self.recording_extractor.set_property("group_name", self.recording_extractor.get_property("group").astype(str)) + def has_probe(self) -> bool: + """ + Check if the recording extractor has probe information. + + Returns + ------- + has_probe : bool + True if the recording extractor has probe information. + """ + return self.recording_extractor.has_probe() + def align_by_interpolation( self, unaligned_timestamps: np.ndarray, diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index 09018c8cc..793e78b97 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -111,13 +111,6 @@ def test_conversion_as_lone_interface(self): self.case = num self.test_kwargs = kwargs self.interface = self.data_interface_cls(**self.test_kwargs) - do_set_probe = isinstance(self.interface, BaseRecordingExtractorInterface) - if do_set_probe: - assert isinstance(self.interface, BaseRecordingExtractorInterface) - self.interface.set_probe( - generate_mock_probe(num_channels=self.interface.recording_extractor.get_num_channels()), - group_mode="by_shank", - ) self.check_metadata_schema_valid() self.check_conversion_options_schema_valid() self.check_metadata() @@ -476,6 +469,31 @@ def test_interface_alignment(self): self.check_nwbfile_temporal_alignment() + def test_conversion_as_lone_interface(self): + interface_kwargs = self.interface_kwargs + if isinstance(interface_kwargs, dict): + interface_kwargs = [interface_kwargs] + for num, kwargs in enumerate(interface_kwargs): + with self.subTest(str(num)): + self.case = num + self.test_kwargs = kwargs + self.interface = self.data_interface_cls(**self.test_kwargs) + assert isinstance(self.interface, BaseRecordingExtractorInterface) + if not self.interface.has_probe(): + self.interface.set_probe( + generate_mock_probe(num_channels=self.interface.recording_extractor.get_num_channels()), + group_mode="by_shank", + ) + self.check_metadata_schema_valid() + self.check_conversion_options_schema_valid() + self.check_metadata() + self.nwbfile_path = str(self.save_directory / f"{self.__class__.__name__}_{num}.nwb") + self.run_conversion(nwbfile_path=self.nwbfile_path) + self.check_read_nwb(nwbfile_path=self.nwbfile_path) + + # Any extra custom checks to run + self.run_custom_checks() + class SortingExtractorInterfaceTestMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): data_interface_cls: BaseSortingExtractorInterface