From 46ddf0acd35412e8a04737b5532971a7a4469dcb Mon Sep 17 00:00:00 2001 From: luiz Date: Wed, 6 Dec 2023 12:39:33 +0100 Subject: [PATCH 1/6] vprobe locations --- requirements.txt | 4 +- .../watters/wattersrecordinginterface.py | 48 +++++++++++-------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/requirements.txt b/requirements.txt index e411ef3..05ad5ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -neuroconv==0.4.4 -spikeinterface==0.98.2 +neuroconv==0.4.7 +spikeinterface==0.99.1 nwbwidgets nwbinspector pre-commit diff --git a/src/jazayeri_lab_to_nwb/watters/wattersrecordinginterface.py b/src/jazayeri_lab_to_nwb/watters/wattersrecordinginterface.py index cae0b91..7ce9e64 100644 --- a/src/jazayeri_lab_to_nwb/watters/wattersrecordinginterface.py +++ b/src/jazayeri_lab_to_nwb/watters/wattersrecordinginterface.py @@ -27,23 +27,32 @@ def add_electrode_locations( if probe_metadata is None: return [] - probe_coord_system = probe_metadata["coordinate_system"] - coord_names = probe_coord_system.split("[")[1].split("]")[0].split(",") + # Add electrodes relative positions, important for sorting algorithms + if "electrodes_locations" in probe_metadata: + locations_array = probe_metadata["electrodes_locations"] + else: + locations_array = [(0, i * 50) for i in range(64)] + + # probe_coord_system = probe_metadata["coordinate_system"] + # coord_names = probe_coord_system.split("[")[1].split("]")[0].split(",") electrode_metadata = [ { - "name": "x", - "description": f"{coord_names[0].strip()} coordinate. {probe_coord_system}", + "name": "rel_x", + "description": "relative x position of electrode", + # "description": f"{coord_names[0].strip()} coordinate. {probe_coord_system}", }, { - "name": "y", - "description": f"{coord_names[1].strip()} coordinate. {probe_coord_system}", + "name": "rel_y", + "description": "relative y position of electrode", + # "description": f"{coord_names[1].strip()} coordinate. {probe_coord_system}", }, ] - if len(coord_names) == 3: + if len(locations_array[0]) == 3: electrode_metadata.append( { - "name": "z", - "description": f"{coord_names[2].strip()} coordinate. {probe_coord_system}", + "name": "rel_z", + "description": "relative y position of electrode", + # "description": f"{coord_names[2].strip()} coordinate. {probe_coord_system}", }, ) @@ -53,22 +62,21 @@ def add_electrode_locations( ids=channel_ids, values=[probe_name] * len(channel_ids), ) - coordinates = probe_metadata["coordinates"] recording_extractor.set_property( - key="x", - values=[coordinates["first_channel"][0], coordinates["last_channel"][0]], - ids=channel_ids[[0, -1]], + key="rel_x", + values=[l[0] for l in locations_array], + ids=channel_ids, ) recording_extractor.set_property( - key="y", - values=[coordinates["first_channel"][1], coordinates["last_channel"][1]], - ids=channel_ids[[0, -1]], + key="rel_y", + values=[l[1] for l in locations_array], + ids=channel_ids, ) - if len(coord_names) == 3: + if len(locations_array[0]) == 3: recording_extractor.set_property( - key="z", - values=[coordinates["first_channel"][2], coordinates["last_channel"][2]], - ids=channel_ids[[0, -1]], + key="rel_z", + values=[l[2] for l in locations_array], + ids=channel_ids, ) return electrode_metadata From be698aa9839b8d7ca645fa1571ef745c4dc632d2 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 7 Dec 2023 12:50:12 +0100 Subject: [PATCH 2/6] Use set_probe and simplify interface by using BinaryRecording --- .../watters/wattersrecordinginterface.py | 134 ++++++------------ 1 file changed, 40 insertions(+), 94 deletions(-) diff --git a/src/jazayeri_lab_to_nwb/watters/wattersrecordinginterface.py b/src/jazayeri_lab_to_nwb/watters/wattersrecordinginterface.py index 7ce9e64..a47a748 100644 --- a/src/jazayeri_lab_to_nwb/watters/wattersrecordinginterface.py +++ b/src/jazayeri_lab_to_nwb/watters/wattersrecordinginterface.py @@ -8,83 +8,14 @@ from neuroconv.datainterfaces.ecephys.baserecordingextractorinterface import BaseRecordingExtractorInterface from neuroconv.utils import FilePathType -from spikeinterface import BaseRecording - - -def add_electrode_locations( - recording_extractor: BaseRecording, - probe_metadata_file: FilePathType, - probe_name: str, - probe_key: str, -) -> list[dict]: - with open(probe_metadata_file, "r") as f: - all_probe_metadata = json.load(f) - probe_metadata = None - for entry in all_probe_metadata: - if entry["label"] == probe_key: - probe_metadata = entry - - if probe_metadata is None: - return [] - # Add electrodes relative positions, important for sorting algorithms - if "electrodes_locations" in probe_metadata: - locations_array = probe_metadata["electrodes_locations"] - else: - locations_array = [(0, i * 50) for i in range(64)] - - # probe_coord_system = probe_metadata["coordinate_system"] - # coord_names = probe_coord_system.split("[")[1].split("]")[0].split(",") - electrode_metadata = [ - { - "name": "rel_x", - "description": "relative x position of electrode", - # "description": f"{coord_names[0].strip()} coordinate. {probe_coord_system}", - }, - { - "name": "rel_y", - "description": "relative y position of electrode", - # "description": f"{coord_names[1].strip()} coordinate. {probe_coord_system}", - }, - ] - if len(locations_array[0]) == 3: - electrode_metadata.append( - { - "name": "rel_z", - "description": "relative y position of electrode", - # "description": f"{coord_names[2].strip()} coordinate. {probe_coord_system}", - }, - ) - - channel_ids = recording_extractor.get_channel_ids() - recording_extractor.set_property( - key="group_name", - ids=channel_ids, - values=[probe_name] * len(channel_ids), - ) - recording_extractor.set_property( - key="rel_x", - values=[l[0] for l in locations_array], - ids=channel_ids, - ) - recording_extractor.set_property( - key="rel_y", - values=[l[1] for l in locations_array], - ids=channel_ids, - ) - if len(locations_array[0]) == 3: - recording_extractor.set_property( - key="rel_z", - values=[l[2] for l in locations_array], - ids=channel_ids, - ) - - return electrode_metadata +import probeinterface as pi +from spikeinterface import BaseRecording class WattersDatRecordingInterface(BaseRecordingExtractorInterface): - ExtractorName = "NumpyRecording" + ExtractorName = "BinaryRecording" def __init__( self, @@ -97,36 +28,55 @@ def __init__( sampling_frequency: float = 30000.0, channel_ids: Optional[list] = None, gain_to_uv: list = [1.0], + offset_to_uv: list = [0.0], probe_metadata_file: Optional[FilePathType] = None, probe_name: str = "vprobe", probe_key: Optional[str] = None, ): - traces = np.memmap(file_path, dtype=dtype, mode="r").reshape(-1, channel_count) source_data = { - "traces_list": [traces], + "file_paths": [file_path], "sampling_frequency": sampling_frequency, + "num_channels": channel_count, "t_starts": [t_start], "channel_ids": channel_ids, + "gain_to_uV": gain_to_uv, + "offset_to_uV": offset_to_uv, + "dtype": dtype, } super().__init__(verbose=verbose, es_key=es_key, **source_data) - if gain_to_uv is not None: - if len(gain_to_uv) == 1: - gain_to_uv = np.full((channel_count,), gain_to_uv[0], dtype=float) - else: - assert len(gain_to_uv) == channel_count, ( - f"There are {channel_count} channels " f"but `gain_to_uv` has length {len(gain_to_uv)}" - ) - gain_to_uv = np.array(gain_to_uv, dtype=float) - self.recording_extractor.set_property("gain_to_uV", gain_to_uv) - self.probe_metadata_file = probe_metadata_file + + # this is used for metadata naming self.probe_name = probe_name - self.probe_key = probe_key - self.electrode_metadata = None - if self.probe_metadata_file is not None and self.probe_key is not None: - self.electrode_metadata = add_electrode_locations( - self.recording_extractor, self.probe_metadata_file, self.probe_name, self.probe_key - ) + # add probe information + probe_metadata = None + if probe_metadata_file is not None and probe_key is not None: + with open(probe_metadata_file, "r") as f: + all_probe_metadata = json.load(f) + for entry in all_probe_metadata: + if entry["label"] == probe_key: + probe_metadata = entry + + if probe_metadata is not None and "electrodes_locations" in probe_metadata: + # Grab electrode position from metadata + locations_array = np.array(probe_metadata["electrodes_locations"]) + ndim = locations_array.shape[1] + probe = pi.Probe(ndim=ndim) + probe.set_contacts(locations_array) + else: + # Generate V-probe geometry: 64 channels arranged vertically with 50 um spacing + probe = pi.generate_linear_probe(num_elec=64, ypitch=50) + probe.name = probe_name + + # set probe to interface recording + self.set_probe(probe) + + # set group_name property to match electrode group name in metadata + self.recording_extractor.set_property( + key="group_name", + values=[probe_name] * len(self.recording_extractor.channel_ids), + ) + def get_metadata(self) -> dict: metadata = super().get_metadata() @@ -147,8 +97,4 @@ def get_metadata(self) -> dict: ] metadata["Ecephys"]["ElectrodeGroup"] = electrode_groups - if self.electrode_metadata is None: - return metadata - - metadata["Ecephys"]["Electrodes"] = self.electrode_metadata return metadata From 7d0040327297f9535c5209511a7ff3ca48846c3b Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Sun, 10 Dec 2023 13:30:12 +0100 Subject: [PATCH 3/6] Fix WattersInterface --- .../watters/wattersrecordinginterface.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/jazayeri_lab_to_nwb/watters/wattersrecordinginterface.py b/src/jazayeri_lab_to_nwb/watters/wattersrecordinginterface.py index a47a748..2302287 100644 --- a/src/jazayeri_lab_to_nwb/watters/wattersrecordinginterface.py +++ b/src/jazayeri_lab_to_nwb/watters/wattersrecordinginterface.py @@ -15,7 +15,7 @@ class WattersDatRecordingInterface(BaseRecordingExtractorInterface): - ExtractorName = "BinaryRecording" + ExtractorName = "BinaryRecordingExtractor" def __init__( self, @@ -27,8 +27,8 @@ def __init__( t_start: float = 0.0, sampling_frequency: float = 30000.0, channel_ids: Optional[list] = None, - gain_to_uv: list = [1.0], - offset_to_uv: list = [0.0], + gain_to_uv: list = 1.0, + offset_to_uv: list = 0.0, probe_metadata_file: Optional[FilePathType] = None, probe_name: str = "vprobe", probe_key: Optional[str] = None, @@ -65,11 +65,12 @@ def __init__( probe.set_contacts(locations_array) else: # Generate V-probe geometry: 64 channels arranged vertically with 50 um spacing - probe = pi.generate_linear_probe(num_elec=64, ypitch=50) + probe = pi.generate_linear_probe(num_elec=channel_count, ypitch=50) + probe.set_device_channel_indices(np.arange(channel_count)) probe.name = probe_name # set probe to interface recording - self.set_probe(probe) + self.set_probe(probe, group_mode="by_probe") # set group_name property to match electrode group name in metadata self.recording_extractor.set_property( From ecb63d6aed8e4b1b39f318fc1a01f0c62f793ce9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 16:49:42 +0000 Subject: [PATCH 4/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- setup.py | 3 ++- src/jazayeri_lab_to_nwb/watters/__init__.py | 9 ++++++--- .../watters/watters_convert_session.py | 7 ++++--- .../watters/wattersbehaviorinterface.py | 10 +++++----- .../watters/wattersnwbconverter.py | 19 +++++++++++-------- .../watters/wattersrecordinginterface.py | 14 +++++++------- .../watters/watterstrialsinterface.py | 8 ++++---- 7 files changed, 39 insertions(+), 31 deletions(-) diff --git a/setup.py b/setup.py index 5018ad7..f8e8892 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from pathlib import Path -from setuptools import setup, find_packages + +from setuptools import find_packages, setup requirements_file_path = Path(__file__).parent / "requirements.txt" with open(requirements_file_path) as file: diff --git a/src/jazayeri_lab_to_nwb/watters/__init__.py b/src/jazayeri_lab_to_nwb/watters/__init__.py index 880f32a..4e94968 100644 --- a/src/jazayeri_lab_to_nwb/watters/__init__.py +++ b/src/jazayeri_lab_to_nwb/watters/__init__.py @@ -1,4 +1,7 @@ -from .wattersbehaviorinterface import WattersEyePositionInterface, WattersPupilSizeInterface -from .watterstrialsinterface import WattersTrialsInterface -from .wattersrecordinginterface import WattersDatRecordingInterface +from .wattersbehaviorinterface import ( + WattersEyePositionInterface, + WattersPupilSizeInterface, +) from .wattersnwbconverter import WattersNWBConverter +from .wattersrecordinginterface import WattersDatRecordingInterface +from .watterstrialsinterface import WattersTrialsInterface diff --git a/src/jazayeri_lab_to_nwb/watters/watters_convert_session.py b/src/jazayeri_lab_to_nwb/watters/watters_convert_session.py index d7c1072..e5e26aa 100644 --- a/src/jazayeri_lab_to_nwb/watters/watters_convert_session.py +++ b/src/jazayeri_lab_to_nwb/watters/watters_convert_session.py @@ -1,16 +1,16 @@ """Primary script to run to convert an entire session for of data using the NWBConverter.""" -import os import datetime import glob import json import logging +import os from pathlib import Path from typing import Union from uuid import uuid4 from zoneinfo import ZoneInfo from neuroconv.tools.data_transfers import automatic_dandi_upload -from neuroconv.utils import load_dict_from_file, dict_deep_update +from neuroconv.utils import dict_deep_update, load_dict_from_file from jazayeri_lab_to_nwb.watters import WattersNWBConverter @@ -64,6 +64,7 @@ def session_to_nwb( """ if dandiset_id is not None: import dandi # check importability + assert os.getenv("DANDI_API_KEY"), ( "Unable to find environment variable 'DANDI_API_KEY'. " "Please retrieve your token from DANDI and set this environment variable." @@ -249,5 +250,5 @@ def session_to_nwb( output_dir_path=output_dir_path, stub_test=stub_test, overwrite=overwrite, - # dandiset_id = "000620", + # dandiset_id = "000620", ) diff --git a/src/jazayeri_lab_to_nwb/watters/wattersbehaviorinterface.py b/src/jazayeri_lab_to_nwb/watters/wattersbehaviorinterface.py index 180e052..c4a0862 100644 --- a/src/jazayeri_lab_to_nwb/watters/wattersbehaviorinterface.py +++ b/src/jazayeri_lab_to_nwb/watters/wattersbehaviorinterface.py @@ -1,13 +1,13 @@ """Primary classes for converting experiment-specific behavior.""" -import numpy as np from pathlib import Path -from pynwb import NWBFile, TimeSeries -from pynwb.behavior import SpatialSeries -from hdmf.backends.hdf5 import H5DataIO +import numpy as np +from hdmf.backends.hdf5 import H5DataIO from neuroconv.basetemporalalignmentinterface import BaseTemporalAlignmentInterface -from neuroconv.utils import DeepDict, FolderPathType, FilePathType from neuroconv.tools.nwb_helpers import get_module +from neuroconv.utils import DeepDict, FilePathType, FolderPathType +from pynwb import NWBFile, TimeSeries +from pynwb.behavior import SpatialSeries class NumpyTemporalAlignmentMixin: diff --git a/src/jazayeri_lab_to_nwb/watters/wattersnwbconverter.py b/src/jazayeri_lab_to_nwb/watters/wattersnwbconverter.py index 96267b7..00ac42e 100644 --- a/src/jazayeri_lab_to_nwb/watters/wattersnwbconverter.py +++ b/src/jazayeri_lab_to_nwb/watters/wattersnwbconverter.py @@ -1,21 +1,24 @@ """Primary NWBConverter class for this dataset.""" import json import logging -import numpy as np -from typing import Optional from pathlib import Path +from typing import Optional +import numpy as np from neuroconv import NWBConverter -from neuroconv.utils import FolderPathType +from neuroconv.basetemporalalignmentinterface import BaseTemporalAlignmentInterface from neuroconv.datainterfaces import ( - SpikeGLXRecordingInterface, KiloSortSortingInterface, + SpikeGLXRecordingInterface, +) +from neuroconv.datainterfaces.ecephys.baserecordingextractorinterface import ( + BaseRecordingExtractorInterface, +) +from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import ( + BaseSortingExtractorInterface, ) -from neuroconv.datainterfaces.ecephys.baserecordingextractorinterface import BaseRecordingExtractorInterface -from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import BaseSortingExtractorInterface -from neuroconv.basetemporalalignmentinterface import BaseTemporalAlignmentInterface from neuroconv.datainterfaces.text.timeintervalsinterface import TimeIntervalsInterface - +from neuroconv.utils import FolderPathType from spikeinterface.core.waveform_tools import has_exceeding_spikes from spikeinterface.curation import remove_excess_spikes diff --git a/src/jazayeri_lab_to_nwb/watters/wattersrecordinginterface.py b/src/jazayeri_lab_to_nwb/watters/wattersrecordinginterface.py index 2302287..5e7159c 100644 --- a/src/jazayeri_lab_to_nwb/watters/wattersrecordinginterface.py +++ b/src/jazayeri_lab_to_nwb/watters/wattersrecordinginterface.py @@ -1,15 +1,16 @@ """Primary class for Watters Plexon probe data.""" -import os import json -import numpy as np -from pynwb import NWBFile +import os from pathlib import Path from typing import Optional, Union -from neuroconv.datainterfaces.ecephys.baserecordingextractorinterface import BaseRecordingExtractorInterface -from neuroconv.utils import FilePathType - +import numpy as np import probeinterface as pi +from neuroconv.datainterfaces.ecephys.baserecordingextractorinterface import ( + BaseRecordingExtractorInterface, +) +from neuroconv.utils import FilePathType +from pynwb import NWBFile from spikeinterface import BaseRecording @@ -78,7 +79,6 @@ def __init__( values=[probe_name] * len(self.recording_extractor.channel_ids), ) - def get_metadata(self) -> dict: metadata = super().get_metadata() metadata["Ecephys"]["Device"] = [ diff --git a/src/jazayeri_lab_to_nwb/watters/watterstrialsinterface.py b/src/jazayeri_lab_to_nwb/watters/watterstrialsinterface.py index fe13f50..79ca06c 100644 --- a/src/jazayeri_lab_to_nwb/watters/watterstrialsinterface.py +++ b/src/jazayeri_lab_to_nwb/watters/watterstrialsinterface.py @@ -1,14 +1,14 @@ """Primary class for converting experiment-specific behavior.""" import json -import numpy as np -import pandas as pd import warnings from pathlib import Path -from pynwb import NWBFile from typing import Optional +import numpy as np +import pandas as pd from neuroconv.datainterfaces.text.timeintervalsinterface import TimeIntervalsInterface -from neuroconv.utils import DeepDict, FolderPathType, FilePathType +from neuroconv.utils import DeepDict, FilePathType, FolderPathType +from pynwb import NWBFile class WattersTrialsInterface(TimeIntervalsInterface): From a826f305ca2d1c2ab90c27b5e87cd0af243dec0c Mon Sep 17 00:00:00 2001 From: luiz Date: Tue, 19 Dec 2023 13:48:52 +0100 Subject: [PATCH 5/6] fix circular import error --- src/jazayeri_lab_to_nwb/watters/__init__.py | 7 ------- .../watters/wattersnwbconverter.py | 13 ++++--------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/src/jazayeri_lab_to_nwb/watters/__init__.py b/src/jazayeri_lab_to_nwb/watters/__init__.py index 4e94968..e69de29 100644 --- a/src/jazayeri_lab_to_nwb/watters/__init__.py +++ b/src/jazayeri_lab_to_nwb/watters/__init__.py @@ -1,7 +0,0 @@ -from .wattersbehaviorinterface import ( - WattersEyePositionInterface, - WattersPupilSizeInterface, -) -from .wattersnwbconverter import WattersNWBConverter -from .wattersrecordinginterface import WattersDatRecordingInterface -from .watterstrialsinterface import WattersTrialsInterface diff --git a/src/jazayeri_lab_to_nwb/watters/wattersnwbconverter.py b/src/jazayeri_lab_to_nwb/watters/wattersnwbconverter.py index 00ac42e..1676882 100644 --- a/src/jazayeri_lab_to_nwb/watters/wattersnwbconverter.py +++ b/src/jazayeri_lab_to_nwb/watters/wattersnwbconverter.py @@ -11,23 +11,18 @@ KiloSortSortingInterface, SpikeGLXRecordingInterface, ) -from neuroconv.datainterfaces.ecephys.baserecordingextractorinterface import ( - BaseRecordingExtractorInterface, -) -from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import ( - BaseSortingExtractorInterface, -) +from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import BaseSortingExtractorInterface from neuroconv.datainterfaces.text.timeintervalsinterface import TimeIntervalsInterface from neuroconv.utils import FolderPathType from spikeinterface.core.waveform_tools import has_exceeding_spikes from spikeinterface.curation import remove_excess_spikes -from . import ( - WattersDatRecordingInterface, +from .wattersbehaviorinterface import ( WattersEyePositionInterface, WattersPupilSizeInterface, - WattersTrialsInterface, ) +from .wattersrecordinginterface import WattersDatRecordingInterface +from .watterstrialsinterface import WattersTrialsInterface class WattersNWBConverter(NWBConverter): From 8d0700dd87a4afd5aae86136bda4adafee74cead Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 12:49:09 +0000 Subject: [PATCH 6/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/jazayeri_lab_to_nwb/watters/wattersnwbconverter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/jazayeri_lab_to_nwb/watters/wattersnwbconverter.py b/src/jazayeri_lab_to_nwb/watters/wattersnwbconverter.py index 1676882..5b96bc9 100644 --- a/src/jazayeri_lab_to_nwb/watters/wattersnwbconverter.py +++ b/src/jazayeri_lab_to_nwb/watters/wattersnwbconverter.py @@ -11,7 +11,9 @@ KiloSortSortingInterface, SpikeGLXRecordingInterface, ) -from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import BaseSortingExtractorInterface +from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import ( + BaseSortingExtractorInterface, +) from neuroconv.datainterfaces.text.timeintervalsinterface import TimeIntervalsInterface from neuroconv.utils import FolderPathType from spikeinterface.core.waveform_tools import has_exceeding_spikes