diff --git a/make_env.yml b/make_env.yml index 897fe00..7a5df21 100644 --- a/make_env.yml +++ b/make_env.yml @@ -3,7 +3,7 @@ channels: - conda-forge - defaults dependencies: -- python>=3.9 +- python>=3.9, <3.13 - pip - pip: - -e . # This calls the setup and therefore requirements minimal diff --git a/pyproject.toml b/pyproject.toml index e1699e8..bae7ce5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,8 +15,11 @@ classifiers = [ ] dependencies = [ - "neuroconv", + "neuroconv[spikegadgets,video]", "nwbinspector", + "pre-commit", + "ipykernel", + "matplotlib", ] [project.urls] diff --git a/src/jadhav_lab_to_nwb/olson_2024/__init__.py b/src/jadhav_lab_to_nwb/olson_2024/__init__.py index 33b9c6c..809f62d 100644 --- a/src/jadhav_lab_to_nwb/olson_2024/__init__.py +++ b/src/jadhav_lab_to_nwb/olson_2024/__init__.py @@ -1,2 +1,3 @@ -from .olson_2024behaviorinterface import Olson2024BehaviorInterface -from .olson_2024nwbconverter import Olson2024NWBConverter +from .olson_2024_behavior_interface import Olson2024BehaviorInterface +from .olson_2024_spike_gadgets_recording_interface import Olson2024SpikeGadgetsRecordingInterface +from .olson_2024_nwbconverter import Olson2024NWBConverter diff --git a/src/jadhav_lab_to_nwb/olson_2024/olson_2024_behaviorinterface.py b/src/jadhav_lab_to_nwb/olson_2024/olson_2024_behavior_interface.py similarity index 93% rename from src/jadhav_lab_to_nwb/olson_2024/olson_2024_behaviorinterface.py rename to src/jadhav_lab_to_nwb/olson_2024/olson_2024_behavior_interface.py index 34acc46..a3d225f 100644 --- a/src/jadhav_lab_to_nwb/olson_2024/olson_2024_behaviorinterface.py +++ b/src/jadhav_lab_to_nwb/olson_2024/olson_2024_behavior_interface.py @@ -4,18 +4,19 @@ from neuroconv.basedatainterface import BaseDataInterface from neuroconv.utils import DeepDict + class Olson2024BehaviorInterface(BaseDataInterface): """Behavior interface for olson_2024 conversion""" keywords = ("behavior",) - + def __init__(self): # This should load the data lazily and prepare variables you need pass def get_metadata(self) -> DeepDict: # Automatically retrieve as much metadata as possible from the source files available - metadata = super().get_metadata() + metadata = super().get_metadata() return metadata diff --git a/src/jadhav_lab_to_nwb/olson_2024/olson_2024_convert_all_sessions.py b/src/jadhav_lab_to_nwb/olson_2024/olson_2024_convert_all_sessions.py index e131f3e..cdd6a48 100644 --- a/src/jadhav_lab_to_nwb/olson_2024/olson_2024_convert_all_sessions.py +++ b/src/jadhav_lab_to_nwb/olson_2024/olson_2024_convert_all_sessions.py @@ -1,6 +1,5 @@ """Primary script to run to convert all sessions in a dataset using session_to_nwb.""" from pathlib import Path -from typing import Union from concurrent.futures import ProcessPoolExecutor, as_completed from pprint import pformat import traceback @@ -11,8 +10,8 @@ def dataset_to_nwb( *, - data_dir_path: Union[str, Path], - output_dir_path: Union[str, Path], + data_dir_path: str | Path, + output_dir_path: str | Path, max_workers: int = 1, verbose: bool = True, ): @@ -20,9 +19,9 @@ def dataset_to_nwb( Parameters ---------- - data_dir_path : Union[str, Path] + data_dir_path : str | Path The path to the directory containing the raw data. - output_dir_path : Union[str, Path] + output_dir_path : str | Path The path to the directory where the NWB files will be saved. max_workers : int, optional The number of workers to use for parallel processing, by default 1 @@ -39,7 +38,7 @@ def dataset_to_nwb( for session_to_nwb_kwargs in session_to_nwb_kwargs_per_session: session_to_nwb_kwargs["output_dir_path"] = output_dir_path session_to_nwb_kwargs["verbose"] = verbose - exception_file_path = data_dir_path / f"ERROR_.txt" # Add error file path here + exception_file_path = data_dir_path / f"ERROR_.txt" # Add error file path here futures.append( executor.submit( safe_session_to_nwb, @@ -51,7 +50,7 @@ def dataset_to_nwb( pass -def safe_session_to_nwb(*, session_to_nwb_kwargs: dict, exception_file_path: Union[Path, str]): +def safe_session_to_nwb(*, session_to_nwb_kwargs: dict, exception_file_path: str | Path): """Convert a session to NWB while handling any errors by recording error messages to the exception_file_path. Parameters @@ -72,13 +71,13 @@ def safe_session_to_nwb(*, session_to_nwb_kwargs: dict, exception_file_path: Uni def get_session_to_nwb_kwargs_per_session( *, - data_dir_path: Union[str, Path], + data_dir_path: str | Path, ): """Get the kwargs for session_to_nwb for each session in the dataset. Parameters ---------- - data_dir_path : Union[str, Path] + data_dir_path : str | Path The path to the directory containing the raw data. Returns @@ -86,11 +85,11 @@ def get_session_to_nwb_kwargs_per_session( list[dict[str, Any]] A list of dictionaries containing the kwargs for session_to_nwb for each session. """ - ##### - # # Implement this function to return the kwargs for session_to_nwb for each session - # This can be a specific list with hard-coded sessions, a path expansion or any conversion specific logic that you might need - ##### - raise NotImplementedError + ##### + # # Implement this function to return the kwargs for session_to_nwb for each session + # This can be a specific list with hard-coded sessions, a path expansion or any conversion specific logic that you might need + ##### + raise NotImplementedError if __name__ == "__main__": diff --git a/src/jadhav_lab_to_nwb/olson_2024/olson_2024_convert_session.py b/src/jadhav_lab_to_nwb/olson_2024/olson_2024_convert_session.py index fd76001..949c04a 100644 --- a/src/jadhav_lab_to_nwb/olson_2024/olson_2024_convert_session.py +++ b/src/jadhav_lab_to_nwb/olson_2024/olson_2024_convert_session.py @@ -1,15 +1,15 @@ """Primary script to run to convert an entire session for of data using the NWBConverter.""" from pathlib import Path -from typing import Union import datetime from zoneinfo import ZoneInfo +import shutil from neuroconv.utils import load_dict_from_file, dict_deep_update from jadhav_lab_to_nwb.olson_2024 import Olson2024NWBConverter -def session_to_nwb(data_dir_path: Union[str, Path], output_dir_path: Union[str, Path], stub_test: bool = False): +def session_to_nwb(data_dir_path: str | Path, output_dir_path: str | Path, stub_test: bool = False): data_dir_path = Path(data_dir_path) output_dir_path = Path(output_dir_path) @@ -17,41 +17,33 @@ def session_to_nwb(data_dir_path: Union[str, Path], output_dir_path: Union[str, output_dir_path = output_dir_path / "nwb_stub" output_dir_path.mkdir(parents=True, exist_ok=True) - session_id = "subject_identifier_usually" + session_id = "sample_session" nwbfile_path = output_dir_path / f"{session_id}.nwb" source_data = dict() conversion_options = dict() - # Add Recording - source_data.update(dict(Recording=dict())) + # Add Ephys + file_path = data_dir_path / f"{data_dir_path.name}.rec" + source_data.update(dict(Recording=dict(file_path=file_path))) conversion_options.update(dict(Recording=dict(stub_test=stub_test))) - # Add Sorting - source_data.update(dict(Sorting=dict())) - conversion_options.update(dict(Sorting=dict())) - - # Add Behavior - source_data.update(dict(Behavior=dict())) - conversion_options.update(dict(Behavior=dict())) + # Add Video + file_paths = [data_dir_path / f"{data_dir_path.name}.1.h264"] + source_data.update(dict(Video=dict(file_paths=file_paths))) + conversion_options.update(dict(Video=dict())) converter = Olson2024NWBConverter(source_data=source_data) # Add datetime to conversion metadata = converter.get_metadata() - datetime.datetime( - year=2020, month=1, day=1, tzinfo=ZoneInfo("US/Eastern") - ) - date = datetime.datetime.today() # TO-DO: Get this from author - metadata["NWBFile"]["session_start_time"] = date + metadata["NWBFile"]["session_start_time"] = datetime.datetime(2023, 5, 3, 11, 26, 42, tzinfo=ZoneInfo("US/Eastern")) # Update default metadata with the editable in the corresponding yaml file editable_metadata_path = Path(__file__).parent / "olson_2024_metadata.yaml" editable_metadata = load_dict_from_file(editable_metadata_path) metadata = dict_deep_update(metadata, editable_metadata) - metadata["Subject"]["subject_id"] = "a_subject_id" # Modify here or in the yaml file - # Run conversion converter.run_conversion(metadata=metadata, nwbfile_path=nwbfile_path, conversion_options=conversion_options) @@ -59,11 +51,17 @@ def session_to_nwb(data_dir_path: Union[str, Path], output_dir_path: Union[str, if __name__ == "__main__": # Parameters for conversion - data_dir_path = Path("/Directory/With/Raw/Formats/") - output_dir_path = Path("~/conversion_nwb/") - stub_test = False - - session_to_nwb(data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - stub_test=stub_test, - ) + data_dir_path = Path( + "/Volumes/T7/CatalystNeuro/Jadhav/SubLearnProject/SL18_D19/SL18_D19_S01_F01_BOX_SLP_20230503_112642" + ) + output_dir_path = Path("/Volumes/T7/CatalystNeuro/Jadhav/conversion_nwb") + stub_test = True + + if output_dir_path.exists(): + shutil.rmtree(output_dir_path, ignore_errors=True) + + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + stub_test=stub_test, + ) diff --git a/src/jadhav_lab_to_nwb/olson_2024/olson_2024_metadata.yaml b/src/jadhav_lab_to_nwb/olson_2024/olson_2024_metadata.yaml index 4141266..3934249 100644 --- a/src/jadhav_lab_to_nwb/olson_2024/olson_2024_metadata.yaml +++ b/src/jadhav_lab_to_nwb/olson_2024/olson_2024_metadata.yaml @@ -1,14 +1,47 @@ NWBFile: - related_publications: - https://doi.org/### or link to APA or MLA citation of the publication session_description: A rich text description of the experiment. Can also just be the abstract of the publication. - institution: Institution where the lab is located + institution: Brandeis University lab: Jadhav experimenter: - Last, First Middle - Last, First Middle + Subject: + description: Long Evans Rat + genotype: Wild Type + sex: M species: Rattus norvegicus - age: TBD # in ISO 8601, such as "P1W2D" - sex: TBD # One of M, F, U, or O + subject_id: SL18 + weight: 467g + +Ecephys: + Device: + - name: ProbeNameTBD + description: ProbeDescriptionTBD + manufacturer: ProbeManufacturerTBD + TrodeGroups: + - name: CA1_R + location: Right hippocampal subfield CA1 + device: ProbeNameTBD + nTrodes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10,11,60,61,62,63,64] + - name: CA1_L + location: Left hippocampal subfield CA1 + device: ProbeNameTBD + nTrodes: [12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27] + - name: SUB_L + location: Left Subiculum + device: ProbeNameTBD + nTrodes: [28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43] + - name: SUB_R + location: Right Subiculum + device: ProbeNameTBD + nTrodes: [44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59] + ElectricalSeries: + - name: ElectricalSeries + description: Raw acquisition of extracellular electrophysiology data recorded by SpikeGadgets. + +Behavior: + Videos: + - name: Video SL18_D19_S01_F01_BOX_SLP_20230503_112642.1 + description: Video of the rat in the box. diff --git a/src/jadhav_lab_to_nwb/olson_2024/olson_2024_notes.md b/src/jadhav_lab_to_nwb/olson_2024/olson_2024_notes.md index 17ec5cf..adc55cb 100644 --- a/src/jadhav_lab_to_nwb/olson_2024/olson_2024_notes.md +++ b/src/jadhav_lab_to_nwb/olson_2024/olson_2024_notes.md @@ -34,3 +34,7 @@ In JLab-Analysis-Suite/SpikeGadgets_Export_Pipeline/PipelineNotes.txt, - Jacob mentions that neuroconv's interface only converts the raw .rec file but nothing else --> will need to extend the interface to cover the rest of the data. + +## Ephys + +- Need probe info (not just control units) diff --git a/src/jadhav_lab_to_nwb/olson_2024/olson_2024_nwbconverter.py b/src/jadhav_lab_to_nwb/olson_2024/olson_2024_nwbconverter.py index 1fe8359..ef5de6b 100644 --- a/src/jadhav_lab_to_nwb/olson_2024/olson_2024_nwbconverter.py +++ b/src/jadhav_lab_to_nwb/olson_2024/olson_2024_nwbconverter.py @@ -1,18 +1,14 @@ """Primary NWBConverter class for this dataset.""" from neuroconv import NWBConverter -from neuroconv.datainterfaces import ( - SpikeGLXRecordingInterface, - PhySortingInterface, -) +from neuroconv.datainterfaces import VideoInterface -from jadhav_lab_to_nwb.olson_2024 import Olson2024BehaviorInterface +from jadhav_lab_to_nwb.olson_2024 import Olson2024BehaviorInterface, Olson2024SpikeGadgetsRecordingInterface class Olson2024NWBConverter(NWBConverter): """Primary conversion class for my extracellular electrophysiology dataset.""" data_interface_classes = dict( - Recording=SpikeGLXRecordingInterface, - Sorting=PhySortingInterface, - Behavior=Olson2024BehaviorInterface, + Recording=Olson2024SpikeGadgetsRecordingInterface, + Video=VideoInterface, ) diff --git a/src/jadhav_lab_to_nwb/olson_2024/olson_2024_spike_gadgets_recording_interface.py b/src/jadhav_lab_to_nwb/olson_2024/olson_2024_spike_gadgets_recording_interface.py new file mode 100644 index 0000000..cab25b8 --- /dev/null +++ b/src/jadhav_lab_to_nwb/olson_2024/olson_2024_spike_gadgets_recording_interface.py @@ -0,0 +1,148 @@ +"""Primary class for converting SpikeGadgets Ephys Recordings.""" +from pynwb.file import NWBFile +from pathlib import Path +from xml.etree import ElementTree +from pydantic import FilePath +import copy +from collections import Counter + +from neuroconv.datainterfaces import SpikeGadgetsRecordingInterface +from neuroconv.utils import DeepDict +from spikeinterface.extractors import SpikeGadgetsRecordingExtractor + + +class Olson2024SpikeGadgetsRecordingInterface(SpikeGadgetsRecordingInterface): + """SpikeGadgets RecordingInterface for olson_2024 conversion.""" + + Extractor = SpikeGadgetsRecordingExtractor + + def __init__(self, file_path: FilePath, **kwargs): + super().__init__(file_path=file_path, **kwargs) + + header_txt = get_spikegadgets_header(file_path) + root = ElementTree.fromstring(header_txt) + sconf = root.find("SpikeConfiguration") + self.hwChan_to_nTrode, self.hwChan_to_hasLFP = {}, {} + for tetrode in sconf.findall("SpikeNTrode"): + nTrode = tetrode.attrib["id"] + lfp_chan = int(tetrode.attrib["LFPChan"]) + for i, electrode in enumerate(tetrode.findall("SpikeChannel")): + hwChan = electrode.attrib["hwChan"] + hasLFP = lfp_chan == i + 1 + self.hwChan_to_nTrode[hwChan] = nTrode + self.hwChan_to_hasLFP[hwChan] = hasLFP + self.nTrode_to_hwChans = {nTrode: [] for nTrode in self.hwChan_to_nTrode.values()} + for hwChan, nTrode in self.hwChan_to_nTrode.items(): + self.nTrode_to_hwChans[nTrode].append(hwChan) + + def get_metadata(self) -> DeepDict: + metadata = super().get_metadata() + return metadata + + def get_metadata_schema(self) -> dict: + metadata_schema = super().get_metadata_schema() + metadata_schema["properties"]["Ecephys"]["properties"]["TrodeGroups"] = { + "type": "array", + "minItems": 1, + "items": { + "required": ["name", "location", "device", "nTrodes"], + "properties": { + "name": {"description": "the name of this Trode group", "pattern": "^[^/]*$", "type": "string"}, + "location": {"description": "description of location of this Trode group", "type": "string"}, + "device": { + "description": "the device that was used to record from this Trode group", + "type": "string", + "target": "pynwb.device.Device", + }, + "nTrodes": { + "description": "the tetrode numbers that belong to this Trode group", + "type": "array", + "items": {"type": "integer"}, + }, + "position": { + "description": "stereotaxic position of this electrode group (x, y, z)", + "type": "array", + }, + }, + "type": "object", + "additionalProperties": False, + "tag": "pynwb.ecephys.ElectrodeGroup", + }, + } + return metadata_schema + + def reformat_metadata(self, metadata: dict) -> dict: + TrodeGroups = metadata["Ecephys"]["TrodeGroups"] + metadata["Ecephys"]["ElectrodeGroup"] = [] + for group in TrodeGroups: + nTrodes = group.pop("nTrodes") + for nTrode in nTrodes: + electrode_group = copy.deepcopy(group) + electrode_group["name"] = f"nTrode{nTrode}" + electrode_group["description"] = f"ElectrodeGroup for tetrode {nTrode}" + metadata["Ecephys"]["ElectrodeGroup"].append(electrode_group) + return metadata + + def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict, **conversion_options): + metadata = self.reformat_metadata(metadata) + channel_ids = self.recording_extractor.get_channel_ids() + channel_names = self.recording_extractor.get_property(key="channel_name", ids=channel_ids) + group_names, chIDs, hasLFPs, locations = [], [], [], [] + for channel_name in channel_names: + hwChan = channel_name.split("hwChan")[-1] + nTrode = self.hwChan_to_nTrode[hwChan] + hasLFP = self.hwChan_to_hasLFP[hwChan] + electrode_group = next( + group for group in metadata["Ecephys"]["ElectrodeGroup"] if group["name"] == f"nTrode{nTrode}" + ) + location = electrode_group["location"] + trode_chan = self.nTrode_to_hwChans[nTrode].index(hwChan) + 1 + + group_names.append(f"nTrode{nTrode}") + hasLFPs.append(hasLFP) + locations.append(location) + chIDs.append(f"nTrode{nTrode}_elec{trode_chan}") + + self.recording_extractor.set_property(key="group_name", ids=channel_ids, values=group_names) + self.recording_extractor.set_property(key="chID", ids=channel_ids, values=chIDs) + self.recording_extractor.set_property(key="hasLFP", ids=channel_ids, values=hasLFPs) + self.recording_extractor.set_property( + key="brain_area", ids=channel_ids, values=locations + ) # brain_area in spikeinterface is location in nwb + + super().add_to_nwbfile(nwbfile=nwbfile, metadata=metadata, **conversion_options) + + +def get_spikegadgets_header(file_path: str | Path): + """Get the header information from a SpikeGadgets .rec file. + + This function reads the .rec file until the "" tag to extract the header information. + + Parameters + ---------- + file_path : str | Path + Path to the .rec file. + + Returns + ------- + str + The header information from the .rec file. + + Raises + ------ + ValueError + If the header does not contain "". + """ + header_size = None + with open(file_path, mode="rb") as f: + while True: + line = f.readline() + if b"" in line: + header_size = f.tell() + break + + if header_size is None: + ValueError("SpikeGadgets: the xml header does not contain ''") + + f.seek(0) + return f.read(header_size).decode("utf8")