Merge pull request #2 from catalystneuro/neuroconv_interfaces

Adds Interfaces for SpikeGadgets and Video

make_env.yml

@@ -3,7 +3,7 @@ channels:
 - conda-forge
 - defaults
 dependencies:
-- python>=3.9
+- python>=3.9, <3.13
 - pip
 - pip:
   - -e .  # This calls the setup and therefore requirements minimal

pyproject.toml

@@ -15,8 +15,11 @@ classifiers = [
 ]
 
 dependencies = [
-  "neuroconv",
+  "neuroconv[spikegadgets,video]",
   "nwbinspector",
+  "pre-commit",
+  "ipykernel",
+  "matplotlib",
 ]
 
 [project.urls]

src/jadhav_lab_to_nwb/olson_2024/__init__.py

@@ -1,2 +1,3 @@
-from .olson_2024behaviorinterface import Olson2024BehaviorInterface
-from .olson_2024nwbconverter import Olson2024NWBConverter
+from .olson_2024_behavior_interface import Olson2024BehaviorInterface
+from .olson_2024_spike_gadgets_recording_interface import Olson2024SpikeGadgetsRecordingInterface
+from .olson_2024_nwbconverter import Olson2024NWBConverter

src/jadhav_lab_to_nwb/olson_2024/olson_2024_behaviorinterface.py → src/jadhav_lab_to_nwb/olson_2024/olson_2024_behavior_interface.py

@@ -4,18 +4,19 @@
 from neuroconv.basedatainterface import BaseDataInterface
 from neuroconv.utils import DeepDict
 
+
 class Olson2024BehaviorInterface(BaseDataInterface):
     """Behavior interface for olson_2024 conversion"""
 
     keywords = ("behavior",)
-    
+
     def __init__(self):
         # This should load the data lazily and prepare variables you need
         pass
 
     def get_metadata(self) -> DeepDict:
         # Automatically retrieve as much metadata as possible from the source files available
-        metadata = super().get_metadata()   
+        metadata = super().get_metadata()
 
         return metadata
 

src/jadhav_lab_to_nwb/olson_2024/olson_2024_convert_all_sessions.py

@@ -1,6 +1,5 @@
 """Primary script to run to convert all sessions in a dataset using session_to_nwb."""
 from pathlib import Path
-from typing import Union
 from concurrent.futures import ProcessPoolExecutor, as_completed
 from pprint import pformat
 import traceback
@@ -11,18 +10,18 @@
 
 def dataset_to_nwb(
     *,
-    data_dir_path: Union[str, Path],
-    output_dir_path: Union[str, Path],
+    data_dir_path: str | Path,
+    output_dir_path: str | Path,
     max_workers: int = 1,
     verbose: bool = True,
 ):
     """Convert the entire dataset to NWB.
 
     Parameters
     ----------
-    data_dir_path : Union[str, Path]
+    data_dir_path : str | Path
         The path to the directory containing the raw data.
-    output_dir_path : Union[str, Path]
+    output_dir_path : str | Path
         The path to the directory where the NWB files will be saved.
     max_workers : int, optional
         The number of workers to use for parallel processing, by default 1
@@ -39,7 +38,7 @@ def dataset_to_nwb(
         for session_to_nwb_kwargs in session_to_nwb_kwargs_per_session:
             session_to_nwb_kwargs["output_dir_path"] = output_dir_path
             session_to_nwb_kwargs["verbose"] = verbose
-            exception_file_path = data_dir_path / f"ERROR_<nwbfile_name>.txt" # Add error file path here
+            exception_file_path = data_dir_path / f"ERROR_<nwbfile_name>.txt"  # Add error file path here
             futures.append(
                 executor.submit(
                     safe_session_to_nwb,
@@ -51,7 +50,7 @@ def dataset_to_nwb(
             pass
 
 
-def safe_session_to_nwb(*, session_to_nwb_kwargs: dict, exception_file_path: Union[Path, str]):
+def safe_session_to_nwb(*, session_to_nwb_kwargs: dict, exception_file_path: str | Path):
     """Convert a session to NWB while handling any errors by recording error messages to the exception_file_path.
 
     Parameters
@@ -72,25 +71,25 @@ def safe_session_to_nwb(*, session_to_nwb_kwargs: dict, exception_file_path: Uni
 
 def get_session_to_nwb_kwargs_per_session(
     *,
-    data_dir_path: Union[str, Path],
+    data_dir_path: str | Path,
 ):
     """Get the kwargs for session_to_nwb for each session in the dataset.
 
     Parameters
     ----------
-    data_dir_path : Union[str, Path]
+    data_dir_path : str | Path
         The path to the directory containing the raw data.
 
     Returns
     -------
     list[dict[str, Any]]
         A list of dictionaries containing the kwargs for session_to_nwb for each session.
     """
-     #####
-     # # Implement this function to return the kwargs for session_to_nwb for each session
-     # This can be a specific list with hard-coded sessions, a path expansion or any conversion specific logic that you might need
-     #####
-    raise NotImplementedError 
+    #####
+    # # Implement this function to return the kwargs for session_to_nwb for each session
+    # This can be a specific list with hard-coded sessions, a path expansion or any conversion specific logic that you might need
+    #####
+    raise NotImplementedError
 
 
 if __name__ == "__main__":

src/jadhav_lab_to_nwb/olson_2024/olson_2024_convert_session.py

@@ -1,69 +1,67 @@
 """Primary script to run to convert an entire session for of data using the NWBConverter."""
 from pathlib import Path
-from typing import Union
 import datetime
 from zoneinfo import ZoneInfo
+import shutil
 
 from neuroconv.utils import load_dict_from_file, dict_deep_update
 
 from jadhav_lab_to_nwb.olson_2024 import Olson2024NWBConverter
 
 
-def session_to_nwb(data_dir_path: Union[str, Path], output_dir_path: Union[str, Path], stub_test: bool = False):
+def session_to_nwb(data_dir_path: str | Path, output_dir_path: str | Path, stub_test: bool = False):
 
     data_dir_path = Path(data_dir_path)
     output_dir_path = Path(output_dir_path)
     if stub_test:
         output_dir_path = output_dir_path / "nwb_stub"
     output_dir_path.mkdir(parents=True, exist_ok=True)
 
-    session_id = "subject_identifier_usually"
+    session_id = "sample_session"
     nwbfile_path = output_dir_path / f"{session_id}.nwb"
 
     source_data = dict()
     conversion_options = dict()
 
-    # Add Recording
-    source_data.update(dict(Recording=dict()))
+    # Add Ephys
+    file_path = data_dir_path / f"{data_dir_path.name}.rec"
+    source_data.update(dict(Recording=dict(file_path=file_path)))
     conversion_options.update(dict(Recording=dict(stub_test=stub_test)))
 
-    # Add Sorting
-    source_data.update(dict(Sorting=dict()))
-    conversion_options.update(dict(Sorting=dict()))
-
-    # Add Behavior
-    source_data.update(dict(Behavior=dict()))
-    conversion_options.update(dict(Behavior=dict()))
+    # Add Video
+    file_paths = [data_dir_path / f"{data_dir_path.name}.1.h264"]
+    source_data.update(dict(Video=dict(file_paths=file_paths)))
+    conversion_options.update(dict(Video=dict()))
 
     converter = Olson2024NWBConverter(source_data=source_data)
 
     # Add datetime to conversion
     metadata = converter.get_metadata()
-    datetime.datetime(
-        year=2020, month=1, day=1, tzinfo=ZoneInfo("US/Eastern")
-    )
-    date = datetime.datetime.today()  # TO-DO: Get this from author
-    metadata["NWBFile"]["session_start_time"] = date
+    metadata["NWBFile"]["session_start_time"] = datetime.datetime(2023, 5, 3, 11, 26, 42, tzinfo=ZoneInfo("US/Eastern"))
 
     # Update default metadata with the editable in the corresponding yaml file
     editable_metadata_path = Path(__file__).parent / "olson_2024_metadata.yaml"
     editable_metadata = load_dict_from_file(editable_metadata_path)
     metadata = dict_deep_update(metadata, editable_metadata)
 
-    metadata["Subject"]["subject_id"] = "a_subject_id"  # Modify here or in the yaml file
-
     # Run conversion
     converter.run_conversion(metadata=metadata, nwbfile_path=nwbfile_path, conversion_options=conversion_options)
 
 
 if __name__ == "__main__":
 
     # Parameters for conversion
-    data_dir_path = Path("/Directory/With/Raw/Formats/")
-    output_dir_path = Path("~/conversion_nwb/")
-    stub_test = False
-
-    session_to_nwb(data_dir_path=data_dir_path,
-                    output_dir_path=output_dir_path,
-                    stub_test=stub_test,
-                    )
+    data_dir_path = Path(
+        "/Volumes/T7/CatalystNeuro/Jadhav/SubLearnProject/SL18_D19/SL18_D19_S01_F01_BOX_SLP_20230503_112642"
+    )
+    output_dir_path = Path("/Volumes/T7/CatalystNeuro/Jadhav/conversion_nwb")
+    stub_test = True
+
+    if output_dir_path.exists():
+        shutil.rmtree(output_dir_path, ignore_errors=True)
+
+    session_to_nwb(
+        data_dir_path=data_dir_path,
+        output_dir_path=output_dir_path,
+        stub_test=stub_test,
+    )

src/jadhav_lab_to_nwb/olson_2024/olson_2024_metadata.yaml

@@ -1,14 +1,47 @@
 NWBFile:
-  related_publications:
-    https://doi.org/### or link to APA or MLA citation of the publication
   session_description:
     A rich text description of the experiment. Can also just be the abstract of the publication.
-  institution: Institution where the lab is located
+  institution: Brandeis University
   lab: Jadhav
   experimenter:
     - Last, First Middle
     - Last, First Middle
+
 Subject:
+  description: Long Evans Rat
+  genotype: Wild Type
+  sex: M
   species: Rattus norvegicus
-  age: TBD  # in ISO 8601, such as "P1W2D"
-  sex: TBD  # One of M, F, U, or O
+  subject_id: SL18
+  weight: 467g
+
+Ecephys:
+  Device:
+    - name: ProbeNameTBD
+      description: ProbeDescriptionTBD
+      manufacturer: ProbeManufacturerTBD
+  TrodeGroups:
+    - name: CA1_R
+      location: Right hippocampal subfield CA1
+      device: ProbeNameTBD
+      nTrodes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10,11,60,61,62,63,64]
+    - name: CA1_L
+      location: Left hippocampal subfield CA1
+      device: ProbeNameTBD
+      nTrodes: [12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27]
+    - name: SUB_L
+      location: Left Subiculum
+      device: ProbeNameTBD
+      nTrodes: [28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43]
+    - name: SUB_R
+      location: Right Subiculum
+      device: ProbeNameTBD
+      nTrodes: [44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59]
+  ElectricalSeries:
+    - name: ElectricalSeries
+      description: Raw acquisition of extracellular electrophysiology data recorded by SpikeGadgets.
+
+Behavior:
+  Videos:
+    - name: Video SL18_D19_S01_F01_BOX_SLP_20230503_112642.1
+      description: Video of the rat in the box.

src/jadhav_lab_to_nwb/olson_2024/olson_2024_notes.md

@@ -34,3 +34,7 @@
 
 In JLab-Analysis-Suite/SpikeGadgets_Export_Pipeline/PipelineNotes.txt,
     - Jacob mentions that neuroconv's interface only converts the raw .rec file but nothing else --> will need to extend the interface to cover the rest of the data.
+
+## Ephys
+
+- Need probe info (not just control units)

src/jadhav_lab_to_nwb/olson_2024/olson_2024_nwbconverter.py

@@ -1,18 +1,14 @@
 """Primary NWBConverter class for this dataset."""
 from neuroconv import NWBConverter
-from neuroconv.datainterfaces import (
-    SpikeGLXRecordingInterface,
-    PhySortingInterface,
-)
+from neuroconv.datainterfaces import VideoInterface
 
-from jadhav_lab_to_nwb.olson_2024 import Olson2024BehaviorInterface
+from jadhav_lab_to_nwb.olson_2024 import Olson2024BehaviorInterface, Olson2024SpikeGadgetsRecordingInterface
 
 
 class Olson2024NWBConverter(NWBConverter):
     """Primary conversion class for my extracellular electrophysiology dataset."""
 
     data_interface_classes = dict(
-        Recording=SpikeGLXRecordingInterface,
-        Sorting=PhySortingInterface,
-        Behavior=Olson2024BehaviorInterface,
+        Recording=Olson2024SpikeGadgetsRecordingInterface,
+        Video=VideoInterface,
     )