Aída's conversion directory

.gitignore

@@ -147,3 +147,5 @@ dmypy.json
 
 # NWB files
 **.nwb
+
+*.out

src/jazayeri_lab_to_nwb/piccato/README.md

@@ -0,0 +1,50 @@
+# Piccato data conversion pipeline
+NWB conversion scripts for Piccato data to the [Neurodata Without Borders](https://nwb-overview.readthedocs.io/) data format.
+
+
+## Usage
+To run a specific conversion, you might need to install first some conversion specific dependencies that are located in each conversion directory:
+
+```
+pip install -r src/jazayeri_lab_to_nwb/piccato/requirements.txt
+```
+
+You can run a specific conversion with the following command:
+```
+python src/jazayeri_lab_to_nwb/piccato/main_convert_session.py $SUBJECT $SESSION
+```
+
+### Piccato working and long-term memory task data
+The conversion function for this experiment, `session_to_nwb`, is found in `src/piccato/main_convert_session.py`. The function takes arguments:
+* `subject` subject name (currently only `'elgar'`.)
+* `session` session date in format `'YYYY-MM-DD'`.
+* `stub_test` indicates whether only a small portion of the data should be saved (mainly used by us for testing purposes).
+* `overwrite` indicates whether to overwrite nwb output files.
+
+The function can be imported in a separate script with and run, or you can run the file directly and specify the arguments in the `if name == "__main__"` block at the bottom.
+
+The function expects the raw data in `data_dir_path` to follow this structure:
+```
+   data_dir_path/
+   ├── behavior_task
+   │   ├── eye.h.json, eye.v.json, etc.
+   │   ├── trials.json
+   ├── raw_data
+   │   ├── behavior
+   │       └── mworks
+   │       └── moog
+   │   ├── spikeglx
+   │       └── */*/*.ap.bin, */*/*.lf.bin, etc.
+   ├── spike_sorting
+   │   ├── spikeglx
+   │       └── kilosort2_5_0
+   ├── phys_metadata.json
+   ├── sync_signals
+       └── spikeglx
+       	   └── transform
+```
+The conversion will try to automatically fetch metadata from the provided data directory. However, some information, such as the subject's name and age, must be specified by the user in the file `src/jazayeri_lab_to_nwb/piccato/metadata.yaml`. If any of the automatically fetched metadata is incorrect, it can also be overriden from this file.
+
+The converted data will be saved in two files, one called `{session_id}_ecephys.nwb`, which contains the raw electrophysiology data from the Neuropixels and V-Probes, and one called `{session_id}_behavior+ecephys.nwb` with behavioral data, trial info, and sorted unit spiking.
+
+If you run into memory issues when writing the `{session_id}_ecephys.nwb` files, you may want to set `buffer_gb` to a value smaller than 1 (its default) in the `conversion_options` dicts for the recording interfaces, i.e. [here](https://github.com/catalystneuro/jazayeri-lab-to-nwb/blob/vprobe_dev/src/jazayeri_lab_to_nwb/watters/main_convert_session.py#L189).

src/jazayeri_lab_to_nwb/piccato/display_interface.py

@@ -0,0 +1,86 @@
+"""Class for converting data about display frames."""
+
+import itertools
+import json
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+import pandas as pd
+from neuroconv.datainterfaces.text.timeintervalsinterface import (
+    TimeIntervalsInterface,
+)
+from neuroconv.utils import FolderPathType
+from pynwb import NWBFile
+
+
+class DisplayInterface(TimeIntervalsInterface):
+    """Class for converting data about display frames.
+
+    All events that occur exactly once per display update are contained in this
+    interface.
+    """
+
+    KEY_MAP = {
+        "frame_closed_loop_gaze_position": "closed_loop_eye_position",
+        "frame_task_phase": "task_phase",
+        "frame_display_times": "start_time",
+    }
+
+    def __init__(self, folder_path: FolderPathType, verbose: bool = True):
+        super().__init__(file_path=folder_path, verbose=verbose)
+
+    def get_metadata(self) -> dict:
+        metadata = super().get_metadata()
+        metadata["TimeIntervals"] = dict(
+            display=dict(
+                table_name="display",
+                table_description="data about each displayed frame",
+            )
+        )
+        return metadata
+
+    def get_timestamps(self) -> np.ndarray:
+        return super(DisplayInterface, self).get_timestamps(
+            column="start_time"
+        )
+
+    def set_aligned_starting_time(self, aligned_starting_time: float) -> None:
+        self.dataframe.start_time += aligned_starting_time
+
+    def _read_file(self, file_path: FolderPathType):
+        # Create dataframe with data for each frame
+        trials = json.load(open(Path(file_path) / "trials.json", "r"))
+        frames = {
+            k_mapped: list(itertools.chain(*[d[k] for d in trials]))
+            for k, k_mapped in DisplayInterface.KEY_MAP.items()
+        }
+
+        return pd.DataFrame(frames)
+
+    def add_to_nwbfile(
+        self,
+        nwbfile: NWBFile,
+        metadata: Optional[dict] = None,
+        tag: str = "display",
+    ):
+        return super(DisplayInterface, self).add_to_nwbfile(
+            nwbfile=nwbfile,
+            metadata=metadata,
+            tag=tag,
+            column_descriptions=self.column_descriptions,
+        )
+
+    @property
+    def column_descriptions(self):
+        column_descriptions = {
+            "closed_loop_eye_position": (
+                "For each frame, the eye position in the close-loop task "
+                "engine. This was used to for real-time eye position "
+                "computations, such as saccade detection and reward delivery."
+            ),
+            "task_phase": "The phase of the task for each frame.",
+            "start_time": "Time of display update for each frame.",
+        }
+
+        return column_descriptions

src/jazayeri_lab_to_nwb/piccato/get_session_paths.py

@@ -0,0 +1,84 @@
+"""Function for getting paths to data on openmind."""
+
+import collections
+import pathlib
+
+SUBJECT_NAME_TO_ID = {
+    "elgar": "elgar",
+}
+
+OM_PATH = "/om2/user/apiccato/phys_preprocessing_open_source/phys_data"
+DANDISET_ID = "000767"
+SessionPaths = collections.namedtuple(
+    "SessionPaths",
+    [
+        "output",
+        "ecephys_data",
+        "behavior_task_data",
+        "session_data",
+        "sync_pulses",
+        "spike_sorting_raw",
+        "postprocessed_data",
+    ],
+)
+
+
+def _get_session_paths_openmind(subject, session):
+    """Get paths to all components of the data on openmind."""
+    # subject_id = SUBJECT_NAME_TO_ID[subject]
+
+    # Path to write output nwb files to
+    output_path = pathlib.Path(
+        f"/om2/user/apiccato/nwb_data/staging/{DANDISET_ID}/sub-{subject}"
+    )
+
+    # Path to the raw data. This is used for reading raw physiology data.
+    ecephys_data_path = pathlib.Path(
+        f"{OM_PATH}/{subject}/{session}/raw_data/"
+    )
+
+    # Path to task and behavior data.
+    behavior_task_data_path = pathlib.Path(
+        f"{OM_PATH}/{subject}/{session}/behavior_task"
+    )
+
+    # Path to sync pulses. This is used for reading timescale transformations
+    # between physiology and mworks data streams.
+    sync_pulses_path = pathlib.Path(
+        f"{OM_PATH}/{subject}/{session}/sync_signals"
+    )
+
+    # Path to spike sorting. This is used for reading spike sorted data.
+    spike_sorting_raw_path = pathlib.Path(
+        f"{OM_PATH}/{subject}/{session}/spike_sorting"
+    )
+
+    session_path = pathlib.Path(f"{OM_PATH}/{subject}/{session}/")
+
+    postprocessed_data_path = pathlib.Path(
+        f"{OM_PATH}/{subject}/{session}/kilosort2_5_0"
+    )
+
+    session_paths = SessionPaths(
+        output=output_path,
+        ecephys_data=ecephys_data_path,
+        session_data=session_path,
+        behavior_task_data=pathlib.Path(behavior_task_data_path),
+        sync_pulses=sync_pulses_path,
+        spike_sorting_raw=spike_sorting_raw_path,
+        postprocessed_data=postprocessed_data_path,
+    )
+
+    return session_paths
+
+
+def get_session_paths(subject, session, repo="openmind"):
+    """Get paths to all components of the data.
+
+    Returns:
+        SessionPaths namedtuple.
+    """
+    if repo == "openmind":
+        return _get_session_paths_openmind(subject=subject, session=session)
+    else:
+        raise ValueError(f"Invalid repo {repo}")