Merge pull request #16 from catalystneuro/watters

Small formatting changes and cleanups.
catalystneuro · Dec 20, 2023 · d8536a5 · d8536a5
2 parents 54b431b + 3c52dc1
commit d8536a5
Show file tree

Hide file tree

Showing 10 changed files with 269 additions and 161 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -9,9 +9,11 @@ repos:
     rev: 22.8.0
     hooks:
     -   id: black
+        args: [--line-length=79]
         exclude: ^docs/
-- repo: https://github.com/pycqa/isort
-  rev: 5.13.1
-  hooks:
-  - id: isort
-    name: isort (python)
+-   repo: https://github.com/pycqa/isort
+    rev: 5.13.1
+    hooks:
+    -   id: isort
+        name: isort (python)
+        args: [--profile=black, --line-length=79]
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,5 +1,5 @@
 [tool.black]
-line-length = 120
+line-length = 79
 target-version = ['py38', 'py39', 'py310']
 include = '\.pyi?$'
 extend-exclude = '''

diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,4 @@
-neuroconv==0.4.7
+neuroconv==0.4.6
 spikeinterface==0.99.1
 nwbwidgets==0.11.3
 nwbinspector==0.4.31

diff --git a/src/jazayeri_lab_to_nwb/watters/display_interface.py b/src/jazayeri_lab_to_nwb/watters/display_interface.py
@@ -1,12 +1,15 @@
 """Class for converting data about display frames."""
+
 import itertools
 import json
 from pathlib import Path
 from typing import Optional
 
 import numpy as np
 import pandas as pd
-from neuroconv.datainterfaces.text.timeintervalsinterface import TimeIntervalsInterface
+from neuroconv.datainterfaces.text.timeintervalsinterface import (
+    TimeIntervalsInterface,
+)
 from neuroconv.utils import FolderPathType
 from pynwb import NWBFile
 
@@ -40,7 +43,9 @@ def get_metadata(self) -> dict:
         return metadata
 
     def get_timestamps(self) -> np.ndarray:
-        return super(DisplayInterface, self).get_timestamps(column="start_time")
+        return super(DisplayInterface, self).get_timestamps(
+            column="start_time"
+        )
 
     def set_aligned_starting_time(self, aligned_starting_time: float) -> None:
         self.dataframe.start_time += aligned_starting_time
@@ -49,15 +54,23 @@ def _read_file(self, file_path: FolderPathType):
         # Create dataframe with data for each frame
         trials = json.load(open(Path(file_path) / "trials.json", "r"))
         frames = {
-            k_mapped: list(itertools.chain(*[d[k] for d in trials])) for k, k_mapped in DisplayInterface.KEY_MAP.items()
+            k_mapped: list(itertools.chain(*[d[k] for d in trials]))
+            for k, k_mapped in DisplayInterface.KEY_MAP.items()
         }
 
         # Serialize object_positions data for hdf5 conversion to work
-        frames["object_positions"] = [json.dumps(x) for x in frames["object_positions"]]
+        frames["object_positions"] = [
+            json.dumps(x) for x in frames["object_positions"]
+        ]
 
         return pd.DataFrame(frames)
 
-    def add_to_nwbfile(self, nwbfile: NWBFile, metadata: Optional[dict] = None, tag: str = "display"):
+    def add_to_nwbfile(
+        self,
+        nwbfile: NWBFile,
+        metadata: Optional[dict] = None,
+        tag: str = "display",
+    ):
         return super(DisplayInterface, self).add_to_nwbfile(
             nwbfile=nwbfile,
             metadata=metadata,

diff --git a/src/jazayeri_lab_to_nwb/watters/get_session_paths.py b/src/jazayeri_lab_to_nwb/watters/get_session_paths.py
@@ -26,28 +26,39 @@ def _get_session_paths_openmind(subject, session):
     subject_id = SUBJECT_NAME_TO_ID[subject]
 
     # Path to write output nwb files to
-    output_path = f"/om/user/nwatters/nwb_data_multi_prediction/staging/sub-{subject}"
+    output_path = (
+        f"/om/user/nwatters/nwb_data_multi_prediction/staging/sub-{subject}"
+    )
 
     # Path to the raw data. This is used for reading raw physiology data.
-    raw_data_path = f"/om4/group/jazlab/nwatters/multi_prediction/phys_data/{subject}/" f"{session}/raw_data"
+    raw_data_path = (
+        f"/om4/group/jazlab/nwatters/multi_prediction/phys_data/{subject}/"
+        f"{session}/raw_data"
+    )
 
     # Path to task and behavior data.
     task_behavior_data_path = (
-        "/om4/group/jazlab/nwatters/multi_prediction/datasets/data_nwb_trials/" f"{subject}/{session}"
+        "/om4/group/jazlab/nwatters/multi_prediction/datasets/data_nwb_trials/"
+        f"{subject}/{session}"
     )
 
     # Path to open-source data. This is used for reading behavior and task data.
     data_open_source_path = (
-        "/om4/group/jazlab/nwatters/multi_prediction/datasets/data_open_source/" f"Subjects/{subject_id}/{session}/001"
+        "/om4/group/jazlab/nwatters/multi_prediction/datasets/data_open_source/"
+        f"Subjects/{subject_id}/{session}/001"
     )
 
     # Path to sync pulses. This is used for reading timescale transformations
     # between physiology and mworks data streams.
-    sync_pulses_path = "/om4/group/jazlab/nwatters/multi_prediction/data_processed/" f"{subject}/{session}/sync_pulses"
+    sync_pulses_path = (
+        "/om4/group/jazlab/nwatters/multi_prediction/data_processed/"
+        f"{subject}/{session}/sync_pulses"
+    )
 
     # Path to spike sorting. This is used for reading spike sorted data.
     spike_sorting_raw_path = (
-        f"/om4/group/jazlab/nwatters/multi_prediction/phys_data/{subject}/" f"{session}/spike_sorting"
+        f"/om4/group/jazlab/nwatters/multi_prediction/phys_data/{subject}/"
+        f"{session}/spike_sorting"
     )
 
     session_paths = SessionPaths(

diff --git a/src/jazayeri_lab_to_nwb/watters/main_convert_session.py b/src/jazayeri_lab_to_nwb/watters/main_convert_session.py
@@ -21,7 +21,6 @@
     See comments below for descriptions of these variables.
 """
 
-import datetime
 import glob
 import json
 import logging
@@ -89,25 +88,38 @@ def _add_v_probe_data(
 
     # Raw data
     recording_file = _get_single_file(probe_data_dir, suffix=".dat")
-    metadata_path = str(session_paths.data_open_source / "probes.metadata.json")
+    metadata_path = str(
+        session_paths.data_open_source / "probes.metadata.json"
+    )
     raw_source_data[f"RecordingVP{probe_num}"] = dict(
         file_path=recording_file,
         probe_metadata_file=metadata_path,
         probe_key=f"probe{(probe_num + 1):02d}",
         probe_name=f"vprobe{probe_num}",
         es_key=f"ElectricalSeriesVP{probe_num}",
     )
-    raw_conversion_options[f"RecordingVP{probe_num}"] = dict(stub_test=stub_test)
+    raw_conversion_options[f"RecordingVP{probe_num}"] = dict(
+        stub_test=stub_test
+    )
 
     # Processed data
-    sorting_path = session_paths.spike_sorting_raw / f"v_probe_{probe_num}" / "ks_3_output_pre_v6_curated"
-    processed_source_data[f"RecordingVP{probe_num}"] = raw_source_data[f"RecordingVP{probe_num}"]
+    sorting_path = (
+        session_paths.spike_sorting_raw
+        / f"v_probe_{probe_num}"
+        / "ks_3_output_pre_v6_curated"
+    )
+    processed_source_data[f"RecordingVP{probe_num}"] = raw_source_data[
+        f"RecordingVP{probe_num}"
+    ]
     processed_source_data[f"SortingVP{probe_num}"] = dict(
-        folder_path=str(sorting_path),
-        keep_good_only=False,
+        folder_path=str(sorting_path), keep_good_only=False
+    )
+    processed_conversion_options[f"RecordingVP{probe_num}"] = dict(
+        stub_test=stub_test, write_electrical_series=False
+    )
+    processed_conversion_options[f"SortingVP{probe_num}"] = dict(
+        stub_test=stub_test, write_as="processing"
     )
-    processed_conversion_options[f"RecordingVP{probe_num}"] = dict(stub_test=stub_test, write_electrical_series=False)
-    processed_conversion_options[f"SortingVP{probe_num}"] = dict(stub_test=stub_test, write_as="processing")
 
 
 def _add_spikeglx_data(
@@ -122,7 +134,11 @@ def _add_spikeglx_data(
     logging.info("Adding SpikeGLX data")
 
     # Raw data
-    spikeglx_dir = [x for x in (session_paths.raw_data / "spikeglx").iterdir() if "settling" not in str(x)]
+    spikeglx_dir = [
+        x
+        for x in (session_paths.raw_data / "spikeglx").iterdir()
+        if "settling" not in str(x)
+    ]
     if len(spikeglx_dir) == 0:
         logging.info("Found no SpikeGLX data")
     elif len(spikeglx_dir) == 1:
@@ -146,11 +162,17 @@ def _add_spikeglx_data(
         folder_path=str(sorting_path),
         keep_good_only=False,
     )
-    processed_conversion_options["SortingNP"] = dict(stub_test=stub_test, write_as="processing")
+    processed_conversion_options["SortingNP"] = dict(
+        stub_test=stub_test, write_as="processing"
+    )
 
 
 def session_to_nwb(
-    subject: str, session: str, stub_test: bool = False, overwrite: bool = True, dandiset_id: Union[str, None] = None
+    subject: str,
+    session: str,
+    stub_test: bool = False,
+    overwrite: bool = True,
+    dandiset_id: Union[str, None] = None,
 ):
     """
     Convert a single session to an NWB file.
@@ -190,7 +212,9 @@ def session_to_nwb(
     logging.info(f"dandiset_id = {dandiset_id}")
 
     # Get paths
-    session_paths = get_session_paths.get_session_paths(subject, session, repo=_REPO)
+    session_paths = get_session_paths.get_session_paths(
+        subject, session, repo=_REPO
+    )
     logging.info(f"session_paths: {session_paths}")
 
     # Get paths for nwb files to write
@@ -199,8 +223,13 @@ def session_to_nwb(
         session_id = f"{session}-stub"
     else:
         session_id = f"{session}"
-    raw_nwb_path = session_paths.output / f"sub-{subject}_ses-{session_id}_ecephys.nwb"
-    processed_nwb_path = session_paths.output / f"sub-{subject}_ses-{session_id}_behavior+ecephys.nwb"
+    raw_nwb_path = (
+        session_paths.output / f"sub-{subject}_ses-{session_id}_ecephys.nwb"
+    )
+    processed_nwb_path = (
+        session_paths.output
+        / f"sub-{subject}_ses-{session_id}_behavior+ecephys.nwb"
+    )
     logging.info(f"raw_nwb_path = {raw_nwb_path}")
     logging.info(f"processed_nwb_path = {processed_nwb_path}")
     logging.info("")
@@ -247,12 +276,16 @@ def session_to_nwb(
 
     # Add trials data
     logging.info("Adding trials data")
-    processed_source_data["Trials"] = dict(folder_path=str(session_paths.task_behavior_data))
+    processed_source_data["Trials"] = dict(
+        folder_path=str(session_paths.task_behavior_data)
+    )
     processed_conversion_options["Trials"] = dict()
 
     # Add display data
     logging.info("Adding display data")
-    processed_source_data["Display"] = dict(folder_path=str(session_paths.task_behavior_data))
+    processed_source_data["Display"] = dict(
+        folder_path=str(session_paths.task_behavior_data)
+    )
     processed_conversion_options["Display"] = dict()
 
     # Create processed data converter
@@ -269,10 +302,14 @@ def session_to_nwb(
     metadata["Subject"]["age"] = _SUBJECT_TO_AGE[subject]
 
     # EcePhys
-    probe_metadata_file = session_paths.data_open_source / "probes.metadata.json"
+    probe_metadata_file = (
+        session_paths.data_open_source / "probes.metadata.json"
+    )
     with open(probe_metadata_file, "r") as f:
         probe_metadata = json.load(f)
-    neuropixel_metadata = [x for x in probe_metadata if x["probe_type"] == "Neuropixels"][0]
+    neuropixel_metadata = [
+        x for x in probe_metadata if x["probe_type"] == "Neuropixels"
+    ][0]
     for entry in metadata["Ecephys"]["ElectrodeGroup"]:
         if entry["device"] == "Neuropixel-Imec":
             # TODO: uncomment when fixed in pynwb
@@ -291,10 +328,10 @@ def session_to_nwb(
 
     # Check if session_start_time was found/set
     if "session_start_time" not in metadata["NWBFile"]:
-        raise ValueError("Session start time was not auto-detected. Please provide it " "in `metadata.yaml`")
-    session_start_time = metadata["NWBFile"]["session_start_time"]
-    metadata["NWBFile"]["session_start_time"] = session_start_time.replace(
-        tzinfo=ZoneInfo("US/Eastern"))
+        raise ValueError(
+            "Session start time was not auto-detected. Please provide it "
+            "in `metadata.yaml`"
+        )
 
     # Run conversion
     logging.info("Running processed conversion")