Merge branch 'main' of https://github.com/datajoint/element-array-ephys

kushalbakshi · Sep 20, 2024 · 9056db8 · 9056db8
2 parents 41e4d6d + 780352b
commit 9056db8
Show file tree

Hide file tree

Showing 11 changed files with 85 additions and 65 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,12 @@
 Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
  [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
 
+## [0.3.5] - 2024-08-16
+
++ Fix - Improve `spikeglx` loader in extracting neuropixels probe type from the meta file
++ Update - Explicit call to `probe.create_neuropixels_probe_types()` to create entries in ProbeType
+
+
 ## [0.3.4] - 2024-03-22
 
 + Add - pytest

diff --git a/element_array_ephys/ephys_acute.py b/element_array_ephys/ephys_acute.py
@@ -188,7 +188,7 @@ def auto_generate_entries(cls, session_key):
 
                 probe_dir = meta_filepath.parent
                 try:
-                    probe_number = re.search("(imec)?\d{1}$", probe_dir.name).group()
+                    probe_number = re.search(r"(imec)?\d{1}$", probe_dir.name).group()
                     probe_number = int(probe_number.replace("imec", ""))
                 except AttributeError:
                     probe_number = meta_fp_idx

diff --git a/element_array_ephys/ephys_no_curation.py b/element_array_ephys/ephys_no_curation.py
@@ -189,7 +189,7 @@ def auto_generate_entries(cls, session_key):
 
                 probe_dir = meta_filepath.parent
                 try:
-                    probe_number = re.search("(imec)?\d{1}$", probe_dir.name).group()
+                    probe_number = re.search(r"(imec)?\d{1}$", probe_dir.name).group()
                     probe_number = int(probe_number.replace("imec", ""))
                 except AttributeError:
                     probe_number = meta_fp_idx

diff --git a/element_array_ephys/probe.py b/element_array_ephys/probe.py
@@ -1,12 +1,10 @@
-"""
-Neuropixels Probes
-"""
-
 import datajoint as dj
 
 from .readers import probe_geometry
 from .readers.probe_geometry import build_electrode_layouts
 
+log = dj.logger
+
 schema = dj.schema()
 
 
@@ -33,20 +31,6 @@ def activate(
         schema_name, create_schema=create_schema, create_tables=create_tables
     )
 
-    # Add neuropixels probes
-    for probe_type in (
-        "neuropixels 1.0 - 3A",
-        "neuropixels 1.0 - 3B",
-        "neuropixels UHD",
-        "neuropixels 2.0 - SS",
-        "neuropixels 2.0 - MS",
-    ):
-        if not (ProbeType & {"probe_type": probe_type}):
-            try:
-                ProbeType.create_neuropixels_probe(probe_type)
-            except dj.errors.DataJointError as e:
-                print(f"Unable to create probe-type: {probe_type}\n{str(e)}")
-
 
 @schema
 class ProbeType(dj.Lookup):
@@ -87,39 +71,10 @@ class Electrode(dj.Part):
 
     @staticmethod
     def create_neuropixels_probe(probe_type: str = "neuropixels 1.0 - 3A"):
-        """
-        Create `ProbeType` and `Electrode` for neuropixels probes:
-        + neuropixels 1.0 - 3A
-        + neuropixels 1.0 - 3B
-        + neuropixels UHD
-        + neuropixels 2.0 - SS
-        + neuropixels 2.0 - MS
-
-        For electrode location, the (0, 0) is the
-         bottom left corner of the probe (ignore the tip portion)
-        Electrode numbering is 0-indexing
-        """
-
-        npx_probes_config = probe_geometry.M
-        npx_probes_config["neuropixels 1.0 - 3A"] = npx_probes_config["3A"]
-        npx_probes_config["neuropixels 1.0 - 3B"] = npx_probes_config["NP1010"]
-        npx_probes_config["neuropixels UHD"] = npx_probes_config["NP1100"]
-        npx_probes_config["neuropixels 2.0 - SS"] = npx_probes_config["NP2000"]
-        npx_probes_config["neuropixels 2.0 - MS"] = npx_probes_config["NP2010"]
-
-        probe_type = {"probe_type": probe_type}
-        probe_params = dict(
-            zip(
-                probe_geometry.geom_param_names,
-                npx_probes_config[probe_type["probe_type"]],
-            )
-        )
-        electrode_layouts = probe_geometry.build_npx_probe(
-            **{**probe_params, **probe_type}
+        log.warning(
+            "Class method `ProbeType.create_neuropixels_probe` is deprecated. Use `create_neuropixels_probe` instead.",
         )
-        with ProbeType.connection.transaction:
-            ProbeType.insert1(probe_type, skip_duplicates=True)
-            ProbeType.Electrode.insert(electrode_layouts, skip_duplicates=True)
+        return create_neuropixels_probe(probe_type)
 
 
 @schema
@@ -171,3 +126,49 @@ class Electrode(dj.Part):
         -> master
         -> ProbeType.Electrode
         """
+
+
+def create_neuropixels_probe_types():
+    # Add neuropixels probes
+    for probe_type in (
+        "neuropixels 1.0 - 3A",
+        "neuropixels 1.0 - 3B",
+        "neuropixels UHD",
+        "neuropixels 2.0 - SS",
+        "neuropixels 2.0 - MS",
+    ):
+        if not (ProbeType & {"probe_type": probe_type}):
+            create_neuropixels_probe(probe_type)
+
+
+def create_neuropixels_probe(probe_type: str = "neuropixels 1.0 - 3A"):
+    """
+    Create `ProbeType` and `Electrode` for neuropixels probes:
+    + neuropixels 1.0 - 3A
+    + neuropixels 1.0 - 3B
+    + neuropixels UHD
+    + neuropixels 2.0 - SS
+    + neuropixels 2.0 - MS
+
+    For electrode location, the (0, 0) is the
+     bottom left corner of the probe (ignore the tip portion)
+    Electrode numbering is 0-indexing
+    """
+    npx_probes_config = probe_geometry.M
+    if probe_type not in npx_probes_config:
+        raise ValueError(
+            f"Probe type {probe_type} not found in probe_geometry configuration. Not a Neuropixels probe?"
+        )
+
+    probe_params = dict(
+        zip(
+            probe_geometry.geom_param_names,
+            npx_probes_config[probe_type],
+        )
+    )
+    electrode_layouts = probe_geometry.build_npx_probe(
+        **{**probe_params, "probe_type": probe_type}
+    )
+    with ProbeType.connection.transaction:
+        ProbeType.insert1({"probe_type": probe_type})
+        ProbeType.Electrode.insert(electrode_layouts)
diff --git a/element_array_ephys/readers/kilosort.py b/element_array_ephys/readers/kilosort.py
@@ -201,14 +201,14 @@ def extract_clustering_info(cluster_output_dir):
         is_curated = bool(np.any(curation_row))
         if creation_time is None and is_curated:
             row_meta = phylog.meta[np.where(curation_row)[0].max()]
-            datetime_str = re.search("\d{2}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}", row_meta)
+            datetime_str = re.search(r"\d{2}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}", row_meta)
             if datetime_str:
                 creation_time = datetime.strptime(
                     datetime_str.group(), "%Y-%m-%d %H:%M:%S"
                 )
             else:
                 creation_time = datetime.fromtimestamp(phylog_filepath.stat().st_ctime)
-                time_str = re.search("\d{2}:\d{2}:\d{2}", row_meta)
+                time_str = re.search(r"\d{2}:\d{2}:\d{2}", row_meta)
                 if time_str:
                     creation_time = datetime.combine(
                         creation_time.date(),

diff --git a/element_array_ephys/readers/kilosort_triggering.py b/element_array_ephys/readers/kilosort_triggering.py
@@ -98,7 +98,7 @@ def __init__(
 
     def parse_input_filename(self):
         meta_filename = next(self._npx_input_dir.glob("*.ap.meta")).name
-        match = re.search("(.*)_g(\d)_t(\d+|cat)\.imec(\d?)\.ap\.meta", meta_filename)
+        match = re.search(r"(.*)_g(\d)_t(\d+|cat)\.imec(\d?)\.ap\.meta", meta_filename)
         session_str, gate_str, trigger_str, probe_str = match.groups()
         return session_str, gate_str, trigger_str, probe_str or "0"
 
@@ -719,7 +719,7 @@ def _get_median_subtraction_duration_from_log(self):
                         ) and previous_line.startswith("Total processing time:"):
                             # regex to search for the processing duration - a float value
                             duration = int(
-                                re.search("\d+\.?\d+", previous_line).group()
+                                re.search(r"\d+\.?\d+", previous_line).group()
                             )
                             return duration
                         previous_line = line

diff --git a/element_array_ephys/readers/probe_geometry.py b/element_array_ephys/readers/probe_geometry.py
@@ -101,6 +101,14 @@
     ]
 )
 
+# additional alias to maintain compatibility with previous naming in the pipeline
+M["neuropixels 1.0 - 3A"] = M["3A"]
+M["neuropixels 1.0 - 3B"] = M["NP1010"]
+M["neuropixels 1.0"] = M["NP1010"]
+M["neuropixels UHD"] = M["NP1100"]
+M["neuropixels 2.0 - SS"] = M["NP2000"]
+M["neuropixels 2.0 - MS"] = M["NP2010"]
+
 
 def build_npx_probe(
     nShank: int,

diff --git a/element_array_ephys/readers/spikeglx.py b/element_array_ephys/readers/spikeglx.py
@@ -262,13 +262,18 @@ def __init__(self, meta_filepath):
         self.fname = meta_filepath
         self.meta = _read_meta(meta_filepath)
 
+        # Get probe part number
+        self.probe_PN = self.meta.get("imDatPrb_pn", "3A")
+
         # Infer npx probe model (e.g. 1.0 (3A, 3B) or 2.0)
-        probe_model = self.meta.get("imDatPrb_type", 1)
-        if probe_model <= 1:
-            if "typeEnabled" in self.meta:
+        probe_model = self.meta.get("imDatPrb_type", 0)
+        if probe_model < 1:
+            if "typeEnabled" in self.meta and self.probe_PN == "3A":
                 self.probe_model = "neuropixels 1.0 - 3A"
-            elif "typeImEnabled" in self.meta:
-                self.probe_model = "neuropixels 1.0 - 3B"
+            elif "typeImEnabled" in self.meta and self.probe_PN == "NP1010":
+                self.probe_model = "neuropixels 1.0"
+            else:
+                self.probe_model = self.probe_PN
         elif probe_model == 1100:
             self.probe_model = "neuropixels UHD"
         elif probe_model == 21:
@@ -293,8 +298,6 @@ def __init__(self, meta_filepath):
                 "Probe Serial Number not found in"
                 ' either "imProbeSN" or "imDatPrb_sn"'
             )
-        # Get probe part number
-        self.probe_PN = self.meta.get("imDatPrb_pn", "3A")
 
         # Parse channel info
         self.chanmap = (

diff --git a/element_array_ephys/version.py b/element_array_ephys/version.py
@@ -1,3 +1,3 @@
 """Package metadata."""
 
-__version__ = "0.3.4"
+__version__ = "0.3.5"
diff --git a/setup.py b/setup.py
@@ -35,7 +35,7 @@
         "plotly",
         "seaborn",
         "spikeinterface",
-        "scikit-image",
+        "scikit-image>=0.20",
         "nbformat>=4.2.0",
         "pyopenephys>=1.1.6",
     ],

diff --git a/tests/tutorial_pipeline.py b/tests/tutorial_pipeline.py
@@ -64,5 +64,7 @@ def get_session_directory(session_key):
 
 ephys.activate(db_prefix + "ephys", db_prefix + "probe", linking_module=__name__)
 
+probe.create_neuropixels_probe_types()
+
 
 __all__ = [""]
Original file line number	Diff line number	Diff line change
Expand Up		@@ -64,5 +64,7 @@ def get_session_directory(session_key):

		ephys.activate(db_prefix + "ephys", db_prefix + "probe", linking_module=__name__)

		probe.create_neuropixels_probe_types()


		__all__ = [""]