From aac673360027f05282efd36c259a4cd53f6e2a15 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 5 Dec 2024 08:57:20 -0600 Subject: [PATCH 1/6] WIP --- neo/rawio/spikeglxrawio.py | 44 ++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py index 2e8f896d8..d6b609b04 100644 --- a/neo/rawio/spikeglxrawio.py +++ b/neo/rawio/spikeglxrawio.py @@ -84,12 +84,9 @@ class SpikeGLXRawIO(BaseRawWithBufferApiIO): ----- * Contrary to other implementations this IO reads the entire folder and subfolders and: deals with several segments based on the `_gt0`, `_gt1`, `_gt2`, etc postfixes - deals with all signals "imec0", "imec1" for neuropixel probes and also - external signal like"nidq". This is the "device" - * For imec device both "ap" and "lf" are extracted so one device have several "streams" - * There are several versions depending the neuropixel probe generation (`1.x`/`2.x`/`3.x`) - * Here, we assume that the `meta` file has the same structure across all generations. - * This IO is developed based on neuropixel generation 2.0, single shank recordings. + deals with all signals coming from different acquisition cards ("imec0", "imec1", etc) in a typical + PXIe chassis and also external signal like"nidq". This is the "device" + * For imec device both "ap" and "lf" are extracted so even a one device setup has several "streams" Examples -------- @@ -125,7 +122,6 @@ def _parse_header(self): stream_names = sorted(list(srates.keys()), key=lambda e: srates[e])[::-1] nb_segment = np.unique([info["seg_index"] for info in self.signals_info_list]).size - # self._memmaps = {} self.signals_info_dict = {} # one unique block self._buffer_descriptions = {0: {}} @@ -166,7 +162,6 @@ def _parse_header(self): stream_id = stream_name - stream_index = stream_names.index(info["stream_name"]) signal_streams.append((stream_name, stream_id, buffer_id)) # add channels to global list @@ -250,7 +245,6 @@ def _parse_header(self): # insert some annotation at some place self._generate_minimal_annotations() self._generate_minimal_annotations() - block_ann = self.raw_annotations["blocks"][0] for seg_index in range(nb_segment): seg_ann = self.raw_annotations["blocks"][0]["segments"][seg_index] @@ -354,17 +348,17 @@ def scan_files(dirname): if len(info_list) == 0: raise FileNotFoundError(f"No appropriate combination of .meta and .bin files were detected in {dirname}") - # the segment index will depend on both 'gate_num' and 'trigger_num' + # the segment index will depend on both 'gate_num' and 'trigger_num' and on "dock_num" # so we order by 'gate_num' then 'trigger_num' # None is before any int def make_key(info): - k0 = info["gate_num"] - if k0 is None: - k0 = -1 - k1 = info["trigger_num"] - if k1 is None: - k1 = -1 - return (k0, k1) + gate_num = info["gate_num"] + if gate_num is None: + gate_num = -1 + trigger_num = info["trigger_num"] + if trigger_num is None: + trigger_num = -1 + return (gate_num, trigger_num) order_key = list({make_key(info) for info in info_list}) order_key = sorted(order_key) @@ -488,9 +482,21 @@ def extract_stream_info(meta_file, meta): else: # NIDQ case has_sync_trace = False - fname = Path(meta_file).stem + + bin_file_path = meta["fileName"] + fname = Path(bin_file_path).stem + + + # First we check if the gate, trigger and dock numbers are present in the meta + + probe_slot = meta.get("imDatPrb_slot", None) + probe_port = meta.get("imDatPrb_port", None) + probe_dock = meta.get("imDatPrb_dock", None) + + run_name, gate_num, trigger_num, device, stream_kind = parse_spikeglx_fname(fname) - + # x = 1 + if "imec" in fname.split(".")[-2]: device = fname.split(".")[-2] stream_kind = fname.split(".")[-1] From 42e128b222a81dabf7183664aac59c12a04da364 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 6 Dec 2024 17:21:07 -0600 Subject: [PATCH 2/6] WIP2 --- neo/rawio/spikeglxrawio.py | 73 +++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py index d6b609b04..4c8bc2957 100644 --- a/neo/rawio/spikeglxrawio.py +++ b/neo/rawio/spikeglxrawio.py @@ -82,11 +82,11 @@ class SpikeGLXRawIO(BaseRawWithBufferApiIO): Notes ----- - * Contrary to other implementations this IO reads the entire folder and subfolders and: - deals with several segments based on the `_gt0`, `_gt1`, `_gt2`, etc postfixes - deals with all signals coming from different acquisition cards ("imec0", "imec1", etc) in a typical - PXIe chassis and also external signal like"nidq". This is the "device" - * For imec device both "ap" and "lf" are extracted so even a one device setup has several "streams" + * This IO reads the entire folder and subfolders locating the `.bin` and `.meta` files + * Handles gates and triggers as segments (based on the `_gt0`, `_gt1`, `_t0` , `_t1` in filenames) + * Handles all signals coming from different acquisition cards ("imec0", "imec1", etc) in a typical + PXIe chassis setup and also external signal like"nidq". + * For imec devices both "ap" and "lf" are extracted so even a one device setup will have several "streams" Examples -------- @@ -348,22 +348,31 @@ def scan_files(dirname): if len(info_list) == 0: raise FileNotFoundError(f"No appropriate combination of .meta and .bin files were detected in {dirname}") - # the segment index will depend on both 'gate_num' and 'trigger_num' and on "dock_num" - # so we order by 'gate_num' then 'trigger_num' - # None is before any int - def make_key(info): - gate_num = info["gate_num"] - if gate_num is None: - gate_num = -1 - trigger_num = info["trigger_num"] - if trigger_num is None: - trigger_num = -1 - return (gate_num, trigger_num) - - order_key = list({make_key(info) for info in info_list}) - order_key = sorted(order_key) - for info in info_list: - info["seg_index"] = order_key.index(make_key(info)) + # This sets non-integers values before integers + normalize = lambda x: x if isinstance(x, int) else -1 + + # Segment index is determined by the gate_num and trigger_num in that order + gate_trigger_tuples = [ + (info_index, (normalize(info["gate_num"]), normalize(info["trigger_num"]))) + for info_index, info in enumerate(info_list) + ] + + sorted_info = sorted(gate_trigger_tuples, key=lambda x: x[1]) + + for seg_index, (info_index, _) in enumerate(sorted_info): + info_list[info_index]["seg_index"] = seg_index + + # Add probe_index + # The logic is that the probe_index is the order of the probe_slot, probe_port, and probe_dock + slot_port_dock_tuples = [ + (info_index, (normalize(info["probe_slot"]), normalize(info["probe_port"]), normalize(info["probe_dock"]))) + for info_index, info in enumerate(info_list) + ] + + # Sorts by the probe_slot, probe_port, and probe_dock tuples + sorted_info = sorted(slot_port_dock_tuples, key=lambda x: x[1]) + for probe_index, (info_index, _) in enumerate(sorted_info): + info_list[info_index]["probe_index"] = probe_index return info_list @@ -482,21 +491,12 @@ def extract_stream_info(meta_file, meta): else: # NIDQ case has_sync_trace = False - + bin_file_path = meta["fileName"] fname = Path(bin_file_path).stem - - - # First we check if the gate, trigger and dock numbers are present in the meta - - probe_slot = meta.get("imDatPrb_slot", None) - probe_port = meta.get("imDatPrb_port", None) - probe_dock = meta.get("imDatPrb_dock", None) - - + run_name, gate_num, trigger_num, device, stream_kind = parse_spikeglx_fname(fname) - # x = 1 - + if "imec" in fname.split(".")[-2]: device = fname.split(".")[-2] stream_kind = fname.split(".")[-1] @@ -556,6 +556,10 @@ def extract_stream_info(meta_file, meta): gain_factor = float(meta["niAiRangeMax"]) / 32768 channel_gains = per_channel_gain * gain_factor + probe_slot = meta.get("imDatPrb_slot", None) + probe_port = meta.get("imDatPrb_port", None) + probe_dock = meta.get("imDatPrb_dock", None) + info = {} info["fname"] = fname info["meta"] = meta @@ -575,6 +579,9 @@ def extract_stream_info(meta_file, meta): info["channel_gains"] = channel_gains info["channel_offsets"] = np.zeros(info["num_chan"]) info["has_sync_trace"] = has_sync_trace + info["probe_slot"] = int(probe_slot) if probe_slot else None + info["probe_port"] = int(probe_port) if probe_port else None + info["probe_dock"] = int(probe_dock) if probe_dock else None if "nidq" in device: info["digital_channels"] = [] From 6e1a5b561a365fe4c9ffc5ef364cd2309d15d06c Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 9 Dec 2024 09:27:54 -0600 Subject: [PATCH 3/6] fix keys --- neo/rawio/spikeglxrawio.py | 51 ++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py index 4c8bc2957..f026c5073 100644 --- a/neo/rawio/spikeglxrawio.py +++ b/neo/rawio/spikeglxrawio.py @@ -352,27 +352,40 @@ def scan_files(dirname): normalize = lambda x: x if isinstance(x, int) else -1 # Segment index is determined by the gate_num and trigger_num in that order - gate_trigger_tuples = [ - (info_index, (normalize(info["gate_num"]), normalize(info["trigger_num"]))) - for info_index, info in enumerate(info_list) - ] + def get_segment_tuple(info): + # Create a key from the normalized gate_num and trigger_num + gate_num = normalize(info.get("gate_num")) + trigger_num = normalize(info.get("trigger_num")) + return (gate_num, trigger_num) - sorted_info = sorted(gate_trigger_tuples, key=lambda x: x[1]) - - for seg_index, (info_index, _) in enumerate(sorted_info): - info_list[info_index]["seg_index"] = seg_index - - # Add probe_index - # The logic is that the probe_index is the order of the probe_slot, probe_port, and probe_dock - slot_port_dock_tuples = [ - (info_index, (normalize(info["probe_slot"]), normalize(info["probe_port"]), normalize(info["probe_dock"]))) - for info_index, info in enumerate(info_list) - ] + unique_segment_tuples = {get_segment_tuple(info) for info in info_list} + sorted_keys = sorted(unique_segment_tuples) + + # Map each unique key to a corresponding index + segment_tuple_to_segment_index = {key: idx for idx, key in enumerate(sorted_keys)} + + for info in info_list: + info["seg_index"] = segment_tuple_to_segment_index[get_segment_tuple(info)] - # Sorts by the probe_slot, probe_port, and probe_dock tuples - sorted_info = sorted(slot_port_dock_tuples, key=lambda x: x[1]) - for probe_index, (info_index, _) in enumerate(sorted_info): - info_list[info_index]["probe_index"] = probe_index + + # Probe index calculation + # This ensures that all nidq entries come before any other keys, which corresponds to index 0. + def get_probe_tuple(info): + slot = normalize(info.get("probe_slot")) + port = normalize(info.get("probe_port")) + dock = normalize(info.get("probe_dock")) + return (slot, port, dock) + + unique_probe_tuples = {get_probe_tuple(info) for info in info_list} + sorted_probe_keys = sorted(unique_probe_tuples) + probe_tuple_to_probe_index = {key: idx for idx, key in enumerate(sorted_probe_keys)} + + for info in info_list: + if info.get("device") == "nidq": + info["device_index"] = 0 # TODO: Handle multi nidq case + else: + info["device_index"] = probe_tuple_to_probe_index[get_probe_tuple(info)] + return info_list From 986f2628b727cc5781a8135ba6ddfa8cbc475b82 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 9 Dec 2024 18:38:02 -0600 Subject: [PATCH 4/6] add test and define new metadata --- neo/rawio/spikeglxrawio.py | 17 ++++++++++++----- neo/test/rawiotest/test_spikeglxrawio.py | 2 ++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py index f026c5073..7f66d1018 100644 --- a/neo/rawio/spikeglxrawio.py +++ b/neo/rawio/spikeglxrawio.py @@ -376,17 +376,26 @@ def get_probe_tuple(info): dock = normalize(info.get("probe_dock")) return (slot, port, dock) - unique_probe_tuples = {get_probe_tuple(info) for info in info_list} + info_list_imec = [info for info in info_list if info.get("device") != "nidq"] + unique_probe_tuples = {get_probe_tuple(info) for info in info_list_imec} sorted_probe_keys = sorted(unique_probe_tuples) probe_tuple_to_probe_index = {key: idx for idx, key in enumerate(sorted_probe_keys)} for info in info_list: if info.get("device") == "nidq": - info["device_index"] = 0 # TODO: Handle multi nidq case + info["device_index"] = "" # TODO: Handle multi nidq case, maybe use meta["typeNiEnabled"] else: info["device_index"] = probe_tuple_to_probe_index[get_probe_tuple(info)] + # Define stream base on device [imec|nidq], device_index and stream_kind [ap|lf] for imec + for info in info_list: + device_kind = info["device_kind"] + device_index = info["device_index"] + stream_kind = f".{info['stream_kind']}" if info["stream_kind"] else "" + stream_name = f"{device_kind}{device_index}{stream_kind}" + info["stream_name"] = stream_name + return info_list @@ -513,7 +522,6 @@ def extract_stream_info(meta_file, meta): if "imec" in fname.split(".")[-2]: device = fname.split(".")[-2] stream_kind = fname.split(".")[-1] - stream_name = device + "." + stream_kind units = "uV" # please note the 1e6 in gain for this uV @@ -553,7 +561,6 @@ def extract_stream_info(meta_file, meta): else: device = fname.split(".")[-1] stream_kind = "" - stream_name = device units = "V" channel_gains = np.ones(num_chan) @@ -586,7 +593,7 @@ def extract_stream_info(meta_file, meta): info["trigger_num"] = trigger_num info["device"] = device info["stream_kind"] = stream_kind - info["stream_name"] = stream_name + info["device_kind"] = meta.get("typeThis", device.split(".")[0]) info["units"] = units info["channel_names"] = [txt.split(";")[0] for txt in meta["snsChanMap"]] info["channel_gains"] = channel_gains diff --git a/neo/test/rawiotest/test_spikeglxrawio.py b/neo/test/rawiotest/test_spikeglxrawio.py index ba65cf83d..ed545d516 100644 --- a/neo/test/rawiotest/test_spikeglxrawio.py +++ b/neo/test/rawiotest/test_spikeglxrawio.py @@ -32,6 +32,8 @@ class TestSpikeGLXRawIO(BaseTestRawIO, unittest.TestCase): "spikeglx/NP2_subset_with_sync", # NP-ultra "spikeglx/np_ultra_stub", + # Filename changed by the user, multi-dock + "spikeglx/multi_probe_multi_dock_multi_shank_filename_without_info", # CatGT "spikeglx/multi_trigger_multi_gate/CatGT/CatGT-A", "spikeglx/multi_trigger_multi_gate/CatGT/CatGT-B", From 069e2dd62790b9d85be90ed981a89bd11eaebc74 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 9 Dec 2024 18:44:58 -0600 Subject: [PATCH 5/6] Update neo/rawio/spikeglxrawio.py Co-authored-by: Alessio Buccino --- neo/rawio/spikeglxrawio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py index 7f66d1018..45b1b426f 100644 --- a/neo/rawio/spikeglxrawio.py +++ b/neo/rawio/spikeglxrawio.py @@ -85,7 +85,7 @@ class SpikeGLXRawIO(BaseRawWithBufferApiIO): * This IO reads the entire folder and subfolders locating the `.bin` and `.meta` files * Handles gates and triggers as segments (based on the `_gt0`, `_gt1`, `_t0` , `_t1` in filenames) * Handles all signals coming from different acquisition cards ("imec0", "imec1", etc) in a typical - PXIe chassis setup and also external signal like"nidq". + PXIe chassis setup and also external signal like "nidq". * For imec devices both "ap" and "lf" are extracted so even a one device setup will have several "streams" Examples From 822f7fa6d1c20d569e36abb06da1b45611d61d00 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 13 Dec 2024 10:36:14 -0600 Subject: [PATCH 6/6] add comments --- neo/rawio/spikeglxrawio.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py index 45b1b426f..6fb1a6071 100644 --- a/neo/rawio/spikeglxrawio.py +++ b/neo/rawio/spikeglxrawio.py @@ -369,13 +369,15 @@ def get_segment_tuple(info): # Probe index calculation - # This ensures that all nidq entries come before any other keys, which corresponds to index 0. + # The calculation is ordered by slot, port, dock in that order, this is the number that appears in the filename + # after imec when using native names (e.g. imec0, imec1, etc.) def get_probe_tuple(info): slot = normalize(info.get("probe_slot")) port = normalize(info.get("probe_port")) dock = normalize(info.get("probe_dock")) return (slot, port, dock) + # TODO: handle one box case info_list_imec = [info for info in info_list if info.get("device") != "nidq"] unique_probe_tuples = {get_probe_tuple(info) for info in info_list_imec} sorted_probe_keys = sorted(unique_probe_tuples) @@ -593,6 +595,7 @@ def extract_stream_info(meta_file, meta): info["trigger_num"] = trigger_num info["device"] = device info["stream_kind"] = stream_kind + # All non-production probes (phase 3B onwards) have "typeThis", otherwise revert to file parsing info["device_kind"] = meta.get("typeThis", device.split(".")[0]) info["units"] = units info["channel_names"] = [txt.split(";")[0] for txt in meta["snsChanMap"]]