Merge pull request #1125 from samuelgarcia/refactor_spikeglx

improve spikeglx file naming

neo/rawio/spikeglxrawio.py

@@ -42,6 +42,7 @@
 imDatPrb_type=24 (NP 2.0, 4-shank)
 
 Author : Samuel Garcia
+Some functions are copied from Graham Findlay
 """
 
 import warnings
@@ -90,6 +91,7 @@ def _parse_header(self):
         for info in self.signals_info_list:
             # key is (seg_index, stream_name)
             key = (info['seg_index'], info['stream_name'])
+            assert key not in self.signals_info_dict
             self.signals_info_dict[key] = info
 
             # create memmap
@@ -166,7 +168,7 @@ def _parse_header(self):
                     # need probeinterface to be installed
                     import probeinterface
                     info = self.signals_info_dict[seg_index, stream_name]
-                    if 'imroTbl' in info['meta'] and info['signal_kind'] == 'ap':
+                    if 'imroTbl' in info['meta'] and info['stream_kind'] == 'ap':
                         # only for ap channel
                         probe = probeinterface.read_spikeglx(info['meta_file'])
                         loc = probe.contact_positions
@@ -233,6 +235,11 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop,
 def scan_files(dirname):
     """
     Scan for pairs of `.bin` and `.meta` files and return information about it.
+
+    After exploring the folder, the segment index (`seg_index`) is construct as follow:
+      * if only one `gate_num=0` then `trigger_num` = `seg_index`
+      * if only one `trigger_num=0` then `gate_num` = `seg_index`
+      * if both are increasing then seg_index increased by gate_num, trigger_num order.
     """
     info_list = []
 
@@ -245,16 +252,99 @@ def scan_files(dirname):
             if meta_filename.exists() and bin_filename.exists():
                 meta = read_meta_file(meta_filename)
                 info = extract_stream_info(meta_filename, meta)
+
                 info['meta_file'] = str(meta_filename)
                 info['bin_file'] = str(bin_filename)
                 info_list.append(info)
 
+            # Let see if this will be anoying or not.
             if bin_filename.stat().st_size != meta['fileSizeBytes']:
                 warnings.warn('.meta file has faulty value for .bin file size on disc')
 
+    # the segment index will depend on both 'gate_num' and 'trigger_num'
+    # so we order by 'gate_num' then 'trigger_num'
+    # None is before any int
+    def make_key(info):
+        k0 = info['gate_num']
+        if k0 is None:
+            k0 = -1
+        k1 = info['trigger_num']
+        if k1 is None:
+            k1 = -1
+        return (k0, k1)
+    order_key = list({make_key(info) for info in info_list})
+    order_key = sorted(order_key)
+    for info in info_list:
+        info['seg_index'] = order_key.index(make_key(info))
+
     return info_list
 
 
+def parse_spikeglx_fname(fname):
+    """
+    Parse recording identifiers from a SpikeGLX style filename.
+
+    spikeglx naming follow this rules:
+    https://github.com/billkarsh/SpikeGLX/blob/master/Markdown/UserManual.md#gates-and-triggers
+
+    Example file name structure:
+    Consider the filenames: `Noise4Sam_g0_t0.nidq.bin` or `Noise4Sam_g0_t0.imec0.lf.bin`
+    The filenames consist of 3 or 4 parts separated by `.`
+    1. "Noise4Sam_g0_t0" will be the `name` variable. This choosen by the user at recording time.
+    2. "_g0_" is the "gate_num"
+    3. "_t0_" is the "trigger_num"
+    4. "nidq" or "imec0" will give the `device`
+    5. "lf" or "ap" will be the `stream_kind`
+        `stream_name` variable is the concatenation of `device.stream_kind`
+
+    This function is copied/modified from Graham Findlay.
+
+    Notes:
+       * Sometimes the original file name is modified by the user and "_gt0_" or "_t0_"
+          are manually removed. In that case gate_name and trigger_num will be None.
+
+    Parameters
+    ---------
+    fname: str
+        The filename to parse without the extension, e.g. "my-run-name_g0_t1.imec2.lf"
+    Returns
+    -------
+    run_name: str
+        The run name, e.g. "my-run-name".
+    gate_num: int or None
+        The gate identifier, e.g. 0.
+    trigger_num: int or None
+        The trigger identifier, e.g. 1.
+    device: str
+        The probe identifier, e.g. "imec2"
+    stream_kind: str or None
+        The data type identifier, "lf" or "ap" or None
+    """
+    r = re.findall(r'(\S*)_g(\d*)_t(\d*)\.(\S*).(ap|lf)', fname)
+    if len(r) == 1:
+        # standard case with probe
+        run_name, gate_num, trigger_num, device, stream_kind = r[0]
+    else:
+        r = re.findall(r'(\S*)_g(\d*)_t(\d*)\.(\S*)', fname)
+        if len(r) == 1:
+            # case for nidaq
+            run_name, gate_num, trigger_num, device = r[0]
+            stream_kind = None
+        else:
+            # the naming do not correspond lets try something more easy
+            r = re.findall(r'(\S*)\.(\S*).(ap|lf)', fname)
+            if len(r) == 1:
+                run_name, device, stream_kind = r[0]
+                gate_num, trigger_num = None, None
+
+    if gate_num is not None:
+        gate_num = int(gate_num)
+    if trigger_num is not None:
+        trigger_num = int(trigger_num)
+
+    return (run_name, gate_num, trigger_num, device, stream_kind)
+
+
 def read_meta_file(meta_file):
     """parse the meta file"""
     with open(meta_file, mode='r') as f:
@@ -281,27 +371,13 @@ def extract_stream_info(meta_file, meta):
     """Extract info from the meta dict"""
 
     num_chan = int(meta['nSavedChans'])
+    fname = Path(meta_file).stem
+    run_name, gate_num, trigger_num, device, stream_kind = parse_spikeglx_fname(fname)
+    device = fname.split('.')[1]
 
-    # Example file name structure:
-    # Consider the filenames: `Noise4Sam_g0_t0.nidq.bin` or `Noise4Sam_g0_t0.imec0.lf.bin`
-    # The filenames consist of 3 or 4 parts separated by `.`
-    # 1. "Noise4Sam_g0_t0" will be the `name` variable. This is chosen by the user
-    #    at recording time.
-    # 2. "_gt0_" will give the `seg_index` (here 0)
-    # 3. "nidq" or "imec0" will give the `device` variable
-    # 4. "lf" or "ap" will be the `signal_kind` variable
-    # `stream_name` variable is the concatenation of `device.signal_kind`
-    name = Path(meta_file).stem
-    r = re.findall(r'_g(\d*)_t', name)
-    if len(r) == 0:
-        # when manual renaming _g0_ can be removed
-        seg_index = 0
-    else:
-        seg_index = int(r[0][0])
-    device = name.split('.')[1]
     if 'imec' in device:
-        signal_kind = name.split('.')[2]
-        stream_name = device + '.' + signal_kind
+        stream_kind = fname.split('.')[2]
+        stream_name = device + '.' + stream_kind
         units = 'uV'
         # please note the 1e6 in gain for this uV
 
@@ -313,16 +389,16 @@ def extract_stream_info(meta_file, meta):
             # https://github.com/billkarsh/SpikeGLX/blob/gh-pages/Support/Metadata_3A.md#imec
             # https://github.com/billkarsh/SpikeGLX/blob/gh-pages/Support/Metadata_3B1.md#imec
             # https://github.com/billkarsh/SpikeGLX/blob/gh-pages/Support/Metadata_3B2.md#imec
-            if signal_kind == 'ap':
+            if stream_kind == 'ap':
                 index_imroTbl = 3
-            elif signal_kind == 'lf':
+            elif stream_kind == 'lf':
                 index_imroTbl = 4
             for c in range(num_chan - 1):
                 v = meta['imroTbl'][c].split(' ')[index_imroTbl]
                 per_channel_gain[c] = 1. / float(v)
             gain_factor = float(meta['imAiRangeMax']) / 512
             channel_gains = gain_factor * per_channel_gain * 1e6
-        elif meta['imDatPrb_type'] in ('21', '24') and signal_kind == 'ap':
+        elif meta['imDatPrb_type'] in ('21', '24') and stream_kind == 'ap':
             # This work with NP 2.0 case with different metadata versions
             # https://github.com/billkarsh/SpikeGLX/blob/gh-pages/Support/Metadata_20.md#channel-entries-by-type
             # https://github.com/billkarsh/SpikeGLX/blob/gh-pages/Support/Metadata_20.md#imec
@@ -334,7 +410,7 @@ def extract_stream_info(meta_file, meta):
             raise NotImplementedError('This meta file version of spikeglx'
                                       'is not implemented')
     else:
-        signal_kind = ''
+        stream_kind = ''
         stream_name = device
         units = 'V'
         channel_gains = np.ones(num_chan)
@@ -352,17 +428,18 @@ def extract_stream_info(meta_file, meta):
         channel_gains = per_channel_gain * gain_factor
 
     info = {}
-    info['name'] = name
+    info['fname'] = fname
     info['meta'] = meta
     for k in ('niSampRate', 'imSampRate'):
         if k in meta:
             info['sampling_rate'] = float(meta[k])
     info['num_chan'] = num_chan
 
     info['sample_length'] = int(meta['fileSizeBytes']) // 2 // num_chan
-    info['seg_index'] = seg_index
+    info['gate_num'] = gate_num
+    info['trigger_num'] = trigger_num
     info['device'] = device
-    info['signal_kind'] = signal_kind
+    info['stream_kind'] = stream_kind
     info['stream_name'] = stream_name
     info['units'] = units
     info['channel_names'] = [txt.split(';')[0] for txt in meta['snsChanMap']]

neo/test/iotest/test_spikeglxio.py

@@ -6,18 +6,13 @@
 
 from neo.io import SpikeGLXIO
 from neo.test.iotest.common_io_test import BaseTestIO
+from neo.test.rawiotest.test_spikeglxrawio import TestSpikeGLXRawIO
 
 
 class TestSpikeGLXIO(BaseTestIO, unittest.TestCase):
     ioclass = SpikeGLXIO
-    entities_to_download = [
-        'spikeglx'
-    ]
-    entities_to_test = [
-        'spikeglx/Noise4Sam_g0',
-        'spikeglx/TEST_20210920_0_g0'
-    ]
-
+    entities_to_download = TestSpikeGLXRawIO.entities_to_download
+    entities_to_test = TestSpikeGLXRawIO.entities_to_test
 
 
 if __name__ == "__main__":

neo/test/rawiotest/test_spikeglxrawio.py

@@ -15,7 +15,21 @@ class TestSpikeGLXRawIO(BaseTestRawIO, unittest.TestCase):
     ]
     entities_to_test = [
         'spikeglx/Noise4Sam_g0',
-        'spikeglx/TEST_20210920_0_g0'
+        'spikeglx/TEST_20210920_0_g0',
+
+        # this is only g0 multi index
+        'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0/5-19-2022-CI0_g0'
+        # this is only g1 multi index
+        'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0/5-19-2022-CI0_g1'
+        # this mix both multi gate and multi trigger (and also multi probe)
+        'spikeglx/sample_data_v2/SpikeGLX/5-19-2022-CI0',
+
+        'spikeglx/sample_data_v2/SpikeGLX/5-19-2022-CI1',
+        'spikeglx/sample_data_v2/SpikeGLX/5-19-2022-CI2',
+        'spikeglx/sample_data_v2/SpikeGLX/5-19-2022-CI3',
+        'spikeglx/sample_data_v2/SpikeGLX/5-19-2022-CI4',
+        'spikeglx/sample_data_v2/SpikeGLX/5-19-2022-CI5',
+
     ]
 
     def test_with_location(self):