Merge branch 'master' into develop

ibllib/__init__.py

@@ -2,7 +2,7 @@
 import logging
 import warnings
 
-__version__ = '2.25.0'
+__version__ = '2.25.2'
 warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib')
 
 # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev

ibllib/ephys/ephysqc.py

@@ -168,7 +168,7 @@ def run(self, update: bool = False, overwrite: bool = True, stream: bool = None,
 
                 if not (np.all(h['x'] == th['x']) and np.all(h['y'] == th['y'])):
                     _logger.critical("Channel geometry seems incorrect")
-                    raise ValueError("Wrong Neuropixel channel mapping used - ABORT")
+                    # raise ValueError("Wrong Neuropixel channel mapping used - ABORT")
 
                 t0s = np.arange(TMIN, sr.rl - SAMPLE_LENGTH, BATCHES_SPACING)
                 all_rms = np.zeros((2, nc, t0s.shape[0]))

ibllib/io/session_params.py

@@ -382,20 +382,25 @@ def get_task_protocol_number(sess_params, task_protocol=None):
         return (next(iter(numbers)) if len(numbers) == 1 else numbers) or None
 
 
-def get_collections(sess_params):
+def get_collections(sess_params, flat=False):
     """
     Find all collections associated with the session.
 
     Parameters
     ----------
     sess_params : dict
         The loaded experiment description map.
+    flat : bool (False)
+        If True, return a flat list of unique collections, otherwise return a map of device/sync/task
 
     Returns
     -------
     dict[str, str]
         A map of device/sync/task and the corresponding collection name.
 
+    list[str]
+        A flat list of unique collection names.
+
     Notes
     -----
     - Assumes only the following data types contained: list, dict, None, str.
@@ -408,12 +413,27 @@ def iter_dict(d):
                 for d in filter(lambda x: isinstance(x, dict), v):
                     iter_dict(d)
             elif isinstance(v, dict) and 'collection' in v:
-                collection_map[k] = v['collection']
+                print(k)
+                # if the key already exists, append the collection name to the list
+                if k in collection_map:
+                    clist = collection_map[k] if isinstance(collection_map[k], list) else [collection_map[k]]
+                    collection_map[k] = list(set(clist + [v['collection']]))
+                else:
+                    collection_map[k] = v['collection']
             elif isinstance(v, dict):
                 iter_dict(v)
 
     iter_dict(sess_params)
-    return collection_map
+    if flat:
+        cflat = []
+        for k, v in collection_map.items():
+            if isinstance(v, list):
+                cflat.extend(v)
+            else:
+                cflat.append(v)
+        return list(set(cflat))
+    else:
+        return collection_map
 
 
 def get_video_compressed(sess_params):

ibllib/tests/test_io.py

@@ -578,6 +578,16 @@ def test_get_collections(self):
         }
         self.assertCountEqual(expected, collections)
 
+    def test_get_collections_repeat_protocols(self):
+        tasks = dict(tasks=[
+            {'passiveChoiceWorld': {'collection': 'raw_passive_data', 'sync_label': 'bpod'}},
+            {'ephysChoiceWorld': {'collection': 'raw_behavior_data', 'sync_label': 'bpod'}},
+            {'passiveChoiceWorld': {'collection': 'raw_passive_data_bis'}}])
+        collections = session_params.get_collections(tasks)
+        self.assertEqual(set(collections['passiveChoiceWorld']), set(['raw_passive_data_bis', 'raw_passive_data']))
+        collections = session_params.get_collections(tasks, flat=True)
+        self.assertEqual(set(collections), set(['raw_passive_data_bis', 'raw_passive_data', 'raw_behavior_data']))
+
 
 class TestRawDaqLoaders(unittest.TestCase):
     """Tests for raw_daq_loaders module"""

release_notes.md

@@ -1,11 +1,16 @@
-## Develop
+## Release Notes 2.25
+
 ### features
 - Training status pipeline now compatible with dynamic pipeline
 - Dynamic DLC task using description file
 - Full photometry lookup table 
 
 ### bugfixes
 - fix for untrainable, unbiasable don't repolulate if already exists
+### 2.25.1
+- relax assertion on Neuropixel channel mappings to allow for personal projects
+### 2.25.2
+- listing of all collections does not skip repeat task protocols anymore for copy/extraction
 
 ## Release Notes 2.23
 ### Release Notes 2.23.1 2023-06-15

requirements.txt

@@ -23,7 +23,7 @@ sparse
 seaborn>=0.9.0
 tqdm>=4.32.1
 # ibl libraries
-ibl-neuropixel>=0.4.0
+ibl-neuropixel>=0.8.1
 iblutil>=1.7.0
 labcams  # widefield extractor
 ONE-api>=2.2