From 509cb8a6529bafbd31e82faba73efe0ae4cf6a14 Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Fri, 24 May 2024 13:36:11 +0300 Subject: [PATCH 1/2] Use correct task for timeline acquisitions in make_pipeline --- ibllib/__init__.py | 2 +- ibllib/pipes/dynamic_pipeline.py | 29 ++++++++++++++++++--------- ibllib/tests/test_dynamic_pipeline.py | 19 ++++++++++++++++++ release_notes.md | 3 +++ 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/ibllib/__init__.py b/ibllib/__init__.py index b823c2288..a8547479f 100644 --- a/ibllib/__init__.py +++ b/ibllib/__init__.py @@ -2,7 +2,7 @@ import logging import warnings -__version__ = '2.35.2' +__version__ = '2.35.3' warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib') # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev diff --git a/ibllib/pipes/dynamic_pipeline.py b/ibllib/pipes/dynamic_pipeline.py index 7024d4ce3..1e9680c7e 100644 --- a/ibllib/pipes/dynamic_pipeline.py +++ b/ibllib/pipes/dynamic_pipeline.py @@ -268,7 +268,12 @@ def make_pipeline(session_path, **pkwargs): compute_status = False else: registration_class = btasks.TrialRegisterRaw - behaviour_class = btasks.ChoiceWorldTrialsNidq + if sync_args['sync_namespace'] == 'timeline': + behaviour_class = btasks.ChoiceWorldTrialsTimeline + elif sync_args['sync_namespace'] in ('spikeglx', None): + behaviour_class = btasks.ChoiceWorldTrialsNidq + else: + raise NotImplementedError(f'No trials task available for sync namespace "{sync_args["sync_namespace"]}"') compute_status = True else: raise NotImplementedError @@ -278,7 +283,7 @@ def make_pipeline(session_path, **pkwargs): tasks[f'Trials_{protocol}_{i:02}'] = type(f'Trials_{protocol}_{i:02}', (behaviour_class,), {})( **kwargs, **sync_kwargs, **task_kwargs, parents=parents) if compute_status: - tasks[f"TrainingStatus_{protocol}_{i:02}"] = type(f'TrainingStatus_{protocol}_{i:02}', ( + tasks[f'TrainingStatus_{protocol}_{i:02}'] = type(f'TrainingStatus_{protocol}_{i:02}', ( btasks.TrainingStatus,), {})(**kwargs, **task_kwargs, parents=[tasks[f'Trials_{protocol}_{i:02}']]) # Ephys tasks @@ -289,7 +294,10 @@ def make_pipeline(session_path, **pkwargs): all_probes = [] register_tasks = [] for pname, probe_info in devices['neuropixel'].items(): - meta_file = spikeglx.glob_ephys_files(Path(session_path).joinpath(probe_info['collection']), ext='meta') + # Glob to support collections such as _00a, _00b. This doesn't fix the issue of NP2.4 + # extractions, however. + probe_collection = next(session_path.glob(probe_info['collection'] + '*')) + meta_file = spikeglx.glob_ephys_files(probe_collection, ext='meta') meta_file = meta_file[0].get('ap') nptype = spikeglx._get_neuropixel_version_from_meta(spikeglx.read_meta_data(meta_file)) nshanks = spikeglx._get_nshanks_from_meta(spikeglx.read_meta_data(meta_file)) @@ -482,12 +490,15 @@ def get_trials_tasks(session_path, one=None): # If experiment description file then use this to make the pipeline if experiment_description is not None: tasks = [] - pipeline = make_pipeline(session_path, one=one) - trials_tasks = [t for t in pipeline.tasks if 'Trials' in t] - for task in trials_tasks: - t = pipeline.tasks.get(task) - t.__init__(session_path, **t.kwargs) - tasks.append(t) + try: + pipeline = make_pipeline(session_path, one=one) + trials_tasks = [t for t in pipeline.tasks if 'Trials' in t] + for task in trials_tasks: + t = pipeline.tasks.get(task) + t.__init__(session_path, **t.kwargs) + tasks.append(t) + except NotImplementedError as ex: + _logger.warning('Failed to get trials tasks: %s', ex) else: # Otherwise default to old way of doing things if one and one.to_eid(session_path): diff --git a/ibllib/tests/test_dynamic_pipeline.py b/ibllib/tests/test_dynamic_pipeline.py index 41420c674..94cbc2a58 100644 --- a/ibllib/tests/test_dynamic_pipeline.py +++ b/ibllib/tests/test_dynamic_pipeline.py @@ -9,6 +9,7 @@ import ibllib.tests import ibllib.pipes.dynamic_pipeline as dyn from ibllib.pipes.tasks import Pipeline, Task +import ibllib.pipes.behavior_tasks as btasks from ibllib.pipes import ephys_preprocessing from ibllib.pipes import training_preprocessing from ibllib.io import session_params @@ -65,6 +66,7 @@ def setUp(self): {'ephysChoiceWorld': {'task_collection': 'raw_task_data_00'}}, {'passiveChoiceWorld': {'task_collection': 'raw_task_data_01'}}, ]} + self.description = description with open(self.session_path_dynamic / '_ibl_experiment.description.yaml', 'w') as fp: yaml.safe_dump(description, fp) @@ -87,8 +89,25 @@ def test_get_trials_tasks(self): one.alyx.cache_mode = None # sneaky hack as this is checked by the pipeline somewhere tasks = dyn.get_trials_tasks(self.session_path_dynamic, one) self.assertEqual(2, len(tasks)) + self.assertIsInstance(tasks[0], btasks.ChoiceWorldTrialsNidq) one.load_datasets.assert_called() # check that description file is checked on disk + # A session with timeline acquisition + self.description['sync']['nidq']['acquisition_software'] = 'timeline' + with open(self.session_path_dynamic / '_ibl_experiment.description.yaml', 'w') as fp: + yaml.safe_dump(self.description, fp) + tasks = dyn.get_trials_tasks(self.session_path_dynamic, one) + self.assertIsInstance(tasks[0], btasks.ChoiceWorldTrialsTimeline) + + # A session with an unknown sync namespace + self.description['sync']['nidq']['acquisition_software'] = 'notepad' + with open(self.session_path_dynamic / '_ibl_experiment.description.yaml', 'w') as fp: + yaml.safe_dump(self.description, fp) + with self.assertLogs(dyn.__name__, 'WARNING') as cm: + self.assertEqual([], dyn.get_trials_tasks(self.session_path_dynamic)) + log_message = cm.records[0].getMessage() + self.assertIn('sync namespace "notepad"', log_message) + # An ephys session tasks = dyn.get_trials_tasks(self.session_path_legacy) self.assertEqual(1, len(tasks)) diff --git a/release_notes.md b/release_notes.md index db09f5e07..69e41573b 100644 --- a/release_notes.md +++ b/release_notes.md @@ -15,6 +15,9 @@ - Support extraction of repNum for advancedChoiceWorld - Support matplotlib v3.9; min slidingRP version now 1.1.1 +#### 2.35.3 +- Use correct task for timeline acquisitions in make_pipeline + ## Release Note 2.34.0 ### features From ced2259847fb2663f460c762649e4317b67abb2a Mon Sep 17 00:00:00 2001 From: olivier Date: Wed, 12 Jun 2024 16:22:58 +0100 Subject: [PATCH 2/2] release 2.36.0 --- ibllib/__init__.py | 2 +- release_notes.md | 6 ++++++ requirements.txt | 6 +++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ibllib/__init__.py b/ibllib/__init__.py index a8547479f..8f7a2aca5 100644 --- a/ibllib/__init__.py +++ b/ibllib/__init__.py @@ -2,7 +2,7 @@ import logging import warnings -__version__ = '2.35.3' +__version__ = '2.36.0' warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib') # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev diff --git a/release_notes.md b/release_notes.md index 69e41573b..07e473596 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,3 +1,9 @@ +## Release Note 2.36.0 + +### features +- Adding spike sorting iblsort task + + ## Release Note 2.35.0 ### features diff --git a/requirements.txt b/requirements.txt index 27ad516d9..e8be00875 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,12 +21,12 @@ sparse seaborn>=0.9.0 tqdm>=4.32.1 # ibl libraries -iblatlas>=0.4.0 +iblatlas>=0.5.3 ibl-neuropixel>=1.0.1 -iblutil>=1.7.0 +iblutil>=1.9.0 mtscomp>=1.0.1 ONE-api~=2.7rc1 -phylib>=2.4 +phylib>=2.6.0 psychofit slidingRP>=1.1.1 # steinmetz lab refractory period metrics pyqt5