From 704a1f339d324f8df3f061e4a6079c66241714c5 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Wed, 13 Dec 2023 10:45:08 +0100 Subject: [PATCH 01/20] change behavior qc to truncate trials up until 400 and check if performance passes --- ibllib/pipes/behavior_tasks.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/ibllib/pipes/behavior_tasks.py b/ibllib/pipes/behavior_tasks.py index 5e7e5d829..1daf04813 100644 --- a/ibllib/pipes/behavior_tasks.py +++ b/ibllib/pipes/behavior_tasks.py @@ -374,17 +374,26 @@ def signature(self): } return signature - def _behaviour_criterion(self, update=True): + def _behaviour_criterion(self, update=True, truncate_to_pass=True): """ Computes and update the behaviour criterion on Alyx """ from brainbox.behavior import training - trials = alfio.load_object(self.session_path.joinpath(self.output_collection), 'trials') + trials = alfio.load_object(self.session_path.joinpath(self.output_collection), 'trials').to_df() good_enough = training.criterion_delay( - n_trials=trials["intervals"].shape[0], + n_trials=trials.shape[0], perf_easy=training.compute_performance_easy(trials), ) + if truncate_to_pass and not good_enough: + n_trials = trials.shape[0] + while not good_enough and n_trials > 400: + n_trials -= 1 + good_enough = training.criterion_delay( + n_trials=n_trials, + perf_easy=training.compute_performance_easy(trials[:n_trials]), + ) + if update: eid = self.one.path2eid(self.session_path, query_type='remote') self.one.alyx.json_field_update( From 8328a128fa9f98611f353f7edf1ede9522601d21 Mon Sep 17 00:00:00 2001 From: Florian Rau Date: Fri, 15 Dec 2023 11:17:09 +0000 Subject: [PATCH 02/20] add @sleepless decorator (#655) * add @sleepless decorator * refactor / add error log for failure * Update misc.py * add test for sleepless decorator --- ibllib/pipes/misc.py | 60 ++++++++++++++++++++++++++++++++------ ibllib/tests/test_pipes.py | 19 ++++++++++++ 2 files changed, 70 insertions(+), 9 deletions(-) diff --git a/ibllib/pipes/misc.py b/ibllib/pipes/misc.py index 39871ad00..44b025f0c 100644 --- a/ibllib/pipes/misc.py +++ b/ibllib/pipes/misc.py @@ -9,8 +9,9 @@ import sys import time import logging +from functools import wraps from pathlib import Path -from typing import Union, List +from typing import Union, List, Callable, Any from inspect import signature import uuid import socket @@ -1148,13 +1149,54 @@ class WindowsInhibitor: ES_CONTINUOUS = 0x80000000 ES_SYSTEM_REQUIRED = 0x00000001 - def __init__(self): - pass + @staticmethod + def _set_thread_execution_state(state: int) -> None: + result = ctypes.windll.kernel32.SetThreadExecutionState(state) + if result == 0: + log.error("Failed to set thread execution state.") - def inhibit(self): - print("Preventing Windows from going to sleep") - ctypes.windll.kernel32.SetThreadExecutionState(WindowsInhibitor.ES_CONTINUOUS | WindowsInhibitor.ES_SYSTEM_REQUIRED) + @staticmethod + def inhibit(quiet: bool = False): + if quiet: + log.debug("Preventing Windows from going to sleep") + else: + print("Preventing Windows from going to sleep") + WindowsInhibitor._set_thread_execution_state(WindowsInhibitor.ES_CONTINUOUS | WindowsInhibitor.ES_SYSTEM_REQUIRED) + + @staticmethod + def uninhibit(quiet: bool = False): + if quiet: + log.debug("Allowing Windows to go to sleep") + else: + print("Allowing Windows to go to sleep") + WindowsInhibitor._set_thread_execution_state(WindowsInhibitor.ES_CONTINUOUS) + + +def sleepless(func: Callable[..., Any]) -> Callable[..., Any]: + """ + Decorator to ensure that the system doesn't enter sleep or idle mode during a long-running task. + + This decorator wraps a function and sets the thread execution state to prevent + the system from entering sleep or idle mode while the decorated function is + running. + + Parameters + ---------- + func : callable + The function to decorate. + + Returns + ------- + callable + The decorated function. + """ - def uninhibit(self): - print("Allowing Windows to go to sleep") - ctypes.windll.kernel32.SetThreadExecutionState(WindowsInhibitor.ES_CONTINUOUS) + @wraps(func) + def inner(*args, **kwargs) -> Any: + if os.name == 'nt': + WindowsInhibitor().inhibit(quiet=True) + result = func(*args, **kwargs) + if os.name == 'nt': + WindowsInhibitor().uninhibit(quiet=True) + return result + return inner diff --git a/ibllib/tests/test_pipes.py b/ibllib/tests/test_pipes.py index ba5c282dd..cbe86462a 100644 --- a/ibllib/tests/test_pipes.py +++ b/ibllib/tests/test_pipes.py @@ -21,6 +21,7 @@ import ibllib.io.extractors.base import ibllib.tests.fixtures.utils as fu from ibllib.pipes import misc +from ibllib.pipes.misc import sleepless from ibllib.tests import TEST_DB import ibllib.pipes.scan_fix_passive_files as fix from ibllib.pipes.base_tasks import RegisterRawDataTask @@ -698,5 +699,23 @@ def test_rename_files(self): self.assertCountEqual(expected, files) +class TestSleeplessDecorator(unittest.TestCase): + + def test_decorator_argument_passing(self): + + def dummy_function(arg1, arg2): + return arg1, arg2 + + # Applying the decorator to the dummy function + decorated_func = sleepless(dummy_function) + + # Check if the function name is maintained + self.assertEqual(decorated_func.__name__, 'dummy_function') + + # Check if arguments are passed correctly + result = decorated_func("test1", "test2") + self.assertEqual(result, ("test1", "test2")) + + if __name__ == '__main__': unittest.main(exit=False, verbosity=2) From 7aaaaa4efa9c1190d744281ac6d905c43ea945b1 Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Fri, 15 Dec 2023 16:20:35 +0200 Subject: [PATCH 03/20] Add get_trials_tasks function Fix tests flake8 Fix test: double temp dir Logging to investigate failing test on Github workflow build Skip test of Github build --- ibllib/io/extractors/base.py | 161 +++++++++++++++++++------ ibllib/io/extractors/camera.py | 10 +- ibllib/pipes/dynamic_pipeline.py | 70 ++++++++++- ibllib/pipes/misc.py | 1 + ibllib/pipes/training_preprocessing.py | 2 +- ibllib/tests/fixtures/utils.py | 22 +++- ibllib/tests/test_dynamic_pipeline.py | 104 +++++++++++++++- 7 files changed, 319 insertions(+), 51 deletions(-) diff --git a/ibllib/io/extractors/base.py b/ibllib/io/extractors/base.py index e49d47980..13f594217 100644 --- a/ibllib/io/extractors/base.py +++ b/ibllib/io/extractors/base.py @@ -1,4 +1,5 @@ """Base Extractor classes. + A module for the base Extractor classes. The Extractor, given a session path, will extract the processed data from raw hardware files and optionally save them. """ @@ -10,7 +11,6 @@ import numpy as np import pandas as pd -from one.alf.files import get_session_path from ibllib.io import raw_data_loaders as raw from ibllib.io.raw_data_loaders import load_settings, _logger @@ -162,7 +162,8 @@ def extract(self, bpod_trials=None, settings=None, **kwargs): def run_extractor_classes(classes, session_path=None, **kwargs): """ - Run a set of extractors with the same inputs + Run a set of extractors with the same inputs. + :param classes: list of Extractor class :param save: True/False :param path_out: (defaults to alf path) @@ -195,12 +196,30 @@ def run_extractor_classes(classes, session_path=None, **kwargs): def _get_task_types_json_config(): + """ + Return the extractor types map. + + This function is only used for legacy sessions, i.e. those without an experiment description + file and will be removed in favor of :func:`_get_task_extractor_map`, which directly returns + the Bpod extractor class name. The experiment description file cuts out the need for pipeline + name identifiers. + + Returns + ------- + Dict[str, str] + A map of task protocol to task extractor identifier, e.g. 'ephys', 'habituation', etc. + + See Also + -------- + _get_task_extractor_map - returns a map of task protocol to Bpod trials extractor class name. + """ with open(Path(__file__).parent.joinpath('extractor_types.json')) as fp: task_types = json.load(fp) try: # look if there are custom extractor types in the personal projects repo import projects.base custom_extractors = Path(projects.base.__file__).parent.joinpath('extractor_types.json') + _logger.debug('Loading extractor types from %s', custom_extractors) with open(custom_extractors) as fp: custom_task_types = json.load(fp) task_types.update(custom_task_types) @@ -210,8 +229,28 @@ def _get_task_types_json_config(): def get_task_protocol(session_path, task_collection='raw_behavior_data'): + """ + Return the task protocol name from task settings. + + If the session path and/or task collection do not exist, the settings file is missing or + otherwise can not be parsed, or if the 'PYBPOD_PROTOCOL' key is absent, None is returned. + A warning is logged if the session path or settings file doesn't exist. An error is logged if + the settings file can not be parsed. + + Parameters + ---------- + session_path : str, pathlib.Path + The absolute session path. + task_collection : str + The session path directory containing the task settings file. + + Returns + ------- + str or None + The Pybpod task protocol name or None if not found. + """ try: - settings = load_settings(get_session_path(session_path), task_collection=task_collection) + settings = load_settings(session_path, task_collection=task_collection) except json.decoder.JSONDecodeError: _logger.error(f'Can\'t read settings for {session_path}') return @@ -223,11 +262,26 @@ def get_task_protocol(session_path, task_collection='raw_behavior_data'): def get_task_extractor_type(task_name): """ - Returns the task type string from the full pybpod task name: - _iblrig_tasks_biasedChoiceWorld3.7.0 returns "biased" - _iblrig_tasks_trainingChoiceWorld3.6.0 returns "training' - :param task_name: - :return: one of ['biased', 'habituation', 'training', 'ephys', 'mock_ephys', 'sync_ephys'] + Returns the task type string from the full pybpod task name. + + Parameters + ---------- + task_name : str + The complete task protocol name from the PYBPOD_PROTOCOL field of the task settings. + + Returns + ------- + str + The extractor type identifier. Examples include 'biased', 'habituation', 'training', + 'ephys', 'mock_ephys' and 'sync_ephys'. + + Examples + -------- + >>> get_task_extractor_type('_iblrig_tasks_biasedChoiceWorld3.7.0') + 'biased' + + >>> get_task_extractor_type('_iblrig_tasks_trainingChoiceWorld3.6.0') + 'training' """ if isinstance(task_name, Path): task_name = get_task_protocol(task_name) @@ -245,16 +299,30 @@ def get_task_extractor_type(task_name): def get_session_extractor_type(session_path, task_collection='raw_behavior_data'): """ - From a session path, loads the settings file, finds the task and checks if extractors exist - task names examples: - :param session_path: - :return: bool + Infer trials extractor type from task settings. + + From a session path, loads the settings file, finds the task and checks if extractors exist. + Examples include 'biased', 'habituation', 'training', 'ephys', 'mock_ephys', and 'sync_ephys'. + Note this should only be used for legacy sessions, i.e. those without an experiment description + file. + + Parameters + ---------- + session_path : str, pathlib.Path + The session path for which to determine the pipeline. + task_collection : str + The session path directory containing the raw task data. + + Returns + ------- + str or False + The task extractor type, e.g. 'biased', 'habituation', 'ephys', or False if unknown. """ - settings = load_settings(session_path, task_collection=task_collection) - if settings is None: - _logger.error(f'ABORT: No data found in "{task_collection}" folder {session_path}') + task_protocol = get_task_protocol(session_path, task_collection=task_collection) + if task_protocol is None: + _logger.error(f'ABORT: No task protocol found in "{task_collection}" folder {session_path}') return False - extractor_type = get_task_extractor_type(settings['PYBPOD_PROTOCOL']) + extractor_type = get_task_extractor_type(task_protocol) if extractor_type: return extractor_type else: @@ -263,9 +331,22 @@ def get_session_extractor_type(session_path, task_collection='raw_behavior_data' def get_pipeline(session_path, task_collection='raw_behavior_data'): """ - Get the pre-processing pipeline name from a session path - :param session_path: - :return: + Get the pre-processing pipeline name from a session path. + + Note this is only suitable for legacy sessions, i.e. those without an experiment description + file. This function will be removed in the future. + + Parameters + ---------- + session_path : str, pathlib.Path + The session path for which to determine the pipeline. + task_collection : str + The session path directory containing the raw task data. + + Returns + ------- + str + The pipeline name inferred from the extractor type, e.g. 'ephys', 'training', 'widefield'. """ stype = get_session_extractor_type(session_path, task_collection=task_collection) return _get_pipeline_from_task_type(stype) @@ -273,18 +354,29 @@ def get_pipeline(session_path, task_collection='raw_behavior_data'): def _get_pipeline_from_task_type(stype): """ - Returns the pipeline from the task type. Some tasks types directly define the pipeline - :param stype: session_type or task extractor type - :return: + Return the pipeline from the task type. + + Some task types directly define the pipeline. Note this is only suitable for legacy sessions, + i.e. those without an experiment description file. This function will be removed in the future. + + Parameters + ---------- + stype : str + The session type or task extractor type, e.g. 'habituation', 'ephys', etc. + + Returns + ------- + str + A task pipeline identifier. """ if stype in ['ephys_biased_opto', 'ephys', 'ephys_training', 'mock_ephys', 'sync_ephys']: return 'ephys' elif stype in ['habituation', 'training', 'biased', 'biased_opto']: return 'training' - elif 'widefield' in stype: + elif isinstance(stype, str) and 'widefield' in stype: return 'widefield' else: - return stype + return stype or '' def _get_task_extractor_map(): @@ -293,7 +385,7 @@ def _get_task_extractor_map(): Returns ------- - dict(str, str) + Dict[str, str] A map of task protocol to Bpod trials extractor class. """ FILENAME = 'task_extractor_map.json' @@ -315,26 +407,26 @@ def get_bpod_extractor_class(session_path, task_collection='raw_behavior_data'): """ Get the Bpod trials extractor class associated with a given Bpod session. + Note that unlike :func:`get_session_extractor_type`, this function maps directly to the Bpod + trials extractor class name. This is hardware invariant and is purly to determine the Bpod only + trials extractor. + Parameters ---------- session_path : str, pathlib.Path The session path containing Bpod behaviour data. task_collection : str - The session_path subfolder containing the Bpod settings file. + The session_path sub-folder containing the Bpod settings file. Returns ------- str The extractor class name. """ - # Attempt to load settings files - settings = load_settings(session_path, task_collection=task_collection) - if settings is None: - raise ValueError(f'No data found in "{task_collection}" folder {session_path}') - # Attempt to get task protocol - protocol = settings.get('PYBPOD_PROTOCOL') + # Attempt to get protocol name from settings file + protocol = get_task_protocol(session_path, task_collection=task_collection) if not protocol: - raise ValueError(f'No task protocol found in {session_path/task_collection}') + raise ValueError(f'No task protocol found in {Path(session_path) / task_collection}') return protocol2extractor(protocol) @@ -342,7 +434,8 @@ def protocol2extractor(protocol): """ Get the Bpod trials extractor class associated with a given Bpod task protocol. - The Bpod task protocol can be found in the 'PYBPOD_PROTOCOL' field of _iblrig_taskSettings.raw.json. + The Bpod task protocol can be found in the 'PYBPOD_PROTOCOL' field of the + _iblrig_taskSettings.raw.json file. Parameters ---------- diff --git a/ibllib/io/extractors/camera.py b/ibllib/io/extractors/camera.py index 93554c86a..a44010821 100644 --- a/ibllib/io/extractors/camera.py +++ b/ibllib/io/extractors/camera.py @@ -1,4 +1,5 @@ """ Camera extractor functions. + This module handles extraction of camera timestamps for both Bpod and DAQ. """ import logging @@ -29,7 +30,7 @@ def extract_camera_sync(sync, chmap=None): """ - Extract camera timestamps from the sync matrix + Extract camera timestamps from the sync matrix. :param sync: dictionary 'times', 'polarities' of fronts detected on sync trace :param chmap: dictionary containing channel indices. Default to constant. @@ -45,7 +46,8 @@ def extract_camera_sync(sync, chmap=None): def get_video_length(video_path): """ - Returns video length + Returns video length. + :param video_path: A path to the video :return: """ @@ -58,9 +60,7 @@ def get_video_length(video_path): class CameraTimestampsFPGA(BaseExtractor): - """ - Extractor for videos using DAQ sync and channel map. - """ + """Extractor for videos using DAQ sync and channel map.""" def __init__(self, label, session_path=None): super().__init__(session_path) diff --git a/ibllib/pipes/dynamic_pipeline.py b/ibllib/pipes/dynamic_pipeline.py index d95497380..ec4228256 100644 --- a/ibllib/pipes/dynamic_pipeline.py +++ b/ibllib/pipes/dynamic_pipeline.py @@ -13,7 +13,7 @@ import spikeglx import ibllib.io.session_params as sess_params -import ibllib.io.extractors.base +from ibllib.io.extractors.base import get_pipeline, get_session_extractor_type import ibllib.pipes.tasks as mtasks import ibllib.pipes.base_tasks as bstasks import ibllib.pipes.widefield_tasks as wtasks @@ -45,7 +45,7 @@ def acquisition_description_legacy_session(session_path, save=False): dict The legacy acquisition description. """ - extractor_type = ibllib.io.extractors.base.get_session_extractor_type(session_path=session_path) + extractor_type = get_session_extractor_type(session_path) etype2protocol = dict(biased='choice_world_biased', habituation='choice_world_habituation', training='choice_world_training', ephys='choice_world_recording') dict_ad = get_acquisition_description(etype2protocol[extractor_type]) @@ -130,7 +130,7 @@ def make_pipeline(session_path, **pkwargs): ---------- session_path : str, Path The absolute session path, i.e. '/path/to/subject/yyyy-mm-dd/nnn'. - **pkwargs + pkwargs Optional arguments passed to the ibllib.pipes.tasks.Pipeline constructor. Returns @@ -147,7 +147,7 @@ def make_pipeline(session_path, **pkwargs): if not acquisition_description: raise ValueError('Experiment description file not found or is empty') devices = acquisition_description.get('devices', {}) - kwargs = {'session_path': session_path} + kwargs = {'session_path': session_path, 'one': pkwargs.get('one')} # Registers the experiment description file tasks['ExperimentDescriptionRegisterRaw'] = type('ExperimentDescriptionRegisterRaw', @@ -430,3 +430,65 @@ def load_pipeline_dict(path): task_list = yaml.full_load(file) return task_list + + +def get_trials_tasks(session_path, one=None): + """ + Return a list of pipeline trials extractor task objects for a given session. + + This function supports both legacy and dynamic pipeline sessions. + + Parameters + ---------- + session_path : str, pathlib.Path + An absolute path to a session. + one : one.api.One + An ONE instance. + + Returns + ------- + list of pipes.tasks.Task + A list of task objects for the provided session. + + """ + # Check for an experiment.description file; ensure downloaded if possible + if one and one.to_eid(session_path): # to_eid returns None if session not registered + one.load_datasets(session_path, ['_ibl_experiment.description'], download_only=True, assert_present=False) + experiment_description = sess_params.read_params(session_path) + + # If experiment description file then use this to make the pipeline + if experiment_description is not None: + tasks = [] + pipeline = make_pipeline(session_path, one=one) + trials_tasks = [t for t in pipeline.tasks if 'Trials' in t] + for task in trials_tasks: + t = pipeline.tasks.get(task) + t.__init__(session_path, **t.kwargs) + tasks.append(t) + else: + # Otherwise default to old way of doing things + pipeline = get_pipeline(session_path) + if pipeline == 'training': + from ibllib.pipes.training_preprocessing import TrainingTrials + tasks = [TrainingTrials(session_path, one=one)] + elif pipeline == 'ephys': + from ibllib.pipes.ephys_preprocessing import EphysTrials + tasks = [EphysTrials(session_path, one=one)] + else: + try: + # try to find a custom extractor in the personal projects extraction class + import projects.base + task_type = get_session_extractor_type(session_path) + assert (PipelineClass := projects.base.get_pipeline(task_type)) + pipeline = PipelineClass(session_path, one=one) + trials_task_name = next((task for task in pipeline.tasks if 'Trials' in task), None) + assert trials_task_name, (f'No "Trials" tasks for custom pipeline ' + f'"{pipeline.name}" with extractor type "{task_type}"') + task = pipeline.tasks.get(trials_task_name) + task(session_path) + tasks = [task] + except (ModuleNotFoundError, AssertionError) as ex: + _logger.warning('Failed to get trials tasks: %s', ex) + tasks = [] + + return tasks diff --git a/ibllib/pipes/misc.py b/ibllib/pipes/misc.py index 39871ad00..27e1df5b3 100644 --- a/ibllib/pipes/misc.py +++ b/ibllib/pipes/misc.py @@ -38,6 +38,7 @@ def subjects_data_folder(folder: Path, rglob: bool = False) -> Path: """Given a root_data_folder will try to find a 'Subjects' data folder. + If Subjects folder is passed will return it directly.""" if not isinstance(folder, Path): folder = Path(folder) diff --git a/ibllib/pipes/training_preprocessing.py b/ibllib/pipes/training_preprocessing.py index db41f8992..ad2172809 100644 --- a/ibllib/pipes/training_preprocessing.py +++ b/ibllib/pipes/training_preprocessing.py @@ -19,7 +19,7 @@ from ibllib.qc.task_extractors import TaskQCExtractor _logger = logging.getLogger(__name__) -warnings.warn('`pipes.training_preprocessing` to be removed in favour of dynamic pipeline') +warnings.warn('`pipes.training_preprocessing` to be removed in favour of dynamic pipeline', FutureWarning) # level 0 diff --git a/ibllib/tests/fixtures/utils.py b/ibllib/tests/fixtures/utils.py index ac7ac5f71..f536875d0 100644 --- a/ibllib/tests/fixtures/utils.py +++ b/ibllib/tests/fixtures/utils.py @@ -216,7 +216,7 @@ def create_fake_raw_behavior_data_folder( ): """Create the folder structure for a raw behaviour session. - Creates a raw_behavior_data folder and optionally, touches some files and writes a experiment + Creates a raw_behavior_data folder and optionally, touches some files and writes an experiment description stub to a `_devices` folder. Parameters @@ -304,8 +304,26 @@ def create_fake_raw_behavior_data_folder( def populate_task_settings(fpath: Path, patch: dict): - with fpath.open("w") as f: + """ + Populate a task settings JSON file. + + Parameters + ---------- + fpath : pathlib.Path + A path to a raw task settings folder or the full settings file path. + patch : dict + The settings dict to write to file. + + Returns + ------- + pathlib.Path + The full settings file path. + """ + if fpath.is_dir(): + fpath /= '_iblrig_taskSettings.raw.json' + with fpath.open('w') as f: json.dump(patch, f, indent=1) + return fpath def create_fake_complete_ephys_session( diff --git a/ibllib/tests/test_dynamic_pipeline.py b/ibllib/tests/test_dynamic_pipeline.py index 8b32b5ff6..41420c674 100644 --- a/ibllib/tests/test_dynamic_pipeline.py +++ b/ibllib/tests/test_dynamic_pipeline.py @@ -1,15 +1,22 @@ import tempfile from pathlib import Path import unittest +from unittest import mock from itertools import chain +import yaml + import ibllib.tests -from ibllib.pipes import dynamic_pipeline +import ibllib.pipes.dynamic_pipeline as dyn +from ibllib.pipes.tasks import Pipeline, Task +from ibllib.pipes import ephys_preprocessing +from ibllib.pipes import training_preprocessing from ibllib.io import session_params +from ibllib.tests.fixtures.utils import populate_task_settings def test_read_write_params_yaml(): - ad = dynamic_pipeline.get_acquisition_description('choice_world_recording') + ad = dyn.get_acquisition_description('choice_world_recording') with tempfile.TemporaryDirectory() as td: session_path = Path(td) session_params.write_params(session_path, ad) @@ -21,14 +28,14 @@ class TestCreateLegacyAcqusitionDescriptions(unittest.TestCase): def test_legacy_biased(self): session_path = Path(ibllib.tests.__file__).parent.joinpath('extractors', 'data', 'session_biased_ge5') - ad = dynamic_pipeline.acquisition_description_legacy_session(session_path) + ad = dyn.acquisition_description_legacy_session(session_path) protocols = list(chain(*map(dict.keys, ad.get('tasks', [])))) self.assertCountEqual(['biasedChoiceWorld'], protocols) self.assertEqual(1, len(ad['devices']['cameras'])) def test_legacy_ephys(self): session_path = Path(ibllib.tests.__file__).parent.joinpath('extractors', 'data', 'session_ephys') - ad_ephys = dynamic_pipeline.acquisition_description_legacy_session(session_path) + ad_ephys = dyn.acquisition_description_legacy_session(session_path) self.assertEqual(2, len(ad_ephys['devices']['neuropixel'])) self.assertEqual(3, len(ad_ephys['devices']['cameras'])) protocols = list(chain(*map(dict.keys, ad_ephys.get('tasks', [])))) @@ -36,7 +43,94 @@ def test_legacy_ephys(self): def test_legacy_training(self): session_path = Path(ibllib.tests.__file__).parent.joinpath('extractors', 'data', 'session_training_ge5') - ad = dynamic_pipeline.acquisition_description_legacy_session(session_path) + ad = dyn.acquisition_description_legacy_session(session_path) protocols = list(chain(*map(dict.keys, ad.get('tasks', [])))) self.assertCountEqual(['trainingChoiceWorld'], protocols) self.assertEqual(1, len(ad['devices']['cameras'])) + + +class TestGetTrialsTasks(unittest.TestCase): + """Test pipes.dynamic_pipeline.get_trials_tasks function.""" + + def setUp(self): + tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(tmpdir.cleanup) + # The github CI root dir contains an alias/symlink so we must resolve it + self.tempdir = Path(tmpdir.name).resolve() + self.session_path_dynamic = self.tempdir / 'subject' / '2023-01-01' / '001' + self.session_path_dynamic.mkdir(parents=True) + description = {'version': '1.0.0', + 'sync': {'nidq': {'collection': 'raw_ephys_data', 'extension': 'bin', 'acquisition_software': 'spikeglx'}}, + 'tasks': [ + {'ephysChoiceWorld': {'task_collection': 'raw_task_data_00'}}, + {'passiveChoiceWorld': {'task_collection': 'raw_task_data_01'}}, + ]} + with open(self.session_path_dynamic / '_ibl_experiment.description.yaml', 'w') as fp: + yaml.safe_dump(description, fp) + + self.session_path_legacy = self.session_path_dynamic.with_name('002') + (collection := self.session_path_legacy.joinpath('raw_behavior_data')).mkdir(parents=True) + self.settings = {'IBLRIG_VERSION': '7.2.2', 'PYBPOD_PROTOCOL': '_iblrig_tasks_ephysChoiceWorld'} + self.settings_path = populate_task_settings(collection, self.settings) + + def test_get_trials_tasks(self): + """Test pipes.dynamic_pipeline.get_trials_tasks function.""" + # A dynamic pipeline session + tasks = dyn.get_trials_tasks(self.session_path_dynamic) + self.assertEqual(2, len(tasks)) + self.assertEqual('raw_task_data_00', tasks[0].collection) + + # Check behaviour with ONE + one = mock.MagicMock() + one.offline = False + one.alyx = mock.MagicMock() + one.alyx.cache_mode = None # sneaky hack as this is checked by the pipeline somewhere + tasks = dyn.get_trials_tasks(self.session_path_dynamic, one) + self.assertEqual(2, len(tasks)) + one.load_datasets.assert_called() # check that description file is checked on disk + + # An ephys session + tasks = dyn.get_trials_tasks(self.session_path_legacy) + self.assertEqual(1, len(tasks)) + self.assertIsInstance(tasks[0], ephys_preprocessing.EphysTrials) + + # A training session + self.settings['PYBPOD_PROTOCOL'] = '_iblrig_tasks_trainingChoiceWorld' + populate_task_settings(self.settings_path, self.settings) + + tasks = dyn.get_trials_tasks(self.session_path_legacy, one=one) + self.assertEqual(1, len(tasks)) + self.assertIsInstance(tasks[0], training_preprocessing.TrainingTrials) + self.assertIs(tasks[0].one, one, 'failed to assign ONE instance to task') + + # A personal project + self.settings['PYBPOD_PROTOCOL'] = '_misc_foobarChoiceWorld' + populate_task_settings(self.settings_path, self.settings) + + m = mock.MagicMock() # Mock the project_extractors repo + m.base.__file__ = str(self.tempdir / 'base.py') + # Create the personal project extractor types map + task_type_map = {'_misc_foobarChoiceWorld': 'foobar'} + extractor_types_path = Path(m.base.__file__).parent.joinpath('extractor_types.json') + populate_task_settings(extractor_types_path, task_type_map) + # Simulate the instantiation of the personal project module's pipeline class + pipeline = mock.Mock(spec=Pipeline) + pipeline.name = 'custom' + task_mock = mock.Mock(spec=Task) + pipeline.tasks = {'RegisterRaw': mock.MagicMock(), 'FooBarTrials': task_mock} + m.base.get_pipeline().return_value = pipeline + with mock.patch.dict('sys.modules', projects=m): + """For unknown reasons this method of mocking the personal projects repo (which is + imported within various functions) fails on the Github test builds. This we check + here and skip the rest of the test if patch didn't work.""" + try: + import projects.base + assert isinstance(projects.base, mock.Mock) + except (AssertionError, ModuleNotFoundError): + self.skipTest('Failed to mock projects module import') + tasks = dyn.get_trials_tasks(self.session_path_legacy) + self.assertEqual(1, len(tasks)) + task_mock.assert_called_once_with(self.session_path_legacy) + # Should handle absent trials tasks + pipeline.tasks.pop('FooBarTrials') + self.assertEqual([], dyn.get_trials_tasks(self.session_path_legacy)) From 3e9fea438d58fd867105ebe2da27b5b9a82bc4a2 Mon Sep 17 00:00:00 2001 From: Gaelle Date: Fri, 22 Dec 2023 13:00:13 +0100 Subject: [PATCH 04/20] data release update --- examples/data_release/data_release_brainwidemap.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/data_release/data_release_brainwidemap.ipynb b/examples/data_release/data_release_brainwidemap.ipynb index 0c3b0a402..851ae3b17 100644 --- a/examples/data_release/data_release_brainwidemap.ipynb +++ b/examples/data_release/data_release_brainwidemap.ipynb @@ -18,7 +18,7 @@ "cell_type": "markdown", "source": [ "## Overview of the Data\n", - "We have released data from 354 Neuropixel recording sessions, which encompass 547 probe insertions, obtained in 115 subjects performing the IBL task across 11 different laboratories. As output of spike-sorting, there are 295501 units; of which 32766 are considered to be of good quality. These units were recorded in overall 194 different brain regions.\n", + "We have released data from 459 Neuropixel recording sessions, which encompass 699 probe insertions, obtained in 139 subjects performing the IBL task across 12 different laboratories. As output of spike-sorting, there are 376730 units; of which 45085 are considered to be of good quality. These units were recorded in overall 138 different brain regions.\n", "\n", "## Data structure and download\n", "The organisation of the data follows the standard IBL data structure.\n", @@ -31,7 +31,7 @@ "\n", "Note:\n", "\n", - "* The tag associated to this release is `2022_Q4_IBL_et_al_BWM`" + "* The tag associated to this release is `Brainwidemap`" ], "metadata": { "collapsed": false From d1812277980ebab38f7f003772fe62c087b6c9cb Mon Sep 17 00:00:00 2001 From: Mayo Faulkner Date: Fri, 22 Dec 2023 12:41:00 +0000 Subject: [PATCH 05/20] change number of parallel workflows --- .github/workflows/ibllib_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ibllib_ci.yml b/.github/workflows/ibllib_ci.yml index 64a98c108..20564820d 100644 --- a/.github/workflows/ibllib_ci.yml +++ b/.github/workflows/ibllib_ci.yml @@ -15,7 +15,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: fail-fast: false # Whether to stop execution of other instances - max-parallel: 4 + max-parallel: 2 matrix: os: ["windows-latest", "ubuntu-latest"] python-version: ["3.8", "3.11"] From a0d62acb7d3928e5db4959d4f36c526770d4fa1d Mon Sep 17 00:00:00 2001 From: Gaelle Date: Wed, 3 Jan 2024 11:31:55 +0100 Subject: [PATCH 06/20] data release update --- examples/data_release/data_release_brainwidemap.ipynb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/data_release/data_release_brainwidemap.ipynb b/examples/data_release/data_release_brainwidemap.ipynb index 851ae3b17..001fed7cf 100644 --- a/examples/data_release/data_release_brainwidemap.ipynb +++ b/examples/data_release/data_release_brainwidemap.ipynb @@ -11,14 +11,14 @@ "source": [ "# Data Release - Brain Wide Map\n", "\n", - "IBL aims to understand the neural basis of decision-making in the mouse by gathering a whole-brain activity map composed of electrophysiological recordings pooled from multiple laboratories. We have systematically recorded from nearly all major brain areas with Neuropixels probes, using a grid system for unbiased sampling and replicating each recording site in at least two laboratories. These data have been used to construct a brain-wide map of activity at single-spike cellular resolution during a [decision-making task]((https://elifesciences.org/articles/63711)). In addition to the map, this data set contains other information gathered during the task: sensory stimuli presented to the mouse; mouse decisions and response times; and mouse pose information from video recordings and DeepLabCut analysis. Please read our accompanying [technical paper](https://doi.org/10.6084/m9.figshare.21400815) for details on the experiment and data processing pipelines. To explore the data, visit [our vizualisation website](https://viz.internationalbrainlab.org/)." + "IBL aims to understand the neural basis of decision-making in the mouse by gathering a whole-brain activity map composed of electrophysiological recordings pooled from multiple laboratories. We have systematically recorded from nearly all major brain areas with Neuropixels probes, using a grid system for unbiased sampling and replicating each recording site in at least two laboratories. These data have been used to construct a brain-wide map of activity at single-spike cellular resolution during a [decision-making task]((https://elifesciences.org/articles/63711)). Please read the associated article [(IBL et al. 2023)](https://www.biorxiv.org/content/10.1101/2023.07.04.547681v2). In addition to the map, this data set contains other information gathered during the task: sensory stimuli presented to the mouse; mouse decisions and response times; and mouse pose information from video recordings and DeepLabCut analysis. Please read our accompanying [technical paper](https://doi.org/10.6084/m9.figshare.21400815) for details on the experiment and data processing pipelines. To explore the data, visit [our vizualisation website](https://viz.internationalbrainlab.org/)." ] }, { "cell_type": "markdown", "source": [ "## Overview of the Data\n", - "We have released data from 459 Neuropixel recording sessions, which encompass 699 probe insertions, obtained in 139 subjects performing the IBL task across 12 different laboratories. As output of spike-sorting, there are 376730 units; of which 45085 are considered to be of good quality. These units were recorded in overall 138 different brain regions.\n", + "We have released data from 459 Neuropixel recording sessions, which encompass 699 probe insertions, obtained in 139 subjects performing the IBL task across 12 different laboratories. As output of spike-sorting, there are 376730 units; of which 45085 are considered to be of good quality. In total, 138 brain regions were recorded in sufficient numbers for inclusion in IBL’s analyses [(IBL et al. 2023)](https://www.biorxiv.org/content/10.1101/2023.07.04.547681v2).\n", "\n", "## Data structure and download\n", "The organisation of the data follows the standard IBL data structure.\n", @@ -31,7 +31,10 @@ "\n", "Note:\n", "\n", - "* The tag associated to this release is `Brainwidemap`" + "* The tag associated to this release is `Brainwidemap`\n", + "\n", + "## Receive updates on the data\n", + "To receive a notification that we released new datasets, please fill up [this form](https://forms.gle/9ex2vL1JwV4QXnf98)\n" ], "metadata": { "collapsed": false From 39c7a1fa291fd57b02d3fc6bab9785c66cde49cf Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Wed, 3 Jan 2024 16:35:02 +0100 Subject: [PATCH 07/20] add to release notes --- release_notes.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/release_notes.md b/release_notes.md index 569aef684..92b771619 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,3 +1,12 @@ +## Release Notes 2.28 + +### features + +### bugfixes + +### other +- Change behavior qc to pass if number of trials > 400 (from start) can be found for which easy trial performance > 0.9 + ## Release Notes 2.27 ### features From d0f202aa195274eb9630b2e03ed3828a0eeb4115 Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Thu, 4 Jan 2024 13:42:24 +0200 Subject: [PATCH 08/20] Issue #701 --- ibllib/io/extractors/ephys_fpga.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibllib/io/extractors/ephys_fpga.py b/ibllib/io/extractors/ephys_fpga.py index 03eb7079c..42d549e51 100644 --- a/ibllib/io/extractors/ephys_fpga.py +++ b/ibllib/io/extractors/ephys_fpga.py @@ -1485,7 +1485,7 @@ def extract_all(session_path, sync_collection='raw_ephys_data', save=True, save_ # Sync Bpod trials to FPGA sync, chmap = get_sync_and_chn_map(session_path, sync_collection) # sync, chmap = get_main_probe_sync(session_path, bin_exists=bin_exists) - trials = FpgaTrials(session_path, bpod_trials=bpod_trials | bpod_wheel) + trials = FpgaTrials(session_path, bpod_trials={**bpod_trials, **bpod_wheel}) # py3.9 -> | outputs, files = trials.extract( save=save, sync=sync, chmap=chmap, path_out=save_path, task_collection=task_collection, protocol_number=protocol_number, **kwargs) From bf1c4cb352a7bee367775ddaee962aa58648ef9d Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Thu, 4 Jan 2024 14:39:24 +0200 Subject: [PATCH 09/20] Update release notes and version --- ibllib/__init__.py | 2 +- release_notes.md | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ibllib/__init__.py b/ibllib/__init__.py index 55e4b2b5d..2cbd2f779 100644 --- a/ibllib/__init__.py +++ b/ibllib/__init__.py @@ -2,7 +2,7 @@ import logging import warnings -__version__ = '2.27' +__version__ = '2.28' warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib') # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev diff --git a/release_notes.md b/release_notes.md index 92b771619..d9a833374 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,8 +1,7 @@ ## Release Notes 2.28 -### features - ### bugfixes +- Fix ibllib.io.extractors.ephys_fpga.extract_all for python 3.8 ### other - Change behavior qc to pass if number of trials > 400 (from start) can be found for which easy trial performance > 0.9 From 32b001010fa4502779a8aaa1032c3d8b706fb741 Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Thu, 4 Jan 2024 17:17:25 +0200 Subject: [PATCH 10/20] Update release notes --- release_notes.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/release_notes.md b/release_notes.md index 604d71a5b..2114de337 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,5 +1,8 @@ ## Release Notes 2.28 +### features +- Added ibllib.pipes.dynamic_pipeline.get_trials_tasks function + ### bugfixes - Fix ibllib.io.extractors.ephys_fpga.extract_all for python 3.8 From ad9f306f3fd775a295af6b5f782076272f110b1e Mon Sep 17 00:00:00 2001 From: Mayo Faulkner Date: Mon, 8 Jan 2024 10:47:10 +0100 Subject: [PATCH 11/20] Auto stash before checking out "origin/develop" add trainingPhaseChoiceWorld to extractor type --- ibllib/io/extractors/extractor_types.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ibllib/io/extractors/extractor_types.json b/ibllib/io/extractors/extractor_types.json index e69e65270..71eea2bcf 100644 --- a/ibllib/io/extractors/extractor_types.json +++ b/ibllib/io/extractors/extractor_types.json @@ -17,5 +17,6 @@ "_habituationChoiceWorld": "habituation", "_trainingChoiceWorld": "training", "ephysMockChoiceWorld": "mock_ephys", - "ephys_certification": "sync_ephys" + "ephys_certification": "sync_ephys", + "trainingPhaseChoiceWorld": "training" } From 550f7830f20e3fd2faa390e6b120b48480cb8713 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Wed, 10 Jan 2024 16:09:25 +0100 Subject: [PATCH 12/20] typo in EphysPostDLC init --- ibllib/pipes/video_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibllib/pipes/video_tasks.py b/ibllib/pipes/video_tasks.py index 7f501a065..d9a312808 100644 --- a/ibllib/pipes/video_tasks.py +++ b/ibllib/pipes/video_tasks.py @@ -467,7 +467,7 @@ class EphysPostDLC(base_tasks.VideoTask): level = 3 force = True - def __int__(self, *args, **kwargs): + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.trials_collection = kwargs.get('trials_collection', 'alf') From 9f43340a5be64131006bdc54154745197688f318 Mon Sep 17 00:00:00 2001 From: k1o0 Date: Thu, 18 Jan 2024 16:48:19 +0200 Subject: [PATCH 13/20] Fix patch_settings for iblrigv8 (#713) * Fix patch_settings for iblrigv8 * Increase coverage --- ibllib/io/raw_data_loaders.py | 19 +++++++------------ ibllib/io/session_params.py | 18 +++++------------- ibllib/tests/test_io.py | 18 +++++++++++++++++- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/ibllib/io/raw_data_loaders.py b/ibllib/io/raw_data_loaders.py index 3adfd3127..6f165479b 100644 --- a/ibllib/io/raw_data_loaders.py +++ b/ibllib/io/raw_data_loaders.py @@ -736,16 +736,6 @@ def _groom_wheel_data_ge5(data, label='file ', path=''): return data -def save_bool(save, dataset_type): - if isinstance(save, bool): - out = save - elif isinstance(save, list): - out = (dataset_type in save) or (Path(dataset_type).stem in save) - if out: - _logger.debug('extracting' + dataset_type) - return out - - def sync_trials_robust(t0, t1, diff_threshold=0.001, drift_threshold_ppm=200, max_shift=5, return_index=False): """ @@ -945,7 +935,7 @@ def patch_settings(session_path, collection='raw_behavior_data', if not settings: raise IOError('Settings file not found') - filename = PureWindowsPath(settings['SETTINGS_FILE_PATH']).name + filename = PureWindowsPath(settings.get('SETTINGS_FILE_PATH', '_iblrig_taskSettings.raw.json')).name file_path = Path(session_path).joinpath(collection, filename) if subject: @@ -955,7 +945,8 @@ def patch_settings(session_path, collection='raw_behavior_data', for k in settings.keys(): if isinstance(settings[k], str): settings[k] = settings[k].replace(f'\\Subjects\\{old_subject}', f'\\Subjects\\{subject}') - settings['SESSION_NAME'] = '\\'.join([subject, *settings['SESSION_NAME'].split('\\')[1:]]) + if 'SESSION_NAME' in settings: + settings['SESSION_NAME'] = '\\'.join([subject, *settings['SESSION_NAME'].split('\\')[1:]]) settings.pop('PYBPOD_SUBJECT_EXTRA', None) # Get rid of Alyx subject info if date: @@ -970,6 +961,10 @@ def patch_settings(session_path, collection='raw_behavior_data', f'\\{settings["SUBJECT_NAME"]}\\{date}' ) settings['SESSION_DATETIME'] = date + settings['SESSION_DATETIME'][10:] + if 'SESSION_END_TIME' in settings: + settings['SESSION_END_TIME'] = date + settings['SESSION_END_TIME'][10:] + if 'SESSION_START_TIME' in settings: + settings['SESSION_START_TIME'] = date + settings['SESSION_START_TIME'][10:] if number: # Patch session number diff --git a/ibllib/io/session_params.py b/ibllib/io/session_params.py index 300a5d6e3..174cf16a4 100644 --- a/ibllib/io/session_params.py +++ b/ibllib/io/session_params.py @@ -30,6 +30,7 @@ from copy import deepcopy from one.converters import ConversionMixin +from iblutil.util import flatten from packaging import version import ibllib.pipes.misc as misc @@ -391,15 +392,15 @@ def get_collections(sess_params, flat=False): sess_params : dict The loaded experiment description map. flat : bool (False) - If True, return a flat list of unique collections, otherwise return a map of device/sync/task + If True, return a flat set of collections, otherwise return a map of device/sync/task Returns ------- dict[str, str] A map of device/sync/task and the corresponding collection name. - list[str] - A flat list of unique collection names. + set[str] + A set of unique collection names. Notes ----- @@ -423,16 +424,7 @@ def iter_dict(d): iter_dict(v) iter_dict(sess_params) - if flat: - cflat = [] - for k, v in collection_map.items(): - if isinstance(v, list): - cflat.extend(v) - else: - cflat.append(v) - return list(set(cflat)) - else: - return collection_map + return set(flatten(collection_map.values())) if flat else collection_map def get_video_compressed(sess_params): diff --git a/ibllib/tests/test_io.py b/ibllib/tests/test_io.py index 7bc75951f..eff04c862 100644 --- a/ibllib/tests/test_io.py +++ b/ibllib/tests/test_io.py @@ -166,11 +166,15 @@ def test_load_encoder_trial_info(self): self.session = Path(__file__).parent.joinpath('extractors', 'data', 'session_biased_ge5') data = raw.load_encoder_trial_info(self.session) self.assertTrue(data is not None) + self.assertIsNone(raw.load_encoder_trial_info(self.session.with_name('empty'))) + self.assertIsNone(raw.load_encoder_trial_info(None)) def test_load_camera_ssv_times(self): session = Path(__file__).parent.joinpath('extractors', 'data', 'session_ephys') with self.assertRaises(ValueError): raw.load_camera_ssv_times(session, 'tail') + with self.assertRaises(FileNotFoundError): + raw.load_camera_ssv_times(session.with_name('foobar'), 'body') bonsai, camera = raw.load_camera_ssv_times(session, 'body') self.assertTrue(bonsai.size == camera.size == 6001) self.assertEqual(bonsai.dtype.str, ' Date: Thu, 18 Jan 2024 17:52:12 +0200 Subject: [PATCH 14/20] Update release notes --- ibllib/__init__.py | 2 +- release_notes.md | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ibllib/__init__.py b/ibllib/__init__.py index 2cbd2f779..770ead37d 100644 --- a/ibllib/__init__.py +++ b/ibllib/__init__.py @@ -2,7 +2,7 @@ import logging import warnings -__version__ = '2.28' +__version__ = '2.28.1' warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib') # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev diff --git a/release_notes.md b/release_notes.md index 2114de337..092b95e7b 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,5 +1,11 @@ ## Release Notes 2.28 +#### 2.28.1 +- Typo in ibllib.pipes.video_tasks.EphysPostDLC class +- ibllib.io.raw_data_loaders.patch_settings works with iblrigv8 settings files + +## Release Notes 2.28 + ### features - Added ibllib.pipes.dynamic_pipeline.get_trials_tasks function From 8ad2ebb5109cd0e4735281040aee7c9803af63d0 Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Thu, 18 Jan 2024 17:52:12 +0200 Subject: [PATCH 15/20] Update release notes --- ibllib/__init__.py | 2 +- release_notes.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ibllib/__init__.py b/ibllib/__init__.py index 2cbd2f779..770ead37d 100644 --- a/ibllib/__init__.py +++ b/ibllib/__init__.py @@ -2,7 +2,7 @@ import logging import warnings -__version__ = '2.28' +__version__ = '2.28.1' warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib') # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev diff --git a/release_notes.md b/release_notes.md index 2114de337..9320ad36c 100644 --- a/release_notes.md +++ b/release_notes.md @@ -9,6 +9,10 @@ ### other - Change behavior qc to pass if number of trials > 400 (from start) can be found for which easy trial performance > 0.9 +#### 2.28.1 +- Typo in ibllib.pipes.video_tasks.EphysPostDLC class +- ibllib.io.raw_data_loaders.patch_settings works with iblrigv8 settings files + ## Release Notes 2.27 ### features From 8c5b0259026d58b09c514a62357086607cb63958 Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Thu, 18 Jan 2024 17:57:18 +0200 Subject: [PATCH 16/20] Update release notes --- release_notes.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/release_notes.md b/release_notes.md index 1e294d902..9320ad36c 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,11 +1,5 @@ ## Release Notes 2.28 -#### 2.28.1 -- Typo in ibllib.pipes.video_tasks.EphysPostDLC class -- ibllib.io.raw_data_loaders.patch_settings works with iblrigv8 settings files - -## Release Notes 2.28 - ### features - Added ibllib.pipes.dynamic_pipeline.get_trials_tasks function From 6a3fa45f7abfd5a65f6fa382e8a7583b09136a57 Mon Sep 17 00:00:00 2001 From: k1o0 Date: Thu, 25 Jan 2024 13:20:36 +0200 Subject: [PATCH 17/20] Fix loading of personal projects extractor map (#718) * Fix loading of personal projects extractor map * flake8 * Fix unrelated test --- ibllib/__init__.py | 2 +- ibllib/io/extractors/base.py | 2 +- ibllib/io/raw_data_loaders.py | 6 +- ibllib/plots/snapshot.py | 11 ++- .../tests/extractors/test_extractors_base.py | 73 +++++++++++++++++++ ibllib/tests/test_plots.py | 4 +- release_notes.md | 3 + 7 files changed, 88 insertions(+), 13 deletions(-) create mode 100644 ibllib/tests/extractors/test_extractors_base.py diff --git a/ibllib/__init__.py b/ibllib/__init__.py index 770ead37d..e10341c08 100644 --- a/ibllib/__init__.py +++ b/ibllib/__init__.py @@ -2,7 +2,7 @@ import logging import warnings -__version__ = '2.28.1' +__version__ = '2.28.2' warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib') # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev diff --git a/ibllib/io/extractors/base.py b/ibllib/io/extractors/base.py index 13f594217..1dd6417e1 100644 --- a/ibllib/io/extractors/base.py +++ b/ibllib/io/extractors/base.py @@ -395,7 +395,7 @@ def _get_task_extractor_map(): # look if there are custom extractor types in the personal projects repo import projects.base custom_extractors = Path(projects.base.__file__).parent.joinpath(FILENAME) - with open(custom_extractors) as fp: + with open(custom_extractors, 'r') as fp: custom_task_types = json.load(fp) task_extractors.update(custom_task_types) except (ModuleNotFoundError, FileNotFoundError): diff --git a/ibllib/io/raw_data_loaders.py b/ibllib/io/raw_data_loaders.py index 6f165479b..c1edd215b 100644 --- a/ibllib/io/raw_data_loaders.py +++ b/ibllib/io/raw_data_loaders.py @@ -3,9 +3,9 @@ # @Author: Niccolò Bonacchi, Miles Wells # @Date: Monday, July 16th 2018, 1:28:46 pm """ -Raw Data Loader functions for PyBpod rig +Raw Data Loader functions for PyBpod rig. -Module contains one loader function per raw datafile +Module contains one loader function per raw datafile. """ import re import json @@ -107,7 +107,7 @@ def load_data(session_path: Union[str, Path], task_collection='raw_behavior_data def load_camera_frameData(session_path, camera: str = 'left', raw: bool = False) -> pd.DataFrame: - """ Loads binary frame data from Bonsai camera recording workflow. + """Loads binary frame data from Bonsai camera recording workflow. Args: session_path (StrPath): Path to session folder diff --git a/ibllib/plots/snapshot.py b/ibllib/plots/snapshot.py index 295ba7cc6..f06f1bf68 100644 --- a/ibllib/plots/snapshot.py +++ b/ibllib/plots/snapshot.py @@ -216,7 +216,7 @@ def register_image(self, image_file, text='', json_field=None, width=None): """ # the protocol is not compatible with byte streaming and json, so serialize the json object here # Make sure that user is logged in, if not, try to log in - assert self.one.alyx.is_logged_in, "No Alyx user is logged in, try running one.alyx.authenticate() first" + assert self.one.alyx.is_logged_in, 'No Alyx user is logged in, try running one.alyx.authenticate() first' note = { 'user': self.one.alyx.user, 'content_type': self.content_type, 'object_id': self.object_id, 'text': text, 'width': width, 'json': json.dumps(json_field)} @@ -232,16 +232,15 @@ def register_image(self, image_file, text='', json_field=None, width=None): # Catch error that results from object_id - content_type mismatch try: note_db = self.one.alyx.rest('notes', 'create', data=note, files={'image': fig_open}) - fig_open.close() return note_db except requests.HTTPError as e: - if "matching query does not exist.'" in str(e): - fig_open.close() + if 'matching query does not exist' in str(e): _logger.error(f'The object_id {self.object_id} does not match an object of type {self.content_type}') _logger.debug(traceback.format_exc()) else: - fig_open.close() - raise + raise e + finally: + fig_open.close() def register_images(self, image_list=None, texts=None, widths=None, jsons=None): """ diff --git a/ibllib/tests/extractors/test_extractors_base.py b/ibllib/tests/extractors/test_extractors_base.py new file mode 100644 index 000000000..f864fcc63 --- /dev/null +++ b/ibllib/tests/extractors/test_extractors_base.py @@ -0,0 +1,73 @@ +import json +import unittest +import tempfile +from pathlib import Path +from unittest.mock import patch, MagicMock + +from ibllib.io.extractors import base + + +class TestExtractorMaps(unittest.TestCase): + """Tests for functions that return Bpod extractor classes.""" + def setUp(self): + # Store original __import__ + self.orig_import = __import__ + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.custom_extractors_path = Path(tmp.name).joinpath('task_extractor_map.json') + self.custom_extractors = {'fooChoiceWorld': 'Bar'} + self.projects = MagicMock() + self.projects.base.__file__ = str(self.custom_extractors_path.with_name('__init__.py')) + with open(self.custom_extractors_path, 'w') as fp: + json.dump(self.custom_extractors, fp) + + def import_mock(self, name, *args): + """Return mock for project_extraction imports.""" + if name == 'projects' or name == 'projects.base': + return self.projects + return self.orig_import(name, *args) + + def test_get_task_extractor_map(self): + """Test ibllib.io.extractors.base._get_task_extractor_map function.""" + # Check the custom map is loaded + with patch('builtins.__import__', side_effect=self.import_mock): + extractors = base._get_task_extractor_map() + self.assertTrue(self.custom_extractors.items() < extractors.items()) + # Test handles case where module not installed + with patch('builtins.__import__', side_effect=ModuleNotFoundError): + extractors = base._get_task_extractor_map() + self.assertFalse(set(self.custom_extractors.items()).issubset(set(extractors.items()))) + # Remove the file and check exception is caught + self.custom_extractors_path.unlink() + extractors = base._get_task_extractor_map() + self.assertFalse(set(self.custom_extractors.items()).issubset(set(extractors.items()))) + + def test_get_bpod_extractor_class(self): + """Test ibllib.io.extractors.base.get_bpod_extractor_class function.""" + # installe + # alf_path = self.custom_extractors_path.parent.joinpath('subject', '2020-01-01', '001', 'raw_task_data_00') + # alf_path.mkdir(parents=True) + settings_file = Path(__file__).parent.joinpath( + 'data', 'session_biased_ge5', 'raw_behavior_data', '_iblrig_taskSettings.raw.json' + ) + # shutil.copy(settings_file, alf_path) + session_path = settings_file.parents[1] + self.assertEqual('BiasedTrials', base.get_bpod_extractor_class(session_path)) + session_path = str(session_path).replace('session_biased_ge5', 'session_training_ge5') + self.assertEqual('TrainingTrials', base.get_bpod_extractor_class(session_path)) + session_path = str(session_path).replace('session_training_ge5', 'foobar') + self.assertRaises(ValueError, base.get_bpod_extractor_class, session_path) + + def test_protocol2extractor(self): + """Test ibllib.io.extractors.base.protocol2extractor function.""" + # Test fuzzy match + (proc, expected), = self.custom_extractors.items() + with patch('builtins.__import__', side_effect=self.import_mock): + extractor = base.protocol2extractor('_mw_' + proc) + self.assertEqual(expected, extractor) + # Test unknown protocol + self.assertRaises(ValueError, base.protocol2extractor, proc) + + +if __name__ == '__main__': + unittest.main() diff --git a/ibllib/tests/test_plots.py b/ibllib/tests/test_plots.py index 1ccef0135..04cf89b99 100644 --- a/ibllib/tests/test_plots.py +++ b/ibllib/tests/test_plots.py @@ -22,7 +22,7 @@ class TestSnapshot(unittest.TestCase): @classmethod def setUpClass(cls): - # Make a small image an store in tmp file + # Make a small image and store in tmp file cls.tmp_dir = tempfile.TemporaryDirectory() cls.img_file = Path(cls.tmp_dir.name).joinpath('test.png') image = Image.new('RGBA', size=(WIDTH, HEIGHT), color=(155, 0, 0)) @@ -40,7 +40,7 @@ def setUpClass(cls): cls.eid = str(eid) def _get_image(self, url): - # This is a bit of a hack because when running a the server locally, the request to the media folder fail + # This is a bit of a hack because when running the server locally, the request to the media folder fails rel_path = urlparse(url).path[1:] try: img_file = list(Path('/var/www/').rglob(rel_path))[0] diff --git a/release_notes.md b/release_notes.md index 9320ad36c..40b777248 100644 --- a/release_notes.md +++ b/release_notes.md @@ -13,6 +13,9 @@ - Typo in ibllib.pipes.video_tasks.EphysPostDLC class - ibllib.io.raw_data_loaders.patch_settings works with iblrigv8 settings files +#### 2.28.2 +- Fix loading of personal projects extractor map + ## Release Notes 2.27 ### features From 669b37be0b1e3b3c5df27d0f6761d3c4536b6b76 Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Thu, 25 Jan 2024 15:59:58 +0200 Subject: [PATCH 18/20] Issue #688 - short circuit push builds if commit already associated with a PR. Credit to https://stackoverflow.com/a/75716200 --- .github/workflows/ibllib_ci.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/ibllib_ci.yml b/.github/workflows/ibllib_ci.yml index 20564820d..ec50517f4 100644 --- a/.github/workflows/ibllib_ci.yml +++ b/.github/workflows/ibllib_ci.yml @@ -10,9 +10,28 @@ on: branches: [ master, develop ] jobs: + detect-outstanding-prs: # Don't run builds for push events if associated with PR + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ github.token }} + outputs: + abort: ${{ steps.debounce.outputs.abort }} + steps: + - name: Debounce + if: github.event_name == 'push' + id: debounce + run: | + pr_branches=$(gh pr list --json headRefName --repo $GITHUB_REPOSITORY) + if [[ $(echo "$pr_branches" | jq -r --arg GITHUB_REF '.[].headRefName | select(. == $GITHUB_REF)') ]]; then + echo "This push is associated with a pull request. Skipping the job." + echo "abort=true" >> "$GITHUB_OUTPUT" + fi + build: name: build (${{ matrix.python-version }}, ${{ matrix.os }}) runs-on: ${{ matrix.os }} + needs: debounce + if: needs.debounce.outputs.abort != 'true' strategy: fail-fast: false # Whether to stop execution of other instances max-parallel: 2 From 31285365466f2ed473b572fcf4b3a97e73c97396 Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Fri, 26 Jan 2024 08:53:06 +0000 Subject: [PATCH 19/20] Raw data loaders in brainbox.io (#709) * spike sorting loader get version method * bugfix in acquisition description if empty protocol * add raw voltage traces loaders in SpikeSortingLoader * remove dangling code and refactor atlas deprecation warnings * add ONE requirement for large ephys files download * bugfix session loader trial collection - second time --- brainbox/io/one.py | 51 ++++++++++++++++++++++++++++++++++++- ibllib/atlas/genes.py | 2 +- ibllib/io/session_params.py | 5 +++- requirements.txt | 2 +- 4 files changed, 56 insertions(+), 4 deletions(-) diff --git a/brainbox/io/one.py b/brainbox/io/one.py index b7fb5e535..4c09579f3 100644 --- a/brainbox/io/one.py +++ b/brainbox/io/one.py @@ -28,6 +28,7 @@ from ibllib.plots import vertical_lines import brainbox.plot +from brainbox.io.spikeglx import Streamer from brainbox.ephys_plots import plot_brain_regions from brainbox.metrics.single_units import quick_unit_metrics from brainbox.behavior.wheel import interpolate_position, velocity_filtered @@ -793,6 +794,7 @@ class SpikeSortingLoader: datasets: list = None # list of all datasets belonging to the session # the following properties are the outcome of a reading function files: dict = None + raw_data_files: list = None # list of raw ap and lf files corresponding to the recording collection: str = '' histology: str = '' # 'alf', 'resolved', 'aligned' or 'traced' spike_sorter: str = 'pykilosort' @@ -829,6 +831,7 @@ def __post_init__(self): if self.atlas is None: self.atlas = AllenAtlas() self.files = {} + self.raw_data_files = [] def _load_object(self, *args, **kwargs): """ @@ -881,6 +884,11 @@ def load_spike_sorting_object(self, obj, *args, **kwargs): self.download_spike_sorting_object(obj, *args, **kwargs) return self._load_object(self.files[obj]) + def get_version(self, spike_sorter='pykilosort'): + collection = self._get_spike_sorting_collection(spike_sorter=spike_sorter) + dset = self.one.alyx.rest('datasets', 'list', session=self.eid, collection=collection, name='spikes.times.npy') + return dset[0]['version'] if len(dset) else 'unknown' + def download_spike_sorting_object(self, obj, spike_sorter='pykilosort', dataset_types=None, collection=None, missing='raise', **kwargs): """ @@ -919,6 +927,46 @@ def download_spike_sorting(self, **kwargs): self.download_spike_sorting_object(obj=obj, **kwargs) self.spike_sorting_path = self.files['spikes'][0].parent + def download_raw_electrophysiology(self, band='ap'): + """ + Downloads raw electrophysiology data files on local disk. + :param band: "ap" (default) or "lf" for LFP band + :return: list of raw data files full paths (ch, meta and cbin files) + """ + raw_data_files = [] + for suffix in [f'*.{band}.ch', f'*.{band}.meta', f'*.{band}.cbin']: + try: + # FIXME: this will fail if multiple LFP segments are found + raw_data_files.append(self.one.load_dataset( + self.eid, + download_only=True, + collection=f'raw_ephys_data/{self.pname}', + dataset=suffix, + check_hash=False, + )) + except ALFObjectNotFound: + _logger.debug(f"{self.session_path} can't locate raw data collection raw_ephys_data/{self.pname}, file {suffix}") + self.raw_data_files = list(set(self.raw_data_files + raw_data_files)) + return raw_data_files + + def raw_electrophysiology(self, stream=True, band='ap', **kwargs): + """ + Returns a reader for the raw electrophysiology data + By default it is a streamer object, but if stream is False, it will return a spikeglx.Reader after having + downloaded the raw data file if necessary + :param stream: + :param band: + :param kwargs: + :return: + """ + if stream: + return Streamer(pid=self.pid, one=self.one, typ=band, **kwargs) + else: + raw_data_files = self.download_raw_electrophysiology(band=band) + cbin_file = next(filter(lambda f: f.name.endswith(f'.{band}.cbin'), raw_data_files), None) + if cbin_file is not None: + return spikeglx.Reader(cbin_file) + def load_channels(self, **kwargs): """ Loads channels @@ -1282,7 +1330,8 @@ def load_trials(self): """ # itiDuration frequently has a mismatched dimension, and we don't need it, exclude using regex self.one.wildcards = False - self.trials = self.one.load_object(self.eid, 'trials', collection='alf', attribute=r'(?!itiDuration).*').to_df() + self.trials = self.one.load_object( + self.eid, 'trials', collection='alf', attribute=r'(?!itiDuration).*').to_df() self.one.wildcards = True self.data_info.loc[self.data_info['name'] == 'trials', 'is_loaded'] = True diff --git a/ibllib/atlas/genes.py b/ibllib/atlas/genes.py index 34ad6c73e..fd4784cba 100644 --- a/ibllib/atlas/genes.py +++ b/ibllib/atlas/genes.py @@ -1,6 +1,6 @@ """Gene expression maps.""" -from iblatlas import genes +from iblatlas.genomics import genes from ibllib.atlas import deprecated_decorator diff --git a/ibllib/io/session_params.py b/ibllib/io/session_params.py index 174cf16a4..8d0d9e16f 100644 --- a/ibllib/io/session_params.py +++ b/ibllib/io/session_params.py @@ -318,7 +318,10 @@ def get_task_protocol(sess_params, task_collection=None): """ collections = get_collections({'tasks': sess_params.get('tasks')}) if task_collection is None: - return set(collections.keys()) # Return all protocols + if len(collections) == 0: + return None + else: + return set(collections.keys()) # Return all protocols else: return next((k for k, v in collections.items() if v == task_collection), None) diff --git a/requirements.txt b/requirements.txt index 242ac9a0b..e3baeede0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,7 @@ tqdm>=4.32.1 ibl-neuropixel>=0.8.1 iblutil>=1.7.0 labcams # widefield extractor -ONE-api>=2.5 +ONE-api>=2.6 slidingRP>=1.0.0 # steinmetz lab refractory period metrics wfield==0.3.7 # widefield extractor frozen for now (2023/07/15) until Joao fixes latest version psychofit From 043441f5481f06af717a547e4b8aafe1c3e9072d Mon Sep 17 00:00:00 2001 From: olivier Date: Fri, 26 Jan 2024 09:14:00 +0000 Subject: [PATCH 20/20] release 2.29.0 --- ibllib/__init__.py | 2 +- release_notes.md | 5 +++++ requirements.txt | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ibllib/__init__.py b/ibllib/__init__.py index e10341c08..aea0d234d 100644 --- a/ibllib/__init__.py +++ b/ibllib/__init__.py @@ -2,7 +2,7 @@ import logging import warnings -__version__ = '2.28.2' +__version__ = '2.29.0' warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib') # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev diff --git a/release_notes.md b/release_notes.md index 40b777248..33bcc69f0 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,3 +1,8 @@ +## Release Notes 2.29 + +### features +- Added raw data loaders and synchronisation tools in brainbox.io.one.SpikeSortingLoader, method `ssl.raw_electrophysiology()` + ## Release Notes 2.28 ### features diff --git a/requirements.txt b/requirements.txt index e3baeede0..37d9af4b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,13 +5,11 @@ flake8>=3.7.8 globus-sdk graphviz matplotlib>=3.0.3 -mtscomp>=1.0.1 numba>=0.56 numpy>=1.18 nptdms opencv-python-headless pandas -phylib>=2.4 pyarrow pynrrd>=0.4.0 pytest @@ -23,11 +21,13 @@ sparse seaborn>=0.9.0 tqdm>=4.32.1 # ibl libraries +iblatlas>=0.4.0 ibl-neuropixel>=0.8.1 iblutil>=1.7.0 labcams # widefield extractor +mtscomp>=1.0.1 ONE-api>=2.6 +phylib>=2.4 +psychofit slidingRP>=1.0.0 # steinmetz lab refractory period metrics wfield==0.3.7 # widefield extractor frozen for now (2023/07/15) until Joao fixes latest version -psychofit -iblatlas