From 9bda4918b111deab30cce0436bcbf3f64b624ca3 Mon Sep 17 00:00:00 2001 From: k1o0 Date: Thu, 4 Jan 2024 17:18:11 +0200 Subject: [PATCH] Develop (#703) * change behavior qc to truncate trials up until 400 and check if performance passes * add test for sleepless decorator * Add get_trials_tasks function * data release update * change number of parallel workflows * Issue #701 --------- Co-authored-by: juhuntenburg Co-authored-by: Florian Rau Co-authored-by: Gaelle Co-authored-by: GaelleChapuis <43007596+GaelleChapuis@users.noreply.github.com> Co-authored-by: Mayo Faulkner --- .github/workflows/ibllib_ci.yml | 2 +- .../data_release_brainwidemap.ipynb | 9 +- ibllib/__init__.py | 2 +- ibllib/io/extractors/base.py | 161 ++++++++++++++---- ibllib/io/extractors/camera.py | 10 +- ibllib/io/extractors/ephys_fpga.py | 2 +- ibllib/pipes/behavior_tasks.py | 15 +- ibllib/pipes/dynamic_pipeline.py | 70 +++++++- ibllib/pipes/misc.py | 61 ++++++- ibllib/pipes/training_preprocessing.py | 2 +- ibllib/tests/fixtures/utils.py | 22 ++- ibllib/tests/test_dynamic_pipeline.py | 104 ++++++++++- ibllib/tests/test_pipes.py | 19 +++ release_notes.md | 11 ++ 14 files changed, 421 insertions(+), 69 deletions(-) diff --git a/.github/workflows/ibllib_ci.yml b/.github/workflows/ibllib_ci.yml index 64a98c108..20564820d 100644 --- a/.github/workflows/ibllib_ci.yml +++ b/.github/workflows/ibllib_ci.yml @@ -15,7 +15,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: fail-fast: false # Whether to stop execution of other instances - max-parallel: 4 + max-parallel: 2 matrix: os: ["windows-latest", "ubuntu-latest"] python-version: ["3.8", "3.11"] diff --git a/examples/data_release/data_release_brainwidemap.ipynb b/examples/data_release/data_release_brainwidemap.ipynb index 0c3b0a402..001fed7cf 100644 --- a/examples/data_release/data_release_brainwidemap.ipynb +++ b/examples/data_release/data_release_brainwidemap.ipynb @@ -11,14 +11,14 @@ "source": [ "# Data Release - Brain Wide Map\n", "\n", - "IBL aims to understand the neural basis of decision-making in the mouse by gathering a whole-brain activity map composed of electrophysiological recordings pooled from multiple laboratories. We have systematically recorded from nearly all major brain areas with Neuropixels probes, using a grid system for unbiased sampling and replicating each recording site in at least two laboratories. These data have been used to construct a brain-wide map of activity at single-spike cellular resolution during a [decision-making task]((https://elifesciences.org/articles/63711)). In addition to the map, this data set contains other information gathered during the task: sensory stimuli presented to the mouse; mouse decisions and response times; and mouse pose information from video recordings and DeepLabCut analysis. Please read our accompanying [technical paper](https://doi.org/10.6084/m9.figshare.21400815) for details on the experiment and data processing pipelines. To explore the data, visit [our vizualisation website](https://viz.internationalbrainlab.org/)." + "IBL aims to understand the neural basis of decision-making in the mouse by gathering a whole-brain activity map composed of electrophysiological recordings pooled from multiple laboratories. We have systematically recorded from nearly all major brain areas with Neuropixels probes, using a grid system for unbiased sampling and replicating each recording site in at least two laboratories. These data have been used to construct a brain-wide map of activity at single-spike cellular resolution during a [decision-making task]((https://elifesciences.org/articles/63711)). Please read the associated article [(IBL et al. 2023)](https://www.biorxiv.org/content/10.1101/2023.07.04.547681v2). In addition to the map, this data set contains other information gathered during the task: sensory stimuli presented to the mouse; mouse decisions and response times; and mouse pose information from video recordings and DeepLabCut analysis. Please read our accompanying [technical paper](https://doi.org/10.6084/m9.figshare.21400815) for details on the experiment and data processing pipelines. To explore the data, visit [our vizualisation website](https://viz.internationalbrainlab.org/)." ] }, { "cell_type": "markdown", "source": [ "## Overview of the Data\n", - "We have released data from 354 Neuropixel recording sessions, which encompass 547 probe insertions, obtained in 115 subjects performing the IBL task across 11 different laboratories. As output of spike-sorting, there are 295501 units; of which 32766 are considered to be of good quality. These units were recorded in overall 194 different brain regions.\n", + "We have released data from 459 Neuropixel recording sessions, which encompass 699 probe insertions, obtained in 139 subjects performing the IBL task across 12 different laboratories. As output of spike-sorting, there are 376730 units; of which 45085 are considered to be of good quality. In total, 138 brain regions were recorded in sufficient numbers for inclusion in IBL’s analyses [(IBL et al. 2023)](https://www.biorxiv.org/content/10.1101/2023.07.04.547681v2).\n", "\n", "## Data structure and download\n", "The organisation of the data follows the standard IBL data structure.\n", @@ -31,7 +31,10 @@ "\n", "Note:\n", "\n", - "* The tag associated to this release is `2022_Q4_IBL_et_al_BWM`" + "* The tag associated to this release is `Brainwidemap`\n", + "\n", + "## Receive updates on the data\n", + "To receive a notification that we released new datasets, please fill up [this form](https://forms.gle/9ex2vL1JwV4QXnf98)\n" ], "metadata": { "collapsed": false diff --git a/ibllib/__init__.py b/ibllib/__init__.py index b55499130..2cbd2f779 100644 --- a/ibllib/__init__.py +++ b/ibllib/__init__.py @@ -2,7 +2,7 @@ import logging import warnings -__version__ = '2.27.1' +__version__ = '2.28' warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib') # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev diff --git a/ibllib/io/extractors/base.py b/ibllib/io/extractors/base.py index e49d47980..13f594217 100644 --- a/ibllib/io/extractors/base.py +++ b/ibllib/io/extractors/base.py @@ -1,4 +1,5 @@ """Base Extractor classes. + A module for the base Extractor classes. The Extractor, given a session path, will extract the processed data from raw hardware files and optionally save them. """ @@ -10,7 +11,6 @@ import numpy as np import pandas as pd -from one.alf.files import get_session_path from ibllib.io import raw_data_loaders as raw from ibllib.io.raw_data_loaders import load_settings, _logger @@ -162,7 +162,8 @@ def extract(self, bpod_trials=None, settings=None, **kwargs): def run_extractor_classes(classes, session_path=None, **kwargs): """ - Run a set of extractors with the same inputs + Run a set of extractors with the same inputs. + :param classes: list of Extractor class :param save: True/False :param path_out: (defaults to alf path) @@ -195,12 +196,30 @@ def run_extractor_classes(classes, session_path=None, **kwargs): def _get_task_types_json_config(): + """ + Return the extractor types map. + + This function is only used for legacy sessions, i.e. those without an experiment description + file and will be removed in favor of :func:`_get_task_extractor_map`, which directly returns + the Bpod extractor class name. The experiment description file cuts out the need for pipeline + name identifiers. + + Returns + ------- + Dict[str, str] + A map of task protocol to task extractor identifier, e.g. 'ephys', 'habituation', etc. + + See Also + -------- + _get_task_extractor_map - returns a map of task protocol to Bpod trials extractor class name. + """ with open(Path(__file__).parent.joinpath('extractor_types.json')) as fp: task_types = json.load(fp) try: # look if there are custom extractor types in the personal projects repo import projects.base custom_extractors = Path(projects.base.__file__).parent.joinpath('extractor_types.json') + _logger.debug('Loading extractor types from %s', custom_extractors) with open(custom_extractors) as fp: custom_task_types = json.load(fp) task_types.update(custom_task_types) @@ -210,8 +229,28 @@ def _get_task_types_json_config(): def get_task_protocol(session_path, task_collection='raw_behavior_data'): + """ + Return the task protocol name from task settings. + + If the session path and/or task collection do not exist, the settings file is missing or + otherwise can not be parsed, or if the 'PYBPOD_PROTOCOL' key is absent, None is returned. + A warning is logged if the session path or settings file doesn't exist. An error is logged if + the settings file can not be parsed. + + Parameters + ---------- + session_path : str, pathlib.Path + The absolute session path. + task_collection : str + The session path directory containing the task settings file. + + Returns + ------- + str or None + The Pybpod task protocol name or None if not found. + """ try: - settings = load_settings(get_session_path(session_path), task_collection=task_collection) + settings = load_settings(session_path, task_collection=task_collection) except json.decoder.JSONDecodeError: _logger.error(f'Can\'t read settings for {session_path}') return @@ -223,11 +262,26 @@ def get_task_protocol(session_path, task_collection='raw_behavior_data'): def get_task_extractor_type(task_name): """ - Returns the task type string from the full pybpod task name: - _iblrig_tasks_biasedChoiceWorld3.7.0 returns "biased" - _iblrig_tasks_trainingChoiceWorld3.6.0 returns "training' - :param task_name: - :return: one of ['biased', 'habituation', 'training', 'ephys', 'mock_ephys', 'sync_ephys'] + Returns the task type string from the full pybpod task name. + + Parameters + ---------- + task_name : str + The complete task protocol name from the PYBPOD_PROTOCOL field of the task settings. + + Returns + ------- + str + The extractor type identifier. Examples include 'biased', 'habituation', 'training', + 'ephys', 'mock_ephys' and 'sync_ephys'. + + Examples + -------- + >>> get_task_extractor_type('_iblrig_tasks_biasedChoiceWorld3.7.0') + 'biased' + + >>> get_task_extractor_type('_iblrig_tasks_trainingChoiceWorld3.6.0') + 'training' """ if isinstance(task_name, Path): task_name = get_task_protocol(task_name) @@ -245,16 +299,30 @@ def get_task_extractor_type(task_name): def get_session_extractor_type(session_path, task_collection='raw_behavior_data'): """ - From a session path, loads the settings file, finds the task and checks if extractors exist - task names examples: - :param session_path: - :return: bool + Infer trials extractor type from task settings. + + From a session path, loads the settings file, finds the task and checks if extractors exist. + Examples include 'biased', 'habituation', 'training', 'ephys', 'mock_ephys', and 'sync_ephys'. + Note this should only be used for legacy sessions, i.e. those without an experiment description + file. + + Parameters + ---------- + session_path : str, pathlib.Path + The session path for which to determine the pipeline. + task_collection : str + The session path directory containing the raw task data. + + Returns + ------- + str or False + The task extractor type, e.g. 'biased', 'habituation', 'ephys', or False if unknown. """ - settings = load_settings(session_path, task_collection=task_collection) - if settings is None: - _logger.error(f'ABORT: No data found in "{task_collection}" folder {session_path}') + task_protocol = get_task_protocol(session_path, task_collection=task_collection) + if task_protocol is None: + _logger.error(f'ABORT: No task protocol found in "{task_collection}" folder {session_path}') return False - extractor_type = get_task_extractor_type(settings['PYBPOD_PROTOCOL']) + extractor_type = get_task_extractor_type(task_protocol) if extractor_type: return extractor_type else: @@ -263,9 +331,22 @@ def get_session_extractor_type(session_path, task_collection='raw_behavior_data' def get_pipeline(session_path, task_collection='raw_behavior_data'): """ - Get the pre-processing pipeline name from a session path - :param session_path: - :return: + Get the pre-processing pipeline name from a session path. + + Note this is only suitable for legacy sessions, i.e. those without an experiment description + file. This function will be removed in the future. + + Parameters + ---------- + session_path : str, pathlib.Path + The session path for which to determine the pipeline. + task_collection : str + The session path directory containing the raw task data. + + Returns + ------- + str + The pipeline name inferred from the extractor type, e.g. 'ephys', 'training', 'widefield'. """ stype = get_session_extractor_type(session_path, task_collection=task_collection) return _get_pipeline_from_task_type(stype) @@ -273,18 +354,29 @@ def get_pipeline(session_path, task_collection='raw_behavior_data'): def _get_pipeline_from_task_type(stype): """ - Returns the pipeline from the task type. Some tasks types directly define the pipeline - :param stype: session_type or task extractor type - :return: + Return the pipeline from the task type. + + Some task types directly define the pipeline. Note this is only suitable for legacy sessions, + i.e. those without an experiment description file. This function will be removed in the future. + + Parameters + ---------- + stype : str + The session type or task extractor type, e.g. 'habituation', 'ephys', etc. + + Returns + ------- + str + A task pipeline identifier. """ if stype in ['ephys_biased_opto', 'ephys', 'ephys_training', 'mock_ephys', 'sync_ephys']: return 'ephys' elif stype in ['habituation', 'training', 'biased', 'biased_opto']: return 'training' - elif 'widefield' in stype: + elif isinstance(stype, str) and 'widefield' in stype: return 'widefield' else: - return stype + return stype or '' def _get_task_extractor_map(): @@ -293,7 +385,7 @@ def _get_task_extractor_map(): Returns ------- - dict(str, str) + Dict[str, str] A map of task protocol to Bpod trials extractor class. """ FILENAME = 'task_extractor_map.json' @@ -315,26 +407,26 @@ def get_bpod_extractor_class(session_path, task_collection='raw_behavior_data'): """ Get the Bpod trials extractor class associated with a given Bpod session. + Note that unlike :func:`get_session_extractor_type`, this function maps directly to the Bpod + trials extractor class name. This is hardware invariant and is purly to determine the Bpod only + trials extractor. + Parameters ---------- session_path : str, pathlib.Path The session path containing Bpod behaviour data. task_collection : str - The session_path subfolder containing the Bpod settings file. + The session_path sub-folder containing the Bpod settings file. Returns ------- str The extractor class name. """ - # Attempt to load settings files - settings = load_settings(session_path, task_collection=task_collection) - if settings is None: - raise ValueError(f'No data found in "{task_collection}" folder {session_path}') - # Attempt to get task protocol - protocol = settings.get('PYBPOD_PROTOCOL') + # Attempt to get protocol name from settings file + protocol = get_task_protocol(session_path, task_collection=task_collection) if not protocol: - raise ValueError(f'No task protocol found in {session_path/task_collection}') + raise ValueError(f'No task protocol found in {Path(session_path) / task_collection}') return protocol2extractor(protocol) @@ -342,7 +434,8 @@ def protocol2extractor(protocol): """ Get the Bpod trials extractor class associated with a given Bpod task protocol. - The Bpod task protocol can be found in the 'PYBPOD_PROTOCOL' field of _iblrig_taskSettings.raw.json. + The Bpod task protocol can be found in the 'PYBPOD_PROTOCOL' field of the + _iblrig_taskSettings.raw.json file. Parameters ---------- diff --git a/ibllib/io/extractors/camera.py b/ibllib/io/extractors/camera.py index 93554c86a..a44010821 100644 --- a/ibllib/io/extractors/camera.py +++ b/ibllib/io/extractors/camera.py @@ -1,4 +1,5 @@ """ Camera extractor functions. + This module handles extraction of camera timestamps for both Bpod and DAQ. """ import logging @@ -29,7 +30,7 @@ def extract_camera_sync(sync, chmap=None): """ - Extract camera timestamps from the sync matrix + Extract camera timestamps from the sync matrix. :param sync: dictionary 'times', 'polarities' of fronts detected on sync trace :param chmap: dictionary containing channel indices. Default to constant. @@ -45,7 +46,8 @@ def extract_camera_sync(sync, chmap=None): def get_video_length(video_path): """ - Returns video length + Returns video length. + :param video_path: A path to the video :return: """ @@ -58,9 +60,7 @@ def get_video_length(video_path): class CameraTimestampsFPGA(BaseExtractor): - """ - Extractor for videos using DAQ sync and channel map. - """ + """Extractor for videos using DAQ sync and channel map.""" def __init__(self, label, session_path=None): super().__init__(session_path) diff --git a/ibllib/io/extractors/ephys_fpga.py b/ibllib/io/extractors/ephys_fpga.py index f19fbdd68..009f68c52 100644 --- a/ibllib/io/extractors/ephys_fpga.py +++ b/ibllib/io/extractors/ephys_fpga.py @@ -1483,7 +1483,7 @@ def extract_all(session_path, sync_collection='raw_ephys_data', save=True, save_ # Sync Bpod trials to FPGA sync, chmap = get_sync_and_chn_map(session_path, sync_collection) # sync, chmap = get_main_probe_sync(session_path, bin_exists=bin_exists) - trials = FpgaTrials(session_path, bpod_trials=bpod_trials | bpod_wheel) + trials = FpgaTrials(session_path, bpod_trials={**bpod_trials, **bpod_wheel}) # py3.9 -> | outputs, files = trials.extract( save=save, sync=sync, chmap=chmap, path_out=save_path, task_collection=task_collection, protocol_number=protocol_number, **kwargs) diff --git a/ibllib/pipes/behavior_tasks.py b/ibllib/pipes/behavior_tasks.py index 5e7e5d829..1daf04813 100644 --- a/ibllib/pipes/behavior_tasks.py +++ b/ibllib/pipes/behavior_tasks.py @@ -374,17 +374,26 @@ def signature(self): } return signature - def _behaviour_criterion(self, update=True): + def _behaviour_criterion(self, update=True, truncate_to_pass=True): """ Computes and update the behaviour criterion on Alyx """ from brainbox.behavior import training - trials = alfio.load_object(self.session_path.joinpath(self.output_collection), 'trials') + trials = alfio.load_object(self.session_path.joinpath(self.output_collection), 'trials').to_df() good_enough = training.criterion_delay( - n_trials=trials["intervals"].shape[0], + n_trials=trials.shape[0], perf_easy=training.compute_performance_easy(trials), ) + if truncate_to_pass and not good_enough: + n_trials = trials.shape[0] + while not good_enough and n_trials > 400: + n_trials -= 1 + good_enough = training.criterion_delay( + n_trials=n_trials, + perf_easy=training.compute_performance_easy(trials[:n_trials]), + ) + if update: eid = self.one.path2eid(self.session_path, query_type='remote') self.one.alyx.json_field_update( diff --git a/ibllib/pipes/dynamic_pipeline.py b/ibllib/pipes/dynamic_pipeline.py index d95497380..ec4228256 100644 --- a/ibllib/pipes/dynamic_pipeline.py +++ b/ibllib/pipes/dynamic_pipeline.py @@ -13,7 +13,7 @@ import spikeglx import ibllib.io.session_params as sess_params -import ibllib.io.extractors.base +from ibllib.io.extractors.base import get_pipeline, get_session_extractor_type import ibllib.pipes.tasks as mtasks import ibllib.pipes.base_tasks as bstasks import ibllib.pipes.widefield_tasks as wtasks @@ -45,7 +45,7 @@ def acquisition_description_legacy_session(session_path, save=False): dict The legacy acquisition description. """ - extractor_type = ibllib.io.extractors.base.get_session_extractor_type(session_path=session_path) + extractor_type = get_session_extractor_type(session_path) etype2protocol = dict(biased='choice_world_biased', habituation='choice_world_habituation', training='choice_world_training', ephys='choice_world_recording') dict_ad = get_acquisition_description(etype2protocol[extractor_type]) @@ -130,7 +130,7 @@ def make_pipeline(session_path, **pkwargs): ---------- session_path : str, Path The absolute session path, i.e. '/path/to/subject/yyyy-mm-dd/nnn'. - **pkwargs + pkwargs Optional arguments passed to the ibllib.pipes.tasks.Pipeline constructor. Returns @@ -147,7 +147,7 @@ def make_pipeline(session_path, **pkwargs): if not acquisition_description: raise ValueError('Experiment description file not found or is empty') devices = acquisition_description.get('devices', {}) - kwargs = {'session_path': session_path} + kwargs = {'session_path': session_path, 'one': pkwargs.get('one')} # Registers the experiment description file tasks['ExperimentDescriptionRegisterRaw'] = type('ExperimentDescriptionRegisterRaw', @@ -430,3 +430,65 @@ def load_pipeline_dict(path): task_list = yaml.full_load(file) return task_list + + +def get_trials_tasks(session_path, one=None): + """ + Return a list of pipeline trials extractor task objects for a given session. + + This function supports both legacy and dynamic pipeline sessions. + + Parameters + ---------- + session_path : str, pathlib.Path + An absolute path to a session. + one : one.api.One + An ONE instance. + + Returns + ------- + list of pipes.tasks.Task + A list of task objects for the provided session. + + """ + # Check for an experiment.description file; ensure downloaded if possible + if one and one.to_eid(session_path): # to_eid returns None if session not registered + one.load_datasets(session_path, ['_ibl_experiment.description'], download_only=True, assert_present=False) + experiment_description = sess_params.read_params(session_path) + + # If experiment description file then use this to make the pipeline + if experiment_description is not None: + tasks = [] + pipeline = make_pipeline(session_path, one=one) + trials_tasks = [t for t in pipeline.tasks if 'Trials' in t] + for task in trials_tasks: + t = pipeline.tasks.get(task) + t.__init__(session_path, **t.kwargs) + tasks.append(t) + else: + # Otherwise default to old way of doing things + pipeline = get_pipeline(session_path) + if pipeline == 'training': + from ibllib.pipes.training_preprocessing import TrainingTrials + tasks = [TrainingTrials(session_path, one=one)] + elif pipeline == 'ephys': + from ibllib.pipes.ephys_preprocessing import EphysTrials + tasks = [EphysTrials(session_path, one=one)] + else: + try: + # try to find a custom extractor in the personal projects extraction class + import projects.base + task_type = get_session_extractor_type(session_path) + assert (PipelineClass := projects.base.get_pipeline(task_type)) + pipeline = PipelineClass(session_path, one=one) + trials_task_name = next((task for task in pipeline.tasks if 'Trials' in task), None) + assert trials_task_name, (f'No "Trials" tasks for custom pipeline ' + f'"{pipeline.name}" with extractor type "{task_type}"') + task = pipeline.tasks.get(trials_task_name) + task(session_path) + tasks = [task] + except (ModuleNotFoundError, AssertionError) as ex: + _logger.warning('Failed to get trials tasks: %s', ex) + tasks = [] + + return tasks diff --git a/ibllib/pipes/misc.py b/ibllib/pipes/misc.py index 39871ad00..37a761f03 100644 --- a/ibllib/pipes/misc.py +++ b/ibllib/pipes/misc.py @@ -9,8 +9,9 @@ import sys import time import logging +from functools import wraps from pathlib import Path -from typing import Union, List +from typing import Union, List, Callable, Any from inspect import signature import uuid import socket @@ -38,6 +39,7 @@ def subjects_data_folder(folder: Path, rglob: bool = False) -> Path: """Given a root_data_folder will try to find a 'Subjects' data folder. + If Subjects folder is passed will return it directly.""" if not isinstance(folder, Path): folder = Path(folder) @@ -1148,13 +1150,54 @@ class WindowsInhibitor: ES_CONTINUOUS = 0x80000000 ES_SYSTEM_REQUIRED = 0x00000001 - def __init__(self): - pass + @staticmethod + def _set_thread_execution_state(state: int) -> None: + result = ctypes.windll.kernel32.SetThreadExecutionState(state) + if result == 0: + log.error("Failed to set thread execution state.") + + @staticmethod + def inhibit(quiet: bool = False): + if quiet: + log.debug("Preventing Windows from going to sleep") + else: + print("Preventing Windows from going to sleep") + WindowsInhibitor._set_thread_execution_state(WindowsInhibitor.ES_CONTINUOUS | WindowsInhibitor.ES_SYSTEM_REQUIRED) + + @staticmethod + def uninhibit(quiet: bool = False): + if quiet: + log.debug("Allowing Windows to go to sleep") + else: + print("Allowing Windows to go to sleep") + WindowsInhibitor._set_thread_execution_state(WindowsInhibitor.ES_CONTINUOUS) + + +def sleepless(func: Callable[..., Any]) -> Callable[..., Any]: + """ + Decorator to ensure that the system doesn't enter sleep or idle mode during a long-running task. - def inhibit(self): - print("Preventing Windows from going to sleep") - ctypes.windll.kernel32.SetThreadExecutionState(WindowsInhibitor.ES_CONTINUOUS | WindowsInhibitor.ES_SYSTEM_REQUIRED) + This decorator wraps a function and sets the thread execution state to prevent + the system from entering sleep or idle mode while the decorated function is + running. + + Parameters + ---------- + func : callable + The function to decorate. + + Returns + ------- + callable + The decorated function. + """ - def uninhibit(self): - print("Allowing Windows to go to sleep") - ctypes.windll.kernel32.SetThreadExecutionState(WindowsInhibitor.ES_CONTINUOUS) + @wraps(func) + def inner(*args, **kwargs) -> Any: + if os.name == 'nt': + WindowsInhibitor().inhibit(quiet=True) + result = func(*args, **kwargs) + if os.name == 'nt': + WindowsInhibitor().uninhibit(quiet=True) + return result + return inner diff --git a/ibllib/pipes/training_preprocessing.py b/ibllib/pipes/training_preprocessing.py index db41f8992..ad2172809 100644 --- a/ibllib/pipes/training_preprocessing.py +++ b/ibllib/pipes/training_preprocessing.py @@ -19,7 +19,7 @@ from ibllib.qc.task_extractors import TaskQCExtractor _logger = logging.getLogger(__name__) -warnings.warn('`pipes.training_preprocessing` to be removed in favour of dynamic pipeline') +warnings.warn('`pipes.training_preprocessing` to be removed in favour of dynamic pipeline', FutureWarning) # level 0 diff --git a/ibllib/tests/fixtures/utils.py b/ibllib/tests/fixtures/utils.py index ac7ac5f71..f536875d0 100644 --- a/ibllib/tests/fixtures/utils.py +++ b/ibllib/tests/fixtures/utils.py @@ -216,7 +216,7 @@ def create_fake_raw_behavior_data_folder( ): """Create the folder structure for a raw behaviour session. - Creates a raw_behavior_data folder and optionally, touches some files and writes a experiment + Creates a raw_behavior_data folder and optionally, touches some files and writes an experiment description stub to a `_devices` folder. Parameters @@ -304,8 +304,26 @@ def create_fake_raw_behavior_data_folder( def populate_task_settings(fpath: Path, patch: dict): - with fpath.open("w") as f: + """ + Populate a task settings JSON file. + + Parameters + ---------- + fpath : pathlib.Path + A path to a raw task settings folder or the full settings file path. + patch : dict + The settings dict to write to file. + + Returns + ------- + pathlib.Path + The full settings file path. + """ + if fpath.is_dir(): + fpath /= '_iblrig_taskSettings.raw.json' + with fpath.open('w') as f: json.dump(patch, f, indent=1) + return fpath def create_fake_complete_ephys_session( diff --git a/ibllib/tests/test_dynamic_pipeline.py b/ibllib/tests/test_dynamic_pipeline.py index 8b32b5ff6..41420c674 100644 --- a/ibllib/tests/test_dynamic_pipeline.py +++ b/ibllib/tests/test_dynamic_pipeline.py @@ -1,15 +1,22 @@ import tempfile from pathlib import Path import unittest +from unittest import mock from itertools import chain +import yaml + import ibllib.tests -from ibllib.pipes import dynamic_pipeline +import ibllib.pipes.dynamic_pipeline as dyn +from ibllib.pipes.tasks import Pipeline, Task +from ibllib.pipes import ephys_preprocessing +from ibllib.pipes import training_preprocessing from ibllib.io import session_params +from ibllib.tests.fixtures.utils import populate_task_settings def test_read_write_params_yaml(): - ad = dynamic_pipeline.get_acquisition_description('choice_world_recording') + ad = dyn.get_acquisition_description('choice_world_recording') with tempfile.TemporaryDirectory() as td: session_path = Path(td) session_params.write_params(session_path, ad) @@ -21,14 +28,14 @@ class TestCreateLegacyAcqusitionDescriptions(unittest.TestCase): def test_legacy_biased(self): session_path = Path(ibllib.tests.__file__).parent.joinpath('extractors', 'data', 'session_biased_ge5') - ad = dynamic_pipeline.acquisition_description_legacy_session(session_path) + ad = dyn.acquisition_description_legacy_session(session_path) protocols = list(chain(*map(dict.keys, ad.get('tasks', [])))) self.assertCountEqual(['biasedChoiceWorld'], protocols) self.assertEqual(1, len(ad['devices']['cameras'])) def test_legacy_ephys(self): session_path = Path(ibllib.tests.__file__).parent.joinpath('extractors', 'data', 'session_ephys') - ad_ephys = dynamic_pipeline.acquisition_description_legacy_session(session_path) + ad_ephys = dyn.acquisition_description_legacy_session(session_path) self.assertEqual(2, len(ad_ephys['devices']['neuropixel'])) self.assertEqual(3, len(ad_ephys['devices']['cameras'])) protocols = list(chain(*map(dict.keys, ad_ephys.get('tasks', [])))) @@ -36,7 +43,94 @@ def test_legacy_ephys(self): def test_legacy_training(self): session_path = Path(ibllib.tests.__file__).parent.joinpath('extractors', 'data', 'session_training_ge5') - ad = dynamic_pipeline.acquisition_description_legacy_session(session_path) + ad = dyn.acquisition_description_legacy_session(session_path) protocols = list(chain(*map(dict.keys, ad.get('tasks', [])))) self.assertCountEqual(['trainingChoiceWorld'], protocols) self.assertEqual(1, len(ad['devices']['cameras'])) + + +class TestGetTrialsTasks(unittest.TestCase): + """Test pipes.dynamic_pipeline.get_trials_tasks function.""" + + def setUp(self): + tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(tmpdir.cleanup) + # The github CI root dir contains an alias/symlink so we must resolve it + self.tempdir = Path(tmpdir.name).resolve() + self.session_path_dynamic = self.tempdir / 'subject' / '2023-01-01' / '001' + self.session_path_dynamic.mkdir(parents=True) + description = {'version': '1.0.0', + 'sync': {'nidq': {'collection': 'raw_ephys_data', 'extension': 'bin', 'acquisition_software': 'spikeglx'}}, + 'tasks': [ + {'ephysChoiceWorld': {'task_collection': 'raw_task_data_00'}}, + {'passiveChoiceWorld': {'task_collection': 'raw_task_data_01'}}, + ]} + with open(self.session_path_dynamic / '_ibl_experiment.description.yaml', 'w') as fp: + yaml.safe_dump(description, fp) + + self.session_path_legacy = self.session_path_dynamic.with_name('002') + (collection := self.session_path_legacy.joinpath('raw_behavior_data')).mkdir(parents=True) + self.settings = {'IBLRIG_VERSION': '7.2.2', 'PYBPOD_PROTOCOL': '_iblrig_tasks_ephysChoiceWorld'} + self.settings_path = populate_task_settings(collection, self.settings) + + def test_get_trials_tasks(self): + """Test pipes.dynamic_pipeline.get_trials_tasks function.""" + # A dynamic pipeline session + tasks = dyn.get_trials_tasks(self.session_path_dynamic) + self.assertEqual(2, len(tasks)) + self.assertEqual('raw_task_data_00', tasks[0].collection) + + # Check behaviour with ONE + one = mock.MagicMock() + one.offline = False + one.alyx = mock.MagicMock() + one.alyx.cache_mode = None # sneaky hack as this is checked by the pipeline somewhere + tasks = dyn.get_trials_tasks(self.session_path_dynamic, one) + self.assertEqual(2, len(tasks)) + one.load_datasets.assert_called() # check that description file is checked on disk + + # An ephys session + tasks = dyn.get_trials_tasks(self.session_path_legacy) + self.assertEqual(1, len(tasks)) + self.assertIsInstance(tasks[0], ephys_preprocessing.EphysTrials) + + # A training session + self.settings['PYBPOD_PROTOCOL'] = '_iblrig_tasks_trainingChoiceWorld' + populate_task_settings(self.settings_path, self.settings) + + tasks = dyn.get_trials_tasks(self.session_path_legacy, one=one) + self.assertEqual(1, len(tasks)) + self.assertIsInstance(tasks[0], training_preprocessing.TrainingTrials) + self.assertIs(tasks[0].one, one, 'failed to assign ONE instance to task') + + # A personal project + self.settings['PYBPOD_PROTOCOL'] = '_misc_foobarChoiceWorld' + populate_task_settings(self.settings_path, self.settings) + + m = mock.MagicMock() # Mock the project_extractors repo + m.base.__file__ = str(self.tempdir / 'base.py') + # Create the personal project extractor types map + task_type_map = {'_misc_foobarChoiceWorld': 'foobar'} + extractor_types_path = Path(m.base.__file__).parent.joinpath('extractor_types.json') + populate_task_settings(extractor_types_path, task_type_map) + # Simulate the instantiation of the personal project module's pipeline class + pipeline = mock.Mock(spec=Pipeline) + pipeline.name = 'custom' + task_mock = mock.Mock(spec=Task) + pipeline.tasks = {'RegisterRaw': mock.MagicMock(), 'FooBarTrials': task_mock} + m.base.get_pipeline().return_value = pipeline + with mock.patch.dict('sys.modules', projects=m): + """For unknown reasons this method of mocking the personal projects repo (which is + imported within various functions) fails on the Github test builds. This we check + here and skip the rest of the test if patch didn't work.""" + try: + import projects.base + assert isinstance(projects.base, mock.Mock) + except (AssertionError, ModuleNotFoundError): + self.skipTest('Failed to mock projects module import') + tasks = dyn.get_trials_tasks(self.session_path_legacy) + self.assertEqual(1, len(tasks)) + task_mock.assert_called_once_with(self.session_path_legacy) + # Should handle absent trials tasks + pipeline.tasks.pop('FooBarTrials') + self.assertEqual([], dyn.get_trials_tasks(self.session_path_legacy)) diff --git a/ibllib/tests/test_pipes.py b/ibllib/tests/test_pipes.py index ba5c282dd..cbe86462a 100644 --- a/ibllib/tests/test_pipes.py +++ b/ibllib/tests/test_pipes.py @@ -21,6 +21,7 @@ import ibllib.io.extractors.base import ibllib.tests.fixtures.utils as fu from ibllib.pipes import misc +from ibllib.pipes.misc import sleepless from ibllib.tests import TEST_DB import ibllib.pipes.scan_fix_passive_files as fix from ibllib.pipes.base_tasks import RegisterRawDataTask @@ -698,5 +699,23 @@ def test_rename_files(self): self.assertCountEqual(expected, files) +class TestSleeplessDecorator(unittest.TestCase): + + def test_decorator_argument_passing(self): + + def dummy_function(arg1, arg2): + return arg1, arg2 + + # Applying the decorator to the dummy function + decorated_func = sleepless(dummy_function) + + # Check if the function name is maintained + self.assertEqual(decorated_func.__name__, 'dummy_function') + + # Check if arguments are passed correctly + result = decorated_func("test1", "test2") + self.assertEqual(result, ("test1", "test2")) + + if __name__ == '__main__': unittest.main(exit=False, verbosity=2) diff --git a/release_notes.md b/release_notes.md index 291fd4055..2114de337 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,3 +1,14 @@ +## Release Notes 2.28 + +### features +- Added ibllib.pipes.dynamic_pipeline.get_trials_tasks function + +### bugfixes +- Fix ibllib.io.extractors.ephys_fpga.extract_all for python 3.8 + +### other +- Change behavior qc to pass if number of trials > 400 (from start) can be found for which easy trial performance > 0.9 + ## Release Notes 2.27 ### features