Training task qc

ibllib/atlas/__init__.py

@@ -1,4 +1,7 @@
-"""A package for working with brain atlases.
+"""(DEPRECATED) A package for working with brain atlases.
+
+For the correct atlas documentation, see
+https://docs.internationalbrainlab.org/_autosummary/iblatlas.html
 
 For examples and tutorials on using the IBL atlas package, see
 https://docs.internationalbrainlab.org/atlas_examples.html

ibllib/io/extractors/base.py

@@ -154,9 +154,9 @@ def extract(self, bpod_trials=None, settings=None, **kwargs):
         if not self.settings:
             self.settings = raw.load_settings(self.session_path, task_collection=self.task_collection)
         if self.settings is None:
-            self.settings = {"IBLRIG_VERSION_TAG": "100.0.0"}
-        elif self.settings.get("IBLRIG_VERSION_TAG", "") == "":
-            self.settings["IBLRIG_VERSION_TAG"] = "100.0.0"
+            self.settings = {"IBLRIG_VERSION": "100.0.0"}
+        elif self.settings.get("IBLRIG_VERSION", "") == "":
+            self.settings["IBLRIG_VERSION"] = "100.0.0"
         return super(BaseBpodTrialsExtractor, self).extract(**kwargs)
 
 

ibllib/io/extractors/biased_trials.py

@@ -1,6 +1,6 @@
 from pathlib import Path, PureWindowsPath
 
-from pkg_resources import parse_version
+from packaging import version
 import numpy as np
 from one.alf.io import AlfBunch
 
@@ -80,8 +80,8 @@ def get_pregenerated_events(bpod_trials, settings):
         pLeft = pLeft[: ntrials]
 
         phase_path = sessions_folder.joinpath(f"session_{num}_stim_phase.npy")
-        is_patched_version = parse_version(
-            settings.get('IBLRIG_VERSION_TAG', 0)) > parse_version('6.4.0')
+        is_patched_version = version.parse(
+            settings.get('IBLRIG_VERSION') or '0') > version.parse('6.4.0')
         if phase_path.exists() and is_patched_version:
             phase = np.load(phase_path)[:ntrials]
 
@@ -211,13 +211,13 @@ def extract_all(session_path, save=False, bpod_trials=False, settings=False, ext
     if not settings:
         settings = raw.load_settings(session_path, task_collection=task_collection)
     if settings is None:
-        settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
+        settings = {'IBLRIG_VERSION': '100.0.0'}
 
-    if settings['IBLRIG_VERSION_TAG'] == '':
-        settings['IBLRIG_VERSION_TAG'] = '100.0.0'
+    if settings['IBLRIG_VERSION'] == '':
+        settings['IBLRIG_VERSION'] = '100.0.0'
 
     # Version check
-    if parse_version(settings['IBLRIG_VERSION_TAG']) >= parse_version('5.0.0'):
+    if version.parse(settings['IBLRIG_VERSION']) >= version.parse('5.0.0'):
         # We now extract a single trials table
         base = [BiasedTrials]
     else:

ibllib/io/extractors/bpod_trials.py

@@ -1,4 +1,5 @@
-"""Trials data extraction from raw Bpod output
+"""Trials data extraction from raw Bpod output.
+
 This module will extract the Bpod trials and wheel data based on the task protocol,
 i.e. habituation, training or biased.
 """
@@ -7,7 +8,7 @@
 from collections import OrderedDict
 import warnings
 
-from pkg_resources import parse_version
+from packaging import version
 from ibllib.io.extractors import habituation_trials, training_trials, biased_trials, opto_trials
 from ibllib.io.extractors.base import get_bpod_extractor_class, protocol2extractor
 from ibllib.io.extractors.habituation_trials import HabituationTrials
@@ -88,8 +89,8 @@ def extract_all(session_path, save=True, bpod_trials=None, settings=None,
         files_wheel = []
         wheel = OrderedDict({k: trials.pop(k) for k in tuple(trials.keys()) if 'wheel' in k})
     elif extractor_type == 'habituation':
-        if settings['IBLRIG_VERSION_TAG'] and \
-                parse_version(settings['IBLRIG_VERSION_TAG']) <= parse_version('5.0.0'):
+        if settings['IBLRIG_VERSION'] and \
+                version.parse(settings['IBLRIG_VERSION']) <= version.parse('5.0.0'):
             _logger.warning('No extraction of legacy habituation sessions')
             return None, None, None
         trials, files_trials = habituation_trials.extract_all(session_path, bpod_trials=bpod_trials, settings=settings, save=save,

ibllib/io/extractors/ephys_passive.py

@@ -93,9 +93,11 @@ def _load_task_protocol(session_path: str, task_collection: str = 'raw_passive_d
     :type session_path: str
     :return: ibl rig task protocol version
     :rtype: str
+
+    FIXME This function has a misleading name
     """
     settings = rawio.load_settings(session_path, task_collection=task_collection)
-    ses_ver = settings["IBLRIG_VERSION_TAG"]
+    ses_ver = settings["IBLRIG_VERSION"]
 
     return ses_ver
 

ibllib/io/extractors/mesoscope.py

@@ -7,7 +7,7 @@
 from one.util import ensure_list
 from one.alf.files import session_path_parts
 import matplotlib.pyplot as plt
-from pkg_resources import parse_version
+from packaging import version
 
 from ibllib.plots.misc import squares, vertical_lines
 from ibllib.io.raw_daq_loaders import (extract_sync_timeline, timeline_get_channel,
@@ -38,16 +38,16 @@ def patch_imaging_meta(meta: dict) -> dict:
         The loaded metadata file, updated to the most recent version.
     """
     # 2023-05-17 (unversioned) adds nFrames, channelSaved keys, MM and Deg keys
-    version = parse_version(meta.get('version') or '0.0.0')
-    if version <= parse_version('0.0.0'):
+    ver = version.parse(meta.get('version') or '0.0.0')
+    if ver <= version.parse('0.0.0'):
         if 'channelSaved' not in meta:
             meta['channelSaved'] = next((x['channelIdx'] for x in meta['FOV'] if 'channelIdx' in x), [])
         fields = ('topLeft', 'topRight', 'bottomLeft', 'bottomRight')
         for fov in meta.get('FOV', []):
             for unit in ('Deg', 'MM'):
                 if unit not in fov:  # topLeftDeg, etc. -> Deg[topLeft]
                     fov[unit] = {f: fov.pop(f + unit, None) for f in fields}
-    elif version == parse_version('0.1.0'):
+    elif ver == version.parse('0.1.0'):
         for fov in meta.get('FOV', []):
             if 'roiUuid' in fov:
                 fov['roiUUID'] = fov.pop('roiUuid')

ibllib/io/extractors/training_trials.py

@@ -1,6 +1,6 @@
 import logging
 import numpy as np
-from pkg_resources import parse_version
+from packaging import version
 from one.alf.io import AlfBunch
 
 import ibllib.io.raw_data_loaders as raw
@@ -216,7 +216,7 @@ def get_feedback_times_ge5(session_path, task_collection='raw_behavior_data', da
 
     def _extract(self):
         # Version check
-        if parse_version(self.settings['IBLRIG_VERSION_TAG']) >= parse_version('5.0.0'):
+        if version.parse(self.settings['IBLRIG_VERSION'] or '100.0.0') >= version.parse('5.0.0'):
             merge = self.get_feedback_times_ge5(self.session_path, task_collection=self.task_collection, data=self.bpod_trials)
         else:
             merge = self.get_feedback_times_lt5(self.session_path, task_collection=self.task_collection, data=self.bpod_trials)
@@ -287,7 +287,7 @@ class GoCueTriggerTimes(BaseBpodTrialsExtractor):
     var_names = 'goCueTrigger_times'
 
     def _extract(self):
-        if parse_version(self.settings['IBLRIG_VERSION_TAG']) >= parse_version('5.0.0'):
+        if version.parse(self.settings['IBLRIG_VERSION'] or '100.0.0') >= version.parse('5.0.0'):
             goCue = np.array([tr['behavior_data']['States timestamps']
                               ['play_tone'][0][0] for tr in self.bpod_trials])
         else:
@@ -361,7 +361,7 @@ class IncludedTrials(BaseBpodTrialsExtractor):
     var_names = 'included'
 
     def _extract(self):
-        if parse_version(self.settings['IBLRIG_VERSION_TAG']) >= parse_version('5.0.0'):
+        if version.parse(self.settings['IBLRIG_VERSION'] or '100.0.0') >= version.parse('5.0.0'):
             trials_included = self.get_included_trials_ge5(
                 data=self.bpod_trials, settings=self.settings)
         else:
@@ -370,7 +370,7 @@ def _extract(self):
 
     @staticmethod
     def get_included_trials_lt5(data=False):
-        trials_included = np.array([True for t in data])
+        trials_included = np.ones(len(data), dtype=bool)
         return trials_included
 
     @staticmethod
@@ -387,7 +387,7 @@ class ItiInTimes(BaseBpodTrialsExtractor):
     var_names = 'itiIn_times'
 
     def _extract(self):
-        if parse_version(self.settings["IBLRIG_VERSION_TAG"]) < parse_version("5.0.0"):
+        if version.parse(self.settings["IBLRIG_VERSION"] or '100.0.0') < version.parse("5.0.0"):
             iti_in = np.ones(len(self.bpod_trials)) * np.nan
         else:
             iti_in = np.array(
@@ -416,7 +416,7 @@ class StimFreezeTriggerTimes(BaseBpodTrialsExtractor):
     var_names = 'stimFreezeTrigger_times'
 
     def _extract(self):
-        if parse_version(self.settings["IBLRIG_VERSION_TAG"]) < parse_version("6.2.5"):
+        if version.parse(self.settings["IBLRIG_VERSION"] or '100.0.0') < version.parse("6.2.5"):
             return np.ones(len(self.bpod_trials)) * np.nan
         freeze_reward = np.array(
             [
@@ -460,9 +460,9 @@ class StimOffTriggerTimes(BaseBpodTrialsExtractor):
     var_names = 'stimOffTrigger_times'
 
     def _extract(self):
-        if parse_version(self.settings["IBLRIG_VERSION_TAG"]) >= parse_version("6.2.5"):
+        if version.parse(self.settings["IBLRIG_VERSION"] or '100.0.0') >= version.parse("6.2.5"):
             stim_off_trigger_state = "hide_stim"
-        elif parse_version(self.settings["IBLRIG_VERSION_TAG"]) >= parse_version("5.0.0"):
+        elif version.parse(self.settings["IBLRIG_VERSION"]) >= version.parse("5.0.0"):
             stim_off_trigger_state = "exit_state"
         else:
             stim_off_trigger_state = "trial_start"
@@ -518,7 +518,7 @@ def _extract(self):
         # Version check
         _logger.warning("Deprecation Warning: this is an old version of stimOn extraction."
                         "From version 5., use StimOnOffFreezeTimes")
-        if parse_version(self.settings['IBLRIG_VERSION_TAG']) >= parse_version('5.0.0'):
+        if version.parse(self.settings['IBLRIG_VERSION'] or '100.0.0') >= version.parse('5.0.0'):
             stimOn_times = self.get_stimOn_times_ge5(self.session_path, data=self.bpod_trials,
                                                      task_collection=self.task_collection)
         else:
@@ -739,11 +739,11 @@ def extract_all(session_path, save=False, bpod_trials=None, settings=None, task_
         bpod_trials = raw.load_data(session_path, task_collection=task_collection)
     if not settings:
         settings = raw.load_settings(session_path, task_collection=task_collection)
-    if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
-        settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
+    if settings is None or settings['IBLRIG_VERSION'] == '':
+        settings = {'IBLRIG_VERSION': '100.0.0'}
 
     # Version check
-    if parse_version(settings['IBLRIG_VERSION_TAG']) >= parse_version('5.0.0'):
+    if version.parse(settings['IBLRIG_VERSION']) >= version.parse('5.0.0'):
         # We now extract a single trials table
         base = [TrainingTrials]
     else:

ibllib/io/raw_data_loaders.py

@@ -7,6 +7,7 @@
 
 Module contains one loader function per raw datafile
 """
+import re
 import json
 import logging
 import wave
@@ -16,7 +17,7 @@
 from typing import Union
 
 from dateutil import parser as dateparser
-from pkg_resources import parse_version
+from packaging import version
 import numpy as np
 import pandas as pd
 
@@ -323,18 +324,47 @@ def _read_settings_json_compatibility_enforced(settings):
             md = json.load(js)
     if 'IS_MOCK' not in md:
         md['IS_MOCK'] = False
+    # Many v < 8 sessions had both version and version tag keys. v > 8 have a version tag.
+    # Some sessions have neither key.  From v8 onwards we will use IBLRIG_VERSION to test rig
+    # version, however some places may still use the version tag.
     if 'IBLRIG_VERSION_TAG' not in md.keys():
         md['IBLRIG_VERSION_TAG'] = md.get('IBLRIG_VERSION', '')
+    if 'IBLRIG_VERSION' not in md.keys():
+        md['IBLRIG_VERSION'] = md['IBLRIG_VERSION_TAG']
+    elif all([md['IBLRIG_VERSION'], md['IBLRIG_VERSION_TAG']]):
+        # This may not be an issue; not sure what the intended difference between these keys was
+        assert md['IBLRIG_VERSION'] == md['IBLRIG_VERSION_TAG'], 'version and version tag mismatch'
+    # Test version can be parsed. If not, log an error and set the version to nothing
+    try:
+        version.parse(md['IBLRIG_VERSION'] or '0')
+    except version.InvalidVersion as ex:
+        _logger.error('%s in iblrig settings, this may affect extraction', ex)
+        # try a more relaxed version parse
+        laxed_parse = re.search(r'^\d+\.\d+\.\d+', md['IBLRIG_VERSION'])
+        # Set the tag as the invalid version
+        md['IBLRIG_VERSION_TAG'] = md['IBLRIG_VERSION']
+        # overwrite version with either successfully parsed one or an empty string
+        md['IBLRIG_VERSION'] = laxed_parse.group() if laxed_parse else ''
+    if 'device_sound' not in md:
+        # sound device must be defined in version 8 and later  # FIXME this assertion will cause tests to break
+        assert version.parse(md['IBLRIG_VERSION'] or '0') < version.parse('8.0.0')
+        # in v7 we must infer the device from the sampling frequency if SD is None
+        if 'sounddevice' in md.get('SD', ''):
+            device = 'xonar'
+        else:
+            freq_map = {192000: 'xonar', 96000: 'harp', 44100: 'sysdefault'}
+            device = freq_map.get(md.get('SOUND_SAMPLE_FREQ'), 'unknown')
+        md['device_sound'] = {'OUTPUT': device}
     # 2018-12-05 Version 3.2.3 fixes (permanent fixes in IBL_RIG from 3.2.4 on)
-    if md['IBLRIG_VERSION_TAG'] == '':
+    if md['IBLRIG_VERSION'] == '':
         pass
-    elif parse_version(md.get('IBLRIG_VERSION_TAG')) >= parse_version('8.0.0'):
+    elif version.parse(md['IBLRIG_VERSION']) >= version.parse('8.0.0'):
         md['SESSION_NUMBER'] = str(md['SESSION_NUMBER']).zfill(3)
         md['PYBPOD_BOARD'] = md['RIG_NAME']
         md['PYBPOD_CREATOR'] = (md['ALYX_USER'], '')
         md['SESSION_DATE'] = md['SESSION_START_TIME'][:10]
         md['SESSION_DATETIME'] = md['SESSION_START_TIME']
-    elif parse_version(md.get('IBLRIG_VERSION_TAG')) <= parse_version('3.2.3'):
+    elif version.parse(md['IBLRIG_VERSION']) <= version.parse('3.2.3'):
         if 'LAST_TRIAL_DATA' in md.keys():
             md.pop('LAST_TRIAL_DATA')
         if 'weighings' in md['PYBPOD_SUBJECT_EXTRA'].keys():
@@ -414,16 +444,16 @@ def load_encoder_events(session_path, task_collection='raw_behavior_data', setti
     path = next(path.glob("_iblrig_encoderEvents.raw*.ssv"), None)
     if not settings:
         settings = load_settings(session_path, task_collection=task_collection)
-    if settings is None or not settings.get('IBLRIG_VERSION_TAG'):
-        settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
+    if settings is None or not settings.get('IBLRIG_VERSION'):
+        settings = {'IBLRIG_VERSION': '100.0.0'}
         # auto-detect old files when version is not labeled
         with open(path) as fid:
             line = fid.readline()
         if line.startswith('Event') and 'StateMachine' in line:
-            settings = {'IBLRIG_VERSION_TAG': '0.0.0'}
+            settings = {'IBLRIG_VERSION': '0.0.0'}
     if not path:
         return None
-    if parse_version(settings['IBLRIG_VERSION_TAG']) >= parse_version('5.0.0'):
+    if version.parse(settings['IBLRIG_VERSION']) >= version.parse('5.0.0'):
         return _load_encoder_events_file_ge5(path)
     else:
         return _load_encoder_events_file_lt5(path)
@@ -518,17 +548,17 @@ def load_encoder_positions(session_path, task_collection='raw_behavior_data', se
     path = next(path.glob("_iblrig_encoderPositions.raw*.ssv"), None)
     if not settings:
         settings = load_settings(session_path, task_collection=task_collection)
-    if settings is None or not settings.get('IBLRIG_VERSION_TAG'):
-        settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
+    if settings is None or not settings.get('IBLRIG_VERSION'):
+        settings = {'IBLRIG_VERSION': '100.0.0'}
         # auto-detect old files when version is not labeled
         with open(path) as fid:
             line = fid.readline()
         if line.startswith('Position'):
-            settings = {'IBLRIG_VERSION_TAG': '0.0.0'}
+            settings = {'IBLRIG_VERSION': '0.0.0'}
     if not path:
         _logger.warning("No data loaded: could not find raw encoderPositions file")
         return None
-    if parse_version(settings['IBLRIG_VERSION_TAG']) >= parse_version('5.0.0'):
+    if version.parse(settings['IBLRIG_VERSION']) >= version.parse('5.0.0'):
         return _load_encoder_positions_file_ge5(path)
     else:
         return _load_encoder_positions_file_lt5(path)

ibllib/io/session_params.py

@@ -30,7 +30,7 @@
 from copy import deepcopy
 
 from one.converters import ConversionMixin
-from pkg_resources import parse_version
+from packaging import version
 
 import ibllib.pipes.misc as misc
 
@@ -71,9 +71,9 @@ def _patch_file(data: dict) -> dict:
         The patched description data.
     """
     if data and (v := data.get('version', '0')) != SPEC_VERSION:
-        if parse_version(v) > parse_version(SPEC_VERSION):
+        if version.parse(v) > version.parse(SPEC_VERSION):
             _logger.warning('Description file generated by more recent code')
-        elif parse_version(v) <= parse_version('0.1.0'):
+        elif version.parse(v) <= version.parse('0.1.0'):
             # Change tasks key from dict to list of dicts
             if 'tasks' in data and isinstance(data['tasks'], dict):
                 data['tasks'] = [{k: v} for k, v in data['tasks'].copy().items()]

ibllib/oneibl/registration.py

@@ -4,7 +4,7 @@
 import logging
 import itertools
 
-from pkg_resources import parse_version
+from packaging import version
 from one.alf.files import get_session_path, folder_parts, get_alf_path
 from one.registration import RegistrationClient, get_dataset_type
 from one.remote.globus import get_local_endpoint_id, get_lab_from_endpoint_id
@@ -230,7 +230,7 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N
             n_trials, n_correct_trials = _get_session_performance(settings, task_data)
 
             # TODO Add task_protocols to Alyx sessions endpoint
-            task_protocols = [md['PYBPOD_PROTOCOL'] + md['IBLRIG_VERSION_TAG'] for md in settings]
+            task_protocols = [md['PYBPOD_PROTOCOL'] + md['IBLRIG_VERSION'] for md in settings]
             # unless specified label the session projects with subject projects
             projects = subject['projects'] if projects is None else projects
             # makes sure projects is a list
@@ -298,7 +298,7 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N
 
         # register all files that match the Alyx patterns and file_list
         if any(settings):
-            rename_files_compatibility(ses_path, settings[0]['IBLRIG_VERSION_TAG'])
+            rename_files_compatibility(ses_path, settings[0]['IBLRIG_VERSION'])
         F = filter(lambda x: self._register_bool(x.name, file_list), self.find_files(ses_path))
         recs = self.register_files(F, created_by=users[0] if users else None, versions=ibllib.__version__)
         return session, recs
@@ -370,7 +370,7 @@ def _alyx_procedure_from_task_type(task_type):
 def rename_files_compatibility(ses_path, version_tag):
     if not version_tag:
         return
-    if parse_version(version_tag) <= parse_version('3.2.3'):
+    if version.parse(version_tag) <= version.parse('3.2.3'):
         task_code = ses_path.glob('**/_ibl_trials.iti_duration.npy')
         for fn in task_code:
             fn.replace(fn.parent.joinpath('_ibl_trials.itiDuration.npy'))