Merge branch 'release/2.32.5'

ibllib/__init__.py

@@ -2,7 +2,7 @@
 import logging
 import warnings
 
-__version__ = '2.32.4'
+__version__ = '2.32.5'
 warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib')
 
 # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev

ibllib/io/session_params.py

@@ -169,9 +169,9 @@ def merge_params(a, b, copy=False):
  if k == 'sync':
  assert k not in a or a[k] == b[k], 'multiple sync fields defined'
  if isinstance(b[k], list):
- prev = a.get(k, [])
+ prev = list(a.get(k, []))
  # For procedures and projects, remove duplicates
- to_add = b[k] if k == 'tasks' else set(prev) ^ set(b[k])
+ to_add = b[k] if k == 'tasks' else set(b[k]) - set(prev)
  a[k] = prev + list(to_add)
  elif isinstance(b[k], dict):
  a[k] = {**a.get(k, {}), **b[k]}

ibllib/oneibl/registration.py

@@ -221,7 +221,7 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N
  # assert len({md['IBLRIG_VERSION_TAG'] for md in settings}) == 1
 
  users = []
- for user in filter(None, map(lambda x: x.get('PYBPOD_CREATOR'), settings)):
+ for user in filter(lambda x: x and x[1], map(lambda x: x.get('PYBPOD_CREATOR'), settings)):
  user = self.assert_exists(user[0], 'users') # user is list of [username, uuid]
  users.append(user['username'])
 
@@ -246,7 +246,7 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N
  if poo_counts:
  json_field['POOP_COUNT'] = int(sum(poo_counts))
 
- if not session: # Create session and weighings
+ if not len(session): # Create session and weighings
  ses_ = {'subject': subject['nickname'],
  'users': users or [subject['responsible_user']],
  'location': settings[0]['PYBPOD_BOARD'],
@@ -273,9 +273,13 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N
  self.register_weight(subject['nickname'], md['SUBJECT_WEIGHT'],
  date_time=md['SESSION_DATETIME'], user=user)
  else: # if session exists update a few key fields
- data = {'procedures': procedures, 'projects': projects}
+ data = {'procedures': procedures, 'projects': projects,
+ 'n_correct_trials': n_correct_trials, 'n_trials': n_trials}
  if task_protocols:
  data['task_protocol'] = '/'.join(task_protocols)
+ if end_time:
+ data['end_time'] = self.ensure_ISO8601(end_time)
+
  session = self.one.alyx.rest('sessions', 'partial_update', id=session_id[0], data=data)
  if json_field:
  session['json'] = self.one.alyx.json_field_update('sessions', session['id'], data=json_field)
@@ -401,14 +405,16 @@ def _get_session_times(fn, md, ses_data):
  """
  if isinstance(md, dict):
  start_time = _start_time = isostr2date(md['SESSION_DATETIME'])
+ end_time = isostr2date(md['SESSION_END_TIME']) if md.get('SESSION_END_TIME') else None
  else:
  start_time = isostr2date(md[0]['SESSION_DATETIME'])
  _start_time = isostr2date(md[-1]['SESSION_DATETIME'])
+ end_time = isostr2date(md[-1]['SESSION_END_TIME']) if md[-1].get('SESSION_END_TIME') else None
  assert isinstance(ses_data, (list, tuple)) and len(ses_data) == len(md)
  assert len(md) == 1 or start_time < _start_time
  ses_data = ses_data[-1]
- if not ses_data:
- return start_time, None
+ if not ses_data or end_time is not None:
+ return start_time, end_time
  c = ses_duration_secs = 0
  for sd in reversed(ses_data):
  ses_duration_secs = (sd['behavior_data']['Trial end timestamp'] -

ibllib/tests/test_io.py

@@ -555,6 +555,23 @@ def test_get_collections_repeat_protocols(self):
  collections = session_params.get_collections(tasks, flat=True)
  self.assertEqual(collections, {'raw_passive_data_bis', 'raw_passive_data', 'raw_behavior_data'})
 
+ def test_merge_params(self):
+ """Test for ibllib.io.session_params.merge_params functions."""
+ a = self.fixture
+ b = {'procedures': ['Imaging', 'Injection'], 'tasks': [{'fooChoiceWorld': {'collection': 'bar'}}]}
+ c = session_params.merge_params(a, b, copy=True)
+ self.assertCountEqual(['Imaging', 'Behavior training/tasks', 'Injection'], c['procedures'])
+ self.assertCountEqual(['passiveChoiceWorld', 'ephysChoiceWorld', 'fooChoiceWorld'], (list(x)[0] for x in c['tasks']))
+ # Ensure a and b not modified
+ self.assertNotEqual(set(c['procedures']), set(a['procedures']))
+ self.assertNotEqual(set(a['procedures']), set(b['procedures']))
+ # Test without copy
+ session_params.merge_params(a, b, copy=False)
+ self.assertCountEqual(['Imaging', 'Behavior training/tasks', 'Injection'], a['procedures'])
+ # Test assertion on duplicate sync
+ b['sync'] = {'foodaq': {'collection': 'raw_sync_data'}}
+ self.assertRaises(AssertionError, session_params.merge_params, a, b)
+
 
 class TestRawDaqLoaders(unittest.TestCase):
  """Tests for raw_daq_loaders module"""

ibllib/tests/test_oneibl.py

@@ -429,16 +429,20 @@ def test_registration_session(self):
  query_type='remote')[0]
  ses_info = self.one.alyx.rest('sessions', 'read', id=eid)
  self.assertTrue(ses_info['procedures'] == ['Behavior training/tasks'])
- self.one.alyx.rest('sessions', 'delete', id=eid)
- # re-register the session as unknown protocol this time
+ # re-register the session as unknown protocol, this time without removing session first
  self.settings['PYBPOD_PROTOCOL'] = 'gnagnagna'
+ # also add an end time
+ start = datetime.datetime.fromisoformat(self.settings['SESSION_DATETIME'])
+ self.settings['SESSION_START_TIME'] = rc.ensure_ISO8601(start)
+ self.settings['SESSION_END_TIME'] = rc.ensure_ISO8601(start + datetime.timedelta(hours=1))
  with open(settings_file, 'w') as fid:
  json.dump(self.settings, fid)
  rc.register_session(self.session_path)
  eid = self.one.search(subject=self.subject, date_range=['2018-04-01', '2018-04-01'],
  query_type='remote')[0]
  ses_info = self.one.alyx.rest('sessions', 'read', id=eid)
  self.assertTrue(ses_info['procedures'] == [])
+ self.assertEqual(self.settings['SESSION_END_TIME'], ses_info['end_time'])
  self.one.alyx.rest('sessions', 'delete', id=eid)
 
  def test_register_chained_session(self):
@@ -457,12 +461,13 @@ def test_register_chained_session(self):
 
  # Save experiment description
  session_params.write_params(self.session_path, experiment_description)
-
+ self.settings['POOP_COUNT'] = 10
  with open(behaviour_paths[1].joinpath('_iblrig_taskSettings.raw.json'), 'w') as fid:
  json.dump(self.settings, fid)
 
  settings = self.settings.copy()
  settings['PYBPOD_PROTOCOL'] = '_iblrig_tasks_passiveChoiceWorld'
+ settings['POOP_COUNT'] = 53
  start_time = (datetime.datetime.fromisoformat(settings['SESSION_DATETIME']) -
  datetime.timedelta(hours=1, minutes=2, seconds=12))
  settings['SESSION_DATETIME'] = start_time.isoformat()
@@ -475,7 +480,9 @@ def test_register_chained_session(self):
  ses_info = self.one.alyx.rest('sessions', 'read', id=session['id'])
  self.assertCountEqual(experiment_description['procedures'], ses_info['procedures'])
  self.assertCountEqual(experiment_description['projects'], ses_info['projects'])
- self.assertCountEqual({'IS_MOCK': False, 'IBLRIG_VERSION': None}, ses_info['json'])
+ # Poo count should be sum of values in both settings files
+ expected = {'IS_MOCK': False, 'IBLRIG_VERSION': '5.4.1', 'POOP_COUNT': 63}
+ self.assertDictEqual(expected, ses_info['json'])
  self.assertEqual('2018-04-01T11:46:14.795526', ses_info['start_time'])
  # Test task protocol
  expected = '_iblrig_tasks_passiveChoiceWorld5.4.1/_iblrig_tasks_ephysChoiceWorld5.4.1'

release_notes.md

@@ -13,6 +13,9 @@
 #### 2.32.4
 - Add support for variations of the biaseCW task in the json task description
 
+#### 2.32.5
+- Minor fixes to IBL registration client, including use of SESSION_END_TIME key
+
 ## Release Notes 2.31
 
 ### features