From 40088c2c7838f5d14343c8ecd83da25530335282 Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Wed, 20 Mar 2024 14:42:23 +0200 Subject: [PATCH] Minor fixes to IBL registration client; related to int-brain-lab/iblrig#621 --- ibllib/__init__.py | 2 +- ibllib/io/session_params.py | 4 ++-- ibllib/oneibl/registration.py | 16 +++++++++++----- ibllib/tests/test_io.py | 17 +++++++++++++++++ ibllib/tests/test_oneibl.py | 15 +++++++++++---- release_notes.md | 3 +++ 6 files changed, 45 insertions(+), 12 deletions(-) diff --git a/ibllib/__init__.py b/ibllib/__init__.py index ebea0b785..4827e8eab 100644 --- a/ibllib/__init__.py +++ b/ibllib/__init__.py @@ -2,7 +2,7 @@ import logging import warnings -__version__ = '2.32.4' +__version__ = '2.32.5' warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib') # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev diff --git a/ibllib/io/session_params.py b/ibllib/io/session_params.py index 8d0d9e16f..e5bbd673f 100644 --- a/ibllib/io/session_params.py +++ b/ibllib/io/session_params.py @@ -169,9 +169,9 @@ def merge_params(a, b, copy=False): if k == 'sync': assert k not in a or a[k] == b[k], 'multiple sync fields defined' if isinstance(b[k], list): - prev = a.get(k, []) + prev = list(a.get(k, [])) # For procedures and projects, remove duplicates - to_add = b[k] if k == 'tasks' else set(prev) ^ set(b[k]) + to_add = b[k] if k == 'tasks' else set(b[k]) - set(prev) a[k] = prev + list(to_add) elif isinstance(b[k], dict): a[k] = {**a.get(k, {}), **b[k]} diff --git a/ibllib/oneibl/registration.py b/ibllib/oneibl/registration.py index 470c8aead..a975e70db 100644 --- a/ibllib/oneibl/registration.py +++ b/ibllib/oneibl/registration.py @@ -221,7 +221,7 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N # assert len({md['IBLRIG_VERSION_TAG'] for md in settings}) == 1 users = [] - for user in filter(None, map(lambda x: x.get('PYBPOD_CREATOR'), settings)): + for user in filter(lambda x: x and x[1], map(lambda x: x.get('PYBPOD_CREATOR'), settings)): user = self.assert_exists(user[0], 'users') # user is list of [username, uuid] users.append(user['username']) @@ -246,7 +246,7 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N if poo_counts: json_field['POOP_COUNT'] = int(sum(poo_counts)) - if not session: # Create session and weighings + if not len(session): # Create session and weighings ses_ = {'subject': subject['nickname'], 'users': users or [subject['responsible_user']], 'location': settings[0]['PYBPOD_BOARD'], @@ -273,9 +273,13 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N self.register_weight(subject['nickname'], md['SUBJECT_WEIGHT'], date_time=md['SESSION_DATETIME'], user=user) else: # if session exists update a few key fields - data = {'procedures': procedures, 'projects': projects} + data = {'procedures': procedures, 'projects': projects, + 'n_correct_trials': n_correct_trials, 'n_trials': n_trials} if task_protocols: data['task_protocol'] = '/'.join(task_protocols) + if end_time: + data['end_time'] = self.ensure_ISO8601(end_time) + session = self.one.alyx.rest('sessions', 'partial_update', id=session_id[0], data=data) if json_field: session['json'] = self.one.alyx.json_field_update('sessions', session['id'], data=json_field) @@ -401,14 +405,16 @@ def _get_session_times(fn, md, ses_data): """ if isinstance(md, dict): start_time = _start_time = isostr2date(md['SESSION_DATETIME']) + end_time = isostr2date(md['SESSION_END_TIME']) if md.get('SESSION_END_TIME') else None else: start_time = isostr2date(md[0]['SESSION_DATETIME']) _start_time = isostr2date(md[-1]['SESSION_DATETIME']) + end_time = isostr2date(md[-1]['SESSION_END_TIME']) if md[-1].get('SESSION_END_TIME') else None assert isinstance(ses_data, (list, tuple)) and len(ses_data) == len(md) assert len(md) == 1 or start_time < _start_time ses_data = ses_data[-1] - if not ses_data: - return start_time, None + if not ses_data or end_time is not None: + return start_time, end_time c = ses_duration_secs = 0 for sd in reversed(ses_data): ses_duration_secs = (sd['behavior_data']['Trial end timestamp'] - diff --git a/ibllib/tests/test_io.py b/ibllib/tests/test_io.py index d08b645e7..d0f459a15 100644 --- a/ibllib/tests/test_io.py +++ b/ibllib/tests/test_io.py @@ -555,6 +555,23 @@ def test_get_collections_repeat_protocols(self): collections = session_params.get_collections(tasks, flat=True) self.assertEqual(collections, {'raw_passive_data_bis', 'raw_passive_data', 'raw_behavior_data'}) + def test_merge_params(self): + """Test for ibllib.io.session_params.merge_params functions.""" + a = self.fixture + b = {'procedures': ['Imaging', 'Injection'], 'tasks': [{'fooChoiceWorld': {'collection': 'bar'}}]} + c = session_params.merge_params(a, b, copy=True) + self.assertCountEqual(['Imaging', 'Behavior training/tasks', 'Injection'], c['procedures']) + self.assertCountEqual(['passiveChoiceWorld', 'ephysChoiceWorld', 'fooChoiceWorld'], (list(x)[0] for x in c['tasks'])) + # Ensure a and b not modified + self.assertNotEqual(set(c['procedures']), set(a['procedures'])) + self.assertNotEqual(set(a['procedures']), set(b['procedures'])) + # Test without copy + session_params.merge_params(a, b, copy=False) + self.assertCountEqual(['Imaging', 'Behavior training/tasks', 'Injection'], a['procedures']) + # Test assertion on duplicate sync + b['sync'] = {'foodaq': {'collection': 'raw_sync_data'}} + self.assertRaises(AssertionError, session_params.merge_params, a, b) + class TestRawDaqLoaders(unittest.TestCase): """Tests for raw_daq_loaders module""" diff --git a/ibllib/tests/test_oneibl.py b/ibllib/tests/test_oneibl.py index 729b391ab..0d17ed520 100644 --- a/ibllib/tests/test_oneibl.py +++ b/ibllib/tests/test_oneibl.py @@ -429,9 +429,12 @@ def test_registration_session(self): query_type='remote')[0] ses_info = self.one.alyx.rest('sessions', 'read', id=eid) self.assertTrue(ses_info['procedures'] == ['Behavior training/tasks']) - self.one.alyx.rest('sessions', 'delete', id=eid) - # re-register the session as unknown protocol this time + # re-register the session as unknown protocol, this time without removing session first self.settings['PYBPOD_PROTOCOL'] = 'gnagnagna' + # also add an end time + start = datetime.datetime.fromisoformat(self.settings['SESSION_DATETIME']) + self.settings['SESSION_START_TIME'] = rc.ensure_ISO8601(start) + self.settings['SESSION_END_TIME'] = rc.ensure_ISO8601(start + datetime.timedelta(hours=1)) with open(settings_file, 'w') as fid: json.dump(self.settings, fid) rc.register_session(self.session_path) @@ -439,6 +442,7 @@ def test_registration_session(self): query_type='remote')[0] ses_info = self.one.alyx.rest('sessions', 'read', id=eid) self.assertTrue(ses_info['procedures'] == []) + self.assertEqual(self.settings['SESSION_END_TIME'], ses_info['end_time']) self.one.alyx.rest('sessions', 'delete', id=eid) def test_register_chained_session(self): @@ -457,12 +461,13 @@ def test_register_chained_session(self): # Save experiment description session_params.write_params(self.session_path, experiment_description) - + self.settings['POOP_COUNT'] = 10 with open(behaviour_paths[1].joinpath('_iblrig_taskSettings.raw.json'), 'w') as fid: json.dump(self.settings, fid) settings = self.settings.copy() settings['PYBPOD_PROTOCOL'] = '_iblrig_tasks_passiveChoiceWorld' + settings['POOP_COUNT'] = 53 start_time = (datetime.datetime.fromisoformat(settings['SESSION_DATETIME']) - datetime.timedelta(hours=1, minutes=2, seconds=12)) settings['SESSION_DATETIME'] = start_time.isoformat() @@ -475,7 +480,9 @@ def test_register_chained_session(self): ses_info = self.one.alyx.rest('sessions', 'read', id=session['id']) self.assertCountEqual(experiment_description['procedures'], ses_info['procedures']) self.assertCountEqual(experiment_description['projects'], ses_info['projects']) - self.assertCountEqual({'IS_MOCK': False, 'IBLRIG_VERSION': None}, ses_info['json']) + # Poo count should be sum of values in both settings files + expected = {'IS_MOCK': False, 'IBLRIG_VERSION': '5.4.1', 'POOP_COUNT': 63} + self.assertDictEqual(expected, ses_info['json']) self.assertEqual('2018-04-01T11:46:14.795526', ses_info['start_time']) # Test task protocol expected = '_iblrig_tasks_passiveChoiceWorld5.4.1/_iblrig_tasks_ephysChoiceWorld5.4.1' diff --git a/release_notes.md b/release_notes.md index 2f98ffaa4..a0e804601 100644 --- a/release_notes.md +++ b/release_notes.md @@ -13,6 +13,9 @@ #### 2.32.4 - Add support for variations of the biaseCW task in the json task description +#### 2.32.5 +- Minor fixes to IBL registion client, including use of SESSION_END_TIME key + ## Release Notes 2.31 ### features