diff --git a/ibllib/__init__.py b/ibllib/__init__.py index 4e328ee70..55e4b2b5d 100644 --- a/ibllib/__init__.py +++ b/ibllib/__init__.py @@ -2,13 +2,12 @@ import logging import warnings -__version__ = '2.26' +__version__ = '2.27' warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib') # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev # who uses the library. However since it can also be provided as an app, the end-users should be provided -# with an useful default logging in standard output without messing with the complex python logging system -# -*- coding:utf-8 -*- +# with a useful default logging in standard output without messing with the complex python logging system USE_LOGGING = True #%(asctime)s,%(msecs)d if USE_LOGGING: diff --git a/ibllib/io/extractors/mesoscope.py b/ibllib/io/extractors/mesoscope.py index 561bb6343..78ed21674 100644 --- a/ibllib/io/extractors/mesoscope.py +++ b/ibllib/io/extractors/mesoscope.py @@ -23,24 +23,34 @@ def patch_imaging_meta(meta: dict) -> dict: """ - Patch imaging meta data for compatibility across versions. + Patch imaging metadata for compatibility across versions. A copy of the dict is NOT returned. Parameters ---------- - dict : dict + meta : dict A folder path that contains a rawImagingData.meta file. Returns ------- dict - The loaded meta data file, updated to the most recent version. + The loaded metadata file, updated to the most recent version. """ - # 2023-05-17 (unversioned) adds nFrames and channelSaved keys - if parse_version(meta.get('version') or '0.0.0') <= parse_version('0.0.0'): + # 2023-05-17 (unversioned) adds nFrames, channelSaved keys, MM and Deg keys + version = parse_version(meta.get('version') or '0.0.0') + if version <= parse_version('0.0.0'): if 'channelSaved' not in meta: meta['channelSaved'] = next((x['channelIdx'] for x in meta['FOV'] if 'channelIdx' in x), []) + fields = ('topLeft', 'topRight', 'bottomLeft', 'bottomRight') + for fov in meta.get('FOV', []): + for unit in ('Deg', 'MM'): + if unit not in fov: # topLeftDeg, etc. -> Deg[topLeft] + fov[unit] = {f: fov.pop(f + unit, None) for f in fields} + elif version == parse_version('0.1.0'): + for fov in meta.get('FOV', []): + if 'roiUuid' in fov: + fov['roiUUID'] = fov.pop('roiUuid') return meta diff --git a/ibllib/io/extractors/video_motion.py b/ibllib/io/extractors/video_motion.py index 981af8a18..4756b2e3a 100644 --- a/ibllib/io/extractors/video_motion.py +++ b/ibllib/io/extractors/video_motion.py @@ -40,11 +40,7 @@ def find_nearest(array, value): class MotionAlignment: - roi = { - 'left': ((800, 1020), (233, 1096)), - 'right': ((426, 510), (104, 545)), - 'body': ((402, 481), (31, 103)) - } + roi = {'left': ((800, 1020), (233, 1096)), 'right': ((426, 510), (104, 545)), 'body': ((402, 481), (31, 103))} def __init__(self, eid=None, one=None, log=logging.getLogger(__name__), **kwargs): self.one = one or ONE() @@ -94,12 +90,9 @@ def line_select_callback(eclick, erelease): return np.array([[x1, x2], [y1, y2]]) plt.imshow(frame) - roi = RectangleSelector(plt.gca(), line_select_callback, - drawtype='box', useblit=True, - button=[1, 3], # don't use middle button - minspanx=5, minspany=5, - spancoords='pixels', - interactive=True) + roi = RectangleSelector(plt.gca(), line_select_callback, drawtype='box', useblit=True, button=[1, 3], + # don't use middle button + minspanx=5, minspany=5, spancoords='pixels', interactive=True) plt.show() ((x1, x2, *_), (y1, *_, y2)) = roi.corners col = np.arange(round(x1), round(x2), dtype=int) @@ -115,14 +108,13 @@ def load_data(self, download=False): self.data.wheel = self.one.load_object(self.eid, 'wheel') self.data.trials = self.one.load_object(self.eid, 'trials') cam = self.one.load(self.eid, ['camera.times'], dclass_output=True) - self.data.camera_times = {vidio.label_from_path(url): ts - for ts, url in zip(cam.data, cam.url)} + self.data.camera_times = {vidio.label_from_path(url): ts for ts, url in zip(cam.data, cam.url)} else: alf_path = self.session_path / 'alf' self.data.wheel = alfio.load_object(alf_path, 'wheel', short_keys=True) self.data.trials = alfio.load_object(alf_path, 'trials') - self.data.camera_times = {vidio.label_from_path(x): alfio.load_file_content(x) - for x in alf_path.glob('*Camera.times*')} + self.data.camera_times = {vidio.label_from_path(x): alfio.load_file_content(x) for x in + alf_path.glob('*Camera.times*')} assert all(x is not None for x in self.data.values()) def _set_eid_or_path(self, session_path_or_eid): @@ -191,8 +183,7 @@ def align_motion(self, period=(-np.inf, np.inf), side='left', sd_thresh=10, disp roi = (*[slice(*r) for r in self.roi[side]], 0) try: # TODO Add function arg to make grayscale - self.alignment.frames = \ - vidio.get_video_frames_preload(camera_path, frame_numbers, mask=roi) + self.alignment.frames = vidio.get_video_frames_preload(camera_path, frame_numbers, mask=roi) assert self.alignment.frames.size != 0 except AssertionError: self.log.error('Failed to open video') @@ -239,8 +230,8 @@ def align_motion(self, period=(-np.inf, np.inf), side='left', sd_thresh=10, disp y = np.pad(self.alignment.df, 1, 'edge') ax[0].plot(x, y, '-x', label='wheel motion energy') thresh = stDev > sd_thresh - ax[0].vlines(x[np.array(np.pad(thresh, 1, 'constant', constant_values=False))], 0, 1, - linewidth=0.5, linestyle=':', label=f'>{sd_thresh} s.d. diff') + ax[0].vlines(x[np.array(np.pad(thresh, 1, 'constant', constant_values=False))], 0, 1, linewidth=0.5, linestyle=':', + label=f'>{sd_thresh} s.d. diff') ax[1].plot(t[interp_mask], np.abs(v[interp_mask])) # Plot other stuff @@ -307,9 +298,7 @@ def init_plot(): data['frame_num'] = 0 mkr = find_nearest(wheel.timestamps[wheel_mask], ts_0) - data['marker'], = ax.plot( - wheel.timestamps[wheel_mask][mkr], - wheel.position[wheel_mask][mkr], 'r-x') + data['marker'], = ax.plot(wheel.timestamps[wheel_mask][mkr], wheel.position[wheel_mask][mkr], 'r-x') ax.set_ylabel('Wheel position (rad))') ax.set_xlabel('Time (s))') return @@ -338,19 +327,13 @@ def animate(i): data['im'].set_data(frame) mkr = find_nearest(wheel.timestamps[wheel_mask], t_x) - data['marker'].set_data( - wheel.timestamps[wheel_mask][mkr], - wheel.position[wheel_mask][mkr] - ) + data['marker'].set_data(wheel.timestamps[wheel_mask][mkr], wheel.position[wheel_mask][mkr]) return data['im'], data['ln'], data['marker'] anim = animation.FuncAnimation(fig, animate, init_func=init_plot, - frames=(range(len(self.alignment.df)) - if save - else cycle(range(60))), - interval=20, blit=False, - repeat=not save, cache_frame_data=False) + frames=(range(len(self.alignment.df)) if save else cycle(range(60))), interval=20, + blit=False, repeat=not save, cache_frame_data=False) anim.running = False def process_key(event): @@ -422,14 +405,12 @@ def fix_keys(alf_object): return ob alf_path = self.session_path.joinpath('alf') - wheel = (fix_keys(alfio.load_object(alf_path, 'wheel')) if location == 'SDSC' - else alfio.load_object(alf_path, 'wheel')) + wheel = (fix_keys(alfio.load_object(alf_path, 'wheel')) if location == 'SDSC' else alfio.load_object(alf_path, 'wheel')) self.wheel_timestamps = wheel.timestamps wheel_pos, self.wheel_time = wh.interpolate_position(wheel.timestamps, wheel.position, freq=1000) self.wheel_vel, _ = wh.velocity_filtered(wheel_pos, 1000) self.camera_times = alfio.load_file_content(next(alf_path.glob(f'_ibl_{self.label}Camera.times*.npy'))) - self.camera_path = str(next(self.session_path.joinpath('raw_video_data').glob( - f'_iblrig_{self.label}Camera.raw*.mp4'))) + self.camera_path = str(next(self.session_path.joinpath('raw_video_data').glob(f'_iblrig_{self.label}Camera.raw*.mp4'))) self.camera_meta = vidio.get_video_meta(self.camera_path) # TODO should read in the description file to get the correct sync location @@ -521,8 +502,7 @@ def compute_motion_energy(self, first, last, wg, iw): while np.any(idx == frames.shape[0] - 1) and counter < 20 and iw != wg.nwin - 1: n_after_offset = (counter + 1) * n_frames last += n_frames - extra_frames = vidio.get_video_frames_preload(cap, frame_numbers=np.arange(last, last + n_frames), - mask=self.mask) + extra_frames = vidio.get_video_frames_preload(cap, frame_numbers=np.arange(last, last + n_frames), mask=self.mask) frames = np.concatenate([frames, extra_frames], axis=0) idx = self.find_contaminated_frames(frames, self.threshold) after_status = True @@ -666,8 +646,7 @@ def extract_times(self, shifts_filt, t_shifts): return new_times @staticmethod - def single_cluster_raster(spike_times, events, trial_idx, dividers, colors, labels, weights=None, fr=True, - norm=False, + def single_cluster_raster(spike_times, events, trial_idx, dividers, colors, labels, weights=None, fr=True, norm=False, axs=None): pre_time = 0.4 post_time = 1 @@ -687,8 +666,7 @@ def single_cluster_raster(spike_times, events, trial_idx, dividers, colors, labe dividers = [0] + dividers + [len(trial_idx)] if axs is None: - fig, axs = plt.subplots(2, 1, figsize=(4, 6), gridspec_kw={'height_ratios': [1, 3], 'hspace': 0}, - sharex=True) + fig, axs = plt.subplots(2, 1, figsize=(4, 6), gridspec_kw={'height_ratios': [1, 3], 'hspace': 0}, sharex=True) else: fig = axs[0].get_figure() @@ -707,8 +685,7 @@ def single_cluster_raster(spike_times, events, trial_idx, dividers, colors, labe psth_div = np.nanmean(psth[t_ids], axis=0) std_div = np.nanstd(psth[t_ids], axis=0) / np.sqrt(len(t_ids)) - axs[0].fill_between(t_psth, psth_div - std_div, - psth_div + std_div, alpha=0.4, color=colors[lid]) + axs[0].fill_between(t_psth, psth_div - std_div, psth_div + std_div, alpha=0.4, color=colors[lid]) axs[0].plot(t_psth, psth_div, alpha=1, color=colors[lid]) lab_max = idx[np.argmax(t_ints)] @@ -726,8 +703,7 @@ def single_cluster_raster(spike_times, events, trial_idx, dividers, colors, labe secax = axs[1].secondary_yaxis('right') secax.set_yticks(label_pos) - secax.set_yticklabels(label, rotation=90, - rotation_mode='anchor', ha='center') + secax.set_yticklabels(label, rotation=90, rotation_mode='anchor', ha='center') for ic, c in enumerate(np.array(colors)[lidx]): secax.get_yticklabels()[ic].set_color(c) @@ -778,8 +754,7 @@ def plot_with_behavior(self): ax02.set_ylabel('Frames') ax02.set_xlabel('Time in session') - ax03.plot(self.camera_times, (self.camera_times - self.new_times) * self.camera_meta['fps'], - 'k', label='extracted - new') + ax03.plot(self.camera_times, (self.camera_times - self.new_times) * self.camera_meta['fps'], 'k', label='extracted - new') ax03.legend() ax03.set_ylim(-5, 5) ax03.set_ylabel('Frames') @@ -792,8 +767,8 @@ def plot_with_behavior(self): ax11.set_title('Wheel') ax12.set_xlabel('Time from first move') - self.single_cluster_raster(self.camera_times, self.trials['firstMovement_times'].values, trial_idx, dividers, - ['g', 'y'], ['left', 'right'], weights=feature_ext, fr=False, axs=[ax21, ax22]) + self.single_cluster_raster(self.camera_times, self.trials['firstMovement_times'].values, trial_idx, dividers, ['g', 'y'], + ['left', 'right'], weights=feature_ext, fr=False, axs=[ax21, ax22]) ax21.sharex(ax22) ax21.set_ylabel('Paw r velocity') ax21.set_title('Extracted times') @@ -808,8 +783,7 @@ def plot_with_behavior(self): ax41.imshow(self.frame_example[0]) rect = matplotlib.patches.Rectangle((self.roi[1][1], self.roi[0][0]), self.roi[1][0] - self.roi[1][1], - self.roi[0][1] - self.roi[0][0], - linewidth=4, edgecolor='g', facecolor='none') + self.roi[0][1] - self.roi[0][0], linewidth=4, edgecolor='g', facecolor='none') ax41.add_patch(rect) ax42.plot(self.all_me) @@ -845,8 +819,7 @@ def plot_without_behavior(self): ax02.set_ylabel('Frames') ax02.set_xlabel('Time in session') - ax03.plot(self.camera_times, (self.camera_times - self.new_times) * self.camera_meta['fps'], - 'k', label='extracted - new') + ax03.plot(self.camera_times, (self.camera_times - self.new_times) * self.camera_meta['fps'], 'k', label='extracted - new') ax03.legend() ax03.set_ylim(-5, 5) ax03.set_ylabel('Frames') @@ -854,8 +827,7 @@ def plot_without_behavior(self): ax04.imshow(self.frame_example[0]) rect = matplotlib.patches.Rectangle((self.roi[1][1], self.roi[0][0]), self.roi[1][0] - self.roi[1][1], - self.roi[0][1] - self.roi[0][0], - linewidth=4, edgecolor='g', facecolor='none') + self.roi[0][1] - self.roi[0][0], linewidth=4, edgecolor='g', facecolor='none') ax04.add_patch(rect) ax05.plot(self.all_me) @@ -866,8 +838,8 @@ def process(self): # Compute the motion energy of the wheel for the whole video wg = WindowGenerator(self.camera_meta['length'], 5000, 4) - out = Parallel(n_jobs=self.nprocess)(delayed(self.compute_motion_energy)(first, last, wg, iw) - for iw, (first, last) in enumerate(wg.firstlast)) + out = Parallel(n_jobs=self.nprocess)( + delayed(self.compute_motion_energy)(first, last, wg, iw) for iw, (first, last) in enumerate(wg.firstlast)) # Concatenate the motion energy into one big array self.all_me = np.array([]) for vals in out[:-1]: @@ -878,11 +850,11 @@ def process(self): to_app = self.times[0] - ((np.arange(int(self.camera_meta['fps'] * toverlap), ) + 1) / self.frate)[::-1] times = np.r_[to_app, self.times] - wg = WindowGenerator(all_me.size - 1, int(self.camera_meta['fps'] * self.twin), - int(self.camera_meta['fps'] * toverlap)) + wg = WindowGenerator(all_me.size - 1, int(self.camera_meta['fps'] * self.twin), int(self.camera_meta['fps'] * toverlap)) - out = Parallel(n_jobs=self.nprocess)(delayed(self.compute_shifts)(times, all_me, first, last, iw, wg) - for iw, (first, last) in enumerate(wg.firstlast)) + out = Parallel(n_jobs=4)( + delayed(self.compute_shifts)(times, all_me, first, last, iw, wg) for iw, (first, last) in enumerate(wg.firstlast) + ) self.shifts = np.array([]) self.t_shifts = np.array([]) @@ -903,11 +875,12 @@ def process(self): if self.upload: fig = self.plot_with_behavior() if self.behavior else self.plot_without_behavior() - save_fig_path = Path(self.session_path.joinpath('snapshot', 'video', 'video_wheel_alignment.png')) + save_fig_path = Path(self.session_path.joinpath('snapshot', 'video', f'video_wheel_alignment_{self.label}.png')) save_fig_path.parent.mkdir(exist_ok=True, parents=True) fig.savefig(save_fig_path) snp = ReportSnapshot(self.session_path, self.eid, content_type='session', one=self.one) snp.outputs = [save_fig_path] snp.register_images(widths=['orig']) + plt.close(fig) return self.new_times diff --git a/ibllib/oneibl/data_handlers.py b/ibllib/oneibl/data_handlers.py index 19c737e15..b41fac1f4 100644 --- a/ibllib/oneibl/data_handlers.py +++ b/ibllib/oneibl/data_handlers.py @@ -131,7 +131,8 @@ def __init__(self, session_path, signatures, one=None): # For cortex lab we need to get the endpoint from the ibl alyx if self.lab == 'cortexlab': - self.globus.add_endpoint(f'flatiron_{self.lab}', alyx=ONE(base_url='https://alyx.internationalbrainlab.org').alyx) + alyx = AlyxClient(base_url='https://alyx.internationalbrainlab.org', cache_rest=None) + self.globus.add_endpoint(f'flatiron_{self.lab}', alyx=alyx) else: self.globus.add_endpoint(f'flatiron_{self.lab}', alyx=self.one.alyx) @@ -140,21 +141,19 @@ def __init__(self, session_path, signatures, one=None): def setUp(self): """Function to download necessary data to run tasks using globus-sdk.""" if self.lab == 'cortexlab': - one = ONE(base_url='https://alyx.internationalbrainlab.org') - df = super().getData(one=one) + df = super().getData(one=ONE(base_url='https://alyx.internationalbrainlab.org')) else: - one = self.one - df = super().getData() + df = super().getData(one=self.one) if len(df) == 0: - # If no datasets found in the cache only work off local file system do not attempt to download any missing data - # using globus + # If no datasets found in the cache only work off local file system do not attempt to + # download any missing data using Globus return # Check for space on local server. If less that 500 GB don't download new data space_free = shutil.disk_usage(self.globus.endpoints['local']['root_path'])[2] if space_free < 500e9: - _logger.warning('Space left on server is < 500GB, wont redownload new data') + _logger.warning('Space left on server is < 500GB, won\'t re-download new data') return rel_sess_path = '/'.join(df.iloc[0]['session_path'].split('/')[-3:]) @@ -190,7 +189,7 @@ def uploadData(self, outputs, version, **kwargs): return register_dataset(outputs, one=self.one, versions=versions, repository=data_repo, **kwargs) def cleanUp(self): - """Clean up, remove the files that were downloaded from globus once task has completed.""" + """Clean up, remove the files that were downloaded from Globus once task has completed.""" for file in self.local_paths: os.unlink(file) diff --git a/ibllib/oneibl/registration.py b/ibllib/oneibl/registration.py index 0996f01e0..554735e15 100644 --- a/ibllib/oneibl/registration.py +++ b/ibllib/oneibl/registration.py @@ -172,31 +172,14 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N # Read in the experiment description file if it exists and get projects and procedures from here experiment_description_file = session_params.read_params(ses_path) + _, subject, date, number, *_ = folder_parts(ses_path) if experiment_description_file is None: collections = ['raw_behavior_data'] else: - projects = experiment_description_file.get('projects', projects) - procedures = experiment_description_file.get('procedures', procedures) - collections = ensure_list(session_params.get_task_collection(experiment_description_file)) - - # read meta data from the rig for the session from the task settings file - task_data = (raw.load_bpod(ses_path, collection) for collection in sorted(collections)) - # Filter collections where settings file was not found - if not (task_data := list(zip(*filter(lambda x: x[0] is not None, task_data)))): - raise ValueError(f'_iblrig_taskSettings.raw.json not found in {ses_path} Abort.') - settings, task_data = task_data - if len(settings) != len(collections): - raise ValueError(f'_iblrig_taskSettings.raw.json not found in {ses_path} Abort.') - - # Do some validation - _, subject, date, number, *_ = folder_parts(ses_path) - assert len({x['SUBJECT_NAME'] for x in settings}) == 1 and settings[0]['SUBJECT_NAME'] == subject - assert len({x['SESSION_DATE'] for x in settings}) == 1 and settings[0]['SESSION_DATE'] == date - assert len({x['SESSION_NUMBER'] for x in settings}) == 1 and settings[0]['SESSION_NUMBER'] == number - assert len({x['IS_MOCK'] for x in settings}) == 1 - assert len({md['PYBPOD_BOARD'] for md in settings}) == 1 - assert len({md.get('IBLRIG_VERSION') for md in settings}) == 1 - # assert len({md['IBLRIG_VERSION_TAG'] for md in settings}) == 1 + # Combine input projects/procedures with those in experiment description + projects = list({*experiment_description_file.get('projects', []), *(projects or [])}) + procedures = list({*experiment_description_file.get('procedures', []), *(procedures or [])}) + collections = session_params.get_task_collection(experiment_description_file) # query Alyx endpoints for subject, error if not found subject = self.assert_exists(subject, 'subjects') @@ -206,31 +189,62 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N date_range=date, number=number, details=True, query_type='remote') - users = [] - for user in filter(None, map(lambda x: x.get('PYBPOD_CREATOR'), settings)): - user = self.assert_exists(user[0], 'users') # user is list of [username, uuid] - users.append(user['username']) - - # extract information about session duration and performance - start_time, end_time = _get_session_times(str(ses_path), settings, task_data) - n_trials, n_correct_trials = _get_session_performance(settings, task_data) - - # TODO Add task_protocols to Alyx sessions endpoint - task_protocols = [md['PYBPOD_PROTOCOL'] + md['IBLRIG_VERSION_TAG'] for md in settings] - # unless specified label the session projects with subject projects - projects = subject['projects'] if projects is None else projects - # makes sure projects is a list - projects = [projects] if isinstance(projects, str) else projects - - # unless specified label the session procedures with task protocol lookup - procedures = procedures or list(set(filter(None, map(self._alyx_procedure_from_task, task_protocols)))) - procedures = [procedures] if isinstance(procedures, str) else procedures - json_fields_names = ['IS_MOCK', 'IBLRIG_VERSION'] - json_field = {k: settings[0].get(k) for k in json_fields_names} - # The poo count field is only updated if the field is defined in at least one of the settings - poo_counts = [md.get('POOP_COUNT') for md in settings if md.get('POOP_COUNT') is not None] - if poo_counts: - json_field['POOP_COUNT'] = int(sum(poo_counts)) + if collections is None: # No task data + assert len(session) != 0, 'no session on Alyx and no tasks in experiment description' + # Fetch the full session JSON and assert that some basic information is present. + # Basically refuse to extract the data if key information is missing + session_details = self.one.alyx.rest('sessions', 'read', id=session_id[0], no_cache=True) + required = ('location', 'start_time', 'lab', 'users') + missing = [k for k in required if not session_details[k]] + assert not any(missing), 'missing session information: ' + ', '.join(missing) + task_protocols = task_data = settings = [] + json_field = None + users = session_details['users'] + else: # Get session info from task data + collections = ensure_list(collections) + # read meta data from the rig for the session from the task settings file + task_data = (raw.load_bpod(ses_path, collection) for collection in sorted(collections)) + # Filter collections where settings file was not found + if not (task_data := list(zip(*filter(lambda x: x[0] is not None, task_data)))): + raise ValueError(f'_iblrig_taskSettings.raw.json not found in {ses_path} Abort.') + settings, task_data = task_data + if len(settings) != len(collections): + raise ValueError(f'_iblrig_taskSettings.raw.json not found in {ses_path} Abort.') + + # Do some validation + assert len({x['SUBJECT_NAME'] for x in settings}) == 1 and settings[0]['SUBJECT_NAME'] == subject['nickname'] + assert len({x['SESSION_DATE'] for x in settings}) == 1 and settings[0]['SESSION_DATE'] == date + assert len({x['SESSION_NUMBER'] for x in settings}) == 1 and settings[0]['SESSION_NUMBER'] == number + assert len({x['IS_MOCK'] for x in settings}) == 1 + assert len({md['PYBPOD_BOARD'] for md in settings}) == 1 + assert len({md.get('IBLRIG_VERSION') for md in settings}) == 1 + # assert len({md['IBLRIG_VERSION_TAG'] for md in settings}) == 1 + + users = [] + for user in filter(None, map(lambda x: x.get('PYBPOD_CREATOR'), settings)): + user = self.assert_exists(user[0], 'users') # user is list of [username, uuid] + users.append(user['username']) + + # extract information about session duration and performance + start_time, end_time = _get_session_times(str(ses_path), settings, task_data) + n_trials, n_correct_trials = _get_session_performance(settings, task_data) + + # TODO Add task_protocols to Alyx sessions endpoint + task_protocols = [md['PYBPOD_PROTOCOL'] + md['IBLRIG_VERSION_TAG'] for md in settings] + # unless specified label the session projects with subject projects + projects = subject['projects'] if projects is None else projects + # makes sure projects is a list + projects = [projects] if isinstance(projects, str) else projects + + # unless specified label the session procedures with task protocol lookup + procedures = procedures or list(set(filter(None, map(self._alyx_procedure_from_task, task_protocols)))) + procedures = [procedures] if isinstance(procedures, str) else procedures + json_fields_names = ['IS_MOCK', 'IBLRIG_VERSION'] + json_field = {k: settings[0].get(k) for k in json_fields_names} + # The poo count field is only updated if the field is defined in at least one of the settings + poo_counts = [md.get('POOP_COUNT') for md in settings if md.get('POOP_COUNT') is not None] + if poo_counts: + json_field['POOP_COUNT'] = int(sum(poo_counts)) if not session: # Create session and weighings ses_ = {'subject': subject['nickname'], @@ -258,9 +272,13 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N user = self.one.alyx.user self.register_weight(subject['nickname'], md['SUBJECT_WEIGHT'], date_time=md['SESSION_DATETIME'], user=user) - else: # if session exists update the JSON field - session = self.one.alyx.rest('sessions', 'read', id=session_id[0], no_cache=True) - self.one.alyx.json_field_update('sessions', session['id'], data=json_field) + else: # if session exists update a few key fields + data = {'procedures': procedures, 'projects': projects} + if task_protocols: + data['task_protocol'] = '/'.join(task_protocols) + session = self.one.alyx.rest('sessions', 'partial_update', id=session_id[0], data=data) + if json_field: + session['json'] = self.one.alyx.json_field_update('sessions', session['id'], data=json_field) _logger.info(session['url'] + ' ') # create associated water administration if not found @@ -279,7 +297,8 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N return session, None # register all files that match the Alyx patterns and file_list - rename_files_compatibility(ses_path, settings[0]['IBLRIG_VERSION_TAG']) + if any(settings): + rename_files_compatibility(ses_path, settings[0]['IBLRIG_VERSION_TAG']) F = filter(lambda x: self._register_bool(x.name, file_list), self.find_files(ses_path)) recs = self.register_files(F, created_by=users[0] if users else None, versions=ibllib.__version__) return session, recs diff --git a/ibllib/pipes/__init__.py b/ibllib/pipes/__init__.py index 2b68cdb04..95e8c6ce9 100644 --- a/ibllib/pipes/__init__.py +++ b/ibllib/pipes/__init__.py @@ -1,8 +1,28 @@ -#!/usr/bin/env python -# -*- coding:utf-8 -*- -# @Author: Niccolò Bonacchi -# @Date: Friday, July 5th 2019, 11:46:37 am -from ibllib.io.flags import FLAG_FILE_NAMES +"""IBL preprocessing pipeline. + +This module concerns the data extraction and preprocessing for IBL data. The lab servers routinely +call `local_server.job_creator` to search for new sessions to extract. The job creator registers +the new session to Alyx (i.e. creates a new session record on the database), if required, then +deduces a set of tasks (a.k.a. the pipeline [*]_) from the 'experiment.description' file at the +root of the session (see `dynamic_pipeline.make_pipeline`). If no file exists one is created, +inferring the acquisition hardware from the task protocol. The new session's pipeline tasks are +then registered for another process (or server) to query. + +Another process calls `local_server.task_queue` to get a list of queued tasks from Alyx, then +`local_server.tasks_runner` to loop through tasks. Each task is run by called +`tasks.run_alyx_task` with a dictionary of task information, including the Task class and its +parameters. + +.. [*] A pipeline is a collection of tasks that depend on one another. A pipeline consists of + tasks associated with the same session path. Unlike pipelines, tasks are represented in Alyx. + A pipeline can be recreated given a list of task dictionaries. The order is defined by the + 'parents' field of each task. + +Notes +----- +All new tasks are subclasses of the base_tasks.DynamicTask class. All others are defunct and shall +be removed in the future. +""" def assign_task(task_deck, session_path, task, **kwargs): diff --git a/ibllib/pipes/dynamic_pipeline.py b/ibllib/pipes/dynamic_pipeline.py index 7c8fd6065..bc2caaf1b 100644 --- a/ibllib/pipes/dynamic_pipeline.py +++ b/ibllib/pipes/dynamic_pipeline.py @@ -1,3 +1,8 @@ +"""Task pipeline creation from an acquisition description. + +The principal function here is `make_pipeline` which reads an `_ibl_experiment.description.yaml` +file and determines the set of tasks required to preprocess the session. +""" import logging import re from collections import OrderedDict @@ -9,7 +14,6 @@ import ibllib.io.session_params as sess_params import ibllib.io.extractors.base -import ibllib.pipes.ephys_preprocessing as epp import ibllib.pipes.tasks as mtasks import ibllib.pipes.base_tasks as bstasks import ibllib.pipes.widefield_tasks as wtasks @@ -307,14 +311,12 @@ def make_pipeline(session_path, **pkwargs): if 'cameras' in devices: cams = list(devices['cameras'].keys()) subset_cams = [c for c in cams if c in ('left', 'right', 'body', 'belly')] - video_kwargs = {'device_collection': 'raw_video_data', - 'cameras': cams} + video_kwargs = {'device_collection': 'raw_video_data', 'cameras': cams} video_compressed = sess_params.get_video_compressed(acquisition_description) if video_compressed: # This is for widefield case where the video is already compressed - tasks[tn] = type((tn := 'VideoConvert'), (vtasks.VideoConvert,), {})( - **kwargs, **video_kwargs) + tasks[tn] = type((tn := 'VideoConvert'), (vtasks.VideoConvert,), {})(**kwargs, **video_kwargs) dlc_parent_task = tasks['VideoConvert'] tasks[tn] = type((tn := f'VideoSyncQC_{sync}'), (vtasks.VideoSyncQcCamlog,), {})( **kwargs, **video_kwargs, **sync_kwargs) @@ -335,11 +337,25 @@ def make_pipeline(session_path, **pkwargs): if sync_kwargs['sync'] != 'bpod': # Here we restrict to videos that we support (left, right or body) + # Currently there is no plan to run DLC on the belly cam + subset_cams = [c for c in cams if c in ('left', 'right', 'body')] video_kwargs['cameras'] = subset_cams tasks[tn] = type((tn := 'DLC'), (vtasks.DLC,), {})( **kwargs, **video_kwargs, parents=[dlc_parent_task]) - tasks['PostDLC'] = type('PostDLC', (epp.EphysPostDLC,), {})( - **kwargs, parents=[tasks['DLC'], tasks[f'VideoSyncQC_{sync}']]) + + # The PostDLC plots require a trials object for QC + # Find the first task that outputs a trials.table dataset + trials_task = ( + t for t in tasks.values() if any('trials.table' in f for f in t.signature.get('output_files', [])) + ) + if trials_task := next(trials_task, None): + parents = [tasks['DLC'], tasks[f'VideoSyncQC_{sync}'], trials_task] + trials_collection = getattr(trials_task, 'output_collection', 'alf') + else: + parents = [tasks['DLC'], tasks[f'VideoSyncQC_{sync}']] + trials_collection = 'alf' + tasks[tn] = type((tn := 'PostDLC'), (vtasks.EphysPostDLC,), {})( + **kwargs, cameras=subset_cams, trials_collection=trials_collection, parents=parents) # Audio tasks if 'microphone' in devices: diff --git a/ibllib/pipes/ephys_preprocessing.py b/ibllib/pipes/ephys_preprocessing.py index 9cef81a34..26cef7050 100644 --- a/ibllib/pipes/ephys_preprocessing.py +++ b/ibllib/pipes/ephys_preprocessing.py @@ -1,3 +1,8 @@ +"""(Deprecated) Electrophysiology data preprocessing tasks. + +These tasks are part of the old pipeline. This module has been replaced by the `ephys_tasks` module +and the dynamic pipeline. +""" import logging import re import shutil diff --git a/ibllib/pipes/local_server.py b/ibllib/pipes/local_server.py index 47f6322b5..e04037b22 100644 --- a/ibllib/pipes/local_server.py +++ b/ibllib/pipes/local_server.py @@ -1,3 +1,9 @@ +"""Lab server pipeline construction and task runner. + +This is the module called by the job services on the lab servers. See +iblscripts/deploy/serverpc/crons for the service scripts that employ this module. +""" +import logging import time from datetime import datetime from pathlib import Path @@ -11,7 +17,6 @@ from one.api import ONE from one.webclient import AlyxClient from one.remote.globus import get_lab_from_endpoint_id, get_local_endpoint_id -from iblutil.util import setup_logger from ibllib.io.extractors.base import get_pipeline, get_task_protocol, get_session_extractor_type from ibllib.pipes import tasks, training_preprocessing, ephys_preprocessing @@ -21,8 +26,10 @@ from ibllib.io.session_params import read_params from ibllib.pipes.dynamic_pipeline import make_pipeline, acquisition_description_legacy_session -_logger = setup_logger(__name__, level='INFO') -LARGE_TASKS = ['EphysVideoCompress', 'TrainingVideoCompress', 'SpikeSorting', 'EphysDLC'] +_logger = logging.getLogger(__name__) +LARGE_TASKS = [ + 'EphysVideoCompress', 'TrainingVideoCompress', 'SpikeSorting', 'EphysDLC', 'MesoscopePreprocess' +] def _get_pipeline_class(session_path, one): @@ -65,7 +72,7 @@ def _get_volume_usage(vol, label=''): def report_health(one): """ - Get a few indicators and label the json field of the corresponding lab with them + Get a few indicators and label the json field of the corresponding lab with them. """ status = {'python_version': sys.version, 'ibllib_version': pkg_resources.get_distribution("ibllib").version, @@ -163,11 +170,20 @@ def job_creator(root_path, one=None, dry=False, rerun=False, max_md5_size=None): def task_queue(mode='all', lab=None, alyx=None): """ Query waiting jobs from the specified Lab - :param mode: Whether to return all waiting tasks, or only small or large (specified in LARGE_TASKS) jobs - :param lab: lab name as per Alyx, otherwise try to infer from local globus install - :param one: ONE instance - ------- + Parameters + ---------- + mode : {'all', 'small', 'large'} + Whether to return all waiting tasks, or only small or large (specified in LARGE_TASKS) jobs. + lab : str + Lab name as per Alyx, otherwise try to infer from local Globus install. + alyx : one.webclient.AlyxClient + An Alyx instance. + + Returns + ------- + list of dict + A list of Alyx tasks associated with `lab` that have a 'Waiting' status. """ alyx = alyx or AlyxClient(cache_rest=None) if lab is None: @@ -207,14 +223,29 @@ def task_queue(mode='all', lab=None, alyx=None): def tasks_runner(subjects_path, tasks_dict, one=None, dry=False, count=5, time_out=None, **kwargs): """ Function to run a list of tasks (task dictionary from Alyx query) on a local server - :param subjects_path: - :param tasks_dict: - :param one: - :param dry: - :param count: maximum number of tasks to run - :param time_out: between each task, if time elapsed is greater than time out, returns (seconds) - :param kwargs: - :return: list of dataset dictionaries + + Parameters + ---------- + subjects_path : str, pathlib.Path + The location of the subject session folders, e.g. '/mnt/s0/Data/Subjects'. + tasks_dict : list of dict + A list of tasks to run. Typically the output of `task_queue`. + one : one.api.OneAlyx + An instance of ONE. + dry : bool, default=False + If true, simply prints the full session paths and task names without running the tasks. + count : int, default=5 + The maximum number of tasks to run from the tasks_dict list. + time_out : float, optional + The time in seconds to run tasks before exiting. If set this will run tasks until the + timeout has elapsed. NB: Only checks between tasks and will not interrupt a running task. + **kwargs + See ibllib.pipes.tasks.run_alyx_task. + + Returns + ------- + list of pathlib.Path + A list of datasets registered to Alyx. """ if one is None: one = ONE(cache_rest=None) diff --git a/ibllib/pipes/mesoscope_tasks.py b/ibllib/pipes/mesoscope_tasks.py index e922eaefb..fee1a9c4a 100644 --- a/ibllib/pipes/mesoscope_tasks.py +++ b/ibllib/pipes/mesoscope_tasks.py @@ -15,6 +15,7 @@ import logging import subprocess import shutil +import uuid from pathlib import Path from itertools import chain from collections import defaultdict, Counter @@ -461,25 +462,35 @@ def _create_db(self, meta): Inputs to suite2p run that deviate from default parameters. """ - # Currently only supporting single plane, assert that this is the case - # FIXME This checks for zstacks but not dual plane mode - if not isinstance(meta['scanImageParams']['hStackManager']['zs'], int): - raise NotImplementedError('Multi-plane imaging not yet supported, data seems to be multi-plane') - # Computing dx and dy - cXY = np.array([fov['topLeftDeg'] for fov in meta['FOV']]) + cXY = np.array([fov['Deg']['topLeft'] for fov in meta['FOV']]) cXY -= np.min(cXY, axis=0) nXnYnZ = np.array([fov['nXnYnZ'] for fov in meta['FOV']]) - sW = np.sqrt(np.sum((np.array([fov['topRightDeg'] for fov in meta['FOV']]) - np.array( - [fov['topLeftDeg'] for fov in meta['FOV']])) ** 2, axis=1)) - sH = np.sqrt(np.sum((np.array([fov['bottomLeftDeg'] for fov in meta['FOV']]) - np.array( - [fov['topLeftDeg'] for fov in meta['FOV']])) ** 2, axis=1)) + + # Currently supporting z-stacks but not supporting dual plane / volumetric imaging, assert that this is not the case + if np.any(nXnYnZ[:, 2] > 1): + raise NotImplementedError('Dual-plane imaging not yet supported, data seems to more than one plane per FOV') + + sW = np.sqrt(np.sum((np.array([fov['Deg']['topRight'] for fov in meta['FOV']]) - np.array( + [fov['Deg']['topLeft'] for fov in meta['FOV']])) ** 2, axis=1)) + sH = np.sqrt(np.sum((np.array([fov['Deg']['bottomLeft'] for fov in meta['FOV']]) - np.array( + [fov['Deg']['topLeft'] for fov in meta['FOV']])) ** 2, axis=1)) pixSizeX = nXnYnZ[:, 0] / sW pixSizeY = nXnYnZ[:, 1] / sH dx = np.round(cXY[:, 0] * pixSizeX).astype(dtype=np.int32) dy = np.round(cXY[:, 1] * pixSizeY).astype(dtype=np.int32) nchannels = len(meta['channelSaved']) if isinstance(meta['channelSaved'], list) else 1 + # Computing number of unique z-planes (slices in tiff) + # FIXME this should work if all FOVs are discrete or if all FOVs are continuous, but may not work for combination of both + slice_ids = [fov['slice_id'] for fov in meta['FOV']] + nplanes = len(set(slice_ids)) + + # Figuring out how many SI Rois we have (one unique ROI may have several FOVs) + # FIXME currently unused + # roiUUIDs = np.array([fov['roiUUID'] for fov in meta['FOV']]) + # nrois = len(np.unique(roiUUIDs)) + db = { 'data_path': sorted(map(str, self.session_path.glob(f'{self.device_collection}'))), 'save_path0': str(self.session_path.joinpath('alf')), @@ -498,13 +509,13 @@ def _create_db(self, meta): 'block_size': [128, 128], 'save_mat': True, # save the data to Fall.mat 'move_bin': True, # move the binary file to save_path - 'scalefactor': 1, # scale manually in x to account for overlap between adjacent ribbons UCL mesoscope 'mesoscan': True, - 'nplanes': 1, + 'nplanes': nplanes, 'nrois': len(meta['FOV']), 'nchannels': nchannels, 'fs': meta['scanImageParams']['hRoiManager']['scanVolumeRate'], 'lines': [list(np.asarray(fov['lineIdx']) - 1) for fov in meta['FOV']], # subtracting 1 to make 0-based + 'slices': slice_ids, # this tells us which FOV corresponds to which tiff slices 'tau': self.get_default_tau(), # deduce the GCamp used from Alyx mouse line (defaults to 1.5; that of GCaMP6s) 'functional_chan': 1, # for now, eventually find(ismember(meta.channelSaved == meta.channelID.green)) 'align_by_chan': 1, # for now, eventually find(ismember(meta.channelSaved == meta.channelID.red)) @@ -691,13 +702,14 @@ def _run(self, *args, provenance=Provenance.ESTIMATE, **kwargs): Notes ----- - Once the FOVs have been registered they cannot be updated with with task. Rerunning this - task will result in an error. + - Once the FOVs have been registered they cannot be updated with this task. Rerunning this + task will result in an error. + - This task modifies the first meta JSON file. All meta files are registered by this task. """ # Load necessary data (filename, collection, _), *_ = self.signature['input_files'] - meta_file = next(self.session_path.glob(f'{collection}/{filename}'), None) - meta = alfio.load_file_content(meta_file) or {} + meta_files = sorted(self.session_path.glob(f'{collection}/{filename}')) + meta = mesoscope.patch_imaging_meta(alfio.load_file_content(meta_files[0]) or {}) nFOV = len(meta.get('FOV', [])) suffix = None if provenance is Provenance.HISTOLOGY else provenance.name.lower() @@ -707,7 +719,7 @@ def _run(self, *args, provenance=Provenance.ESTIMATE, **kwargs): mean_image_mlapdv, mean_image_ids = self.project_mlapdv(meta) # Save the meta data file with new coordinate fields - with open(meta_file, 'w') as fp: + with open(meta_files[0], 'w') as fp: json.dump(meta, fp) # Save the mean image datasets @@ -736,7 +748,47 @@ def _run(self, *args, provenance=Provenance.ESTIMATE, **kwargs): # Register FOVs in Alyx self.register_fov(meta, suffix) - return sorted([meta_file, *roi_files, *mean_image_files]) + return sorted([*meta_files, *roi_files, *mean_image_files]) + + def update_surgery_json(self, meta, normal_vector): + """ + Update surgery JSON with surface normal vector. + + Adds the key 'surface_normal_unit_vector' to the most recent surgery JSON, containing the + provided three element vector. The recorded craniotomy center must match the coordinates + in the provided meta file. + + Parameters + ---------- + meta : dict + The imaging meta data file containing the 'centerMM' key. + normal_vector : array_like + A three element unit vector normal to the surface of the craniotomy center. + + Returns + ------- + dict + The updated surgery record, or None if no surgeries found. + """ + if not self.one or self.one.offline: + _logger.warning('failed to update surgery JSON: ONE offline') + return + # Update subject JSON with unit normal vector of craniotomy centre (used in histology) + subject = self.one.path2ref(self.session_path, parse=False)['subject'] + surgeries = self.one.alyx.rest('surgeries', 'list', subject=subject, procedure='craniotomy') + if not surgeries: + _logger.error(f'Surgery not found for subject "{subject}"') + return + surgery = surgeries[0] # Check most recent surgery in list + center = (meta['centerMM']['ML'], meta['centerMM']['AP']) + match = (k for k, v in surgery['json'].items() if + str(k).startswith('craniotomy') and np.allclose(v['center'], center)) + if (key := next(match, None)) is None: + _logger.error('Failed to update surgery JSON: no matching craniotomy found') + return surgery + data = {key: {**surgery['json'][key], 'surface_normal_unit_vector': tuple(normal_vector)}} + surgery['json'] = self.one.alyx.json_field_update('subjects', subject, data=data) + return surgery def roi_mlapdv(self, nFOV: int, suffix=None): """ @@ -755,9 +807,9 @@ def roi_mlapdv(self, nFOV: int, suffix=None): Returns ------- - dict of int: numpy.array + dict of int : numpy.array A map of field of view to ROI MLAPDV coordinates. - dict of int: numpy.array + dict of int : numpy.array A map of field of view to ROI brain location IDs. """ all_mlapdv = {} @@ -842,8 +894,11 @@ def register_fov(self, meta: dict, suffix: str = None) -> (list, list): slice_counts = Counter(f['roiUUID'] for f in meta.get('FOV', [])) # Create a new stack in Alyx for all stacks containing more than one slice. # Map of ScanImage ROI UUID to Alyx ImageStack UUID. - stack_ids = {i: self.one.alyx.rest('imaging-stack', 'create', data={'name': i})['id'] - for i in slice_counts if slice_counts[i] > 1} + if dry: + stack_ids = {i: uuid.uuid4() for i in slice_counts if slice_counts[i] > 1} + else: + stack_ids = {i: self.one.alyx.rest('imaging-stack', 'create', data={'name': i})['id'] + for i in slice_counts if slice_counts[i] > 1} for i, fov in enumerate(meta.get('FOV', [])): assert set(fov.keys()) >= {'MLAPDV', 'nXnYnZ', 'roiUUID'} @@ -962,6 +1017,9 @@ def project_mlapdv(self, meta, atlas=None): # Get the surface normal unit vector of dorsal triangle normal_vector = surface_normal(dorsal_triangle) + # Update the surgery JSON field with normal unit vector, for use in histology alignment + self.update_surgery_json(meta, normal_vector) + # find the coordDV that sits on the triangular face and had [coordML, coordAP] coordinates; # the three vertices defining the triangle face_vertices = points[dorsal_connectivity_list[face_ind, :], :] @@ -1005,13 +1063,6 @@ def project_mlapdv(self, meta, atlas=None): # xx and yy are in mm in coverslip space points = ((0, fov['nXnYnZ'][0] - 1), (0, fov['nXnYnZ'][1] - 1)) - if 'MM' not in fov: - fov['MM'] = { - 'topLeft': fov.pop('topLeftMM'), - 'topRight': fov.pop('topRightMM'), - 'bottomLeft': fov.pop('bottomLeftMM'), - 'bottomRight': fov.pop('bottomRightMM') - } # The four corners of the FOV, determined by taking the center of the craniotomy in MM, # the x-y coordinates of the imaging window center (from the tiled reference image) in # galvanometer units, and the x-y coordinates of the FOV center in galvanometer units. diff --git a/ibllib/pipes/misc.py b/ibllib/pipes/misc.py index d3911533f..39871ad00 100644 --- a/ibllib/pipes/misc.py +++ b/ibllib/pipes/misc.py @@ -1,3 +1,4 @@ +"""Miscellaneous pipeline utility functions.""" import ctypes import hashlib import json diff --git a/ibllib/pipes/purge_rig_data.py b/ibllib/pipes/purge_rig_data.py index abe0251da..9b7afba05 100644 --- a/ibllib/pipes/purge_rig_data.py +++ b/ibllib/pipes/purge_rig_data.py @@ -1,13 +1,12 @@ -#!/usr/bin/env python -# -*- coding:utf-8 -*- -# @Author: Niccolò Bonacchi -# @Date: Thursday, March 28th 2019, 7:53:44 pm """ -Purge data from RIG +Purge data from acquisition PC. + +Steps: + - Find all files by rglob - Find all sessions of the found files - Check Alyx if corresponding datasetTypes have been registered as existing -sessions and files on Flatiron + sessions and files on Flatiron - Delete local raw file if found on Flatiron """ import argparse diff --git a/ibllib/pipes/tasks.py b/ibllib/pipes/tasks.py index b6e632579..25a645385 100644 --- a/ibllib/pipes/tasks.py +++ b/ibllib/pipes/tasks.py @@ -1,3 +1,4 @@ +"""The abstract Pipeline and Task superclasses and concrete task runner.""" from pathlib import Path import abc import logging @@ -602,22 +603,39 @@ def name(self): def run_alyx_task(tdict=None, session_path=None, one=None, job_deck=None, max_md5_size=None, machine=None, clobber=True, location='server', mode='log'): """ - Runs a single Alyx job and registers output datasets - :param tdict: - :param session_path: - :param one: - :param job_deck: optional list of job dictionaries belonging to the session. Needed - to check dependency status if the jdict has a parent field. If jdict has a parent and - job_deck is not entered, will query the database - :param max_md5_size: in bytes, if specified, will not compute the md5 checksum above a given - filesize to save time - :param machine: string identifying the machine the task is run on, optional - :param clobber: bool, if True any existing logs are overwritten, default is True - :param location: where you are running the task, 'server' - local lab server, 'remote' - any - compute node/ computer, 'SDSC' - flatiron compute node, 'AWS' - using data from aws s3 - :param mode: str ('log' or 'raise') behaviour to adopt if an error occured. If 'raise', it - will Raise the error at the very end of this function (ie. after having labeled the tasks) - :return: + Runs a single Alyx job and registers output datasets. + + Parameters + ---------- + tdict : dict + An Alyx task dictionary to instantiate and run. + session_path : str, pathlib.Path + A session path containing the task input data. + one : one.api.OneAlyx + An instance of ONE. + job_deck : list of dict, optional + A list of all tasks in the same pipeline. If None, queries Alyx to get this. + max_md5_size : int, optional + An optional maximum file size in bytes. Files with sizes larger than this will not have + their MD5 checksum calculated to save time. + machine : str, optional + A string identifying the machine the task is run on. + clobber : bool, default=True + If true any existing logs are overwritten on Alyx. + location : {'remote', 'server', 'sdsc', 'aws'} + Where you are running the task, 'server' - local lab server, 'remote' - any + compute node/ computer, 'sdsc' - Flatiron compute node, 'aws' - using data from AWS S3 + node. + mode : {'log', 'raise}, default='log' + Behaviour to adopt if an error occurred. If 'raise', it will raise the error at the very + end of this function (i.e. after having labeled the tasks). + + Returns + ------- + Task + The instantiated task object that was run. + list of pathlib.Path + A list of registered datasets. """ registered_dsets = [] # here we need to check parents' status, get the job_deck if not available diff --git a/ibllib/pipes/training_preprocessing.py b/ibllib/pipes/training_preprocessing.py index b47adcc65..db41f8992 100644 --- a/ibllib/pipes/training_preprocessing.py +++ b/ibllib/pipes/training_preprocessing.py @@ -1,3 +1,9 @@ +"""(Deprecated) Training data preprocessing tasks. + +These tasks are part of the old pipeline. This module has been replaced by the dynamic pipeline +and the `behavior_tasks` module. +""" + import logging from collections import OrderedDict from one.alf.files import session_path_parts diff --git a/ibllib/pipes/video_tasks.py b/ibllib/pipes/video_tasks.py index eaf00aaa0..7f501a065 100644 --- a/ibllib/pipes/video_tasks.py +++ b/ibllib/pipes/video_tasks.py @@ -1,9 +1,13 @@ import logging import subprocess -import cv2 import traceback from pathlib import Path +import cv2 +import pandas as pd +import numpy as np + +from ibllib.qc.dlc import DlcQC from ibllib.io import ffmpeg, raw_daq_loaders from ibllib.pipes import base_tasks from ibllib.io.video import get_video_meta @@ -11,6 +15,9 @@ from ibllib.qc.camera import run_all_qc as run_camera_qc from ibllib.misc import check_nvidia_driver from ibllib.io.video import label_from_path, assert_valid_label +from ibllib.plots.snapshot import ReportSnapshot +from ibllib.plots.figures import dlc_qc_plot +from brainbox.behavior.dlc import likelihood_threshold, get_licks, get_pupil_diameter, get_smooth_pupil_diameter _logger = logging.getLogger('ibllib') @@ -48,9 +55,9 @@ def assert_expected_outputs(self, raise_error=True): required = any('Camera.frameData' in x or 'Camera.timestamps' in x for x in map(str, files)) if not (everything_is_fine and required): for out in self.outputs: - _logger.error(f"{out}") + _logger.error(f'{out}') if raise_error: - raise FileNotFoundError("Missing outputs after task completion") + raise FileNotFoundError('Missing outputs after task completion') return everything_is_fine, files @@ -120,7 +127,7 @@ def _run(self): # convert the avi files to mp4 avi_file = next(self.session_path.joinpath(self.device_collection).glob(f'{cam}_cam*.avi')) mp4_file = self.session_path.joinpath(self.device_collection, f'_iblrig_{cam}Camera.raw.mp4') - command2run = f"ffmpeg -i {str(avi_file)} -c:v copy -c:a copy -y {str(mp4_file)}" + command2run = f'ffmpeg -i {str(avi_file)} -c:v copy -c:a copy -y {str(mp4_file)}' process = subprocess.Popen(command2run, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) info, error = process.communicate() @@ -191,7 +198,7 @@ def _run(self, qc=True, **kwargs): class VideoSyncQcBpod(base_tasks.VideoTask): """ Task to sync camera timestamps to main DAQ timestamps - N.B Signatures only reflect new daq naming convention, non compatible with ephys when not running on server + N.B Signatures only reflect new daq naming convention, non-compatible with ephys when not running on server """ priority = 40 job_size = 'small' @@ -241,7 +248,7 @@ def _run(self, **kwargs): class VideoSyncQcNidq(base_tasks.VideoTask): """ Task to sync camera timestamps to main DAQ timestamps - N.B Signatures only reflect new daq naming convention, non compatible with ephys when not running on server + N.B Signatures only reflect new daq naming convention, non-compatible with ephys when not running on server """ priority = 40 job_size = 'small' @@ -328,19 +335,19 @@ def _check_dlcenv(self): f'Scripts run_dlc.sh and run_dlc.py do not exist in {self.scripts}' assert len(list(self.scripts.rglob('run_motion.*'))) == 2, \ f'Scripts run_motion.sh and run_motion.py do not exist in {self.scripts}' - assert self.dlcenv.exists(), f"DLC environment does not exist in assumed location {self.dlcenv}" + assert self.dlcenv.exists(), f'DLC environment does not exist in assumed location {self.dlcenv}' command2run = f"source {self.dlcenv}; python -c 'import iblvideo; print(iblvideo.__version__)'" process = subprocess.Popen( command2run, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - executable="/bin/bash" + executable='/bin/bash' ) info, error = process.communicate() if process.returncode != 0: raise AssertionError(f"DLC environment check failed\n{error.decode('utf-8')}") - version = info.decode("utf-8").strip().split('\n')[-1] + version = info.decode('utf-8').strip().split('\n')[-1] return version @staticmethod @@ -378,11 +385,11 @@ def _run(self, cams=None, overwrite=False): file_mp4 = next(self.session_path.joinpath('raw_video_data').glob(f'_iblrig_{cam}Camera.raw*.mp4')) if not file_mp4.exists(): # In this case we set the status to Incomplete. - _logger.error(f"No raw video file available for {cam}, skipping.") + _logger.error(f'No raw video file available for {cam}, skipping.') self.status = -3 continue if not self._video_intact(file_mp4): - _logger.error(f"Corrupt raw video file {file_mp4}") + _logger.error(f'Corrupt raw video file {file_mp4}') self.status = -1 continue # Check that dlc environment is ok, shell scripts exists, and get iblvideo version, GPU addressable @@ -398,13 +405,13 @@ def _run(self, cams=None, overwrite=False): shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - executable="/bin/bash", + executable='/bin/bash', ) info, error = process.communicate() # info_str = info.decode("utf-8").strip() # _logger.info(info_str) if process.returncode != 0: - error_str = error.decode("utf-8").strip() + error_str = error.decode('utf-8').strip() _logger.error(f'DLC failed for {cam}Camera.\n\n' f'++++++++ Output of subprocess for debugging ++++++++\n\n' f'{error_str}\n' @@ -423,13 +430,13 @@ def _run(self, cams=None, overwrite=False): shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - executable="/bin/bash", + executable='/bin/bash', ) info, error = process.communicate() - # info_str = info.decode("utf-8").strip() + # info_str = info.decode('utf-8').strip() # _logger.info(info_str) if process.returncode != 0: - error_str = error.decode("utf-8").strip() + error_str = error.decode('utf-8').strip() _logger.error(f'Motion energy failed for {cam}Camera.\n\n' f'++++++++ Output of subprocess for debugging ++++++++\n\n' f'{error_str}\n' @@ -440,7 +447,7 @@ def _run(self, cams=None, overwrite=False): f'{cam}Camera.ROIMotionEnergy*.npy'))) actual_outputs.append(next(self.session_path.joinpath('alf').glob( f'{cam}ROIMotionEnergy.position*.npy'))) - except BaseException: + except Exception: _logger.error(traceback.format_exc()) self.status = -1 continue @@ -450,3 +457,156 @@ def _run(self, cams=None, overwrite=False): actual_outputs = None self.status = -1 return actual_outputs + + +class EphysPostDLC(base_tasks.VideoTask): + """ + The post_dlc task takes dlc traces as input and computes useful quantities, as well as qc. + """ + io_charge = 90 + level = 3 + force = True + + def __int__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.trials_collection = kwargs.get('trials_collection', 'alf') + + @property + def signature(self): + return { + 'input_files': [(f'_ibl_{cam}Camera.dlc.pqt', 'alf', True) for cam in self.cameras] + + [(f'_ibl_{cam}Camera.times.npy', 'alf', True) for cam in self.cameras] + + # the following are required for the DLC plot only + # they are not strictly required, some plots just might be skipped + # In particular the raw videos don't need to be downloaded as they can be streamed + [(f'_iblrig_{cam}Camera.raw.mp4', self.device_collection, True) for cam in self.cameras] + + [(f'{cam}ROIMotionEnergy.position.npy', 'alf', False) for cam in self.cameras] + + # The trials table is used in the DLC QC, however this is not an essential dataset + [('_ibl_trials.table.pqt', self.trials_collection, False)], + 'output_files': [(f'_ibl_{cam}Camera.features.pqt', 'alf', True) for cam in self.cameras] + + [('licks.times.npy', 'alf', True)] + } + + def _run(self, overwrite=True, run_qc=True, plot_qc=True): + """ + Run the PostDLC task. Returns a list of file locations for the output files in signature. The created plot + (dlc_qc_plot.png) is not returned, but saved in session_path/snapshots and uploaded to Alyx as a note. + + :param overwrite: bool, whether to recompute existing output files (default is False). + Note that the dlc_qc_plot will be (re-)computed even if overwrite = False + :param run_qc: bool, whether to run the DLC QC (default is True) + :param plot_qc: book, whether to create the dlc_qc_plot (default is True) + + """ + # Check if output files exist locally + exist, output_files = self.assert_expected(self.signature['output_files'], silent=True) + if exist and not overwrite: + _logger.warning('EphysPostDLC outputs exist and overwrite=False, skipping computations of outputs.') + else: + if exist and overwrite: + _logger.warning('EphysPostDLC outputs exist and overwrite=True, overwriting existing outputs.') + # Find all available DLC files + dlc_files = list(Path(self.session_path).joinpath('alf').glob('_ibl_*Camera.dlc.*')) + for dlc_file in dlc_files: + _logger.debug(dlc_file) + output_files = [] + combined_licks = [] + + for dlc_file in dlc_files: + # Catch unforeseen exceptions and move on to next cam + try: + cam = label_from_path(dlc_file) + # load dlc trace and camera times + dlc = pd.read_parquet(dlc_file) + dlc_thresh = likelihood_threshold(dlc, 0.9) + # try to load respective camera times + try: + dlc_t = np.load(next(Path(self.session_path).joinpath('alf').glob(f'_ibl_{cam}Camera.times.*npy'))) + times = True + if dlc_t.shape[0] == 0: + _logger.error(f'camera.times empty for {cam} camera. ' + f'Computations using camera.times will be skipped') + self.status = -1 + times = False + elif dlc_t.shape[0] < len(dlc_thresh): + _logger.error(f'Camera times shorter than DLC traces for {cam} camera. ' + f'Computations using camera.times will be skipped') + self.status = -1 + times = 'short' + except StopIteration: + self.status = -1 + times = False + _logger.error(f'No camera.times for {cam} camera. ' + f'Computations using camera.times will be skipped') + # These features are only computed from left and right cam + if cam in ('left', 'right'): + features = pd.DataFrame() + # If camera times are available, get the lick time stamps for combined array + if times is True: + _logger.info(f'Computing lick times for {cam} camera.') + combined_licks.append(get_licks(dlc_thresh, dlc_t)) + elif times is False: + _logger.warning(f'Skipping lick times for {cam} camera as no camera.times available') + elif times == 'short': + _logger.warning(f'Skipping lick times for {cam} camera as camera.times are too short') + # Compute pupil diameter, raw and smoothed + _logger.info(f'Computing raw pupil diameter for {cam} camera.') + features['pupilDiameter_raw'] = get_pupil_diameter(dlc_thresh) + try: + _logger.info(f'Computing smooth pupil diameter for {cam} camera.') + features['pupilDiameter_smooth'] = get_smooth_pupil_diameter(features['pupilDiameter_raw'], + cam) + except Exception: + _logger.error(f'Computing smooth pupil diameter for {cam} camera failed, saving all NaNs.') + _logger.error(traceback.format_exc()) + features['pupilDiameter_smooth'] = np.nan + # Save to parquet + features_file = Path(self.session_path).joinpath('alf', f'_ibl_{cam}Camera.features.pqt') + features.to_parquet(features_file) + output_files.append(features_file) + + # For all cams, compute DLC QC if times available + if run_qc is True and times in [True, 'short']: + # Setting download_data to False because at this point the data should be there + qc = DlcQC(self.session_path, side=cam, one=self.one, download_data=False) + qc.run(update=True) + else: + if times is False: + _logger.warning(f'Skipping QC for {cam} camera as no camera.times available') + if not run_qc: + _logger.warning(f'Skipping QC for {cam} camera as run_qc=False') + + except Exception: + _logger.error(traceback.format_exc()) + self.status = -1 + continue + + # Combined lick times + if len(combined_licks) > 0: + lick_times_file = Path(self.session_path).joinpath('alf', 'licks.times.npy') + np.save(lick_times_file, sorted(np.concatenate(combined_licks))) + output_files.append(lick_times_file) + else: + _logger.warning('No lick times computed for this session.') + + if plot_qc: + _logger.info('Creating DLC QC plot') + try: + session_id = self.one.path2eid(self.session_path) + fig_path = self.session_path.joinpath('snapshot', 'dlc_qc_plot.png') + if not fig_path.parent.exists(): + fig_path.parent.mkdir(parents=True, exist_ok=True) + fig = dlc_qc_plot(self.session_path, one=self.one, cameras=self.cameras, device_collection=self.device_collection, + trials_collection=self.trials_collection) + fig.savefig(fig_path) + fig.clf() + snp = ReportSnapshot(self.session_path, session_id, one=self.one) + snp.outputs = [fig_path] + snp.register_images(widths=['orig'], + function=str(dlc_qc_plot.__module__) + '.' + str(dlc_qc_plot.__name__)) + except Exception: + _logger.error('Could not create and/or upload DLC QC Plot') + _logger.error(traceback.format_exc()) + self.status = -1 + + return output_files diff --git a/ibllib/plots/figures.py b/ibllib/plots/figures.py index 17762ce01..34a444e9b 100644 --- a/ibllib/plots/figures.py +++ b/ibllib/plots/figures.py @@ -669,7 +669,8 @@ def raw_destripe(raw, fs, t0, i_plt, n_plt, return fig, axs -def dlc_qc_plot(session_path, one=None): +def dlc_qc_plot(session_path, one=None, device_collection='raw_video_data', + cameras=('left', 'right', 'body'), trials_collection='alf'): """ Creates DLC QC plot. Data is searched first locally, then on Alyx. Panels that lack required data are skipped. @@ -707,14 +708,13 @@ def dlc_qc_plot(session_path, one=None): if one.alyx.base_url == 'https://alyx.cortexlab.net': one = ONE(base_url='https://alyx.internationalbrainlab.org') data = {} - cams = ['left', 'right', 'body'] session_path = Path(session_path) # Load data for each camera - for cam in cams: + for cam in cameras: # Load a single frame for each video # Check if video data is available locally,if yes, load a single frame - video_path = session_path.joinpath('raw_video_data', f'_iblrig_{cam}Camera.raw.mp4') + video_path = session_path.joinpath(device_collection, f'_iblrig_{cam}Camera.raw.mp4') if video_path.exists(): data[f'{cam}_frame'] = get_video_frame(video_path, frame_number=5 * 60 * SAMPLING[cam])[:, :, 0] # If not, try to stream a frame (try three times) @@ -725,7 +725,7 @@ def dlc_qc_plot(session_path, one=None): try: data[f'{cam}_frame'] = get_video_frame(video_url, frame_number=5 * 60 * SAMPLING[cam])[:, :, 0] break - except BaseException: + except Exception: if tries < 2: tries += 1 logger.info(f"Streaming {cam} video failed, retrying x{tries}") @@ -757,19 +757,20 @@ def dlc_qc_plot(session_path, one=None): data[f'{cam}_{feat}'] = None # If we have no frame and/or no DLC and/or no times for all cams, raise an error, something is really wrong - assert any([data[f'{cam}_frame'] is not None for cam in cams]), "No camera data could be loaded, aborting." - assert any([data[f'{cam}_dlc'] is not None for cam in cams]), "No DLC data could be loaded, aborting." - assert any([data[f'{cam}_times'] is not None for cam in cams]), "No camera times data could be loaded, aborting." + assert any(data[f'{cam}_frame'] is not None for cam in cameras), "No camera data could be loaded, aborting." + assert any(data[f'{cam}_dlc'] is not None for cam in cameras), "No DLC data could be loaded, aborting." + assert any(data[f'{cam}_times'] is not None for cam in cameras), "No camera times data could be loaded, aborting." # Load session level data for alf_object in ['trials', 'wheel', 'licks']: try: - data[f'{alf_object}'] = alfio.load_object(session_path.joinpath('alf'), alf_object) # load locally + data[f'{alf_object}'] = alfio.load_object(session_path.joinpath(trials_collection), alf_object) # load locally continue except ALFObjectNotFound: pass try: - data[f'{alf_object}'] = one.load_object(one.path2eid(session_path), alf_object) # then try from alyx + # then try from alyx + data[f'{alf_object}'] = one.load_object(one.path2eid(session_path), alf_object, collection=trials_collection) except ALFObjectNotFound: logger.warning(f"Could not load {alf_object} object, some plots have to be skipped.") data[f'{alf_object}'] = None @@ -786,7 +787,7 @@ def dlc_qc_plot(session_path, one=None): # Make a list of panels, if inputs are missing, instead input a text to display panels = [] # Panel A, B, C: Trace on frame - for cam in cams: + for cam in cameras: if data[f'{cam}_frame'] is not None and data[f'{cam}_dlc'] is not None: panels.append((plot_trace_on_frame, {'frame': data[f'{cam}_frame'], 'dlc_df': data[f'{cam}_dlc'], 'cam': cam})) @@ -795,15 +796,14 @@ def dlc_qc_plot(session_path, one=None): # If trials data is not there, we cannot plot any of the trial average plots, skip all remaining panels if data['trials'] is None: - panels.extend([(None, 'No trial data,\ncannot compute trial avgs') for i in range(7)]) + panels.extend([(None, 'No trial data,\ncannot compute trial avgs')] * 7) else: # Panel D: Motion energy - camera_dict = {'left': {'motion_energy': data['left_ROIMotionEnergy'], 'times': data['left_times']}, - 'right': {'motion_energy': data['right_ROIMotionEnergy'], 'times': data['right_times']}, - 'body': {'motion_energy': data['body_ROIMotionEnergy'], 'times': data['body_times']}} - for cam in ['left', 'right', 'body']: # Remove cameras where we don't have motion energy AND camera times - if camera_dict[cam]['motion_energy'] is None or camera_dict[cam]['times'] is None: - _ = camera_dict.pop(cam) + camera_dict = {} + for cam in cameras: # Remove cameras where we don't have motion energy AND camera times + d = {'motion_energy': data.get(f'{cam}_ROIMotionEnergy'), 'times': data.get(f'{cam}_times')} + if not any(x is None for x in d.values()): + camera_dict[cam] = d if len(camera_dict) > 0: panels.append((plot_motion_energy_hist, {'camera_dict': camera_dict, 'trials_df': data['trials']})) else: @@ -833,7 +833,7 @@ def dlc_qc_plot(session_path, one=None): 'trials_df': data['trials'], 'feature': 'nose_tip', 'legend': False, 'cam': cam})) else: - panels.extend([(None, 'Data missing or corrupt\nSpeed histograms') for i in range(2)]) + panels.extend([(None, 'Data missing or corrupt\nSpeed histograms')] * 2) # Panel H and I: Lick plots if data['licks'] and data['licks'].times.shape[0] > 0: @@ -846,7 +846,7 @@ def dlc_qc_plot(session_path, one=None): # Try if all data is there for left cam first, otherwise right for cam in ['left', 'right']: fail = False - if (data[f'{cam}_times'] is not None and data[f'{cam}_features'] is not None + if (data.get(f'{cam}_times') is not None and data.get(f'{cam}_features') is not None and len(data[f'{cam}_times']) >= len(data[f'{cam}_features']) and not np.all(np.isnan(data[f'{cam}_features'].pupilDiameter_smooth))): break @@ -872,7 +872,7 @@ def dlc_qc_plot(session_path, one=None): else: try: panel[0](**panel[1]) - except BaseException: + except Exception: logger.error(f'Error in {panel[0].__name__}\n' + traceback.format_exc()) ax.text(.5, .5, f'Error while plotting\n{panel[0].__name__}', color='r', fontweight='bold', fontsize=12, horizontalalignment='center', verticalalignment='center', transform=ax.transAxes) diff --git a/ibllib/tests/test_mesoscope.py b/ibllib/tests/test_mesoscope.py index 4579d202b..b828386db 100644 --- a/ibllib/tests/test_mesoscope.py +++ b/ibllib/tests/test_mesoscope.py @@ -35,23 +35,22 @@ def test_meta(self): """ expected = { 'data_path': [str(self.img_path)], + 'save_path0': str(self.session_path.joinpath('alf')), 'fast_disk': '', + 'look_one_level_down': False, 'num_workers': -1, - 'save_path0': str(self.session_path.joinpath('alf')), - 'move_bin': True, + 'num_workers_roi': -1, 'keep_movie_raw': False, 'delete_bin': False, 'batch_size': 500, - 'combined': False, - 'look_one_level_down': False, - 'num_workers_roi': -1, 'nimg_init': 400, + 'combined': False, 'nonrigid': True, 'maxregshift': 0.05, 'denoise': 1, 'block_size': [128, 128], 'save_mat': True, - 'scalefactor': 1, + 'move_bin': True, 'mesoscan': True, 'nplanes': 1, 'tau': 1.5, @@ -61,6 +60,7 @@ def test_meta(self): 'nchannels': 1, 'fs': 6.8, 'lines': [[3, 4, 5]], + 'slices': [0], 'dx': np.array([0], dtype=int), 'dy': np.array([0], dtype=int), } @@ -69,7 +69,7 @@ def test_meta(self): 'scanImageParams': {'hStackManager': {'zs': 320}, 'hRoiManager': {'scanVolumeRate': 6.8}}, 'FOV': [{'topLeftDeg': [-1, 1.3], 'topRightDeg': [3, 1.3], 'bottomLeftDeg': [-1, 5.2], - 'nXnYnZ': [512, 512, 1], 'channelIdx': 2, 'lineIdx': [4, 5, 6]}] + 'nXnYnZ': [512, 512, 1], 'channelIdx': 2, 'lineIdx': [4, 5, 6], 'slice_id': 0}] } with open(self.img_path.joinpath('_ibl_rawImagingData.meta.json'), 'w') as f: json.dump(meta, f) @@ -150,6 +150,41 @@ def test_nearest_neighbour_1d(self): np.testing.assert_array_equal(val, [1., 1., 1., 3., 3., 2., 5., 5.]) np.testing.assert_array_equal(ind, [1, 1, 1, 4, 4, 0, 3, 3]) + def test_update_surgery_json(self): + """Test for MesoscopeFOV.update_surgery_json method. + + Here we mock the Alyx object and simply check the method's calls. + """ + one = ONE(**TEST_DB) + task = MesoscopeFOV('/foo/bar/subject/2020-01-01/001', one=one) + record = {'json': {'craniotomy_00': {'center': [1., -3.]}, 'craniotomy_01': {'center': [2.7, -1.3]}}} + normal_vector = np.array([0.5, 1., 0.]) + meta = {'centerMM': {'ML': 2.7, 'AP': -1.30000000001}} + with mock.patch.object(one.alyx, 'rest', return_value=[record, {}]), \ + mock.patch.object(one.alyx, 'json_field_update') as mock_rest: + task.update_surgery_json(meta, normal_vector) + expected = {'craniotomy_01': {'center': [2.7, -1.3], + 'surface_normal_unit_vector': (0.5, 1., 0.)}} + mock_rest.assert_called_once_with('subjects', 'subject', data=expected) + + # Check errors and warnings + # No matching craniotomy center + with self.assertLogs('ibllib.pipes.mesoscope_tasks', 'ERROR'), \ + mock.patch.object(one.alyx, 'rest', return_value=[record, {}]): + task.update_surgery_json({'centerMM': {'ML': 0., 'AP': 0.}}, normal_vector) + # No matching surgery records + with self.assertLogs('ibllib.pipes.mesoscope_tasks', 'ERROR'), \ + mock.patch.object(one.alyx, 'rest', return_value=[]): + task.update_surgery_json(meta, normal_vector) + # ONE offline + one.mode = 'local' + try: + with self.assertLogs('ibllib.pipes.mesoscope_tasks', 'WARNING'): + task.update_surgery_json(meta, normal_vector) + finally: + # ONE function is cached so we must reset the mode for other tests + one.mode = 'auto' + class TestRegisterFOV(unittest.TestCase): """Test for MesoscopeFOV.register_fov method.""" @@ -173,7 +208,7 @@ def test_register_fov(self): 'bottomLeft': [2317.3, -2181.4, -466.3], 'bottomRight': [2862.7, -2206.9, -679.4], 'center': [2596.1, -1900.5, -588.6]} meta = {'FOV': [{'MLAPDV': mlapdv, 'nXnYnZ': [512, 512, 1], 'roiUUID': 0}]} - with unittest.mock.patch.object(self.one.alyx, 'rest') as mock_rest: + with unittest.mock.patch.object(task.one.alyx, 'rest') as mock_rest: task.register_fov(meta, 'estimate') calls = mock_rest.call_args_list self.assertEqual(3, len(calls)) @@ -197,8 +232,8 @@ def test_register_fov(self): # Check dry mode with suffix input = None for file in self.session_path.joinpath('alf', 'FOV_00').glob('mpciMeanImage.*'): file.replace(file.with_name(file.name.replace('_estimate', ''))) - self.one.mode = 'local' - with unittest.mock.patch.object(self.one.alyx, 'rest') as mock_rest: + task.one.mode = 'local' + with unittest.mock.patch.object(task.one.alyx, 'rest') as mock_rest: out = task.register_fov(meta, None) mock_rest.assert_not_called() self.assertEqual(1, len(out)) @@ -206,3 +241,10 @@ def test_register_fov(self): locations = out[0]['location'] self.assertEqual(1, len(locations)) self.assertEqual('L', locations[0].get('provenance', 'L')) + + def tearDown(self) -> None: + """ + The ONE function is cached and therefore the One object persists beyond this test. + Here we return the mode back to the default after testing behaviour in offline mode. + """ + self.one.mode = 'auto'