From 159330d454c996234f6574568178b050c6f7102c Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Tue, 5 Sep 2023 09:57:24 +0200 Subject: [PATCH 01/10] Neuropixels - handle sync channel --- neo/rawio/openephysbinaryrawio.py | 6 +++- neo/rawio/spikeglxrawio.py | 50 ++++++++++++++++++++----------- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index 650c96672..3eee8f48c 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -138,7 +138,8 @@ def _parse_header(self): channel_names = [ch["channel_name"] for ch in d["channels"]] # if there is a sync channel and it should not be loaded, # find the right channel index and slice the memmap - if any(["SYNC" in ch for ch in channel_names]) and \ + has_sync_channel = any(["SYNC" in ch for ch in channel_names]) + if has_sync_channel and \ not self.load_sync_channel: sync_channel_name = [ch for ch in channel_names if "SYNC" in ch][0] sync_channel_index = channel_names.index(sync_channel_name) @@ -150,6 +151,9 @@ def _parse_header(self): raise NotImplementedError("SYNC channel removal is only supported " "when the sync channel is in the last " "position") + if not has_sync_channel and self.load_sync_channel: + raise ValueError("SYNC channel is not present in the recording. " + "Set load_sync_channel to False") d['memmap'] = memmap_sigs diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py index 495b428b2..f9fc7cdaa 100644 --- a/neo/rawio/spikeglxrawio.py +++ b/neo/rawio/spikeglxrawio.py @@ -124,8 +124,16 @@ def _parse_header(self): signal_channels.append((chan_name, chan_id, info['sampling_rate'], 'int16', info['units'], info['channel_gains'][local_chan], info['channel_offsets'][local_chan], stream_id)) - if not self.load_sync_channel: + if not self.load_sync_channel and info['has_sync_trace']: signal_channels = signal_channels[:-1] + # make memmap view + for segment_index in range(nb_segment): + if (segment_index, stream_name) in self._memmaps: + memmap = self._memmaps[(segment_index, stream_name)] + self._memmaps[(segment_index, stream_name)] = memmap[:, :-1] + elif self.load_sync_channel and not info['has_sync_trace']: + raise ValueError("SYNC channel is not present in the recording. " + "Set load_sync_channel to False") signal_streams = np.array(signal_streams, dtype=_signal_stream_dtype) signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype) @@ -201,25 +209,29 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, stream_index, channel_indexes): stream_id = self.header['signal_streams'][stream_index]['id'] memmap = self._memmaps[seg_index, stream_id] + + # since we cut the memmap, we can simplify the channel selection if channel_indexes is None: - if self.load_sync_channel: - channel_selection = slice(None) - else: - channel_selection = slice(-1) + channel_selection = slice(None) + # if self.load_sync_channel: + # channel_selection = slice(None) + # else: + # channel_selection = slice(-1) elif isinstance(channel_indexes, slice): - if self.load_sync_channel: - # simple - channel_selection = channel_indexes - else: - # more tricky because negative - sl_start = channel_indexes.start - sl_stop = channel_indexes.stop - sl_step = channel_indexes.step - if sl_stop is not None and sl_stop < 0: - sl_stop = sl_stop - 1 - elif sl_stop is None: - sl_stop = -1 - channel_selection = slice(sl_start, sl_stop, sl_step) + channel_selection = channel_indexes + # if self.load_sync_channel: + # # simple + # channel_selection = channel_indexes + # else: + # # more tricky because negative + # sl_start = channel_indexes.start + # sl_stop = channel_indexes.stop + # sl_step = channel_indexes.step + # if sl_stop is not None and sl_stop < 0: + # sl_stop = sl_stop - 1 + # elif sl_stop is None: + # sl_stop = -1 + # channel_selection = slice(sl_start, sl_stop, sl_step) elif not isinstance(channel_indexes, slice): if np.all(np.diff(channel_indexes) == 1): # consecutive channel then slice this avoid a copy (because of ndarray.take(...) @@ -372,6 +384,7 @@ def extract_stream_info(meta_file, meta): """Extract info from the meta dict""" num_chan = int(meta['nSavedChans']) + ap, lf, sy = [int(s) for s in meta["snsApLfSy"].split(",")] fname = Path(meta_file).stem run_name, gate_num, trigger_num, device, stream_kind = parse_spikeglx_fname(fname) @@ -447,5 +460,6 @@ def extract_stream_info(meta_file, meta): info['channel_names'] = [txt.split(';')[0] for txt in meta['snsChanMap']] info['channel_gains'] = channel_gains info['channel_offsets'] = np.zeros(info['num_chan']) + info['has_sync_trace'] = sy == 1 return info From 0970fb96203f0cbe82361f0c064ca33352659b65 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 7 Sep 2023 11:52:53 +0200 Subject: [PATCH 02/10] Slice memmap at get_analogsignal_chunk --- neo/rawio/openephysbinaryrawio.py | 178 +++++++++++++++--------------- neo/rawio/spikeglxrawio.py | 17 ++- 2 files changed, 95 insertions(+), 100 deletions(-) diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index 3eee8f48c..826e10565 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -91,13 +91,13 @@ def _parse_header(self): self._sig_streams[block_index][seg_index] = {} self._evt_streams[block_index][seg_index] = {} for stream_index, stream_name in enumerate(sig_stream_names): - d = all_streams[block_index][seg_index]['continuous'][stream_name] - d['stream_name'] = stream_name - self._sig_streams[block_index][seg_index][stream_index] = d + info_cnt = all_streams[block_index][seg_index]['continuous'][stream_name] + info_cnt['stream_name'] = stream_name + self._sig_streams[block_index][seg_index][stream_index] = info_cnt for i, stream_name in enumerate(event_stream_names): - d = all_streams[block_index][seg_index]['events'][stream_name] - d['stream_name'] = stream_name - self._evt_streams[block_index][seg_index][i] = d + info_evt = all_streams[block_index][seg_index]['events'][stream_name] + info_evt['stream_name'] = stream_name + self._evt_streams[block_index][seg_index][i] = info_evt # signals zone # create signals channel map: several channel per stream @@ -105,9 +105,9 @@ def _parse_header(self): for stream_index, stream_name in enumerate(sig_stream_names): # stream_index is the index in vector sytream names stream_id = str(stream_index) - d = self._sig_streams[0][0][stream_index] + info = self._sig_streams[0][0][stream_index] new_channels = [] - for chan_info in d['channels']: + for chan_info in info['channels']: chan_id = chan_info['channel_name'] if "SYNC" in chan_id and not self.load_sync_channel: continue @@ -117,7 +117,7 @@ def _parse_header(self): else: units = chan_info["units"] new_channels.append((chan_info['channel_name'], - chan_id, float(d['sample_rate']), d['dtype'], units, + chan_id, float(info['sample_rate']), info['dtype'], units, chan_info['bit_volts'], 0., stream_id)) signal_channels.extend(new_channels) signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype) @@ -131,60 +131,48 @@ def _parse_header(self): # create memmap for signals for block_index in range(nb_block): for seg_index in range(nb_segment_per_block[block_index]): - for stream_index, d in self._sig_streams[block_index][seg_index].items(): - num_channels = len(d['channels']) - memmap_sigs = np.memmap(d['raw_filename'], d['dtype'], + for stream_index, info in self._sig_streams[block_index][seg_index].items(): + num_channels = len(info['channels']) + memmap_sigs = np.memmap(info['raw_filename'], info['dtype'], order='C', mode='r').reshape(-1, num_channels) - channel_names = [ch["channel_name"] for ch in d["channels"]] - # if there is a sync channel and it should not be loaded, - # find the right channel index and slice the memmap + channel_names = [ch["channel_name"] for ch in info["channels"]] + + # check sync channel vlaidity has_sync_channel = any(["SYNC" in ch for ch in channel_names]) - if has_sync_channel and \ - not self.load_sync_channel: - sync_channel_name = [ch for ch in channel_names if "SYNC" in ch][0] - sync_channel_index = channel_names.index(sync_channel_name) - - # only sync channel in last position is supported to keep memmap - if sync_channel_index == num_channels - 1: - memmap_sigs = memmap_sigs[:, :-1] - else: - raise NotImplementedError("SYNC channel removal is only supported " - "when the sync channel is in the last " - "position") if not has_sync_channel and self.load_sync_channel: raise ValueError("SYNC channel is not present in the recording. " "Set load_sync_channel to False") - d['memmap'] = memmap_sigs + info['memmap'] = memmap_sigs # events zone # channel map: one channel one stream event_channels = [] for stream_ind, stream_name in enumerate(event_stream_names): - d = self._evt_streams[0][0][stream_ind] - if 'states' in d: + info = self._evt_streams[0][0][stream_ind] + if 'states' in info: evt_channel_type = "epoch" else: evt_channel_type = "event" - event_channels.append((d['channel_name'], d['channel_name'], evt_channel_type)) + event_channels.append((info['channel_name'], info['channel_name'], evt_channel_type)) event_channels = np.array(event_channels, dtype=_event_channel_dtype) # create memmap for events for block_index in range(nb_block): for seg_index in range(nb_segment_per_block[block_index]): - for stream_index, d in self._evt_streams[block_index][seg_index].items(): + for stream_index, info in self._evt_streams[block_index][seg_index].items(): for name in _possible_event_stream_names: - if name + '_npy' in d: - data = np.load(d[name + '_npy'], mmap_mode='r') - d[name] = data + if name + '_npy' in info: + data = np.load(info[name + '_npy'], mmap_mode='r') + info[name] = data # check that events have timestamps - assert 'timestamps' in d, "Event stream does not have timestamps!" + assert 'timestamps' in info, "Event stream does not have timestamps!" # Updates for OpenEphys v0.6: # In new vesion (>=0.6) timestamps.npy is now called sample_numbers.npy # The timestamps are already in seconds, so that event times don't require scaling # see https://open-ephys.github.io/gui-docs/User-Manual/Recording-data/Binary-format.html#events - if 'sample_numbers' in d: + if 'sample_numbers' in info: self._use_direct_evt_timestamps = True else: self._use_direct_evt_timestamps = False @@ -193,26 +181,26 @@ def _parse_header(self): # of event (ttl, text, binary) # and this is transform into unicode # all theses data are put in event array annotations - if 'text' in d: + if 'text' in info: # text case - d['labels'] = d['text'].astype('U') - elif 'metadata' in d: + info['labels'] = info['text'].astype('U') + elif 'metadata' in info: # binary case - d['labels'] = d['channels'].astype('U') - elif 'channels' in d: + info['labels'] = info['channels'].astype('U') + elif 'channels' in info: # ttl case use channels - d['labels'] = d['channels'].astype('U') - elif 'states' in d: + info['labels'] = info['channels'].astype('U') + elif 'states' in info: # ttl case use states - d['labels'] = d['states'].astype('U') + info['labels'] = info['states'].astype('U') else: raise ValueError(f'There is no possible labels for this event: {stream_name}') # # If available, use 'states' to compute event duration - if 'states' in d and d["states"].size: - states = d["states"] - timestamps = d["timestamps"] - labels = d["labels"] + if 'states' in info and info["states"].size: + states = info["states"] + timestamps = info["timestamps"] + labels = info["labels"] rising = np.where(states > 0)[0] falling = np.where(states < 0)[0] @@ -228,12 +216,12 @@ def _parse_header(self): if len(rising) == len(falling): durations = timestamps[falling] - timestamps[rising] - d["rising"] = rising - d["timestamps"] = timestamps[rising] - d["labels"] = labels[rising] - d["durations"] = durations + info["rising"] = rising + info["timestamps"] = timestamps[rising] + info["labels"] = labels[rising] + info["durations"] = durations else: - d["durations"] = None + info["durations"] = None # no spike read yet # can be implemented on user demand @@ -250,9 +238,9 @@ def _parse_header(self): global_t_stop = None # loop over signals - for stream_index, d in self._sig_streams[block_index][seg_index].items(): - t_start = d['t_start'] - dur = d['memmap'].shape[0] / float(d['sample_rate']) + for stream_index, info in self._sig_streams[block_index][seg_index].items(): + t_start = info['t_start'] + dur = info['memmap'].shape[0] / float(info['sample_rate']) t_stop = t_start + dur if global_t_start is None or global_t_start > t_start: global_t_start = t_start @@ -261,15 +249,15 @@ def _parse_header(self): # loop over events for stream_index, stream_name in enumerate(event_stream_names): - d = self._evt_streams[block_index][seg_index][stream_index] - if d['timestamps'].size == 0: + info = self._evt_streams[block_index][seg_index][stream_index] + if info['timestamps'].size == 0: continue - t_start = d['timestamps'][0] - t_stop = d['timestamps'][-1] + t_start = info['timestamps'][0] + t_stop = info['timestamps'][-1] if not self._use_direct_evt_timestamps: - t_start /= d['sample_rate'] - t_stop /= d['sample_rate'] + t_start /= info['sample_rate'] + t_stop /= info['sample_rate'] if global_t_start is None or global_t_start > t_start: global_t_start = t_start @@ -298,35 +286,35 @@ def _parse_header(self): # array annotations for signal channels for stream_index, stream_name in enumerate(sig_stream_names): sig_ann = seg_ann['signals'][stream_index] - d = self._sig_streams[0][0][stream_index] + info = self._sig_streams[0][0][stream_index] for k in ('identifier', 'history', 'source_processor_index', 'recorded_processor_index'): - if k in d['channels'][0]: - values = np.array([chan_info[k] for chan_info in d['channels']]) + if k in info['channels'][0]: + values = np.array([chan_info[k] for chan_info in info['channels']]) sig_ann['__array_annotations__'][k] = values # array annotations for event channels # use other possible data in _possible_event_stream_names for stream_index, stream_name in enumerate(event_stream_names): ev_ann = seg_ann['events'][stream_index] - d = self._evt_streams[0][0][stream_index] - if 'rising' in d: - selected_indices = d["rising"] + info = self._evt_streams[0][0][stream_index] + if 'rising' in info: + selected_indices = info["rising"] else: selected_indices = None for k in _possible_event_stream_names: if k in ('timestamps', 'rising'): continue - if k in d: + if k in info: # split custom dtypes into separate annotations - if d[k].dtype.names: - for name in d[k].dtype.names: - arr_ann = d[k][name].flatten() + if info[k].dtype.names: + for name in info[k].dtype.names: + arr_ann = info[k][name].flatten() if selected_indices is not None: arr_ann = arr_ann[selected_indices] ev_ann['__array_annotations__'][name] = arr_ann else: - arr_ann = d[k] + arr_ann = info[k] if selected_indices is not None: arr_ann = arr_ann[selected_indices] ev_ann['__array_annotations__'][k] = arr_ann @@ -356,7 +344,15 @@ def _get_signal_t_start(self, block_index, seg_index, stream_index): def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, stream_index, channel_indexes): + info = self._sig_streams[block_index][seg_index][stream_index] sigs = self._sig_streams[block_index][seg_index][stream_index]['memmap'] + + # take care of SYNC channel + channel_names = [ch["channel_name"] for ch in info["channels"]] + has_sync_channel = any(["SYNC" in ch for ch in channel_names]) + if not self.load_sync_channel and has_sync_channel: + sigs = sigs[:, :-1] + sigs = sigs[i_start:i_stop, :] if channel_indexes is not None: sigs = sigs[:, channel_indexes] @@ -380,15 +376,15 @@ def _event_count(self, block_index, seg_index, event_channel_index): return timestamps.size def _get_event_timestamps(self, block_index, seg_index, event_channel_index, t_start, t_stop): - d = self._evt_streams[block_index][seg_index][event_channel_index] - timestamps = d['timestamps'] - durations = d["durations"] - labels = d['labels'] + info = self._evt_streams[block_index][seg_index][event_channel_index] + timestamps = info['timestamps'] + durations = info["durations"] + labels = info['labels'] # slice it if needed if t_start is not None: if not self._use_direct_evt_timestamps: - ind_start = int(t_start * d['sample_rate']) + ind_start = int(t_start * info['sample_rate']) mask = timestamps >= ind_start else: mask = timestamps >= t_start @@ -396,7 +392,7 @@ def _get_event_timestamps(self, block_index, seg_index, event_channel_index, t_s labels = labels[mask] if t_stop is not None: if not self._use_direct_evt_timestamps: - ind_stop = int(t_stop * d['sample_rate']) + ind_stop = int(t_stop * info['sample_rate']) mask = timestamps < ind_stop else: mask = timestamps < t_stop @@ -405,17 +401,17 @@ def _get_event_timestamps(self, block_index, seg_index, event_channel_index, t_s return timestamps, durations, labels def _rescale_event_timestamp(self, event_timestamps, dtype, event_channel_index): - d = self._evt_streams[0][0][event_channel_index] + info = self._evt_streams[0][0][event_channel_index] if not self._use_direct_evt_timestamps: - event_times = event_timestamps.astype(dtype) / float(d['sample_rate']) + event_times = event_timestamps.astype(dtype) / float(info['sample_rate']) else: event_times = event_timestamps.astype(dtype) return event_times def _rescale_epoch_duration(self, raw_duration, dtype, event_channel_index): - d = self._evt_streams[0][0][event_channel_index] + info = self._evt_streams[0][0][event_channel_index] if not self._use_direct_evt_timestamps: - durations = raw_duration.astype(dtype) / float(d['sample_rate']) + durations = raw_duration.astype(dtype) / float(info['sample_rate']) else: durations = raw_duration.astype(dtype) return durations @@ -516,26 +512,26 @@ def explore_folder(dirname, experiment_names=None): for d in rec_structure['continuous']: # when multi Record Node the stream name also contains # the node name to make it unique - oe_stream_name = Path(d["folder_name"]).name # remove trailing slash + oe_stream_name = Path(info["folder_name"]).name # remove trailing slash if len(node_name) > 0: stream_name = node_name + '#' + oe_stream_name else: stream_name = oe_stream_name - raw_filename = recording_folder / 'continuous' / d['folder_name'] / 'continuous.dat' + raw_filename = recording_folder / 'continuous' / info['folder_name'] / 'continuous.dat' # Updates for OpenEphys v0.6: # In new vesion (>=0.6) timestamps.npy is now called sample_numbers.npy # see https://open-ephys.github.io/gui-docs/User-Manual/Recording-data/Binary-format.html#continuous - sample_numbers = recording_folder / 'continuous' / d['folder_name'] / \ + sample_numbers = recording_folder / 'continuous' / info['folder_name'] / \ 'sample_numbers.npy' if sample_numbers.is_file(): timestamp_file = sample_numbers else: - timestamp_file = recording_folder / 'continuous' / d['folder_name'] / \ + timestamp_file = recording_folder / 'continuous' / info['folder_name'] / \ 'timestamps.npy' timestamps = np.load(str(timestamp_file), mmap_mode='r') timestamp0 = timestamps[0] - t_start = timestamp0 / d['sample_rate'] + t_start = timestamp0 / info['sample_rate'] # TODO for later : gap checking signal_stream = d.copy() @@ -549,12 +545,12 @@ def explore_folder(dirname, experiment_names=None): if (root / 'events').exists() and len(rec_structure['events']) > 0: recording['streams']['events'] = {} for d in rec_structure['events']: - oe_stream_name = Path(d["folder_name"]).name # remove trailing slash + oe_stream_name = Path(info["folder_name"]).name # remove trailing slash stream_name = node_name + '#' + oe_stream_name event_stream = d.copy() for name in _possible_event_stream_names: - npy_filename = root / 'events' / d['folder_name'] / f'{name}.npy' + npy_filename = root / 'events' / info['folder_name'] / f'{name}.npy' if npy_filename.is_file(): event_stream[f'{name}_npy'] = str(npy_filename) diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py index f9fc7cdaa..6cee0a308 100644 --- a/neo/rawio/spikeglxrawio.py +++ b/neo/rawio/spikeglxrawio.py @@ -88,7 +88,6 @@ def _parse_header(self): # sort stream_name by higher sampling rate first srates = {info['stream_name']: info['sampling_rate'] for info in self.signals_info_list} stream_names = sorted(list(srates.keys()), key=lambda e: srates[e])[::-1] - nb_segment = np.unique([info['seg_index'] for info in self.signals_info_list]).size self._memmaps = {} @@ -124,14 +123,8 @@ def _parse_header(self): signal_channels.append((chan_name, chan_id, info['sampling_rate'], 'int16', info['units'], info['channel_gains'][local_chan], info['channel_offsets'][local_chan], stream_id)) - if not self.load_sync_channel and info['has_sync_trace']: - signal_channels = signal_channels[:-1] - # make memmap view - for segment_index in range(nb_segment): - if (segment_index, stream_name) in self._memmaps: - memmap = self._memmaps[(segment_index, stream_name)] - self._memmaps[(segment_index, stream_name)] = memmap[:, :-1] - elif self.load_sync_channel and not info['has_sync_trace']: + # check sync channel validity + if self.load_sync_channel and not info['has_sync_trace']: raise ValueError("SYNC channel is not present in the recording. " "Set load_sync_channel to False") @@ -209,6 +202,12 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, stream_index, channel_indexes): stream_id = self.header['signal_streams'][stream_index]['id'] memmap = self._memmaps[seg_index, stream_id] + stream_name = self.header['signal_streams']['name'][stream_index] + + # take care of sync channel + info = self.signals_info_dict[0, stream_name] + if not self.load_sync_channel and info['has_sync_trace']: + memmap = memmap[:, :-1] # since we cut the memmap, we can simplify the channel selection if channel_indexes is None: From e1a6cd5e522170dc0626614a6be3728e595de35e Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 7 Sep 2023 17:16:46 +0200 Subject: [PATCH 03/10] Extend NP2 models and final fixes --- neo/rawio/openephysbinaryrawio.py | 12 +++---- neo/rawio/spikeglxrawio.py | 54 +++++++++++++------------------ 2 files changed, 28 insertions(+), 38 deletions(-) diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index 826e10565..32e6b8309 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -137,9 +137,9 @@ def _parse_header(self): order='C', mode='r').reshape(-1, num_channels) channel_names = [ch["channel_name"] for ch in info["channels"]] - # check sync channel vlaidity + # check sync channel validity (only for AP and LF) has_sync_channel = any(["SYNC" in ch for ch in channel_names]) - if not has_sync_channel and self.load_sync_channel: + if not has_sync_channel and self.load_sync_channel and "NI-DAQ" not in info["stream_name"]: raise ValueError("SYNC channel is not present in the recording. " "Set load_sync_channel to False") info['memmap'] = memmap_sigs @@ -509,7 +509,7 @@ def explore_folder(dirname, experiment_names=None): if (recording_folder / 'continuous').exists() and len(rec_structure['continuous']) > 0: recording['streams']['continuous'] = {} - for d in rec_structure['continuous']: + for info in rec_structure['continuous']: # when multi Record Node the stream name also contains # the node name to make it unique oe_stream_name = Path(info["folder_name"]).name # remove trailing slash @@ -534,7 +534,7 @@ def explore_folder(dirname, experiment_names=None): t_start = timestamp0 / info['sample_rate'] # TODO for later : gap checking - signal_stream = d.copy() + signal_stream = info.copy() signal_stream['raw_filename'] = str(raw_filename) signal_stream['dtype'] = 'int16' signal_stream['timestamp0'] = timestamp0 @@ -544,11 +544,11 @@ def explore_folder(dirname, experiment_names=None): if (root / 'events').exists() and len(rec_structure['events']) > 0: recording['streams']['events'] = {} - for d in rec_structure['events']: + for info in rec_structure['events']: oe_stream_name = Path(info["folder_name"]).name # remove trailing slash stream_name = node_name + '#' + oe_stream_name - event_stream = d.copy() + event_stream = info.copy() for name in _possible_event_stream_names: npy_filename = root / 'events' / info['folder_name'] / f'{name}.npy' if npy_filename.is_file(): diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py index 6cee0a308..42ff04e4e 100644 --- a/neo/rawio/spikeglxrawio.py +++ b/neo/rawio/spikeglxrawio.py @@ -121,12 +121,15 @@ def _parse_header(self): chan_name = info['channel_names'][local_chan] chan_id = f'{stream_name}#{chan_name}' signal_channels.append((chan_name, chan_id, info['sampling_rate'], 'int16', - info['units'], info['channel_gains'][local_chan], - info['channel_offsets'][local_chan], stream_id)) + info['units'], info['channel_gains'][local_chan], + info['channel_offsets'][local_chan], stream_id)) # check sync channel validity - if self.load_sync_channel and not info['has_sync_trace']: - raise ValueError("SYNC channel is not present in the recording. " - "Set load_sync_channel to False") + if "nidq" not in stream_name: + if not self.load_sync_channel and info['has_sync_trace']: + signal_channels = signal_channels[:-1] + if self.load_sync_channel and not info['has_sync_trace']: + raise ValueError("SYNC channel is not present in the recording. " + "Set load_sync_channel to False") signal_streams = np.array(signal_streams, dtype=_signal_stream_dtype) signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype) @@ -212,25 +215,8 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, # since we cut the memmap, we can simplify the channel selection if channel_indexes is None: channel_selection = slice(None) - # if self.load_sync_channel: - # channel_selection = slice(None) - # else: - # channel_selection = slice(-1) elif isinstance(channel_indexes, slice): channel_selection = channel_indexes - # if self.load_sync_channel: - # # simple - # channel_selection = channel_indexes - # else: - # # more tricky because negative - # sl_start = channel_indexes.start - # sl_stop = channel_indexes.stop - # sl_step = channel_indexes.step - # if sl_stop is not None and sl_stop < 0: - # sl_stop = sl_stop - 1 - # elif sl_stop is None: - # sl_stop = -1 - # channel_selection = slice(sl_start, sl_stop, sl_step) elif not isinstance(channel_indexes, slice): if np.all(np.diff(channel_indexes) == 1): # consecutive channel then slice this avoid a copy (because of ndarray.take(...) @@ -297,7 +283,7 @@ def parse_spikeglx_fname(fname): Parse recording identifiers from a SpikeGLX style filename. spikeglx naming follow this rules: - https://github.com/billkarsh/SpikeGLX/blob/master/Markdown/UserManual.md#gates-and-triggers + https://github.com/billkarsh/SpikeGLX/blob/15ec8898e17829f9f08c226bf04f46281f106e5f/Markdown/UserManual.md#gates-and-triggers Example file name structure: Consider the filenames: `Noise4Sam_g0_t0.nidq.bin` or `Noise4Sam_g0_t0.imec0.lf.bin` @@ -383,7 +369,13 @@ def extract_stream_info(meta_file, meta): """Extract info from the meta dict""" num_chan = int(meta['nSavedChans']) - ap, lf, sy = [int(s) for s in meta["snsApLfSy"].split(",")] + if "snsApLfSy" in meta: + # AP and LF meta have this field + ap, lf, sy = [int(s) for s in meta["snsApLfSy"].split(",")] + has_sync_trace = sy == 1 + else: + # NIDQ case + has_sync_trace = False fname = Path(meta_file).stem run_name, gate_num, trigger_num, device, stream_kind = parse_spikeglx_fname(fname) @@ -399,9 +391,7 @@ def extract_stream_info(meta_file, meta): per_channel_gain = np.ones(num_chan, dtype='float64') if 'imDatPrb_type' not in meta or meta['imDatPrb_type'] == '0' or meta['imDatPrb_type'] in ('1015', '1022', '1030', '1031', '1032'): # This work with NP 1.0 case with different metadata versions - # https://github.com/billkarsh/SpikeGLX/blob/gh-pages/Support/Metadata_3A.md#imec - # https://github.com/billkarsh/SpikeGLX/blob/gh-pages/Support/Metadata_3B1.md#imec - # https://github.com/billkarsh/SpikeGLX/blob/gh-pages/Support/Metadata_3B2.md#imec + # https://github.com/billkarsh/SpikeGLX/blob/15ec8898e17829f9f08c226bf04f46281f106e5f/Markdown/Metadata_30.md if stream_kind == 'ap': index_imroTbl = 3 elif stream_kind == 'lf': @@ -411,11 +401,11 @@ def extract_stream_info(meta_file, meta): per_channel_gain[c] = 1. / float(v) gain_factor = float(meta['imAiRangeMax']) / 512 channel_gains = gain_factor * per_channel_gain * 1e6 - elif meta['imDatPrb_type'] in ('21', '24') and stream_kind == 'ap': + elif meta['imDatPrb_type'] in ('21', '24', '2003', '2004', '2013', '2014'): # This work with NP 2.0 case with different metadata versions - # https://github.com/billkarsh/SpikeGLX/blob/gh-pages/Support/Metadata_20.md#channel-entries-by-type - # https://github.com/billkarsh/SpikeGLX/blob/gh-pages/Support/Metadata_20.md#imec - # https://github.com/billkarsh/SpikeGLX/blob/gh-pages/Support/Metadata_30.md#imec + # https://github.com/billkarsh/SpikeGLX/blob/15ec8898e17829f9f08c226bf04f46281f106e5f/Markdown/Metadata_30.md#imec + # We allow also LF streams for NP2.0 because CatGT can produce them + # See: https://github.com/SpikeInterface/spikeinterface/issues/1949 per_channel_gain[:-1] = 1 / 80. gain_factor = float(meta['imAiRangeMax']) / 8192 channel_gains = gain_factor * per_channel_gain * 1e6 @@ -459,6 +449,6 @@ def extract_stream_info(meta_file, meta): info['channel_names'] = [txt.split(';')[0] for txt in meta['snsChanMap']] info['channel_gains'] = channel_gains info['channel_offsets'] = np.zeros(info['num_chan']) - info['has_sync_trace'] = sy == 1 + info['has_sync_trace'] = has_sync_trace return info From b4182e2e2a73fb4a9de947ce74dd38eab9b03dfe Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Mon, 18 Sep 2023 13:29:43 +0200 Subject: [PATCH 04/10] Extend tests with new GIN datasets --- .../rawiotest/test_openephysbinaryrawio.py | 46 +++++++++-- neo/test/rawiotest/test_spikeglxrawio.py | 82 ++++++++++++++----- 2 files changed, 102 insertions(+), 26 deletions(-) diff --git a/neo/test/rawiotest/test_openephysbinaryrawio.py b/neo/test/rawiotest/test_openephysbinaryrawio.py index 27fd33011..c4a5bff3b 100644 --- a/neo/test/rawiotest/test_openephysbinaryrawio.py +++ b/neo/test/rawiotest/test_openephysbinaryrawio.py @@ -6,16 +6,48 @@ class TestOpenEphysBinaryRawIO(BaseTestRawIO, unittest.TestCase): rawioclass = OpenEphysBinaryRawIO - entities_to_download = [ - 'openephysbinary' - ] + entities_to_download = ["openephysbinary"] entities_to_test = [ - 'openephysbinary/v0.5.3_two_neuropixels_stream', - 'openephysbinary/v0.4.4.1_with_video_tracking', - 'openephysbinary/v0.5.x_two_nodes', - 'openephysbinary/v0.6.x_neuropixels_multiexp_multistream', + "openephysbinary/v0.5.3_two_neuropixels_stream", + "openephysbinary/v0.4.4.1_with_video_tracking", + "openephysbinary/v0.5.x_two_nodes", + "openephysbinary/v0.6.x_neuropixels_multiexp_multistream", + "openephysbinary/v0.6.x_neuropixels_with_sync", ] + def test_sync(self): + rawio_with_sync = OpenEphysBinaryRawIO( + self.get_local_path("openephysbinary/v0.6.x_neuropixels_with_sync"), load_sync_channel=True + ) + rawio_with_sync.parse_header() + stream_name = [s_name for s_name in rawio_with_sync.header["signal_streams"]["name"] if "AP" in s_name][0] + stream_index = list(rawio_with_sync.header["signal_streams"]["name"]).index(stream_name) + + # AP stream has 385 channels + chunk = rawio_with_sync.get_analogsignal_chunk( + block_index=0, seg_index=0, i_start=0, i_stop=100, stream_index=stream_index + ) + assert chunk.shape[1] == 385 + + rawio_no_sync = OpenEphysBinaryRawIO( + self.get_local_path("openephysbinary/v0.6.x_neuropixels_with_sync"), load_sync_channel=False + ) + rawio_no_sync.parse_header() + + # AP stream has 384 channels + chunk = rawio_no_sync.get_analogsignal_chunk( + block_index=0, seg_index=0, i_start=0, i_stop=100, stream_index=stream_index + ) + assert chunk.shape[1] == 384 + + def test_no_sync(self): + # requesting sync channel when there is none raises an error + with self.assertRaises(ValueError): + rawio_no_sync = OpenEphysBinaryRawIO( + self.get_local_path("openephysbinary/v0.6.x_neuropixels_multiexp_multistream"), load_sync_channel=True + ) + rawio_no_sync.parse_header() + if __name__ == "__main__": unittest.main() diff --git a/neo/test/rawiotest/test_spikeglxrawio.py b/neo/test/rawiotest/test_spikeglxrawio.py index e4fd3a810..8b36dc0e7 100644 --- a/neo/test/rawiotest/test_spikeglxrawio.py +++ b/neo/test/rawiotest/test_spikeglxrawio.py @@ -10,37 +10,81 @@ class TestSpikeGLXRawIO(BaseTestRawIO, unittest.TestCase): rawioclass = SpikeGLXRawIO - entities_to_download = [ - 'spikeglx' - ] + entities_to_download = ["spikeglx"] entities_to_test = [ - 'spikeglx/Noise4Sam_g0', - 'spikeglx/TEST_20210920_0_g0', - + "spikeglx/Noise4Sam_g0", + "spikeglx/TEST_20210920_0_g0", # this is only g0 multi index - 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0/5-19-2022-CI0_g0', + "spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0/5-19-2022-CI0_g0", # this is only g1 multi index - 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0/5-19-2022-CI0_g1', + "spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0/5-19-2022-CI0_g1", # this mix both multi gate and multi trigger (and also multi probe) - 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0', - - 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI1', - 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI2', - 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI3', - 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI4', - 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI5', - + "spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0", + "spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI1", + "spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI2", + "spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI3", + "spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI4", + "spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI5", + # different sync/sybset options with commercial NP2 + "spikeglx/NP2_with_sync", + "spikeglx/NP2_no_sync", + "spikeglx/NP2_subset_with_sync", ] def test_with_location(self): - rawio = SpikeGLXRawIO(self.get_local_path('spikeglx/Noise4Sam_g0'), load_channel_location=True) + rawio = SpikeGLXRawIO(self.get_local_path("spikeglx/Noise4Sam_g0"), load_channel_location=True) rawio.parse_header() # one of the stream have channel location have_location = [] - for sig_anotations in rawio.raw_annotations['blocks'][0]['segments'][0]['signals']: - have_location.append('channel_location_0' in sig_anotations['__array_annotations__']) + for sig_anotations in rawio.raw_annotations["blocks"][0]["segments"][0]["signals"]: + have_location.append("channel_location_0" in sig_anotations["__array_annotations__"]) assert any(have_location) + def test_sync(self): + rawio_with_sync = SpikeGLXRawIO(self.get_local_path("spikeglx/NP2_with_sync"), load_sync_channel=True) + rawio_with_sync.parse_header() + stream_index = list(rawio_with_sync.header["signal_streams"]["name"]).index("imec0.ap") + + # AP stream has 385 channels + chunk = rawio_with_sync.get_analogsignal_chunk( + block_index=0, seg_index=0, i_start=0, i_stop=100, stream_index=stream_index + ) + assert chunk.shape[1] == 385 + + rawio_no_sync = SpikeGLXRawIO(self.get_local_path("spikeglx/NP2_with_sync"), load_sync_channel=False) + rawio_no_sync.parse_header() + + # AP stream has 384 channels + chunk = rawio_no_sync.get_analogsignal_chunk( + block_index=0, seg_index=0, i_start=0, i_stop=100, stream_index=stream_index + ) + assert chunk.shape[1] == 384 + + def test_no_sync(self): + # requesting sync channel when there is none raises an error + with self.assertRaises(ValueError): + rawio_no_sync = SpikeGLXRawIO(self.get_local_path("spikeglx/NP2_no_sync"), load_sync_channel=True) + rawio_no_sync.parse_header() + + def test_subset_with_sync(self): + rawio_sub = SpikeGLXRawIO(self.get_local_path("spikeglx/NP2_subset_with_sync"), load_sync_channel=True) + rawio_sub.parse_header() + stream_index = list(rawio_sub.header["signal_streams"]["name"]).index("imec0.ap") + + # AP stream has 121 channels + chunk = rawio_sub.get_analogsignal_chunk( + block_index=0, seg_index=0, i_start=0, i_stop=100, stream_index=stream_index + ) + assert chunk.shape[1] == 121 + + rawio_sub_no_sync = SpikeGLXRawIO(self.get_local_path("spikeglx/NP2_subset_with_sync"), load_sync_channel=False) + rawio_sub_no_sync.parse_header() + # AP stream has 120 channels + chunk = rawio_sub_no_sync.get_analogsignal_chunk( + block_index=0, seg_index=0, i_start=0, i_stop=100, stream_index=stream_index + ) + assert chunk.shape[1] == 120 + if __name__ == "__main__": unittest.main() From 84bd438261c8b4d5c0cc3e2d2a2dd1f07390fc28 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Mon, 18 Sep 2023 14:50:32 +0200 Subject: [PATCH 05/10] Fix Open Ephys array annotations when not loading sync channel --- neo/rawio/openephysbinaryrawio.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index ca24d37b2..3ac15fcbf 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -294,10 +294,16 @@ def _parse_header(self): for stream_index, stream_name in enumerate(sig_stream_names): sig_ann = seg_ann['signals'][stream_index] info = self._sig_streams[0][0][stream_index] + channel_names = [ch["channel_name"] for ch in info["channels"]] + + # check sync channel validity (only for AP and LF) + has_sync_channel = any(["SYNC" in ch for ch in channel_names]) for k in ('identifier', 'history', 'source_processor_index', 'recorded_processor_index'): if k in info['channels'][0]: values = np.array([chan_info[k] for chan_info in info['channels']]) + if has_sync_channel: + values = values[:-1] sig_ann['__array_annotations__'][k] = values # array annotations for event channels From b99037a17d36929ae8a393525eb35dde78a399a6 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Fri, 3 Nov 2023 10:49:12 +0100 Subject: [PATCH 06/10] Pre-load has_sync_trace in OpenEphys --- neo/rawio/openephysbinaryrawio.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index 3ac15fcbf..d788236c6 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -101,6 +101,10 @@ def _parse_header(self): info_cnt = all_streams[block_index][seg_index]['continuous'][stream_name] info_cnt['stream_name'] = stream_name self._sig_streams[block_index][seg_index][stream_index] = info_cnt + + # check for SYNC channel for Neuropixels streams + has_sync_trace = any(["SYNC" in ch["channel_name"] for ch in info_cnt["channels"]]) + self._sig_streams[block_index][seg_index][stream_index]['has_sync_trace'] = has_sync_trace for i, stream_name in enumerate(event_stream_names): info_evt = all_streams[block_index][seg_index]['events'][stream_name] info_evt['stream_name'] = stream_name @@ -142,11 +146,10 @@ def _parse_header(self): num_channels = len(info['channels']) memmap_sigs = np.memmap(info['raw_filename'], info['dtype'], order='C', mode='r').reshape(-1, num_channels) - channel_names = [ch["channel_name"] for ch in info["channels"]] + has_sync_trace = self._sig_streams[block_index][seg_index][stream_index]['has_sync_trace'] # check sync channel validity (only for AP and LF) - has_sync_channel = any(["SYNC" in ch for ch in channel_names]) - if not has_sync_channel and self.load_sync_channel and "NI-DAQ" not in info["stream_name"]: + if not has_sync_trace and self.load_sync_channel and "NI-DAQ" not in info["stream_name"]: raise ValueError("SYNC channel is not present in the recording. " "Set load_sync_channel to False") info['memmap'] = memmap_sigs @@ -293,16 +296,14 @@ def _parse_header(self): # array annotations for signal channels for stream_index, stream_name in enumerate(sig_stream_names): sig_ann = seg_ann['signals'][stream_index] - info = self._sig_streams[0][0][stream_index] - channel_names = [ch["channel_name"] for ch in info["channels"]] + info = self._sig_streams[block_index][seg_index][stream_index] + has_sync_trace = self._sig_streams[block_index][seg_index][stream_index]['has_sync_trace'] - # check sync channel validity (only for AP and LF) - has_sync_channel = any(["SYNC" in ch for ch in channel_names]) for k in ('identifier', 'history', 'source_processor_index', 'recorded_processor_index'): if k in info['channels'][0]: values = np.array([chan_info[k] for chan_info in info['channels']]) - if has_sync_channel: + if has_sync_trace: values = values[:-1] sig_ann['__array_annotations__'][k] = values @@ -357,13 +358,10 @@ def _get_signal_t_start(self, block_index, seg_index, stream_index): def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, stream_index, channel_indexes): - info = self._sig_streams[block_index][seg_index][stream_index] sigs = self._sig_streams[block_index][seg_index][stream_index]['memmap'] + has_sync_trace = self._sig_streams[block_index][seg_index][stream_index]['has_sync_trace'] - # take care of SYNC channel - channel_names = [ch["channel_name"] for ch in info["channels"]] - has_sync_channel = any(["SYNC" in ch for ch in channel_names]) - if not self.load_sync_channel and has_sync_channel: + if not self.load_sync_channel and has_sync_trace: sigs = sigs[:, :-1] sigs = sigs[i_start:i_stop, :] From 03a388601d929045fc84f4f2260d5a7bc8219294 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Fri, 3 Nov 2023 11:04:18 +0100 Subject: [PATCH 07/10] PEP 8 fixes --- neo/rawio/openephysbinaryrawio.py | 19 +++++++++++++------ neo/rawio/spikeglxrawio.py | 6 ++++-- .../rawiotest/test_openephysbinaryrawio.py | 12 ++++++++---- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index d788236c6..a59137696 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -103,8 +103,10 @@ def _parse_header(self): self._sig_streams[block_index][seg_index][stream_index] = info_cnt # check for SYNC channel for Neuropixels streams - has_sync_trace = any(["SYNC" in ch["channel_name"] for ch in info_cnt["channels"]]) - self._sig_streams[block_index][seg_index][stream_index]['has_sync_trace'] = has_sync_trace + has_sync_trace = any(["SYNC" in ch["channel_name"] + for ch in info_cnt["channels"]]) + self._sig_streams[block_index][seg_index][stream_index]['has_sync_trace'] \ + = has_sync_trace for i, stream_name in enumerate(event_stream_names): info_evt = all_streams[block_index][seg_index]['events'][stream_name] info_evt['stream_name'] = stream_name @@ -146,10 +148,12 @@ def _parse_header(self): num_channels = len(info['channels']) memmap_sigs = np.memmap(info['raw_filename'], info['dtype'], order='C', mode='r').reshape(-1, num_channels) - has_sync_trace = self._sig_streams[block_index][seg_index][stream_index]['has_sync_trace'] + has_sync_trace = \ + self._sig_streams[block_index][seg_index][stream_index]['has_sync_trace'] # check sync channel validity (only for AP and LF) - if not has_sync_trace and self.load_sync_channel and "NI-DAQ" not in info["stream_name"]: + if not has_sync_trace and self.load_sync_channel \ + and "NI-DAQ" not in info["stream_name"]: raise ValueError("SYNC channel is not present in the recording. " "Set load_sync_channel to False") info['memmap'] = memmap_sigs @@ -204,7 +208,9 @@ def _parse_header(self): # ttl case use states info['labels'] = info['states'].astype('U') else: - raise ValueError(f'There is no possible labels for this event: {stream_name}') + raise ValueError( + f'There is no possible labels for this event!' + ) # # If available, use 'states' to compute event duration if 'states' in info and info["states"].size: @@ -297,7 +303,8 @@ def _parse_header(self): for stream_index, stream_name in enumerate(sig_stream_names): sig_ann = seg_ann['signals'][stream_index] info = self._sig_streams[block_index][seg_index][stream_index] - has_sync_trace = self._sig_streams[block_index][seg_index][stream_index]['has_sync_trace'] + has_sync_trace = \ + self._sig_streams[block_index][seg_index][stream_index]['has_sync_trace'] for k in ('identifier', 'history', 'source_processor_index', 'recorded_processor_index'): diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py index 42ff04e4e..15f969e5d 100644 --- a/neo/rawio/spikeglxrawio.py +++ b/neo/rawio/spikeglxrawio.py @@ -185,7 +185,8 @@ def _parse_header(self): # one fake channel for "sys0" loc = np.concatenate((loc, [[0., 0.]]), axis=0) for ndim in range(loc.shape[1]): - sig_ann['__array_annotations__'][f'channel_location_{ndim}'] = loc[:, ndim] + sig_ann['__array_annotations__'][f'channel_location_{ndim}'] = \ + loc[:, ndim] def _segment_t_start(self, block_index, seg_index): return 0. @@ -389,7 +390,8 @@ def extract_stream_info(meta_file, meta): # metad['imroTbl'] contain two gain per channel AP and LF # except for the last fake channel per_channel_gain = np.ones(num_chan, dtype='float64') - if 'imDatPrb_type' not in meta or meta['imDatPrb_type'] == '0' or meta['imDatPrb_type'] in ('1015', '1022', '1030', '1031', '1032'): + if 'imDatPrb_type' not in meta or meta['imDatPrb_type'] == '0' or meta['imDatPrb_type'] \ + in ('1015', '1022', '1030', '1031', '1032'): # This work with NP 1.0 case with different metadata versions # https://github.com/billkarsh/SpikeGLX/blob/15ec8898e17829f9f08c226bf04f46281f106e5f/Markdown/Metadata_30.md if stream_kind == 'ap': diff --git a/neo/test/rawiotest/test_openephysbinaryrawio.py b/neo/test/rawiotest/test_openephysbinaryrawio.py index c4a5bff3b..093f86f9f 100644 --- a/neo/test/rawiotest/test_openephysbinaryrawio.py +++ b/neo/test/rawiotest/test_openephysbinaryrawio.py @@ -17,10 +17,12 @@ class TestOpenEphysBinaryRawIO(BaseTestRawIO, unittest.TestCase): def test_sync(self): rawio_with_sync = OpenEphysBinaryRawIO( - self.get_local_path("openephysbinary/v0.6.x_neuropixels_with_sync"), load_sync_channel=True + self.get_local_path("openephysbinary/v0.6.x_neuropixels_with_sync"), + load_sync_channel=True ) rawio_with_sync.parse_header() - stream_name = [s_name for s_name in rawio_with_sync.header["signal_streams"]["name"] if "AP" in s_name][0] + stream_name = [s_name for s_name in rawio_with_sync.header["signal_streams"]["name"] + if "AP" in s_name][0] stream_index = list(rawio_with_sync.header["signal_streams"]["name"]).index(stream_name) # AP stream has 385 channels @@ -30,7 +32,8 @@ def test_sync(self): assert chunk.shape[1] == 385 rawio_no_sync = OpenEphysBinaryRawIO( - self.get_local_path("openephysbinary/v0.6.x_neuropixels_with_sync"), load_sync_channel=False + self.get_local_path("openephysbinary/v0.6.x_neuropixels_with_sync"), + load_sync_channel=False ) rawio_no_sync.parse_header() @@ -44,7 +47,8 @@ def test_no_sync(self): # requesting sync channel when there is none raises an error with self.assertRaises(ValueError): rawio_no_sync = OpenEphysBinaryRawIO( - self.get_local_path("openephysbinary/v0.6.x_neuropixels_multiexp_multistream"), load_sync_channel=True + self.get_local_path("openephysbinary/v0.6.x_neuropixels_multiexp_multistream"), + load_sync_channel=True ) rawio_no_sync.parse_header() From 3c4ae5491be6e68297a5e96f6617c0e7bf91c0db Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Fri, 3 Nov 2023 11:32:03 +0100 Subject: [PATCH 08/10] Force download of maxwell hdf5 plugin --- neo/test/iotest/test_maxwellio.py | 2 +- neo/test/rawiotest/test_maxwellrawio.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/test/iotest/test_maxwellio.py b/neo/test/iotest/test_maxwellio.py index c1ac30059..4a29ceda7 100644 --- a/neo/test/iotest/test_maxwellio.py +++ b/neo/test/iotest/test_maxwellio.py @@ -18,7 +18,7 @@ class TestMaxwellIO(BaseTestIO, unittest.TestCase, ): ] def setUp(self): - auto_install_maxwell_hdf5_compression_plugin(force_download=False) + auto_install_maxwell_hdf5_compression_plugin(force_download=True) BaseTestIO.setUp(self) diff --git a/neo/test/rawiotest/test_maxwellrawio.py b/neo/test/rawiotest/test_maxwellrawio.py index 0c90ec306..c7c16c309 100644 --- a/neo/test/rawiotest/test_maxwellrawio.py +++ b/neo/test/rawiotest/test_maxwellrawio.py @@ -17,7 +17,7 @@ class TestMaxwellRawIO(BaseTestRawIO, unittest.TestCase, ): ] def setUp(self): - auto_install_maxwell_hdf5_compression_plugin(force_download=False) + auto_install_maxwell_hdf5_compression_plugin(force_download=True) BaseTestRawIO.setUp(self) From d98e468b8cc19d01db7a73b2654f0d05e7402da7 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Fri, 3 Nov 2023 11:45:59 +0100 Subject: [PATCH 09/10] Fix Maxwell tests --- neo/test/iotest/test_maxwellio.py | 11 ++++++++++- neo/test/rawiotest/test_maxwellrawio.py | 10 +++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/neo/test/iotest/test_maxwellio.py b/neo/test/iotest/test_maxwellio.py index 4a29ceda7..2bdf4f567 100644 --- a/neo/test/iotest/test_maxwellio.py +++ b/neo/test/iotest/test_maxwellio.py @@ -6,6 +6,8 @@ from neo.rawio.maxwellrawio import auto_install_maxwell_hdf5_compression_plugin +ON_GITHUB = bool(os.getenv('GITHUB_ACTIONS')) + class TestMaxwellIO(BaseTestIO, unittest.TestCase, ): ioclass = MaxwellIO @@ -18,7 +20,14 @@ class TestMaxwellIO(BaseTestIO, unittest.TestCase, ): ] def setUp(self): - auto_install_maxwell_hdf5_compression_plugin(force_download=True) + if ON_GITHUB: + # set a custom existing path for the hdf5 plugin + hdf5_plugin_path = '/home/runner/work/hdf5_plugin_path_maxwell' + os.environ['HDF5_PLUGIN_PATH'] = hdf5_plugin_path + else: + hdf5_plugin_path = None + auto_install_maxwell_hdf5_compression_plugin(hdf5_plugin_path=hdf5_plugin_path, + force_download=False) BaseTestIO.setUp(self) diff --git a/neo/test/rawiotest/test_maxwellrawio.py b/neo/test/rawiotest/test_maxwellrawio.py index c7c16c309..e67c8f87d 100644 --- a/neo/test/rawiotest/test_maxwellrawio.py +++ b/neo/test/rawiotest/test_maxwellrawio.py @@ -5,6 +5,7 @@ auto_install_maxwell_hdf5_compression_plugin) from neo.test.rawiotest.common_rawio_test import BaseTestRawIO +ON_GITHUB = bool(os.getenv('GITHUB_ACTIONS')) class TestMaxwellRawIO(BaseTestRawIO, unittest.TestCase, ): rawioclass = MaxwellRawIO @@ -17,7 +18,14 @@ class TestMaxwellRawIO(BaseTestRawIO, unittest.TestCase, ): ] def setUp(self): - auto_install_maxwell_hdf5_compression_plugin(force_download=True) + if ON_GITHUB: + # set a custom existing path for the hdf5 plugin + hdf5_plugin_path = '/home/runner/work/hdf5_plugin_path_maxwell' + os.environ['HDF5_PLUGIN_PATH'] = hdf5_plugin_path + else: + hdf5_plugin_path = None + auto_install_maxwell_hdf5_compression_plugin(hdf5_plugin_path=hdf5_plugin_path, + force_download=False) BaseTestRawIO.setUp(self) From 3e3392c19744ea7d4ddddaeb8da84c6749a66f4c Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Mon, 6 Nov 2023 12:12:12 +0100 Subject: [PATCH 10/10] Revert maxwell changes --- neo/test/iotest/test_maxwellio.py | 11 +---------- neo/test/rawiotest/test_maxwellrawio.py | 10 +--------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/neo/test/iotest/test_maxwellio.py b/neo/test/iotest/test_maxwellio.py index 2bdf4f567..c1ac30059 100644 --- a/neo/test/iotest/test_maxwellio.py +++ b/neo/test/iotest/test_maxwellio.py @@ -6,8 +6,6 @@ from neo.rawio.maxwellrawio import auto_install_maxwell_hdf5_compression_plugin -ON_GITHUB = bool(os.getenv('GITHUB_ACTIONS')) - class TestMaxwellIO(BaseTestIO, unittest.TestCase, ): ioclass = MaxwellIO @@ -20,14 +18,7 @@ class TestMaxwellIO(BaseTestIO, unittest.TestCase, ): ] def setUp(self): - if ON_GITHUB: - # set a custom existing path for the hdf5 plugin - hdf5_plugin_path = '/home/runner/work/hdf5_plugin_path_maxwell' - os.environ['HDF5_PLUGIN_PATH'] = hdf5_plugin_path - else: - hdf5_plugin_path = None - auto_install_maxwell_hdf5_compression_plugin(hdf5_plugin_path=hdf5_plugin_path, - force_download=False) + auto_install_maxwell_hdf5_compression_plugin(force_download=False) BaseTestIO.setUp(self) diff --git a/neo/test/rawiotest/test_maxwellrawio.py b/neo/test/rawiotest/test_maxwellrawio.py index e67c8f87d..0c90ec306 100644 --- a/neo/test/rawiotest/test_maxwellrawio.py +++ b/neo/test/rawiotest/test_maxwellrawio.py @@ -5,7 +5,6 @@ auto_install_maxwell_hdf5_compression_plugin) from neo.test.rawiotest.common_rawio_test import BaseTestRawIO -ON_GITHUB = bool(os.getenv('GITHUB_ACTIONS')) class TestMaxwellRawIO(BaseTestRawIO, unittest.TestCase, ): rawioclass = MaxwellRawIO @@ -18,14 +17,7 @@ class TestMaxwellRawIO(BaseTestRawIO, unittest.TestCase, ): ] def setUp(self): - if ON_GITHUB: - # set a custom existing path for the hdf5 plugin - hdf5_plugin_path = '/home/runner/work/hdf5_plugin_path_maxwell' - os.environ['HDF5_PLUGIN_PATH'] = hdf5_plugin_path - else: - hdf5_plugin_path = None - auto_install_maxwell_hdf5_compression_plugin(hdf5_plugin_path=hdf5_plugin_path, - force_download=False) + auto_install_maxwell_hdf5_compression_plugin(force_download=False) BaseTestRawIO.setUp(self)