diff --git a/neo/rawio/axonrawio.py b/neo/rawio/axonrawio.py index 1bca18485..7efa824e1 100644 --- a/neo/rawio/axonrawio.py +++ b/neo/rawio/axonrawio.py @@ -158,8 +158,8 @@ def _parse_header(self): # Get raw data by segment # self._raw_signals = {} self._t_starts = {} - self._buffer_descriptions = {0 :{}} - self._stream_buffer_slice = {stream_id : None} + self._buffer_descriptions = {0: {}} + self._stream_buffer_slice = {stream_id: None} pos = 0 for seg_index in range(nb_segment): length = episode_array[seg_index]["len"] @@ -174,12 +174,12 @@ def _parse_header(self): self._buffer_descriptions[0][seg_index] = {} self._buffer_descriptions[0][seg_index][buffer_id] = { - "type" : "raw", - "file_path" : str(self.filename), - "dtype" : str(sig_dtype), + "type": "raw", + "file_path": str(self.filename), + "dtype": str(sig_dtype), "order": "C", - "file_offset" : head_offset + pos * sig_dtype.itemsize, - "shape" : (int(length // nbchannel), int(nbchannel)), + "file_offset": head_offset + pos * sig_dtype.itemsize, + "shape": (int(length // nbchannel), int(nbchannel)), } pos += length @@ -239,7 +239,9 @@ def _parse_header(self): else: gain, offset = 1.0, 0.0 - signal_channels.append((name, str(chan_id), self._sampling_rate, sig_dtype, units, gain, offset, stream_id, buffer_id)) + signal_channels.append( + (name, str(chan_id), self._sampling_rate, sig_dtype, units, gain, offset, stream_id, buffer_id) + ) signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype) @@ -313,7 +315,6 @@ def _get_signal_t_start(self, block_index, seg_index, stream_index): def _get_analogsignal_buffer_description(self, block_index, seg_index, buffer_id): return self._buffer_descriptions[block_index][seg_index][buffer_id] - def _event_count(self, block_index, seg_index, event_channel_index): return self._raw_ev_timestamps.size diff --git a/neo/rawio/baserawio.py b/neo/rawio/baserawio.py index 34c3e85a3..1d6f74df7 100644 --- a/neo/rawio/baserawio.py +++ b/neo/rawio/baserawio.py @@ -1375,7 +1375,6 @@ def _get_analogsignal_buffer_description(self, block_index, seg_index, buffer_id raise (NotImplementedError) - class BaseRawWithBufferApiIO(BaseRawIO): """ Generic class for reader that support "buffer api". @@ -1402,7 +1401,7 @@ def _get_signal_size(self, block_index, seg_index, stream_index): buffer_desc = self.get_analogsignal_buffer_description(block_index, seg_index, buffer_id) # some hdf5 revert teh buffer time_axis = buffer_desc.get("time_axis", 0) - return buffer_desc['shape'][time_axis] + return buffer_desc["shape"][time_axis] def _get_analogsignal_chunk( self, @@ -1413,42 +1412,47 @@ def _get_analogsignal_chunk( stream_index: int, channel_indexes: list[int] | None, ): - + stream_id = self.header["signal_streams"][stream_index]["id"] buffer_id = self.header["signal_streams"][stream_index]["buffer_id"] - - buffer_slice = self._stream_buffer_slice[stream_id] + buffer_slice = self._stream_buffer_slice[stream_id] buffer_desc = self.get_analogsignal_buffer_description(block_index, seg_index, buffer_id) i_start = i_start or 0 - i_stop = i_stop or buffer_desc['shape'][0] + i_stop = i_stop or buffer_desc["shape"][0] - if buffer_desc['type'] == "raw": + if buffer_desc["type"] == "raw": - # open files on demand and keep reference to opened file - if not hasattr(self, '_memmap_analogsignal_buffers'): + # open files on demand and keep reference to opened file + if not hasattr(self, "_memmap_analogsignal_buffers"): self._memmap_analogsignal_buffers = {} if block_index not in self._memmap_analogsignal_buffers: self._memmap_analogsignal_buffers[block_index] = {} if seg_index not in self._memmap_analogsignal_buffers[block_index]: self._memmap_analogsignal_buffers[block_index][seg_index] = {} if buffer_id not in self._memmap_analogsignal_buffers[block_index][seg_index]: - fid = open(buffer_desc['file_path'], mode='rb') + fid = open(buffer_desc["file_path"], mode="rb") self._memmap_analogsignal_buffers[block_index][seg_index][buffer_id] = fid else: fid = self._memmap_analogsignal_buffers[block_index][seg_index][buffer_id] - - num_channels = buffer_desc['shape'][1] - - raw_sigs = get_memmap_chunk_from_opened_file(fid, num_channels, i_start, i_stop, np.dtype(buffer_desc['dtype']), file_offset=buffer_desc['file_offset']) - - elif buffer_desc['type'] == 'hdf5': + num_channels = buffer_desc["shape"][1] + + raw_sigs = get_memmap_chunk_from_opened_file( + fid, + num_channels, + i_start, + i_stop, + np.dtype(buffer_desc["dtype"]), + file_offset=buffer_desc["file_offset"], + ) + + elif buffer_desc["type"] == "hdf5": - # open files on demand and keep reference to opened file - if not hasattr(self, '_hdf5_analogsignal_buffers'): + # open files on demand and keep reference to opened file + if not hasattr(self, "_hdf5_analogsignal_buffers"): self._hdf5_analogsignal_buffers = {} if block_index not in self._hdf5_analogsignal_buffers: self._hdf5_analogsignal_buffers[block_index] = {} @@ -1456,14 +1460,15 @@ def _get_analogsignal_chunk( self._hdf5_analogsignal_buffers[block_index][seg_index] = {} if buffer_id not in self._hdf5_analogsignal_buffers[block_index][seg_index]: import h5py - h5file = h5py.File(buffer_desc['file_path'], mode="r") + + h5file = h5py.File(buffer_desc["file_path"], mode="r") self._hdf5_analogsignal_buffers[block_index][seg_index][buffer_id] = h5file else: h5file = self._hdf5_analogsignal_buffers[block_index][seg_index][buffer_id] hdf5_path = buffer_desc["hdf5_path"] full_raw_sigs = h5file[hdf5_path] - + time_axis = buffer_desc.get("time_axis", 0) if time_axis == 0: raw_sigs = full_raw_sigs[i_start:i_stop, :] @@ -1475,31 +1480,28 @@ def _get_analogsignal_chunk( if buffer_slice is not None: raw_sigs = raw_sigs[:, buffer_slice] - - else: raise NotImplementedError() # this is a pre slicing when the stream do not contain all channels (for instance spikeglx when load_sync_channel=False) if buffer_slice is not None: raw_sigs = raw_sigs[:, buffer_slice] - + # channel slice requested if channel_indexes is not None: raw_sigs = raw_sigs[:, channel_indexes] - return raw_sigs def __del__(self): - if hasattr(self, '_memmap_analogsignal_buffers'): + if hasattr(self, "_memmap_analogsignal_buffers"): for block_index in self._memmap_analogsignal_buffers.keys(): for seg_index in self._memmap_analogsignal_buffers[block_index].keys(): for buffer_id, fid in self._memmap_analogsignal_buffers[block_index][seg_index].items(): fid.close() del self._memmap_analogsignal_buffers - if hasattr(self, '_hdf5_analogsignal_buffers'): + if hasattr(self, "_hdf5_analogsignal_buffers"): for block_index in self._hdf5_analogsignal_buffers.keys(): for seg_index in self._hdf5_analogsignal_buffers[block_index].keys(): for buffer_id, h5_file in self._hdf5_analogsignal_buffers[block_index][seg_index].items(): diff --git a/neo/rawio/brainvisionrawio.py b/neo/rawio/brainvisionrawio.py index cd03e67d0..7246d8d02 100644 --- a/neo/rawio/brainvisionrawio.py +++ b/neo/rawio/brainvisionrawio.py @@ -80,19 +80,18 @@ def _parse_header(self): sig_dtype = np.dtype(fmts[fmt]) - stream_id = "0" buffer_id = "0" - self._buffer_descriptions = {0 :{0 : {}}} + self._buffer_descriptions = {0: {0: {}}} self._stream_buffer_slice = {} shape = get_memmap_shape(binary_filename, sig_dtype, num_channels=nb_channel, offset=0) self._buffer_descriptions[0][0][buffer_id] = { - "type" : "raw", - "file_path" : binary_filename, - "dtype" : str(sig_dtype), + "type": "raw", + "file_path": binary_filename, + "dtype": str(sig_dtype), "order": "C", - "file_offset" : 0, - "shape" : shape, + "file_offset": 0, + "shape": shape, } self._stream_buffer_slice[stream_id] = None diff --git a/neo/rawio/edfrawio.py b/neo/rawio/edfrawio.py index 5f791f24c..7e02cd356 100644 --- a/neo/rawio/edfrawio.py +++ b/neo/rawio/edfrawio.py @@ -173,14 +173,17 @@ def _parse_header(self): for array_key in array_keys: array_anno = {array_key: [h[array_key] for h in self.signal_headers]} seg_ann["signals"].append({"__array_annotations__": array_anno}) - + # We store the following attributes for rapid access without needing the reader - + self._t_stop = self.edf_reader.datarecord_duration * self.edf_reader.datarecords_in_file # use sample count of first signal in stream - self._stream_index_samples = {stream_index : self.edf_reader.getNSamples()[chidx][0] for stream_index, chidx in self.stream_idx_to_chidx.items()} + self._stream_index_samples = { + stream_index: self.edf_reader.getNSamples()[chidx][0] + for stream_index, chidx in self.stream_idx_to_chidx.items() + } self._number_of_events = len(self.edf_reader.readAnnotations()[0]) - + self.close() def _get_stream_channels(self, stream_index): diff --git a/neo/rawio/elanrawio.py b/neo/rawio/elanrawio.py index 06c928145..79b5ea349 100644 --- a/neo/rawio/elanrawio.py +++ b/neo/rawio/elanrawio.py @@ -163,7 +163,7 @@ def _parse_header(self): buffer_id = "0" signal_buffers = np.array([("Signals", buffer_id)], dtype=_signal_buffer_dtype) signal_streams = np.array([("Signals", stream_id, buffer_id)], dtype=_signal_stream_dtype) - + sig_channels = [] for c, chan_info in enumerate(channel_infos): chan_name = chan_info["label"] @@ -197,16 +197,16 @@ def _parse_header(self): sig_channels = np.array(sig_channels, dtype=_signal_channel_dtype) # raw data - self._buffer_descriptions = {0 :{0 : {}}} + self._buffer_descriptions = {0: {0: {}}} self._stream_buffer_slice = {} shape = get_memmap_shape(self.filename, sig_dtype, num_channels=nb_channel + 2, offset=0) self._buffer_descriptions[0][0][buffer_id] = { - "type" : "raw", - "file_path" : self.filename, - "dtype" : sig_dtype, + "type": "raw", + "file_path": self.filename, + "dtype": sig_dtype, "order": "C", - "file_offset" : 0, - "shape" : shape, + "file_offset": 0, + "shape": shape, } self._stream_buffer_slice["0"] = slice(0, -2) diff --git a/neo/rawio/maxwellrawio.py b/neo/rawio/maxwellrawio.py index 790f2d19a..bdae1ccc4 100644 --- a/neo/rawio/maxwellrawio.py +++ b/neo/rawio/maxwellrawio.py @@ -119,12 +119,12 @@ def _parse_header(self): # create signal channels max_sig_length = 0 - self._buffer_descriptions = {0 :{0 :{}}} + self._buffer_descriptions = {0: {0: {}}} self._stream_buffer_slice = {} sig_channels = [] well_indices_to_remove = [] for stream_index, stream_id in enumerate(signal_streams["id"]): - + if int(version) == 20160704: sr = 20000.0 settings = h5file["settings"] @@ -163,14 +163,14 @@ def _parse_header(self): continue self._stream_buffer_slice[stream_id] = None - + buffer_id = stream_id shape = h5file[hdf5_path].shape self._buffer_descriptions[0][0][buffer_id] = { - "type" : "hdf5", - "file_path" : str(self.filename), - "hdf5_path" : hdf5_path, - "shape" : shape, + "type": "hdf5", + "file_path": str(self.filename), + "hdf5_path": hdf5_path, + "shape": shape, "time_axis": 1, } self._stream_buffer_slice[stream_id] = slice(None) @@ -232,7 +232,9 @@ def _get_signal_t_start(self, block_index, seg_index, stream_index): def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, stream_index, channel_indexes): try: - return super()._get_analogsignal_chunk(block_index, seg_index, i_start, i_stop, stream_index, channel_indexes) + return super()._get_analogsignal_chunk( + block_index, seg_index, i_start, i_stop, stream_index, channel_indexes + ) except OSError as e: print("*" * 10) print(_hdf_maxwell_error) diff --git a/neo/rawio/micromedrawio.py b/neo/rawio/micromedrawio.py index 1f807f0d3..52dd222bd 100644 --- a/neo/rawio/micromedrawio.py +++ b/neo/rawio/micromedrawio.py @@ -46,14 +46,13 @@ class MicromedRawIO(BaseRawWithBufferApiIO): extensions = ["trc", "TRC"] rawmode = "one-file" - def __init__(self, filename=""): BaseRawWithBufferApiIO.__init__(self) self.filename = filename def _parse_header(self): - self._buffer_descriptions = {0 :{ 0: {}}} + self._buffer_descriptions = {0: {0: {}}} with open(self.filename, "rb") as fid: f = StructFile(fid) @@ -106,16 +105,14 @@ def _parse_header(self): buffer_id = "0" stream_id = "0" self._buffer_descriptions[0][0][buffer_id] = { - "type" : "raw", - "file_path" : str(self.filename), - "dtype" : sig_dtype, + "type": "raw", + "file_path": str(self.filename), + "dtype": sig_dtype, "order": "C", - "file_offset" : 0, - "shape" : signal_shape, + "file_offset": 0, + "shape": signal_shape, } - - # Reading Code Info zname2, pos, length = zones["ORDER"] f.seek(pos) @@ -144,12 +141,12 @@ def _parse_header(self): sampling_rate *= Rate_Min chan_id = str(c) - - signal_channels.append((chan_name, chan_id, sampling_rate, sig_dtype, units, gain, offset, stream_id, buffer_id)) - + signal_channels.append( + (chan_name, chan_id, sampling_rate, sig_dtype, units, gain, offset, stream_id, buffer_id) + ) signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype) - + self._stream_buffer_slice = {"0": slice(None)} signal_buffers = np.array([("Signals", buffer_id)], dtype=_signal_buffer_dtype) signal_streams = np.array([("Signals", stream_id, buffer_id)], dtype=_signal_stream_dtype) diff --git a/neo/rawio/neuronexusrawio.py b/neo/rawio/neuronexusrawio.py index 1858b2409..9a3e26167 100644 --- a/neo/rawio/neuronexusrawio.py +++ b/neo/rawio/neuronexusrawio.py @@ -136,14 +136,14 @@ def _parse_header(self): # the will cretae a memory map with teh generic mechanism buffer_id = "0" - self._buffer_descriptions = {0 :{0 :{}}} + self._buffer_descriptions = {0: {0: {}}} self._buffer_descriptions[0][0][buffer_id] = { - "type" : "raw", - "file_path" : str(binary_file), - "dtype" : BINARY_DTYPE, + "type": "raw", + "file_path": str(binary_file), + "dtype": BINARY_DTYPE, "order": "C", - "file_offset" : 0, - "shape" : (self._n_samples, self._n_channels), + "file_offset": 0, + "shape": (self._n_samples, self._n_channels), } # Make the memory map for timestamp self._timestamps = np.memmap( diff --git a/neo/rawio/neuroscoperawio.py b/neo/rawio/neuroscoperawio.py index 0090687df..0a303335b 100644 --- a/neo/rawio/neuroscoperawio.py +++ b/neo/rawio/neuroscoperawio.py @@ -112,16 +112,16 @@ def _parse_header(self): shape = get_memmap_shape(self.data_file_path, sig_dtype, num_channels=nb_channel, offset=0) buffer_id = "0" stream_id = "0" - self._buffer_descriptions = {0: {0:{}}} + self._buffer_descriptions = {0: {0: {}}} self._buffer_descriptions[0][0][buffer_id] = { - "type" : "raw", - "file_path" : str(self.data_file_path), - "dtype" : sig_dtype, + "type": "raw", + "file_path": str(self.data_file_path), + "dtype": sig_dtype, "order": "C", - "file_offset" : 0, - "shape" : shape, + "file_offset": 0, + "shape": shape, } - self._stream_buffer_slice = {stream_id : None} + self._stream_buffer_slice = {stream_id: None} # one unique stream and buffer signal_buffers = np.array([("Signals", buffer_id)], dtype=_signal_buffer_dtype) diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index ade2c2d1b..957c195c9 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -173,15 +173,14 @@ def _parse_header(self): num_channels = len(info["channels"]) stream_id = str(stream_index) buffer_id = str(stream_index) - shape = get_memmap_shape(info["raw_filename"], info["dtype"], num_channels=num_channels, - offset=0) + shape = get_memmap_shape(info["raw_filename"], info["dtype"], num_channels=num_channels, offset=0) self._buffer_descriptions[block_index][seg_index][buffer_id] = { - "type" : "raw", - "file_path" : str(info["raw_filename"]), - "dtype" : info["dtype"], + "type": "raw", + "file_path": str(info["raw_filename"]), + "dtype": info["dtype"], "order": "C", - "file_offset" : 0, - "shape" : shape, + "file_offset": 0, + "shape": shape, } has_sync_trace = self._sig_streams[block_index][seg_index][stream_index]["has_sync_trace"] @@ -462,7 +461,6 @@ def _get_analogsignal_buffer_description(self, block_index, seg_index, buffer_id return self._buffer_descriptions[block_index][seg_index][buffer_id] - _possible_event_stream_names = ( "timestamps", "sample_numbers", diff --git a/neo/rawio/rawbinarysignalrawio.py b/neo/rawio/rawbinarysignalrawio.py index feef2f98b..b9df92c2c 100644 --- a/neo/rawio/rawbinarysignalrawio.py +++ b/neo/rawio/rawbinarysignalrawio.py @@ -85,17 +85,16 @@ def _parse_header(self): buffer_id = "0" stream_id = "0" shape = get_memmap_shape(self.filename, self.dtype, num_channels=self.nb_channel, offset=self.bytesoffset) - self._buffer_descriptions = {0:{0:{}}} + self._buffer_descriptions = {0: {0: {}}} self._buffer_descriptions[0][0][buffer_id] = { - "type" : "raw", - "file_path" : str(self.filename), - "dtype" : "uint16", + "type": "raw", + "file_path": str(self.filename), + "dtype": "uint16", "order": "C", - "file_offset" : self.bytesoffset, - "shape" : shape, + "file_offset": self.bytesoffset, + "shape": shape, } - self._stream_buffer_slice = {stream_id : None} - + self._stream_buffer_slice = {stream_id: None} else: # The the neo.io.RawBinarySignalIO is used for write_segment diff --git a/neo/rawio/rawmcsrawio.py b/neo/rawio/rawmcsrawio.py index cba8baae6..52a83c561 100644 --- a/neo/rawio/rawmcsrawio.py +++ b/neo/rawio/rawmcsrawio.py @@ -26,6 +26,7 @@ ) from .utils import get_memmap_shape + class RawMCSRawIO(BaseRawWithBufferApiIO): """ Class for reading an mcs file converted by the MC_DataToo binary converter @@ -65,16 +66,16 @@ def _parse_header(self): # ) file_offset = info["header_size"] shape = get_memmap_shape(self.filename, self.dtype, num_channels=self.nb_channel, offset=file_offset) - self._buffer_descriptions = {0:{0:{}}} + self._buffer_descriptions = {0: {0: {}}} self._buffer_descriptions[0][0][buffer_id] = { - "type" : "raw", - "file_path" : str(self.filename), - "dtype" : "uint16", + "type": "raw", + "file_path": str(self.filename), + "dtype": "uint16", "order": "C", - "file_offset" : file_offset, - "shape" : shape, + "file_offset": file_offset, + "shape": shape, } - self._stream_buffer_slice = {stream_id : None} + self._stream_buffer_slice = {stream_id: None} sig_channels = [] for c in range(self.nb_channel): @@ -121,7 +122,7 @@ def _segment_t_start(self, block_index, seg_index): def _segment_t_stop(self, block_index, seg_index): # t_stop = self._raw_signals.shape[0] / self.sampling_rate sig_size = self.get_signal_size(block_index, seg_index, 0) - t_stop = sig_size / self.sampling_rate + t_stop = sig_size / self.sampling_rate return t_stop # def _get_signal_size(self, block_index, seg_index, stream_index): diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py index 3a66a9878..2e8f896d8 100644 --- a/neo/rawio/spikeglxrawio.py +++ b/neo/rawio/spikeglxrawio.py @@ -125,11 +125,10 @@ def _parse_header(self): stream_names = sorted(list(srates.keys()), key=lambda e: srates[e])[::-1] nb_segment = np.unique([info["seg_index"] for info in self.signals_info_list]).size - # self._memmaps = {} self.signals_info_dict = {} # one unique block - self._buffer_descriptions = {0 :{}} + self._buffer_descriptions = {0: {}} self._stream_buffer_slice = {} for info in self.signals_info_list: seg_index, stream_name = info["seg_index"], info["stream_name"] @@ -143,17 +142,15 @@ def _parse_header(self): if seg_index not in self._buffer_descriptions[0]: self._buffer_descriptions[block_index][seg_index] = {} - + self._buffer_descriptions[block_index][seg_index][buffer_id] = { - "type" : "raw", - "file_path" : info["bin_file"], - "dtype" : "int16", + "type": "raw", + "file_path": info["bin_file"], + "dtype": "int16", "order": "C", - "file_offset" : 0, - "shape" : get_memmap_shape(info["bin_file"], "int16", num_channels=info["num_chan"], offset=0), + "file_offset": 0, + "shape": get_memmap_shape(info["bin_file"], "int16", num_channels=info["num_chan"], offset=0), } - - # create channel header signal_buffers = [] @@ -189,7 +186,7 @@ def _parse_header(self): buffer_id, ) ) - + # all channel by dafult unless load_sync_channel=False self._stream_buffer_slice[stream_id] = None # check sync channel validity @@ -327,8 +324,6 @@ def _rescale_epoch_duration(self, raw_duration, dtype, event_channel_index): def _get_analogsignal_buffer_description(self, block_index, seg_index, buffer_id): return self._buffer_descriptions[block_index][seg_index][buffer_id] - - def scan_files(dirname): diff --git a/neo/rawio/utils.py b/neo/rawio/utils.py index 0365da625..bed9763e5 100644 --- a/neo/rawio/utils.py +++ b/neo/rawio/utils.py @@ -1,9 +1,10 @@ import mmap import numpy as np + def get_memmap_shape(filename, dtype, num_channels=None, offset=0): dtype = np.dtype(dtype) - with open(filename, mode='rb') as f: + with open(filename, mode="rb") as f: f.seek(0, 2) flen = f.tell() bytes = flen - offset @@ -17,7 +18,7 @@ def get_memmap_shape(filename, dtype, num_channels=None, offset=0): return shape -def get_memmap_chunk_from_opened_file(fid, num_channels, start, stop, dtype, file_offset=0): +def get_memmap_chunk_from_opened_file(fid, num_channels, start, stop, dtype, file_offset=0): """ Utility function to get a chunk as a memmap array directly from an opened file. Using this instead memmap can avoid memmory consumption when multiprocessing. @@ -41,7 +42,7 @@ def get_memmap_chunk_from_opened_file(fid, num_channels, start, stop, dtype, fi # Adjust the length so it includes the extra data from rounding down # the memmap offset to a multiple of ALLOCATIONGRANULARITY length += start_offset - + memmap_obj = mmap.mmap(fid.fileno(), length=length, access=mmap.ACCESS_READ, offset=memmap_offset) arr = np.ndarray( @@ -52,5 +53,3 @@ def get_memmap_chunk_from_opened_file(fid, num_channels, start, stop, dtype, fi ) return arr - - diff --git a/neo/rawio/winedrrawio.py b/neo/rawio/winedrrawio.py index 78d0db67e..4439562c8 100644 --- a/neo/rawio/winedrrawio.py +++ b/neo/rawio/winedrrawio.py @@ -65,17 +65,16 @@ def _parse_header(self): # one unique block with one unique segment # one unique buffer splited in several streams buffer_id = "0" - self._buffer_descriptions = {0 :{0 :{}}} + self._buffer_descriptions = {0: {0: {}}} self._buffer_descriptions[0][0][buffer_id] = { - "type" : "raw", - "file_path" : str(self.filename), - "dtype" : "int16", + "type": "raw", + "file_path": str(self.filename), + "dtype": "int16", "order": "C", - "file_offset" : int(header["NBH"]), - "shape" : (header["NP"] // header["NC"], header["NC"]), + "file_offset": int(header["NBH"]), + "shape": (header["NP"] // header["NC"], header["NC"]), } - DT = header["DT"] if "TU" in header: if header["TU"] == "ms": diff --git a/neo/rawio/winwcprawio.py b/neo/rawio/winwcprawio.py index 36049ae8b..0c262a9d1 100644 --- a/neo/rawio/winwcprawio.py +++ b/neo/rawio/winwcprawio.py @@ -49,7 +49,7 @@ def _parse_header(self): # one unique block with several segments # one unique buffer splited in several streams - self._buffer_descriptions = {0 :{}} + self._buffer_descriptions = {0: {}} with open(self.filename, "rb") as fid: @@ -97,12 +97,12 @@ def _parse_header(self): buffer_id = "0" self._buffer_descriptions[0][seg_index] = {} self._buffer_descriptions[0][seg_index][buffer_id] = { - "type" : "raw", - "file_path" : str(self.filename), - "dtype" : "int16", + "type": "raw", + "file_path": str(self.filename), + "dtype": "int16", "order": "C", - "file_offset" : ind0, - "shape" : (NP, NC), + "file_offset": ind0, + "shape": (NP, NC), } all_sampling_interval.append(analysisHeader["SamplingInterval"]) diff --git a/neo/rawio/xarray_utils.py b/neo/rawio/xarray_utils.py index 99c868b8d..cd3ed23fa 100644 --- a/neo/rawio/xarray_utils.py +++ b/neo/rawio/xarray_utils.py @@ -15,8 +15,6 @@ import base64 - - def to_zarr_v2_reference(rawio_reader, block_index=0, seg_index=0, buffer_id=None): """ Transform the buffer_description_api into a dict ready for the xarray API 'reference://' @@ -24,7 +22,7 @@ def to_zarr_v2_reference(rawio_reader, block_index=0, seg_index=0, buffer_id=Non See https://fsspec.github.io/kerchunk/spec.html See https://docs.xarray.dev/en/latest/user-guide/io.html#kerchunk - + See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html @@ -41,7 +39,6 @@ def to_zarr_v2_reference(rawio_reader, block_index=0, seg_index=0, buffer_id=Non buffer_name = signal_buffers["name"][buffer_index] - rfs = dict() rfs["version"] = 1 rfs["refs"] = dict() @@ -49,13 +46,12 @@ def to_zarr_v2_reference(rawio_reader, block_index=0, seg_index=0, buffer_id=Non zattrs = dict(name=buffer_name) rfs["refs"][".zattrs"] = zattrs - - descr = rawio_reader.get_analogsignal_buffer_description(block_index=block_index, seg_index=seg_index, - buffer_id=buffer_id) + descr = rawio_reader.get_analogsignal_buffer_description( + block_index=block_index, seg_index=seg_index, buffer_id=buffer_id + ) if descr["type"] == "raw": - # channel : small enough can be internal with base64 mask = rawio_reader.header["signal_channels"]["buffer_id"] == buffer_id channels = rawio_reader.header["signal_channels"][mask] @@ -73,7 +69,7 @@ def to_zarr_v2_reference(rawio_reader, block_index=0, seg_index=0, buffer_id=Non zarr_format=2, ) zattrs = dict( - _ARRAY_DIMENSIONS=['channel'], + _ARRAY_DIMENSIONS=["channel"], ) rfs["refs"]["channel/.zattrs"] = zattrs rfs["refs"]["channel/.zarray"] = zarray @@ -91,22 +87,22 @@ def to_zarr_v2_reference(rawio_reader, block_index=0, seg_index=0, buffer_id=Non zarr_format=2, ) zattrs = dict( - _ARRAY_DIMENSIONS=['time', 'channel'], + _ARRAY_DIMENSIONS=["time", "channel"], name=buffer_name, ) - units = np.unique(channels['units']) + units = np.unique(channels["units"]) if units.size == 1: - zattrs['units'] = units[0] - gain = np.unique(channels['gain']) - offset = np.unique(channels['offset']) + zattrs["units"] = units[0] + gain = np.unique(channels["gain"]) + offset = np.unique(channels["offset"]) if gain.size == 1 and offset.size: - zattrs['scale_factor'] = gain[0] - zattrs['add_offset'] = offset[0] - zattrs['sampling_rate'] = float(channels['sampling_rate'][0]) + zattrs["scale_factor"] = gain[0] + zattrs["add_offset"] = offset[0] + zattrs["sampling_rate"] = float(channels["sampling_rate"][0]) # unique big chunk # TODO later : optional split in several small chunks - array_size = np.prod(descr["shape"], dtype='int64') + array_size = np.prod(descr["shape"], dtype="int64") chunk_size = array_size * dtype.itemsize rfs["refs"]["traces/0.0"] = [str(descr["file_path"]), descr["file_offset"], chunk_size] rfs["refs"]["traces/.zarray"] = zarray @@ -117,19 +113,18 @@ def to_zarr_v2_reference(rawio_reader, block_index=0, seg_index=0, buffer_id=Non else: raise ValueError(f"buffer_description type not handled {descr['type']}") - # TODO later channel array_annotations + # TODO later channel array_annotations return rfs - def to_xarray_dataset(rawio_reader, block_index=0, seg_index=0, buffer_id=None): """ Utils fonction that transorm an instance a rawio into a xarray.Dataset with lazy access. This works only for rawio class that return True with has_buffer_description_api() and hinerits from BaseRawWithBufferApiIO. - + Note : the original idea of the function is from Ben Dichter in this page https://gist.github.com/bendichter/30a9afb34b2178098c99f3b01fe72e75 @@ -152,6 +147,7 @@ def to_xarray_dataset(rawio_reader, block_index=0, seg_index=0, buffer_id=None): ) return ds + def to_xarray_datatree(rawio_reader): """ Expose a neo.rawio reader class to a xarray DataTree to lazily read signals. @@ -166,19 +162,19 @@ def to_xarray_datatree(rawio_reader): except: raise ImportError("use xarray dev branch or pip install xarray-datatree") - signal_buffers = rawio_reader.header['signal_buffers'] + signal_buffers = rawio_reader.header["signal_buffers"] buffer_ids = signal_buffers["id"] tree = DataTree(name="root") - num_block = rawio_reader.header['nb_block'] + num_block = rawio_reader.header["nb_block"] for block_index in range(num_block): - block = DataTree(name=f'block{block_index}', parent=tree) - num_seg = rawio_reader.header['nb_segment'][block_index] + block = DataTree(name=f"block{block_index}", parent=tree) + num_seg = rawio_reader.header["nb_segment"][block_index] for seg_index in range(num_seg): - segment = DataTree(name=f'segment{seg_index}', parent=block) + segment = DataTree(name=f"segment{seg_index}", parent=block) for buffer_id in buffer_ids: ds = to_xarray_dataset(rawio_reader, block_index=block_index, seg_index=seg_index, buffer_id=buffer_id) - DataTree(data=ds, name=ds.attrs['name'], parent=segment) + DataTree(data=ds, name=ds.attrs["name"], parent=segment) return tree diff --git a/neo/test/rawiotest/rawio_compliance.py b/neo/test/rawiotest/rawio_compliance.py index 263c22997..611b0fd48 100644 --- a/neo/test/rawiotest/rawio_compliance.py +++ b/neo/test/rawiotest/rawio_compliance.py @@ -489,7 +489,6 @@ def check_buffer_api(reader): nb_block = reader.block_count() nb_event_channel = reader.event_channels_count() - for block_index in range(nb_block): nb_seg = reader.segment_count(block_index) for seg_index in range(nb_seg): @@ -501,6 +500,7 @@ def check_buffer_api(reader): try: import xarray + HAVE_XARRAY = True except ImportError: HAVE_XARRAY = False @@ -509,5 +509,6 @@ def check_buffer_api(reader): # this test quickly the experimental xaray_utils.py if xarray is present on the system # this is not the case for the CI from neo.rawio.xarray_utils import to_xarray_dataset + ds = to_xarray_dataset(reader, block_index=block_index, seg_index=seg_index, buffer_id=buffer_id) assert isinstance(ds, xarray.Dataset)