From 8ed3297122dec0dce632737ebf4b2f28856f35cd Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Mon, 9 Sep 2024 12:43:54 +0200 Subject: [PATCH 1/5] WIP add micromed segment and related t_start --- neo/rawio/micromedrawio.py | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/neo/rawio/micromedrawio.py b/neo/rawio/micromedrawio.py index ffcbdfb87..1ab51a9b7 100644 --- a/neo/rawio/micromedrawio.py +++ b/neo/rawio/micromedrawio.py @@ -97,6 +97,26 @@ def _parse_header(self): -1, Num_Chan ) + # "TRONCA" zone define segments + zname2, pos, length = zones["TRONCA"] + f.seek(pos) + max_segments = 100 + info_segments = [] + for i in range(max_segments): + seg_time = np.frombuffer(f.read(8), dtype="u8")[0] + seg_size = np.frombuffer(f.read(8), dtype="u8")[0] + + if seg_time == 0: + break + else: + info_segments.append((seg_time, seg_size)) + + if len(info_segments) == 0: + info_segments = [(0, 0)] + + if len(info_segments) > 1: + raise RuntimeError("Neo do not support more than one segment at the moment") + # Reading Code Info zname2, pos, length = zones["ORDER"] f.seek(pos) @@ -134,6 +154,11 @@ def _parse_header(self): assert np.unique(signal_channels["sampling_rate"]).size == 1 self._sampling_rate = float(np.unique(signal_channels["sampling_rate"])[0]) + # TODO change this when multi segment handling + self._global_t_start = info_segments[0][0] / self._sampling_rate + + + # Event channels event_channels = [] event_channels.append(("Trigger", "", "event")) @@ -194,11 +219,12 @@ def _source_name(self): return self.filename def _segment_t_start(self, block_index, seg_index): - return 0.0 + # return 0.0 + return self._global_t_start def _segment_t_stop(self, block_index, seg_index): t_stop = self._raw_signals.shape[0] / self._sampling_rate - return t_stop + return t_stop + self.segment_t_start(block_index, seg_index) def _get_signal_size(self, block_index, seg_index, stream_index): assert stream_index == 0 @@ -206,7 +232,8 @@ def _get_signal_size(self, block_index, seg_index, stream_index): def _get_signal_t_start(self, block_index, seg_index, stream_index): assert stream_index == 0 - return 0.0 + # return 0.0 + return self._global_t_start def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, stream_index, channel_indexes): if channel_indexes is None: @@ -249,7 +276,8 @@ def _get_event_timestamps(self, block_index, seg_index, event_channel_index, t_s def _rescale_event_timestamp(self, event_timestamps, dtype, event_channel_index): event_times = event_timestamps.astype(dtype) / self._sampling_rate - return event_times + event_times += self._global_t_start + return event_times def _rescale_epoch_duration(self, raw_duration, dtype, event_channel_index): durations = raw_duration.astype(dtype) / self._sampling_rate From ccf210373353b9faa1aee780d8c9fb92bb20ceaf Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Fri, 20 Sep 2024 17:14:24 +0200 Subject: [PATCH 2/5] TRONCA u8 to u4 --- neo/rawio/micromedrawio.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/neo/rawio/micromedrawio.py b/neo/rawio/micromedrawio.py index 1ab51a9b7..39bcfd2b1 100644 --- a/neo/rawio/micromedrawio.py +++ b/neo/rawio/micromedrawio.py @@ -97,25 +97,29 @@ def _parse_header(self): -1, Num_Chan ) + print(self._raw_signals.shape) + # "TRONCA" zone define segments zname2, pos, length = zones["TRONCA"] f.seek(pos) max_segments = 100 info_segments = [] for i in range(max_segments): - seg_time = np.frombuffer(f.read(8), dtype="u8")[0] - seg_size = np.frombuffer(f.read(8), dtype="u8")[0] + seg_time = np.frombuffer(f.read(4), dtype="u4")[0] + seg_size = np.frombuffer(f.read(4), dtype="u4")[0] + print(seg_time, seg_size) if seg_time == 0: break else: info_segments.append((seg_time, seg_size)) + print(info_segments) if len(info_segments) == 0: info_segments = [(0, 0)] - if len(info_segments) > 1: - raise RuntimeError("Neo do not support more than one segment at the moment") + # if len(info_segments) > 1: + # raise RuntimeError("Neo do not support more than one segment at the moment") # Reading Code Info zname2, pos, length = zones["ORDER"] From 5342c41ec4877a57b95dd8810766df6198cc8fd1 Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Mon, 23 Sep 2024 11:23:33 +0200 Subject: [PATCH 3/5] micromed multi segment wip --- neo/rawio/micromedrawio.py | 57 +++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/neo/rawio/micromedrawio.py b/neo/rawio/micromedrawio.py index 39bcfd2b1..2f9796ff2 100644 --- a/neo/rawio/micromedrawio.py +++ b/neo/rawio/micromedrawio.py @@ -97,26 +97,24 @@ def _parse_header(self): -1, Num_Chan ) - print(self._raw_signals.shape) - # "TRONCA" zone define segments zname2, pos, length = zones["TRONCA"] f.seek(pos) max_segments = 100 - info_segments = [] + self.info_segments = [] for i in range(max_segments): - seg_time = np.frombuffer(f.read(4), dtype="u4")[0] - seg_size = np.frombuffer(f.read(4), dtype="u4")[0] - print(seg_time, seg_size) - - if seg_time == 0: + seg_start = int(np.frombuffer(f.read(4), dtype="u4")[0]) + trace_offset = int(np.frombuffer(f.read(4), dtype="u4")[0]) + if seg_start == 0 and trace_offset == 0: break else: - info_segments.append((seg_time, seg_size)) - print(info_segments) + self.info_segments.append((seg_start, trace_offset)) - if len(info_segments) == 0: - info_segments = [(0, 0)] + if len(self.info_segments) == 0: + # one unique segment = general case + self.info_segments.append((0, 0)) + + # if len(info_segments) > 1: # raise RuntimeError("Neo do not support more than one segment at the moment") @@ -159,9 +157,18 @@ def _parse_header(self): self._sampling_rate = float(np.unique(signal_channels["sampling_rate"])[0]) # TODO change this when multi segment handling - self._global_t_start = info_segments[0][0] / self._sampling_rate + seg_limits = [trace_offset for seg_start, trace_offset in self.info_segments] + [self._raw_signals.shape[0]] + nb_segment = len(self.info_segments) + self._t_starts = [] + self._seg_raw_signals = [] + for seg_index in range(nb_segment): + seg_start, trace_offset = self.info_segments[seg_index] + self._t_starts.append(seg_start / self._sampling_rate) + + start = seg_limits[seg_index] + stop = seg_limits[seg_index + 1] + self._seg_raw_signals.append(self._raw_signals[start:stop]) - # Event channels event_channels = [] @@ -199,7 +206,7 @@ def _parse_header(self): # fille into header dict self.header = {} self.header["nb_block"] = 1 - self.header["nb_segment"] = [1] + self.header["nb_segment"] = [nb_segment] self.header["signal_streams"] = signal_streams self.header["signal_channels"] = signal_channels self.header["spike_channels"] = spike_channels @@ -223,26 +230,24 @@ def _source_name(self): return self.filename def _segment_t_start(self, block_index, seg_index): - # return 0.0 - return self._global_t_start + return self._t_starts[seg_index] def _segment_t_stop(self, block_index, seg_index): - t_stop = self._raw_signals.shape[0] / self._sampling_rate - return t_stop + self.segment_t_start(block_index, seg_index) + duration = self._seg_raw_signals[seg_index].shape[0] / self._sampling_rate + return duration + self.segment_t_start(block_index, seg_index) def _get_signal_size(self, block_index, seg_index, stream_index): assert stream_index == 0 - return self._raw_signals.shape[0] + return self._seg_raw_signals[seg_index].shape[0] def _get_signal_t_start(self, block_index, seg_index, stream_index): assert stream_index == 0 - # return 0.0 - return self._global_t_start + return self._t_starts[seg_index] def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, stream_index, channel_indexes): - if channel_indexes is None: - channel_indexes = slice(channel_indexes) - raw_signals = self._raw_signals[slice(i_start, i_stop), channel_indexes] + raw_signals = self._seg_raw_signals[seg_index][slice(i_start, i_stop), :] + if channel_indexes is not None: + raw_signals = raw_signals[:, channel_indexes] return raw_signals def _spike_count(self, block_index, seg_index, unit_index): @@ -280,7 +285,7 @@ def _get_event_timestamps(self, block_index, seg_index, event_channel_index, t_s def _rescale_event_timestamp(self, event_timestamps, dtype, event_channel_index): event_times = event_timestamps.astype(dtype) / self._sampling_rate - event_times += self._global_t_start + # event_times += self._global_t_start return event_times def _rescale_epoch_duration(self, raw_duration, dtype, event_channel_index): From 8b48f4b5805fd8b02e96f9d58f7bdbe6175fcdf7 Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Fri, 25 Oct 2024 08:48:06 +0200 Subject: [PATCH 4/5] Handle events offset when multi segment --- neo/rawio/micromedrawio.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/neo/rawio/micromedrawio.py b/neo/rawio/micromedrawio.py index 2f9796ff2..ed532d7e0 100644 --- a/neo/rawio/micromedrawio.py +++ b/neo/rawio/micromedrawio.py @@ -156,7 +156,6 @@ def _parse_header(self): assert np.unique(signal_channels["sampling_rate"]).size == 1 self._sampling_rate = float(np.unique(signal_channels["sampling_rate"])[0]) - # TODO change this when multi segment handling seg_limits = [trace_offset for seg_start, trace_offset in self.info_segments] + [self._raw_signals.shape[0]] nb_segment = len(self.info_segments) self._t_starts = [] @@ -191,13 +190,17 @@ def _parse_header(self): dtype = np.dtype(ev_dtype) rawevent = np.memmap(self.filename, dtype=dtype, mode="r", offset=pos, shape=length // dtype.itemsize) - keep = ( - (rawevent["start"] >= rawevent["start"][0]) - & (rawevent["start"] < self._raw_signals.shape[0]) - & (rawevent["start"] != 0) - ) - rawevent = rawevent[keep] - self._raw_events.append(rawevent) + # important : all events timing are related to the first segment t_start + self._raw_events.append([]) + for seg_index in range(nb_segment): + left_lim = seg_limits[seg_index] + right_lim = seg_limits[seg_index + 1] + keep = ( + (rawevent["start"] >= left_lim) + & (rawevent["start"] < right_lim) + & (rawevent["start"] != 0) + ) + self._raw_events[-1].append(rawevent[keep]) # No spikes spike_channels = [] @@ -254,22 +257,26 @@ def _spike_count(self, block_index, seg_index, unit_index): return 0 def _event_count(self, block_index, seg_index, event_channel_index): - n = self._raw_events[event_channel_index].size + n = self._raw_events[event_channel_index][seg_index].size return n def _get_event_timestamps(self, block_index, seg_index, event_channel_index, t_start, t_stop): - raw_event = self._raw_events[event_channel_index] + raw_event = self._raw_events[event_channel_index][seg_index] + + # important : all events timing are related to the first segment t_start + seg_start0, _ = self.info_segments[0] if t_start is not None: - keep = raw_event["start"] >= int(t_start * self._sampling_rate) + keep = raw_event["start"] + seg_start0 >= int(t_start * self._sampling_rate) raw_event = raw_event[keep] if t_stop is not None: - keep = raw_event["start"] <= int(t_stop * self._sampling_rate) + keep = raw_event["start"] + seg_start0 <= int(t_stop * self._sampling_rate) raw_event = raw_event[keep] - timestamp = raw_event["start"] + timestamp = raw_event["start"] + seg_start0 + if event_channel_index < 2: durations = None else: @@ -285,8 +292,7 @@ def _get_event_timestamps(self, block_index, seg_index, event_channel_index, t_s def _rescale_event_timestamp(self, event_timestamps, dtype, event_channel_index): event_times = event_timestamps.astype(dtype) / self._sampling_rate - # event_times += self._global_t_start - return event_times + return event_times def _rescale_epoch_duration(self, raw_duration, dtype, event_channel_index): durations = raw_duration.astype(dtype) / self._sampling_rate From dff7de8c1fcbac7ec4c6bfeb4ef99b23bdf2d55d Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Tue, 29 Oct 2024 09:30:23 +0100 Subject: [PATCH 5/5] test for micromed multi segment --- neo/test/rawiotest/test_micromedrawio.py | 38 +++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/neo/test/rawiotest/test_micromedrawio.py b/neo/test/rawiotest/test_micromedrawio.py index c74ea857c..85e68a347 100644 --- a/neo/test/rawiotest/test_micromedrawio.py +++ b/neo/test/rawiotest/test_micromedrawio.py @@ -8,6 +8,7 @@ from neo.test.rawiotest.common_rawio_test import BaseTestRawIO +import numpy as np class TestMicromedRawIO( BaseTestRawIO, @@ -15,7 +16,42 @@ class TestMicromedRawIO( ): rawioclass = MicromedRawIO entities_to_download = ["micromed"] - entities_to_test = ["micromed/File_micromed_1.TRC"] + entities_to_test = [ + "micromed/File_micromed_1.TRC", + "micromed/File_mircomed2.TRC", + "micromed/File_mircomed2_2segments.TRC", + ] + + def test_micromed_multi_segments(self): + file_full = self.get_local_path("micromed/File_mircomed2.TRC") + file_splitted = self.get_local_path("micromed/File_mircomed2_2segments.TRC") + + # the second file contains 2 pieces of the first file + # so it is 2 segments with the same traces but reduced + # note that traces in the splited can differ at the very end of the cut + + reader1 = MicromedRawIO(file_full) + reader1.parse_header() + assert reader1.segment_count(block_index=0) == 1 + assert reader1.get_signal_t_start(block_index=0, seg_index=0, stream_index=0) == 0. + traces1 = reader1.get_analogsignal_chunk(stream_index=0) + + reader2 = MicromedRawIO(file_splitted) + reader2.parse_header() + print(reader2) + assert reader2.segment_count(block_index=0) == 2 + + # check that pieces of the second file is equal to the first file (except a truncation at the end) + for seg_index in range(2): + t_start = reader2.get_signal_t_start(block_index=0, seg_index=seg_index, stream_index=0) + assert t_start > 0 + sr = reader2.get_signal_sampling_rate(stream_index=0) + ind_start = int(t_start * sr) + traces1_chunk = traces1[ind_start: ind_start+traces2.shape[0]] + traces2 = reader2.get_analogsignal_chunk(block_index=0, seg_index=seg_index, stream_index=0) + # we remove the last 100 sample because tools that cut traces is truncating the last buffer + assert np.array_equal(traces2[:-100], traces1_chunk[:-100]) + if __name__ == "__main__":