From 034f96ec0aa04927649cdec88c58aaa74cd833c4 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Sat, 6 Apr 2024 11:14:21 -0400 Subject: [PATCH 01/15] initial fixes --- neo/rawio/intanrawio.py | 141 ++++++++++++++++++++++++++++++++-------- 1 file changed, 115 insertions(+), 26 deletions(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index 756acb88a..819b701ab 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -95,10 +95,26 @@ def _parse_header(self): raise FileNotFoundError(f"{filename} does not exist") if self.filename.endswith(".rhs"): - self.file_format = "header-attached" - self._global_info, self._ordered_channels, data_dtype, header_size, self._block_size = read_rhs( - self.filename - ) + if filename.name == "info.rhs": + if any((filename.parent / file).exists for file in one_file_per_signal_filenames): + self.file_format = "one-file-per-signal" + raw_file_paths_dict = create_one_file_per_signal_dict(dirname=filename.parent, rhs=True) + else: + self.file_format = "one-file-per-channel" + raw_file_paths_dict = create_one_file_per_channel_dict(dirname=filename.parent, rhs=True) + else: + self.file_format = "header-attached" + + ( + self._global_info, + self._ordered_channels, + data_dtype, + header_size, + self._block_size, + channel_number_dict, + ) = read_rhs(self.filename) + + # 3 possibilities for rhd files, one combines the header and the data in the same file with suffix `rhd` while # the other two separates the data from the header which is always called `info.rhd` # attached to the actual binary file with data @@ -107,11 +123,11 @@ def _parse_header(self): # first we have one-file-per-signal which is where one neo stream/file is saved as .dat files if any((filename.parent / file).exists() for file in one_file_per_signal_filenames): self.file_format = "one-file-per-signal" - raw_file_paths_dict = create_one_file_per_signal_dict(filename.parent) + raw_file_paths_dict = create_one_file_per_signal_dict(dirname=filename.parent) # then there is one-file-per-channel where each channel in a neo stream is in its own .dat file else: self.file_format = "one-file-per-channel" - raw_file_paths_dict = create_one_file_per_channel_dict(filename.parent) + raw_file_paths_dict = create_one_file_per_channel_dict(dirname=filename.parent) # finally the format with the header-attached to the binary file as one giant file else: self.file_format = "header-attached" @@ -435,13 +451,15 @@ def read_variable_header(f, header): ] -def read_rhs(filename): +def read_rhs(filename, file_format: str): BLOCK_SIZE = 128 # sample per block with open(filename, mode="rb") as f: global_info = read_variable_header(f, rhs_global_header) channels_by_type = {k: [] for k in [0, 3, 4, 5, 6]} + if not file_format == "header-attached": + data_dtype = {k: [] for k in range(7)} # 5 streams + 6 for timestamps for not header attached for g in range(global_info["nb_signal_group"]): group_info = read_variable_header(f, rhs_signal_group_header) @@ -452,6 +470,10 @@ def read_rhs(filename): if bool(chan_info["channel_enabled"]): channels_by_type[chan_info["signal_type"]].append(chan_info) + + # useful dictionary for knowing the number of channels for non-header attached formats + channel_number_dict = {i: len(channels_by_type[i]) for i in range(6)} + header_size = f.tell() sr = global_info["sampling_rate"] @@ -469,7 +491,14 @@ def read_rhs(filename): chan_info["offset"] = -32768 * 0.195 chan_info["dtype"] = "uint16" ordered_channels.append(chan_info) - data_dtype += [(name, "uint16", BLOCK_SIZE)] + if file_format == "header-attached": + data_dtype += [(name, "uint16", BLOCK_SIZE)] + else: + data_dtype[0] = "int16" + + + # TODO @Heberto I need to read about this section more.... I'm not + # sure how it works if bool(global_info["dc_amplifier_data_saved"]): for chan_info in channels_by_type[0]: @@ -500,6 +529,9 @@ def read_rhs(filename): ordered_channels.append(chan_info_stim) data_dtype += [(name + "_STIM", "uint16", BLOCK_SIZE)] + # TODO @Heberto to update to version 3 we also likely need to add in the + # supply and aux voltages. + # 3: Analog input channel. # 4: Analog output channel. for sig_type in [ @@ -514,23 +546,46 @@ def read_rhs(filename): chan_info["offset"] = -32768 * 0.0003125 chan_info["dtype"] = "uint16" ordered_channels.append(chan_info) - data_dtype += [(name, "uint16", BLOCK_SIZE)] + if file_format == "header-attached": + data_dtype += [(name, "uint16", BLOCK_SIZE)] + else: + data_dtype[sig_type] = "unit16" # 5: Digital input channel. # 6: Digital output channel. for sig_type in [5, 6]: - # at the moment theses channel are not in sig channel list - # but they are in the raw memamp if len(channels_by_type[sig_type]) > 0: - name = {5: "DIGITAL-IN", 6: "DIGITAL-OUT"}[sig_type] - data_dtype += [(name, "uint16", BLOCK_SIZE)] + name = {4: "DIGITAL-IN", 5: "DIGITAL-OUT"}[sig_type] + chan_info = channels_by_type[sig_type][0] + # So currently until we have get_digitalsignal_chunk we need to do a tiny hack to + # make this memory map work correctly. So since our digital data is not organized + # by channel like analog/ADC are we have to overwrite the native name to create + # a single permanent name that we can find with channel id + chan_info["native_channel_name"] = name # overwite to allow memmap to work + chan_info["sampling_rate"] = sr + chan_info["units"] = "TTL" # arbitrary units TTL for logic + chan_info["gain"] = 1.0 + chan_info["offset"] = 0.0 + chan_info["dtype"] = "uint16" + ordered_channels.append(chan_info) + if file_format == "header-attached": + data_dtype += [(name, "uint16", BLOCK_SIZE)] + else: + data_dtype[sig_type] = "uint16" - if bool(global_info["notch_filter_mode"]) and global_info["major_version"] >= 3: - global_info["notch_filter_applied"] = True + if global_info["notch_filter_mode"] == 2 and global_info["major_version"] >= V("3.0"): + global_info["notch_filter"] = "60Hz" + elif global_info["notch_filter_mode"] == 1 and global_info["major_version"]>= V("3.0"): + global_info["notch_filter"] = "50Hz" else: - global_info["notch_filter_applied"] = False + global_info["notch_filter"] = False - return global_info, ordered_channels, data_dtype, header_size, BLOCK_SIZE + if not file_format == "header-attached": + # filter out dtypes without any values + data_dtype = {k: v for (k, v) in data_dtype.items() if len(v) > 0} + channel_number_dict = {k: v for (k, v) in channel_number_dict.items() if v > 0} + + return global_info, ordered_channels, data_dtype, header_size, BLOCK_SIZE, channel_number_dict ############### @@ -808,8 +863,11 @@ def read_rhd(filename, file_format: str): return global_info, ordered_channels, data_dtype, header_size, BLOCK_SIZE, channel_number_dict -########################### -# RHD Zone for Binary Files +########################################################################## +# RHX Zone for Binary Files +# This zone gives the headerless binary files for both rhs and rhd header files +# This occurs with the new rhx version of the intan recording software as an +# optional software that can be turned # For One File Per Signal one_file_per_signal_filenames = [ @@ -822,13 +880,29 @@ def read_rhd(filename, file_format: str): ] -def create_one_file_per_signal_dict(dirname): - """Function for One File Per Signal Type""" +def create_one_file_per_signal_dict(dirname, rhs: bool=False): + """Function for One File Per Signal Type + + Parameters + ---------- + dirname: pathlib.Path + The folder to explore + rhs: bool, default: False + Whether this is an rhd or an rhs file + """ + + # if rhs we have an extra stream to add + if rhs: + one_file_per_signal_filenames.insert(4, "analogout.dat") + raw_file_paths_dict = {} for raw_index, raw_file in enumerate(one_file_per_signal_filenames): if Path(dirname / raw_file).is_file(): raw_file_paths_dict[raw_index] = Path(dirname / raw_file) - raw_file_paths_dict[6] = Path(dirname / "time.dat") + if rhs: + raw_file_paths_dict[7] = Path(dirname / "time.dat") + else: + raw_file_paths_dict[6] = Path(dirname / "time.dat") return raw_file_paths_dict @@ -838,19 +912,34 @@ def create_one_file_per_signal_dict(dirname): "amp", "aux", "vdd", - "board-ANALOG", + "board-ANALOG-IN", "board-DIGITAL-IN", "board-DIGITAL-OUT", ] -def create_one_file_per_channel_dict(dirname): - """Utility function for One File Per Channel""" +def create_one_file_per_channel_dict(dirname, rhs: bool=False): + """Utility function for One File Per Channel + + Parameters + ---------- + dirname: pathlib.Path + The folder to explore + rhs: bool, default: False + Whether this is an rhd or an rhs file + """ + # if rhs we have an extra stream to add + if rhs: + possible_raw_file_prefixes.insert(4, "board-ANALOG-OUT") + file_names = dirname.glob("**/*.dat") files = [file for file in file_names if file.is_file()] raw_file_paths_dict = {} for raw_index, prefix in enumerate(possible_raw_file_prefixes): raw_file_paths_dict[raw_index] = [file for file in files if prefix in file.name] - raw_file_paths_dict[6] = [Path(dirname / "time.dat")] + if rhs: + raw_file_paths_dict[7] = [Path(dirname / "time.dat")] + else: + raw_file_paths_dict[6] = [Path(dirname / "time.dat")] return raw_file_paths_dict From 35ba654778ab8ca967eab75564bfd69ed9ac9b64 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Sat, 6 Apr 2024 11:29:31 -0400 Subject: [PATCH 02/15] oops --- neo/rawio/intanrawio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index 819b701ab..3d0bec073 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -112,7 +112,7 @@ def _parse_header(self): header_size, self._block_size, channel_number_dict, - ) = read_rhs(self.filename) + ) = read_rhs(self.filename, self.file_format) # 3 possibilities for rhd files, one combines the header and the data in the same file with suffix `rhd` while From 5b554017c21a4fef047acc8adf3aade066d4fc7c Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Mon, 8 Apr 2024 10:23:04 -0400 Subject: [PATCH 03/15] add in dtypes and channel numbers for rhs --- neo/rawio/intanrawio.py | 105 ++++++++++++++++++++++++++-------------- 1 file changed, 70 insertions(+), 35 deletions(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index 3d0bec073..7ef734874 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -22,6 +22,7 @@ import os from collections import OrderedDict from packaging.version import Version as V +import warnings import numpy as np @@ -114,7 +115,6 @@ def _parse_header(self): channel_number_dict, ) = read_rhs(self.filename, self.file_format) - # 3 possibilities for rhd files, one combines the header and the data in the same file with suffix `rhd` while # the other two separates the data from the header which is always called `info.rhd` # attached to the actual binary file with data @@ -457,9 +457,12 @@ def read_rhs(filename, file_format: str): with open(filename, mode="rb") as f: global_info = read_variable_header(f, rhs_global_header) + # channels_by_type is simpler than data_dtype because 0 contains 0, 10 and 11 internally channels_by_type = {k: [] for k in [0, 3, 4, 5, 6]} if not file_format == "header-attached": - data_dtype = {k: [] for k in range(7)} # 5 streams + 6 for timestamps for not header attached + # data_dtype for rhs is complicated. There is not 1, 2 (supply and aux), + # but there are dc-amp (10) and stim (11). we make timestamps (7) + data_dtype = {k: [] for k in [0, 3, 4, 5, 6, 7, 10, 11]} for g in range(global_info["nb_signal_group"]): group_info = read_variable_header(f, rhs_signal_group_header) @@ -470,9 +473,8 @@ def read_rhs(filename, file_format: str): if bool(chan_info["channel_enabled"]): channels_by_type[chan_info["signal_type"]].append(chan_info) - # useful dictionary for knowing the number of channels for non-header attached formats - channel_number_dict = {i: len(channels_by_type[i]) for i in range(6)} + channel_number_dict = {i: len(channels_by_type[i]) for i in [0, 3, 4, 5, 6]} header_size = f.tell() @@ -480,7 +482,11 @@ def read_rhs(filename, file_format: str): # construct dtype by re-ordering channels by types ordered_channels = [] - data_dtype = [("timestamp", "int32", BLOCK_SIZE)] + if file_format == "header-attached": + data_dtype = [("timestamp", "int32", BLOCK_SIZE)] + else: + data_dtype[7] = "int32" + channel_number_dict[7] = 1 # 0: RHS2000 amplifier channel. for chan_info in channels_by_type[0]: @@ -488,19 +494,23 @@ def read_rhs(filename, file_format: str): chan_info["sampling_rate"] = sr chan_info["units"] = "uV" chan_info["gain"] = 0.195 - chan_info["offset"] = -32768 * 0.195 - chan_info["dtype"] = "uint16" + if file_format == "header-attached": + chan_info["offset"] = -32768 * 0.195 + else: + chan_info["offset"] = 0.0 + if file_format == "header-attached": + chan_info["dtype"] = "uint16" + else: + chan_info["dtype"] = "int16" ordered_channels.append(chan_info) if file_format == "header-attached": data_dtype += [(name, "uint16", BLOCK_SIZE)] else: - data_dtype[0] = "int16" - - - # TODO @Heberto I need to read about this section more.... I'm not - # sure how it works + data_dtype[0] = "int16" if bool(global_info["dc_amplifier_data_saved"]): + # if we have dc amp we need to grab the correct number of channels + channel_number_dict[10] = channel_number_dict[0] for chan_info in channels_by_type[0]: name = chan_info["native_channel_name"] chan_info_dc = dict(chan_info) @@ -512,25 +522,35 @@ def read_rhs(filename, file_format: str): chan_info_dc["signal_type"] = 10 # put it in another group chan_info_dc["dtype"] = "uint16" ordered_channels.append(chan_info_dc) - data_dtype += [(name + "_DC", "uint16", BLOCK_SIZE)] + if file_format == "header-attached": + data_dtype += [(name + "_DC", "uint16", BLOCK_SIZE)] + else: + data_dtype[10] = "unit16" + # I can't seem to get stim files to generate for one-file-per-channel + # so let's skip for now and can be given on request - for chan_info in channels_by_type[0]: - name = chan_info["native_channel_name"] - chan_info_stim = dict(chan_info) - chan_info_stim["native_channel_name"] = name + "_STIM" - chan_info_stim["sampling_rate"] = sr - # stim channel are coplicated because they are coded - # with bits, they do not fit the gain/offset rawio strategy - chan_info_stim["units"] = "" - chan_info_stim["gain"] = 1.0 - chan_info_stim["offset"] = 0.0 - chan_info_stim["signal_type"] = 11 # put it in another group - chan_info_stim["dtype"] = "uint16" - ordered_channels.append(chan_info_stim) - data_dtype += [(name + "_STIM", "uint16", BLOCK_SIZE)] - - # TODO @Heberto to update to version 3 we also likely need to add in the - # supply and aux voltages. + if file_format != "one-file-per-channel": + for chan_info in channels_by_type[0]: + name = chan_info["native_channel_name"] + chan_info_stim = dict(chan_info) + chan_info_stim["native_channel_name"] = name + "_STIM" + chan_info_stim["sampling_rate"] = sr + # stim channel are complicated because they are coded + # with bits, they do not fit the gain/offset rawio strategy + chan_info_stim["units"] = "" + chan_info_stim["gain"] = 1.0 + chan_info_stim["offset"] = 0.0 + chan_info_stim["signal_type"] = 11 # put it in another group + chan_info_stim["dtype"] = "uint16" + ordered_channels.append(chan_info_stim) + if file_format == "header-attached": + data_dtype += [(name + "_STIM", "uint16", BLOCK_SIZE)] + else: + data_dtype[11] == "unit16" + else: + warnings.warn('Stim not implemented for one-file-per-channel due to lack of test files') + + # No supply or aux for rhs files (ie no stream 1 and 2) # 3: Analog input channel. # 4: Analog output channel. @@ -575,7 +595,7 @@ def read_rhs(filename, file_format: str): if global_info["notch_filter_mode"] == 2 and global_info["major_version"] >= V("3.0"): global_info["notch_filter"] = "60Hz" - elif global_info["notch_filter_mode"] == 1 and global_info["major_version"]>= V("3.0"): + elif global_info["notch_filter_mode"] == 1 and global_info["major_version"] >= V("3.0"): global_info["notch_filter"] = "50Hz" else: global_info["notch_filter"] = False @@ -880,9 +900,9 @@ def read_rhd(filename, file_format: str): ] -def create_one_file_per_signal_dict(dirname, rhs: bool=False): +def create_one_file_per_signal_dict(dirname, rhs: bool = False): """Function for One File Per Signal Type - + Parameters ---------- dirname: pathlib.Path @@ -892,6 +912,7 @@ def create_one_file_per_signal_dict(dirname, rhs: bool=False): """ # if rhs we have an extra stream to add + if rhs: one_file_per_signal_filenames.insert(4, "analogout.dat") @@ -904,6 +925,13 @@ def create_one_file_per_signal_dict(dirname, rhs: bool=False): else: raw_file_paths_dict[6] = Path(dirname / "time.dat") + if rhs: + # 10 and 11 are hardcoded in the rhs_reader above so hardcoded here too + if Path(dirname / "dcamplifier.dat").is_file(): + raw_file_paths_dict[10] = Path(dirname / "dcamplifier.dat") + if Path(dirname / "stim.dat").is_file(): + raw_file_paths_dict[11] = Path(dirname / "stim.dat") + return raw_file_paths_dict @@ -918,9 +946,9 @@ def create_one_file_per_signal_dict(dirname, rhs: bool=False): ] -def create_one_file_per_channel_dict(dirname, rhs: bool=False): +def create_one_file_per_channel_dict(dirname, rhs: bool = False): """Utility function for One File Per Channel - + Parameters ---------- dirname: pathlib.Path @@ -942,4 +970,11 @@ def create_one_file_per_channel_dict(dirname, rhs: bool=False): else: raw_file_paths_dict[6] = [Path(dirname / "time.dat")] + if rhs: + # 10 and 11 are hardcoded in the rhs reader so hardcoded here + raw_file_paths_dict[10] = [file for file in files if "dc-" in file.name] + # we can find the files, but I can see how to read them out of header + # so for now we don't expose the stim files in one-file-per-channel + raw_file_paths_dict[11] = [file for file in files if "stim-" in file.name] + return raw_file_paths_dict From d55f3aa2ffc403475fd40bec467c0f07d968162c Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Tue, 9 Apr 2024 13:21:10 -0400 Subject: [PATCH 04/15] add test files in tests more fixes --- neo/rawio/intanrawio.py | 22 ++++++++++++++++++---- neo/test/iotest/test_intanio.py | 2 ++ neo/test/rawiotest/test_intanrawio.py | 2 ++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index 7ef734874..a3f3591a0 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -211,7 +211,10 @@ def _parse_header(self): signal_streams = np.zeros(stream_ids.size, dtype=_signal_stream_dtype) signal_streams["id"] = stream_ids for stream_index, stream_id in enumerate(stream_ids): - signal_streams["name"][stream_index] = stream_type_to_name.get(int(stream_id), "") + if self.filename.endswith('.rhd'): + signal_streams["name"][stream_index] = stream_type_to_name_rhd.get(int(stream_id), "") + else: + signal_streams["name"][stream_index] = stream_type_to_name_rhs.get(int(stream_id), "") self._max_sampling_rate = np.max(signal_channels["sampling_rate"]) @@ -450,6 +453,16 @@ def read_variable_header(f, header): ("electrode_impedance_phase", "float32"), ] +stream_type_to_name_rhs= { + 0: "RHS2000 amplifier channel", + 3: "USB board ADC input channel", + 4: "USB board ADC output channel", + 5: "USB board digital input channel", + 6: "USB board digital output channel", + 10: "DC Amplifier channel", + 11: "Stim channel" +} + def read_rhs(filename, file_format: str): BLOCK_SIZE = 128 # sample per block @@ -530,6 +543,7 @@ def read_rhs(filename, file_format: str): # so let's skip for now and can be given on request if file_format != "one-file-per-channel": + channel_number_dict[11] = channel_number_dict[0] # should be one stim / amplifier channel for chan_info in channels_by_type[0]: name = chan_info["native_channel_name"] chan_info_stim = dict(chan_info) @@ -548,7 +562,7 @@ def read_rhs(filename, file_format: str): else: data_dtype[11] == "unit16" else: - warnings.warn('Stim not implemented for one-file-per-channel due to lack of test files') + warnings.warn("Stim not implemented for `one-file-per-channel` due to lack of test files") # No supply or aux for rhs files (ie no stream 1 and 2) @@ -575,7 +589,7 @@ def read_rhs(filename, file_format: str): # 6: Digital output channel. for sig_type in [5, 6]: if len(channels_by_type[sig_type]) > 0: - name = {4: "DIGITAL-IN", 5: "DIGITAL-OUT"}[sig_type] + name = {5: "DIGITAL-IN", 6: "DIGITAL-OUT"}[sig_type] chan_info = channels_by_type[sig_type][0] # So currently until we have get_digitalsignal_chunk we need to do a tiny hack to # make this memory map work correctly. So since our digital data is not organized @@ -675,7 +689,7 @@ def read_rhs(filename, file_format: str): ("electrode_impedance_phase", "float32"), ] -stream_type_to_name = { +stream_type_to_name_rhd = { 0: "RHD2000 amplifier channel", 1: "RHD2000 auxiliary input channel", 2: "RHD2000 supply voltage channel", diff --git a/neo/test/iotest/test_intanio.py b/neo/test/iotest/test_intanio.py index a91138384..211d600de 100644 --- a/neo/test/iotest/test_intanio.py +++ b/neo/test/iotest/test_intanio.py @@ -19,6 +19,8 @@ class TestIntanIO( "intan/intan_rhd_test_1.rhd", "intan/intan_fpc_test_231117_052630/info.rhd", "intan/intan_fps_test_231117_052500/info.rhd", + "intan/intan_fpc_rhs_test_240329_091536/info.rhs", + "intan/intan_fps_rhs_test_240329_091637/info.rhs", ] diff --git a/neo/test/rawiotest/test_intanrawio.py b/neo/test/rawiotest/test_intanrawio.py index 794480787..f2e39600d 100644 --- a/neo/test/rawiotest/test_intanrawio.py +++ b/neo/test/rawiotest/test_intanrawio.py @@ -16,6 +16,8 @@ class TestIntanRawIO( "intan/intan_rhd_test_1.rhd", "intan/intan_fpc_test_231117_052630/info.rhd", "intan/intan_fps_test_231117_052500/info.rhd", + "intan/intan_fpc_rhs_test_240329_091536/info.rhs", + "intan/intan_fps_rhs_test_240329_091637/info.rhs", ] From f82eb61a210bce22ccf4644884eb5036af552d54 Mon Sep 17 00:00:00 2001 From: Zach McKenzie <92116279+zm711@users.noreply.github.com> Date: Thu, 2 May 2024 16:44:32 -0400 Subject: [PATCH 05/15] Fix naming of test files --- neo/test/iotest/test_intanio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/test/iotest/test_intanio.py b/neo/test/iotest/test_intanio.py index 211d600de..bd18359a2 100644 --- a/neo/test/iotest/test_intanio.py +++ b/neo/test/iotest/test_intanio.py @@ -19,8 +19,8 @@ class TestIntanIO( "intan/intan_rhd_test_1.rhd", "intan/intan_fpc_test_231117_052630/info.rhd", "intan/intan_fps_test_231117_052500/info.rhd", - "intan/intan_fpc_rhs_test_240329_091536/info.rhs", - "intan/intan_fps_rhs_test_240329_091637/info.rhs", + "intan/intan_fpc_rhs_test_240329_091537/info.rhs", + "intan/intan_fps_rhs_test_240329_091636/info.rhs", ] From fbb2ebff130f40c5db58fc29f751fbcf3de2f2a8 Mon Sep 17 00:00:00 2001 From: Zach McKenzie <92116279+zm711@users.noreply.github.com> Date: Thu, 2 May 2024 20:11:00 -0400 Subject: [PATCH 06/15] Update neo/test/rawiotest/test_intanrawio.py --- neo/test/rawiotest/test_intanrawio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/test/rawiotest/test_intanrawio.py b/neo/test/rawiotest/test_intanrawio.py index f2e39600d..398aa3566 100644 --- a/neo/test/rawiotest/test_intanrawio.py +++ b/neo/test/rawiotest/test_intanrawio.py @@ -16,8 +16,8 @@ class TestIntanRawIO( "intan/intan_rhd_test_1.rhd", "intan/intan_fpc_test_231117_052630/info.rhd", "intan/intan_fps_test_231117_052500/info.rhd", - "intan/intan_fpc_rhs_test_240329_091536/info.rhs", - "intan/intan_fps_rhs_test_240329_091637/info.rhs", + "intan/intan_fpc_rhs_test_240329_091537/info.rhs", + "intan/intan_fps_rhs_test_240329_091636/info.rhs", ] From 13ab6c5e96a914e33644a55b519a16ff7e6f8218 Mon Sep 17 00:00:00 2001 From: Zach McKenzie <92116279+zm711@users.noreply.github.com> Date: Thu, 2 May 2024 20:21:02 -0400 Subject: [PATCH 07/15] Fix pointers to test files --- neo/test/iotest/test_intanio.py | 2 +- neo/test/rawiotest/test_intanrawio.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/test/iotest/test_intanio.py b/neo/test/iotest/test_intanio.py index bd18359a2..7d15a5f99 100644 --- a/neo/test/iotest/test_intanio.py +++ b/neo/test/iotest/test_intanio.py @@ -19,7 +19,7 @@ class TestIntanIO( "intan/intan_rhd_test_1.rhd", "intan/intan_fpc_test_231117_052630/info.rhd", "intan/intan_fps_test_231117_052500/info.rhd", - "intan/intan_fpc_rhs_test_240329_091537/info.rhs", + "intan/intan_fpc_rhs_test_240329_091637/info.rhs", "intan/intan_fps_rhs_test_240329_091636/info.rhs", ] diff --git a/neo/test/rawiotest/test_intanrawio.py b/neo/test/rawiotest/test_intanrawio.py index 398aa3566..1666efc69 100644 --- a/neo/test/rawiotest/test_intanrawio.py +++ b/neo/test/rawiotest/test_intanrawio.py @@ -16,7 +16,7 @@ class TestIntanRawIO( "intan/intan_rhd_test_1.rhd", "intan/intan_fpc_test_231117_052630/info.rhd", "intan/intan_fps_test_231117_052500/info.rhd", - "intan/intan_fpc_rhs_test_240329_091537/info.rhs", + "intan/intan_fpc_rhs_test_240329_091637/info.rhs", "intan/intan_fps_rhs_test_240329_091636/info.rhs", ] From e01f638b5800ceaa48bde1abd06eb037a2147700 Mon Sep 17 00:00:00 2001 From: Zach McKenzie <92116279+zm711@users.noreply.github.com> Date: Thu, 2 May 2024 20:34:47 -0400 Subject: [PATCH 08/15] oops --- neo/rawio/intanrawio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index a3f3591a0..b96a38514 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -97,7 +97,7 @@ def _parse_header(self): if self.filename.endswith(".rhs"): if filename.name == "info.rhs": - if any((filename.parent / file).exists for file in one_file_per_signal_filenames): + if any((filename.parent / file).exists() for file in one_file_per_signal_filenames): self.file_format = "one-file-per-signal" raw_file_paths_dict = create_one_file_per_signal_dict(dirname=filename.parent, rhs=True) else: From 78700d7c32b8a8b951e27e0f2ef386c9c9466cf5 Mon Sep 17 00:00:00 2001 From: Zach McKenzie <92116279+zm711@users.noreply.github.com> Date: Thu, 2 May 2024 20:51:04 -0400 Subject: [PATCH 09/15] dtype fixes --- neo/rawio/intanrawio.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index b96a38514..7a766d79a 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -538,7 +538,7 @@ def read_rhs(filename, file_format: str): if file_format == "header-attached": data_dtype += [(name + "_DC", "uint16", BLOCK_SIZE)] else: - data_dtype[10] = "unit16" + data_dtype[10] = "uint16" # I can't seem to get stim files to generate for one-file-per-channel # so let's skip for now and can be given on request @@ -560,7 +560,7 @@ def read_rhs(filename, file_format: str): if file_format == "header-attached": data_dtype += [(name + "_STIM", "uint16", BLOCK_SIZE)] else: - data_dtype[11] == "unit16" + data_dtype[11] == "uint16" else: warnings.warn("Stim not implemented for `one-file-per-channel` due to lack of test files") @@ -583,7 +583,7 @@ def read_rhs(filename, file_format: str): if file_format == "header-attached": data_dtype += [(name, "uint16", BLOCK_SIZE)] else: - data_dtype[sig_type] = "unit16" + data_dtype[sig_type] = "uint16" # 5: Digital input channel. # 6: Digital output channel. From fa49d3952134e638a4c220d0e91f656261c75d78 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Fri, 3 May 2024 18:02:41 -0400 Subject: [PATCH 10/15] split rhs and rhd + fix timestamps --- neo/rawio/intanrawio.py | 176 +++++++++++++++++++++++++++------------- 1 file changed, 121 insertions(+), 55 deletions(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index 7da9397e7..6fe63c48e 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -98,12 +98,12 @@ def _parse_header(self): if self.filename.endswith(".rhs"): if filename.name == "info.rhs": - if any((filename.parent / file).exists() for file in one_file_per_signal_filenames): + if any((filename.parent / file).exists() for file in one_file_per_signal_filenames_rhs): self.file_format = "one-file-per-signal" - raw_file_paths_dict = create_one_file_per_signal_dict(dirname=filename.parent, rhs=True) + raw_file_paths_dict = create_one_file_per_signal_dict_rhs(dirname=filename.parent) else: self.file_format = "one-file-per-channel" - raw_file_paths_dict = create_one_file_per_channel_dict(dirname=filename.parent, rhs=True) + raw_file_paths_dict = create_one_file_per_channel_dict_rhs(dirname=filename.parent) else: self.file_format = "header-attached" @@ -122,13 +122,13 @@ def _parse_header(self): elif self.filename.endswith(".rhd"): if filename.name == "info.rhd": # first we have one-file-per-signal which is where one neo stream/file is saved as .dat files - if any((filename.parent / file).exists() for file in one_file_per_signal_filenames): + if any((filename.parent / file).exists() for file in one_file_per_signal_filenames_rhd): self.file_format = "one-file-per-signal" - raw_file_paths_dict = create_one_file_per_signal_dict(dirname=filename.parent) + raw_file_paths_dict = create_one_file_per_signal_dict_rhd(dirname=filename.parent) # then there is one-file-per-channel where each channel in a neo stream is in its own .dat file else: self.file_format = "one-file-per-channel" - raw_file_paths_dict = create_one_file_per_channel_dict(dirname=filename.parent) + raw_file_paths_dict = create_one_file_per_channel_dict_rhd(dirname=filename.parent) # finally the format with the header-attached to the binary file as one giant file else: self.file_format = "header-attached" @@ -175,7 +175,8 @@ def _parse_header(self): # check timestamp continuity if self.file_format == "header-attached": timestamp = self._raw_data["timestamp"].flatten() - # timestamps are always the last stream + + # timestamps are always 6 for rhd and 7 for rhs elif self.file_format == "one-file-per-signal": time_stream_index = max(self._raw_data.keys()) timestamp = self._raw_data[time_stream_index] @@ -213,7 +214,7 @@ def _parse_header(self): signal_streams = np.zeros(stream_ids.size, dtype=_signal_stream_dtype) signal_streams["id"] = stream_ids for stream_index, stream_id in enumerate(stream_ids): - if self.filename.endswith('.rhd'): + if self.filename.endswith(".rhd"): signal_streams["name"][stream_index] = stream_type_to_name_rhd.get(int(stream_id), "") else: signal_streams["name"][stream_index] = stream_type_to_name_rhs.get(int(stream_id), "") @@ -455,14 +456,14 @@ def read_variable_header(f, header): ("electrode_impedance_phase", "float32"), ] -stream_type_to_name_rhs= { +stream_type_to_name_rhs = { 0: "RHS2000 amplifier channel", 3: "USB board ADC input channel", 4: "USB board ADC output channel", 5: "USB board digital input channel", 6: "USB board digital output channel", 10: "DC Amplifier channel", - 11: "Stim channel" + 11: "Stim channel", } @@ -501,8 +502,8 @@ def read_rhs(filename, file_format: str): if file_format == "header-attached": data_dtype = [("timestamp", "int32", BLOCK_SIZE)] else: - data_dtype[7] = "int32" - channel_number_dict[7] = 1 + data_dtype[15] = "int32" + channel_number_dict[15] = 1 # 0: RHS2000 amplifier channel. for chan_info in channels_by_type[0]: @@ -546,7 +547,7 @@ def read_rhs(filename, file_format: str): # so let's skip for now and can be given on request if file_format != "one-file-per-channel": - channel_number_dict[11] = channel_number_dict[0] # should be one stim / amplifier channel + channel_number_dict[11] = channel_number_dict[0] # should be one stim / amplifier channel for chan_info in channels_by_type[0]: name = chan_info["native_channel_name"] chan_info_stim = dict(chan_info) @@ -902,12 +903,11 @@ def read_rhd(filename, file_format: str): ########################################################################## # RHX Zone for Binary Files -# This zone gives the headerless binary files for both rhs and rhd header files -# This occurs with the new rhx version of the intan recording software as an -# optional software that can be turned +# This section provides all possible headerless binary files in both the rhs and rhd +# formats. -# For One File Per Signal -one_file_per_signal_filenames = [ +# RHD Binary Files for One File Per Signal +one_file_per_signal_filenames_rhd = [ "amplifier.dat", "auxiliary.dat", "supply.dat", @@ -917,43 +917,73 @@ def read_rhd(filename, file_format: str): ] -def create_one_file_per_signal_dict(dirname, rhs: bool = False): +def create_one_file_per_signal_dict_rhd(dirname): """Function for One File Per Signal Type Parameters ---------- dirname: pathlib.Path The folder to explore - rhs: bool, default: False - Whether this is an rhd or an rhs file + + Returns + ------- + raw_files_paths_dict: dict + A dict of all the file paths """ - # if rhs we have an extra stream to add + raw_file_paths_dict = {} + for raw_index, raw_file in enumerate(one_file_per_signal_filenames_rhd): + if Path(dirname / raw_file).is_file(): + raw_file_paths_dict[raw_index] = Path(dirname / raw_file) + + raw_file_paths_dict[6] = Path(dirname / "time.dat") + + return raw_file_paths_dict + + +# RHS Binary Files for One File Per Signal +one_file_per_signal_filenames_rhs = [ + "amplifier.dat", + "auxiliary.dat", + "supply.dat", + "analogin.dat", + "analogout.dat" "digitalin.dat", + "digitalout.dat", +] + + +def create_one_file_per_signal_dict_rhs(dirname): + """Function for One File Per Signal Type + + Parameters + ---------- + dirname: pathlib.Path + The folder to explore - if rhs: - one_file_per_signal_filenames.insert(4, "analogout.dat") + Returns + ------- + raw_files_paths_dict: dict + A dict of all the file paths + """ raw_file_paths_dict = {} - for raw_index, raw_file in enumerate(one_file_per_signal_filenames): + for raw_index, raw_file in enumerate(one_file_per_signal_filenames_rhs): if Path(dirname / raw_file).is_file(): raw_file_paths_dict[raw_index] = Path(dirname / raw_file) - if rhs: - raw_file_paths_dict[7] = Path(dirname / "time.dat") - else: - raw_file_paths_dict[6] = Path(dirname / "time.dat") - if rhs: - # 10 and 11 are hardcoded in the rhs_reader above so hardcoded here too - if Path(dirname / "dcamplifier.dat").is_file(): - raw_file_paths_dict[10] = Path(dirname / "dcamplifier.dat") - if Path(dirname / "stim.dat").is_file(): - raw_file_paths_dict[11] = Path(dirname / "stim.dat") + # we need time to be the last value + raw_file_paths_dict[15] = Path(dirname / "time.dat") + # 10 and 11 are hardcoded in the rhs_reader above so hardcoded here too + if Path(dirname / "dcamplifier.dat").is_file(): + raw_file_paths_dict[10] = Path(dirname / "dcamplifier.dat") + if Path(dirname / "stim.dat").is_file(): + raw_file_paths_dict[11] = Path(dirname / "stim.dat") return raw_file_paths_dict -# For One File Per Channel -possible_raw_file_prefixes = [ +# RHD Binary Files for One File Per Channel +possible_raw_file_prefixes_rhd = [ "amp", "aux", "vdd", @@ -963,35 +993,71 @@ def create_one_file_per_signal_dict(dirname, rhs: bool = False): ] -def create_one_file_per_channel_dict(dirname, rhs: bool = False): +def create_one_file_per_channel_dict_rhd(dirname): """Utility function for One File Per Channel Parameters ---------- dirname: pathlib.Path The folder to explore - rhs: bool, default: False - Whether this is an rhd or an rhs file + + Returns + ------- + raw_files_paths_dict: dict + A dict of all the file paths """ - # if rhs we have an extra stream to add - if rhs: - possible_raw_file_prefixes.insert(4, "board-ANALOG-OUT") file_names = dirname.glob("**/*.dat") files = [file for file in file_names if file.is_file()] raw_file_paths_dict = {} - for raw_index, prefix in enumerate(possible_raw_file_prefixes): + for raw_index, prefix in enumerate(possible_raw_file_prefixes_rhd): raw_file_paths_dict[raw_index] = [file for file in files if prefix in file.name] - if rhs: - raw_file_paths_dict[7] = [Path(dirname / "time.dat")] - else: - raw_file_paths_dict[6] = [Path(dirname / "time.dat")] - - if rhs: - # 10 and 11 are hardcoded in the rhs reader so hardcoded here - raw_file_paths_dict[10] = [file for file in files if "dc-" in file.name] - # we can find the files, but I can see how to read them out of header - # so for now we don't expose the stim files in one-file-per-channel - raw_file_paths_dict[11] = [file for file in files if "stim-" in file.name] + + raw_file_paths_dict[6] = [Path(dirname / "time.dat")] + + return raw_file_paths_dict + + +# RHS Binary Files for One File Per Channel +possible_raw_file_prefixes_rhs = [ + "amp", + "aux", + "vdd", + "board-ANALOG-IN", + "board-ANALOG-OUT", + "board-DIGITAL-IN", + "board-DIGITAL-OUT", +] + + +def create_one_file_per_channel_dict_rhs( + dirname, +): + """Utility function for One File Per Channel + + Parameters + ---------- + dirname: pathlib.Path + The folder to explore + + Returns + ------- + raw_files_paths_dict: dict + A dict of all the file paths + """ + + file_names = dirname.glob("**/*.dat") + files = [file for file in file_names if file.is_file()] + raw_file_paths_dict = {} + for raw_index, prefix in enumerate(possible_raw_file_prefixes_rhs): + raw_file_paths_dict[raw_index] = [file for file in files if prefix in file.name] + + # we need time to be the last value + raw_file_paths_dict[15] = [Path(dirname / "time.dat")] + # 10 and 11 are hardcoded in the rhs reader so hardcoded here + raw_file_paths_dict[10] = [file for file in files if "dc-" in file.name] + # we can find the files, but I can see how to read them out of header + # so for now we don't expose the stim files in one-file-per-channel + raw_file_paths_dict[11] = [file for file in files if "stim-" in file.name] return raw_file_paths_dict From 10bf3c61b84bd7d98a029347ca2a6f2421d1c21c Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Fri, 3 May 2024 18:44:52 -0400 Subject: [PATCH 11/15] put streams in correct order --- neo/rawio/intanrawio.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index 6fe63c48e..9b875d9b0 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -212,7 +212,11 @@ def _parse_header(self): stream_ids = np.unique(signal_channels["stream_id"]) signal_streams = np.zeros(stream_ids.size, dtype=_signal_stream_dtype) - signal_streams["id"] = stream_ids + + # we need to sort the data because the string of 10 is mis-sorted. + stream_ids_sorted = sorted([int(stream_id) for stream_id in stream_ids]) + signal_streams["id"] = [str(stream_id) for stream_id in stream_ids_sorted] + for stream_index, stream_id in enumerate(stream_ids): if self.filename.endswith(".rhd"): signal_streams["name"][stream_index] = stream_type_to_name_rhd.get(int(stream_id), "") From cb869c3cf83c719aac6dbb9734b04d899de70de1 Mon Sep 17 00:00:00 2001 From: Zach McKenzie <92116279+zm711@users.noreply.github.com> Date: Fri, 3 May 2024 18:52:12 -0400 Subject: [PATCH 12/15] fix test file naming. --- neo/test/iotest/test_intanio.py | 2 +- neo/test/rawiotest/test_intanrawio.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/test/iotest/test_intanio.py b/neo/test/iotest/test_intanio.py index 7d15a5f99..bedd65bf8 100644 --- a/neo/test/iotest/test_intanio.py +++ b/neo/test/iotest/test_intanio.py @@ -20,7 +20,7 @@ class TestIntanIO( "intan/intan_fpc_test_231117_052630/info.rhd", "intan/intan_fps_test_231117_052500/info.rhd", "intan/intan_fpc_rhs_test_240329_091637/info.rhs", - "intan/intan_fps_rhs_test_240329_091636/info.rhs", + "intan/intan_fps_rhs_test_240329_091536/info.rhs", ] diff --git a/neo/test/rawiotest/test_intanrawio.py b/neo/test/rawiotest/test_intanrawio.py index 1666efc69..3f5ee3c7c 100644 --- a/neo/test/rawiotest/test_intanrawio.py +++ b/neo/test/rawiotest/test_intanrawio.py @@ -17,7 +17,7 @@ class TestIntanRawIO( "intan/intan_fpc_test_231117_052630/info.rhd", "intan/intan_fps_test_231117_052500/info.rhd", "intan/intan_fpc_rhs_test_240329_091637/info.rhs", - "intan/intan_fps_rhs_test_240329_091636/info.rhs", + "intan/intan_fps_rhs_test_240329_091536/info.rhs", ] From ae41690477a3530c89fbfb18901f7bc983157990 Mon Sep 17 00:00:00 2001 From: Zach McKenzie <92116279+zm711@users.noreply.github.com> Date: Sat, 4 May 2024 18:03:28 -0400 Subject: [PATCH 13/15] typo fix --- neo/rawio/intanrawio.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index 9b875d9b0..12621f9c4 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -176,7 +176,7 @@ def _parse_header(self): if self.file_format == "header-attached": timestamp = self._raw_data["timestamp"].flatten() - # timestamps are always 6 for rhd and 7 for rhs + # timestamps are always last stream for headerless binary files elif self.file_format == "one-file-per-signal": time_stream_index = max(self._raw_data.keys()) timestamp = self._raw_data[time_stream_index] @@ -951,7 +951,8 @@ def create_one_file_per_signal_dict_rhd(dirname): "auxiliary.dat", "supply.dat", "analogin.dat", - "analogout.dat" "digitalin.dat", + "analogout.dat", + "digitalin.dat", "digitalout.dat", ] From 987e07b4bf2418d0999613176e552d2ec5c2f954 Mon Sep 17 00:00:00 2001 From: Zach McKenzie <92116279+zm711@users.noreply.github.com> Date: Sat, 4 May 2024 18:31:05 -0400 Subject: [PATCH 14/15] make sure stream_ids are sorted --- neo/rawio/intanrawio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index 12621f9c4..36f4dc1de 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -217,7 +217,7 @@ def _parse_header(self): stream_ids_sorted = sorted([int(stream_id) for stream_id in stream_ids]) signal_streams["id"] = [str(stream_id) for stream_id in stream_ids_sorted] - for stream_index, stream_id in enumerate(stream_ids): + for stream_index, stream_id in enumerate(stream_ids_sorted): if self.filename.endswith(".rhd"): signal_streams["name"][stream_index] = stream_type_to_name_rhd.get(int(stream_id), "") else: From 6c5743f998470147d7bae9cd82c202ecfccd6843 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Sat, 4 May 2024 19:04:45 -0400 Subject: [PATCH 15/15] fix one-file-per-signal --- neo/rawio/intanrawio.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index 36f4dc1de..b4c7dc320 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -481,8 +481,8 @@ def read_rhs(filename, file_format: str): channels_by_type = {k: [] for k in [0, 3, 4, 5, 6]} if not file_format == "header-attached": # data_dtype for rhs is complicated. There is not 1, 2 (supply and aux), - # but there are dc-amp (10) and stim (11). we make timestamps (7) - data_dtype = {k: [] for k in [0, 3, 4, 5, 6, 7, 10, 11]} + # but there are dc-amp (10) and stim (11). we make timestamps (15) + data_dtype = {k: [] for k in [0, 3, 4, 5, 6, 10, 11, 15]} for g in range(global_info["nb_signal_group"]): group_info = read_variable_header(f, rhs_signal_group_header) @@ -568,7 +568,7 @@ def read_rhs(filename, file_format: str): if file_format == "header-attached": data_dtype += [(name + "_STIM", "uint16", BLOCK_SIZE)] else: - data_dtype[11] == "uint16" + data_dtype[11] = "uint16" else: warnings.warn("Stim not implemented for `one-file-per-channel` due to lack of test files") @@ -952,7 +952,7 @@ def create_one_file_per_signal_dict_rhd(dirname): "supply.dat", "analogin.dat", "analogout.dat", - "digitalin.dat", + "digitalin.dat", "digitalout.dat", ]