From 6077740757f494a457818a64894d27abc9f70668 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 6 May 2024 11:50:18 -0600 Subject: [PATCH 01/14] open the file intan --- neo/rawio/intanrawio.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index 756acb88a..823b228a0 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -42,7 +42,9 @@ class IntanRawIO(BaseRawIO): Parameters ---------- filename: str, default: '' - name of the 'rhd' or 'rhs' data file + name of the 'rhd' or 'rhs' data file + strict_mode_for_timestamps: bool, default: True + If True, the reader will raise an error if timestamps are not continuous. Notes ----- @@ -79,10 +81,11 @@ class IntanRawIO(BaseRawIO): extensions = ["rhd", "rhs", "dat"] rawmode = "one-file" - def __init__(self, filename=""): + def __init__(self, filename="", strict_mode_for_timestamps=True): BaseRawIO.__init__(self) self.filename = filename + self.strict_mode_for_timestamps = strict_mode_for_timestamps def _source_name(self): return self.filename @@ -165,10 +168,15 @@ def _parse_header(self): elif self.file_format == "one-file-per-channel": time_stream_index = max(self._raw_data.keys()) timestamp = self._raw_data[time_stream_index][0] - - assert np.all(np.diff(timestamp) == 1), ( - "Timestamp have gaps, this could be due " "to a corrupted file or an inappropriate file merge" - ) + + if self.strict_mode_for_timestamps: + timestamps_are_not_contigious = np.any(np.diff(timestamp) != 1) + if timestamps_are_not_contigious: + error_msg = ( + "Timestamps are not continuous, this could be due to a corrupted file or an inappropriate file merge. " + "Set strict_mode_for_timestamps to False to ignore this error and open the file." + ) + raise ValueError(error_msg) # signals signal_channels = [] From c38c300883d4c74fa352ec20ad1f9fe13c095e3a Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 6 May 2024 14:17:40 -0600 Subject: [PATCH 02/14] add attribute and logic --- neo/rawio/intanrawio.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index 194018c3d..caa696c69 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -44,9 +44,11 @@ class IntanRawIO(BaseRawIO): ---------- filename: str, default: '' name of the 'rhd' or 'rhs' data file - strict_mode_for_timestamps: bool, default: True - If True, the reader will raise an error if timestamps are not continuous. - + load_unsafe_data: bool, default: False + If True, data that violates integrity assumptions will be loaded. At the moment the only integrity + check we perform is that timestamps are continuous. Setting this to True will ignore this check and set + the attribute `unsafe_timestamps` to True if the timestamps are not continous. This attribute can be checked + after parsing the header to see if the timestamps are continuous or not. Notes ----- * Intan reader can handle two file formats 'rhd' and 'rhs'. It will automatically @@ -82,11 +84,13 @@ class IntanRawIO(BaseRawIO): extensions = ["rhd", "rhs", "dat"] rawmode = "one-file" - def __init__(self, filename="", strict_mode_for_timestamps=True): + def __init__(self, filename="", load_unsafe_data=True): BaseRawIO.__init__(self) self.filename = filename - self.strict_mode_for_timestamps = strict_mode_for_timestamps + self.load_unsafe_data = load_unsafe_data + self.unsafe_timestamps = False + def _source_name(self): return self.filename @@ -170,12 +174,13 @@ def _parse_header(self): time_stream_index = max(self._raw_data.keys()) timestamp = self._raw_data[time_stream_index][0] - if self.strict_mode_for_timestamps: - timestamps_are_not_contigious = np.any(np.diff(timestamp) != 1) - if timestamps_are_not_contigious: + timestamps_are_not_contigious = np.any(np.diff(timestamp) != 1) + if timestamps_are_not_contigious: + self.unsafe_timestamps = True + if not self.load_unsafe_data: error_msg = ( "Timestamps are not continuous, this could be due to a corrupted file or an inappropriate file merge. " - "Set strict_mode_for_timestamps to False to ignore this error and open the file." + "Initialize the reader with `load_unsafe_data=True` to ignore this error and open the file." ) raise NeoReadWriteError(error_msg) From a44ab4a5ec6440ff296bcc0f57766bb57337f5a7 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Fri, 10 May 2024 16:13:37 -0400 Subject: [PATCH 03/14] test adding 3.12 --- .github/workflows/core-test.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/core-test.yml b/.github/workflows/core-test.yml index d5c1efa82..f654dcdc1 100644 --- a/.github/workflows/core-test.yml +++ b/.github/workflows/core-test.yml @@ -26,17 +26,32 @@ jobs: matrix: os: ["ubuntu-latest", "windows-latest"] # "macos-latest", - python-version: ['3.8', '3.9', '3.10', '3.11'] - numpy-version: ['1.20.3', '1.21.6', '1.22.4', '1.23.5', '1.24.1', '1.25.1'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + numpy-version: ['1.20.3', '1.21.6', '1.22.4', '1.23.5', '1.24.1', '1.25.1', '1.26.4'] exclude: - python-version: '3.8' numpy-version: '1.25.1' + - python-version: '3.8' + numpy-version: '1.26.4' - python-version: '3.10' numpy-version: '1.20.3' - python-version: '3.11' numpy-version: '1.20.3' - python-version: '3.11' numpy-version: '1.21.6' + - python-version: '3.12' + numpy-version: '1.20.3' + # python 3.12 only works on latest numpy + - python-version: '3.12' + numpy-version: '1.21.6' + - python-version: '3.12' + numpy-version: '1.22.4' + - python-version: '3.12' + numpy-version: '1.23.5' + - python-version: '3.12' + numpy-version: '1.24.1' + - python-version: '3.12' + numpy-version: '1.25.1' steps: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 From 381bd4bd06c22443e49a83a4c2b93689c01e8258 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Fri, 10 May 2024 16:23:32 -0400 Subject: [PATCH 04/14] matrix for io tests --- .github/workflows/io-test.yml | 2 +- environment_testing.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/io-test.yml b/.github/workflows/io-test.yml index 3688562da..a936684ff 100644 --- a/.github/workflows/io-test.yml +++ b/.github/workflows/io-test.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: true matrix: - python-version: ['3.9', ] + python-version: ['3.9', '3.10', '3.11'] defaults: # by default run in bash mode (required for conda usage) run: diff --git a/environment_testing.yml b/environment_testing.yml index 7e5aee66a..8ff8bd984 100644 --- a/environment_testing.yml +++ b/environment_testing.yml @@ -2,6 +2,5 @@ name: neo-test-env channels: - conda-forge dependencies: - - python=3.9 - datalad - pip From 46e60464cd6ebe19e05ca99df1ac355efbbd7c63 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Fri, 10 May 2024 16:41:21 -0400 Subject: [PATCH 05/14] fix python on testing --- .github/workflows/io-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/io-test.yml b/.github/workflows/io-test.yml index a936684ff..e2b7147e2 100644 --- a/.github/workflows/io-test.yml +++ b/.github/workflows/io-test.yml @@ -13,7 +13,7 @@ concurrency: # Cancel previous workflows on the same pull request jobs: build-and-test: - name: Test on (${{ inputs.os }}) + name: Test on (${{ inputs.os }}) (${{ matrix.python-version}}) runs-on: ${{ inputs.os }} strategy: fail-fast: true @@ -75,6 +75,7 @@ jobs: if: steps.cache-conda-env.outputs.cache-hit != 'true' run: | conda env update --name neo-test-env --file environment_testing.yml --prune + conda update python= ${{ matrix.python-version }} - name: Configure git run: | From 88eabf9a86ec95c497a3974ca32366c126133520 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Fri, 10 May 2024 16:44:42 -0400 Subject: [PATCH 06/14] oops --- .github/workflows/io-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/io-test.yml b/.github/workflows/io-test.yml index e2b7147e2..933fbdba1 100644 --- a/.github/workflows/io-test.yml +++ b/.github/workflows/io-test.yml @@ -75,7 +75,7 @@ jobs: if: steps.cache-conda-env.outputs.cache-hit != 'true' run: | conda env update --name neo-test-env --file environment_testing.yml --prune - conda update python= ${{ matrix.python-version }} + conda update python=${{ matrix.python-version }} - name: Configure git run: | From 2eca60dfe3916780dedf5895d48dfcba08e803dc Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Fri, 10 May 2024 16:46:39 -0400 Subject: [PATCH 07/14] try with conda install --- .github/workflows/io-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/io-test.yml b/.github/workflows/io-test.yml index 933fbdba1..974d8b70a 100644 --- a/.github/workflows/io-test.yml +++ b/.github/workflows/io-test.yml @@ -75,7 +75,7 @@ jobs: if: steps.cache-conda-env.outputs.cache-hit != 'true' run: | conda env update --name neo-test-env --file environment_testing.yml --prune - conda update python=${{ matrix.python-version }} + conda install python=${{ matrix.python-version }} - name: Configure git run: | From 32c5310b812de28bdff304bf91dcd6ef38d68e67 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 13 May 2024 15:38:30 -0600 Subject: [PATCH 08/14] add zach suggestion --- neo/rawio/intanrawio.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index ca583ddae..ecc960580 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -19,8 +19,6 @@ """ from pathlib import Path -import os -from collections import OrderedDict from packaging.version import Version as V import warnings @@ -45,7 +43,7 @@ class IntanRawIO(BaseRawIO): ---------- filename: str, default: '' name of the 'rhd' or 'rhs' data file - load_unsafe_data: bool, default: False + load_data_unsafely: bool, default: False If True, data that violates integrity assumptions will be loaded. At the moment the only integrity check we perform is that timestamps are continuous. Setting this to True will ignore this check and set the attribute `unsafe_timestamps` to True if the timestamps are not continous. This attribute can be checked @@ -85,11 +83,11 @@ class IntanRawIO(BaseRawIO): extensions = ["rhd", "rhs", "dat"] rawmode = "one-file" - def __init__(self, filename="", load_unsafe_data=True): + def __init__(self, filename="", load_data_unsafely=True): BaseRawIO.__init__(self) self.filename = filename - self.load_unsafe_data = load_unsafe_data + self.load_data_unsafely = load_data_unsafely self.unsafe_timestamps = False @@ -194,10 +192,10 @@ def _parse_header(self): timestamps_are_not_contigious = np.any(np.diff(timestamp) != 1) if timestamps_are_not_contigious: self.unsafe_timestamps = True - if not self.load_unsafe_data: + if not self.load_data_unsafely: error_msg = ( "Timestamps are not continuous, this could be due to a corrupted file or an inappropriate file merge. " - "Initialize the reader with `load_unsafe_data=True` to ignore this error and open the file." + "Initialize the reader with `load_data_unsafely=True` to ignore this error and open the file." ) raise NeoReadWriteError(error_msg) From 2919a6d3d9ecec8358f940cda950869044f45eda Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 13 May 2024 16:19:49 -0600 Subject: [PATCH 09/14] fix typos notiched by Zach --- neo/rawio/intanrawio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index ecc960580..b4fe462a8 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -189,8 +189,8 @@ def _parse_header(self): time_stream_index = max(self._raw_data.keys()) timestamp = self._raw_data[time_stream_index][0] - timestamps_are_not_contigious = np.any(np.diff(timestamp) != 1) - if timestamps_are_not_contigious: + timestamps_are_not_contiguous = np.any(np.diff(timestamp) != 1) + if timestamps_are_not_contiguous: self.unsafe_timestamps = True if not self.load_data_unsafely: error_msg = ( From d306ac7d59ed03b81d64bf53a2c8a093a28a10fd Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 13 May 2024 16:33:57 -0600 Subject: [PATCH 10/14] improve error docstring and change default --- neo/rawio/intanrawio.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index b4fe462a8..9ec76a316 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -83,7 +83,7 @@ class IntanRawIO(BaseRawIO): extensions = ["rhd", "rhs", "dat"] rawmode = "one-file" - def __init__(self, filename="", load_data_unsafely=True): + def __init__(self, filename="", load_data_unsafely=False): BaseRawIO.__init__(self) self.filename = filename @@ -189,13 +189,15 @@ def _parse_header(self): time_stream_index = max(self._raw_data.keys()) timestamp = self._raw_data[time_stream_index][0] - timestamps_are_not_contiguous = np.any(np.diff(timestamp) != 1) + discontinuous_timestamps = np.diff(timestamp) != 1 + timestamps_are_not_contiguous = np.any(discontinuous_timestamps) if timestamps_are_not_contiguous: self.unsafe_timestamps = True if not self.load_data_unsafely: error_msg = ( "Timestamps are not continuous, this could be due to a corrupted file or an inappropriate file merge. " - "Initialize the reader with `load_data_unsafely=True` to ignore this error and open the file." + "Initialize the reader with `load_data_unsafely=True` to ignore this error and open the file. \n" + f"Indexes of discontinuous timestamps: {timestamp[discontinuous_timestamps]}" ) raise NeoReadWriteError(error_msg) From 21ac9ed50aa8704e31ca630c3e09a511d0c880a4 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 13 May 2024 16:37:52 -0600 Subject: [PATCH 11/14] another damn spelling story --- neo/rawio/intanrawio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index 9ec76a316..445a57a78 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -197,7 +197,7 @@ def _parse_header(self): error_msg = ( "Timestamps are not continuous, this could be due to a corrupted file or an inappropriate file merge. " "Initialize the reader with `load_data_unsafely=True` to ignore this error and open the file. \n" - f"Indexes of discontinuous timestamps: {timestamp[discontinuous_timestamps]}" + f"Timestamps around discontinuities: {timestamp[discontinuous_timestamps]}" ) raise NeoReadWriteError(error_msg) From 8fc19553bf2bfeb95af56ba013115ee0b1e44b6f Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 30 May 2024 02:37:08 -0600 Subject: [PATCH 12/14] naming --- neo/rawio/intanrawio.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index 445a57a78..2aea8f6ce 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -43,10 +43,10 @@ class IntanRawIO(BaseRawIO): ---------- filename: str, default: '' name of the 'rhd' or 'rhs' data file - load_data_unsafely: bool, default: False + ignore_integrity_checks: bool, default: False If True, data that violates integrity assumptions will be loaded. At the moment the only integrity check we perform is that timestamps are continuous. Setting this to True will ignore this check and set - the attribute `unsafe_timestamps` to True if the timestamps are not continous. This attribute can be checked + the attribute `discontinous_timestamps` to True if the timestamps are not continous. This attribute can be checked after parsing the header to see if the timestamps are continuous or not. Notes ----- @@ -83,12 +83,12 @@ class IntanRawIO(BaseRawIO): extensions = ["rhd", "rhs", "dat"] rawmode = "one-file" - def __init__(self, filename="", load_data_unsafely=False): + def __init__(self, filename="", ignore_integrity_checks=False): BaseRawIO.__init__(self) self.filename = filename - self.load_data_unsafely = load_data_unsafely - self.unsafe_timestamps = False + self.ignore_integrity_checks = ignore_integrity_checks + self.discontinous_timestamps = False def _source_name(self): @@ -192,11 +192,11 @@ def _parse_header(self): discontinuous_timestamps = np.diff(timestamp) != 1 timestamps_are_not_contiguous = np.any(discontinuous_timestamps) if timestamps_are_not_contiguous: - self.unsafe_timestamps = True - if not self.load_data_unsafely: + self.discontinous_timestamps = True + if not self.ignore_integrity_checks: error_msg = ( "Timestamps are not continuous, this could be due to a corrupted file or an inappropriate file merge. " - "Initialize the reader with `load_data_unsafely=True` to ignore this error and open the file. \n" + "Initialize the reader with `ignore_integrity_checks=True` to ignore this error and open the file. \n" f"Timestamps around discontinuities: {timestamp[discontinuous_timestamps]}" ) raise NeoReadWriteError(error_msg) From dc04d48f3b00b1f7b25c7d4c870f74a27508fe68 Mon Sep 17 00:00:00 2001 From: Garcia Samuel Date: Thu, 30 May 2024 10:40:19 +0200 Subject: [PATCH 13/14] Update .github/workflows/io-test.yml --- .github/workflows/io-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/io-test.yml b/.github/workflows/io-test.yml index 974d8b70a..8522e7e4e 100644 --- a/.github/workflows/io-test.yml +++ b/.github/workflows/io-test.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: true matrix: - python-version: ['3.9', '3.10', '3.11'] + python-version: ['3.9', '3.11'] defaults: # by default run in bash mode (required for conda usage) run: From 7a53c7eb08fc59f853747ddacec4d96a5db56063 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 30 May 2024 03:50:15 -0600 Subject: [PATCH 14/14] yet another typo --- neo/rawio/intanrawio.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py index 2aea8f6ce..eaac62870 100644 --- a/neo/rawio/intanrawio.py +++ b/neo/rawio/intanrawio.py @@ -46,7 +46,7 @@ class IntanRawIO(BaseRawIO): ignore_integrity_checks: bool, default: False If True, data that violates integrity assumptions will be loaded. At the moment the only integrity check we perform is that timestamps are continuous. Setting this to True will ignore this check and set - the attribute `discontinous_timestamps` to True if the timestamps are not continous. This attribute can be checked + the attribute `discontinuous_timestamps` to True if the timestamps are not continous. This attribute can be checked after parsing the header to see if the timestamps are continuous or not. Notes ----- @@ -88,7 +88,7 @@ def __init__(self, filename="", ignore_integrity_checks=False): BaseRawIO.__init__(self) self.filename = filename self.ignore_integrity_checks = ignore_integrity_checks - self.discontinous_timestamps = False + self.discontinuous_timestamps = False def _source_name(self): @@ -192,7 +192,7 @@ def _parse_header(self): discontinuous_timestamps = np.diff(timestamp) != 1 timestamps_are_not_contiguous = np.any(discontinuous_timestamps) if timestamps_are_not_contiguous: - self.discontinous_timestamps = True + self.discontinuous_timestamps = True if not self.ignore_integrity_checks: error_msg = ( "Timestamps are not continuous, this could be due to a corrupted file or an inappropriate file merge. "