From 540a1abe06a0e88a6c4710e41878c270a32e310c Mon Sep 17 00:00:00 2001 From: Luigi Petrucco Date: Mon, 2 Dec 2024 14:51:12 +0100 Subject: [PATCH 1/7] solution first draft --- neo/rawio/openephysbinaryrawio.py | 61 ++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index 27e3a80c9..f53fe4287 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -217,7 +217,6 @@ def _parse_header(self): if name + "_npy" in info: data = np.load(info[name + "_npy"], mmap_mode="r") info[name] = data - # check that events have timestamps assert "timestamps" in info, "Event stream does not have timestamps!" # Updates for OpenEphys v0.6: @@ -253,30 +252,50 @@ def _parse_header(self): # 'states' was introduced in OpenEphys v0.6. For previous versions, events used 'channel_states' if "states" in info or "channel_states" in info: states = info["channel_states"] if "channel_states" in info else info["states"] + if states.size > 0: timestamps = info["timestamps"] labels = info["labels"] - rising = np.where(states > 0)[0] - falling = np.where(states < 0)[0] - - # infer durations + + # Identify unique channels based on state values + channels = np.unique(np.abs(states)) + + rising_indices = [] + falling_indices = [] + + for channel in channels: + # Find rising and falling edges for each channel + rising = np.where(states == channel)[0] + falling = np.where(states == -channel)[0] + + # Ensure each rising has a corresponding falling + if rising.size > 0 and falling.size > 0: + if rising[0] > falling[0]: + falling = falling[1:] + if rising.size > falling.size: + rising = rising[:-1] + + rising_indices.extend(rising) + falling_indices.extend(falling) + + rising_indices = np.array(rising_indices) + falling_indices = np.array(falling_indices) + + # Sort the indices to maintain chronological order + sorted_order = np.argsort(rising_indices) + rising_indices = rising_indices[sorted_order] + falling_indices = falling_indices[sorted_order] + durations = None - if len(states) > 0: - # make sure first event is rising and last is falling - if states[0] < 0: - falling = falling[1:] - if states[-1] > 0: - rising = rising[:-1] - - if len(rising) == len(falling): - durations = timestamps[falling] - timestamps[rising] - if not self._use_direct_evt_timestamps: - timestamps = timestamps / info["sample_rate"] - durations = durations / info["sample_rate"] - - info["rising"] = rising - info["timestamps"] = timestamps[rising] - info["labels"] = labels[rising] + if len(rising_indices) == len(falling_indices): + durations = timestamps[falling_indices] - timestamps[rising_indices] + if not self._use_direct_evt_timestamps: + timestamps = timestamps / info["sample_rate"] + durations = durations / info["sample_rate"] + + info["rising"] = rising_indices + info["timestamps"] = timestamps[rising_indices] + info["labels"] = labels[rising_indices] info["durations"] = durations # no spike read yet From ffcc39423ea8c872be1f4df8aaeb4867e6497a52 Mon Sep 17 00:00:00 2001 From: Luigi Petrucco Date: Mon, 2 Dec 2024 15:04:57 +0100 Subject: [PATCH 2/7] test events reading --- .../rawiotest/test_openephysbinaryrawio.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/neo/test/rawiotest/test_openephysbinaryrawio.py b/neo/test/rawiotest/test_openephysbinaryrawio.py index 7df22e93d..0959ddf62 100644 --- a/neo/test/rawiotest/test_openephysbinaryrawio.py +++ b/neo/test/rawiotest/test_openephysbinaryrawio.py @@ -3,6 +3,8 @@ from neo.rawio.openephysbinaryrawio import OpenEphysBinaryRawIO from neo.test.rawiotest.common_rawio_test import BaseTestRawIO +import numpy as np + class TestOpenEphysBinaryRawIO(BaseTestRawIO, unittest.TestCase): rawioclass = OpenEphysBinaryRawIO @@ -57,6 +59,25 @@ def test_missing_folders(self): ) rawio.parse_header() + def test_multiple_ttl_events_parsing(self): + rawio = OpenEphysBinaryRawIO( + self.get_local_path("openephysbinary/v0.6.x_neuropixels_with_sync"), load_sync_channel=False + ) + rawio.parse_header() + rawio.header = rawio.header + # Testing co + # This is the TTL events from the NI Board channel + ttl_events = rawio._evt_streams[0][0][1] + assert "rising" in ttl_events.keys() + assert "labels" in ttl_events.keys() + assert "durations" in ttl_events.keys() + assert "timestamps" in ttl_events.keys() + + # Check that durations of different event streams are correctly parsed: + assert np.allclose(ttl_events["durations"][ttl_events["labels"] == "1"], 0.5, atol=0.001) + assert np.allclose(ttl_events["durations"][ttl_events["labels"] == "6"], 0.025, atol=0.001) + assert np.allclose(ttl_events["durations"][ttl_events["labels"] == "7"], 0.016666, atol=0.001) + if __name__ == "__main__": unittest.main() From 4da8bcaa5b621c0c5a663397d338ae772e05ee24 Mon Sep 17 00:00:00 2001 From: Luigi Petrucco Date: Mon, 2 Dec 2024 15:05:44 +0100 Subject: [PATCH 3/7] blacked --- neo/rawio/openephysbinaryrawio.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index f53fe4287..fe6120aec 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -252,47 +252,47 @@ def _parse_header(self): # 'states' was introduced in OpenEphys v0.6. For previous versions, events used 'channel_states' if "states" in info or "channel_states" in info: states = info["channel_states"] if "channel_states" in info else info["states"] - + if states.size > 0: timestamps = info["timestamps"] labels = info["labels"] - + # Identify unique channels based on state values channels = np.unique(np.abs(states)) - + rising_indices = [] falling_indices = [] - + for channel in channels: # Find rising and falling edges for each channel rising = np.where(states == channel)[0] falling = np.where(states == -channel)[0] - + # Ensure each rising has a corresponding falling if rising.size > 0 and falling.size > 0: if rising[0] > falling[0]: falling = falling[1:] if rising.size > falling.size: rising = rising[:-1] - + rising_indices.extend(rising) falling_indices.extend(falling) - + rising_indices = np.array(rising_indices) falling_indices = np.array(falling_indices) - + # Sort the indices to maintain chronological order sorted_order = np.argsort(rising_indices) rising_indices = rising_indices[sorted_order] falling_indices = falling_indices[sorted_order] - + durations = None if len(rising_indices) == len(falling_indices): durations = timestamps[falling_indices] - timestamps[rising_indices] if not self._use_direct_evt_timestamps: timestamps = timestamps / info["sample_rate"] durations = durations / info["sample_rate"] - + info["rising"] = rising_indices info["timestamps"] = timestamps[rising_indices] info["labels"] = labels[rising_indices] From c9311c6f3be4b5046e3df923be69f9b612dff5bc Mon Sep 17 00:00:00 2001 From: Luigi Petrucco Date: Tue, 3 Dec 2024 16:46:15 +0100 Subject: [PATCH 4/7] more comments --- neo/rawio/openephysbinaryrawio.py | 5 +++++ neo/test/rawiotest/common_rawio_test.py | 10 +++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index fe6120aec..297514b4e 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -263,6 +263,11 @@ def _parse_header(self): rising_indices = [] falling_indices = [] + # all channels are packed into the same `states` array. + # So the states array includes positive and negative values for each channel: + # for example channel one rising would be +1 and channel one falling would be -1, + # channel two rising would be +2 and channel two falling would be -2, etc. + # This is the case for sure for version >= 0.6.x. for channel in channels: # Find rising and falling edges for each channel rising = np.where(states == channel)[0] diff --git a/neo/test/rawiotest/common_rawio_test.py b/neo/test/rawiotest/common_rawio_test.py index 488cb9fbf..10c7eb2fc 100644 --- a/neo/test/rawiotest/common_rawio_test.py +++ b/neo/test/rawiotest/common_rawio_test.py @@ -66,11 +66,11 @@ def setUpClass(cls): """ cls.shortname = cls.rawioclass.__name__.lower().replace("rawio", "") - if HAVE_DATALAD and cls.use_network: - for remote_path in cls.entities_to_download: - download_dataset(repo=repo_for_test, remote_path=remote_path) - else: - raise unittest.SkipTest("Requires datalad download of data from the web") + # if HAVE_DATALAD and cls.use_network: + # for remote_path in cls.entities_to_download: + # download_dataset(repo=repo_for_test, remote_path=remote_path) + # else: + # raise unittest.SkipTest("Requires datalad download of data from the web") def get_local_base_folder(self): return get_local_testing_data_folder() From c3bea9fe0ee6a4a7ea7868022b707819dc7e9600 Mon Sep 17 00:00:00 2001 From: Luigi Petrucco Date: Tue, 3 Dec 2024 16:47:24 +0100 Subject: [PATCH 5/7] revert erroneous commit --- neo/test/rawiotest/common_rawio_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/neo/test/rawiotest/common_rawio_test.py b/neo/test/rawiotest/common_rawio_test.py index 10c7eb2fc..488cb9fbf 100644 --- a/neo/test/rawiotest/common_rawio_test.py +++ b/neo/test/rawiotest/common_rawio_test.py @@ -66,11 +66,11 @@ def setUpClass(cls): """ cls.shortname = cls.rawioclass.__name__.lower().replace("rawio", "") - # if HAVE_DATALAD and cls.use_network: - # for remote_path in cls.entities_to_download: - # download_dataset(repo=repo_for_test, remote_path=remote_path) - # else: - # raise unittest.SkipTest("Requires datalad download of data from the web") + if HAVE_DATALAD and cls.use_network: + for remote_path in cls.entities_to_download: + download_dataset(repo=repo_for_test, remote_path=remote_path) + else: + raise unittest.SkipTest("Requires datalad download of data from the web") def get_local_base_folder(self): return get_local_testing_data_folder() From 44d48078df739bd446c52906169da99f70ece48e Mon Sep 17 00:00:00 2001 From: Luigi Petrucco Date: Fri, 6 Dec 2024 17:45:15 +0100 Subject: [PATCH 6/7] skipping channels with bad edge detections --- neo/rawio/openephysbinaryrawio.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index 297514b4e..ef3fbdd88 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -263,9 +263,9 @@ def _parse_header(self): rising_indices = [] falling_indices = [] - # all channels are packed into the same `states` array. + # all channels are packed into the same `states` array. # So the states array includes positive and negative values for each channel: - # for example channel one rising would be +1 and channel one falling would be -1, + # for example channel one rising would be +1 and channel one falling would be -1, # channel two rising would be +2 and channel two falling would be -2, etc. # This is the case for sure for version >= 0.6.x. for channel in channels: @@ -280,6 +280,15 @@ def _parse_header(self): if rising.size > falling.size: rising = rising[:-1] + # ensure that the number of rising and falling edges are the same: + if len(rising) != len(falling): + warn( + f"Channel {channel} has {len(rising)} rising edges and " + f"{len(falling)} falling edges. The number of rising and " + f"falling edges should be equal. Skipping events from this channel." + ) + continue + rising_indices.extend(rising) falling_indices.extend(falling) @@ -292,11 +301,11 @@ def _parse_header(self): falling_indices = falling_indices[sorted_order] durations = None - if len(rising_indices) == len(falling_indices): - durations = timestamps[falling_indices] - timestamps[rising_indices] - if not self._use_direct_evt_timestamps: - timestamps = timestamps / info["sample_rate"] - durations = durations / info["sample_rate"] + # if len(rising_indices) == len(falling_indices): + durations = timestamps[falling_indices] - timestamps[rising_indices] + if not self._use_direct_evt_timestamps: + timestamps = timestamps / info["sample_rate"] + durations = durations / info["sample_rate"] info["rising"] = rising_indices info["timestamps"] = timestamps[rising_indices] From a1af1c6bf2a1f6a247142bb31ee8f38d5acbaed6 Mon Sep 17 00:00:00 2001 From: Zach McKenzie <92116279+zm711@users.noreply.github.com> Date: Fri, 6 Dec 2024 17:17:48 -0500 Subject: [PATCH 7/7] fix doc typo --- neo/rawio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/rawio/__init__.py b/neo/rawio/__init__.py index 37eab2c59..9b9a86a60 100644 --- a/neo/rawio/__init__.py +++ b/neo/rawio/__init__.py @@ -29,7 +29,7 @@ * :attr:`MicromedRawIO` * :attr:`NeuralynxRawIO` * :attr:`NeuroExplorerRawIO` -* :attr:`NeuroNexusRawIO +* :attr:`NeuroNexusRawIO` * :attr:`NeuroScopeRawIO` * :attr:`NIXRawIO` * :attr:`OpenEphysRawIO`