From f498179431d23291f72bbea40bc6a95b65dc5913 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Thu, 22 Jun 2023 10:29:01 +0200 Subject: [PATCH 01/13] Add plexon2 recording, sorting and event support --- .../extractors/neoextractors/__init__.py | 7 +- .../extractors/neoextractors/plexon2.py | 102 ++++++++++++++++++ .../extractors/tests/test_neoextractors.py | 18 ++++ 3 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 src/spikeinterface/extractors/neoextractors/plexon2.py diff --git a/src/spikeinterface/extractors/neoextractors/__init__.py b/src/spikeinterface/extractors/neoextractors/__init__.py index 0d9da1960a..4c12017328 100644 --- a/src/spikeinterface/extractors/neoextractors/__init__.py +++ b/src/spikeinterface/extractors/neoextractors/__init__.py @@ -25,6 +25,8 @@ read_openephys_event, ) from .plexon import PlexonRecordingExtractor, PlexonSortingExtractor, read_plexon, read_plexon_sorting +from .plexon2 import (Plexon2SortingExtractor, Plexon2RecordingExtractor, Plexon2EventExtractor, + read_plexon2, read_plexon2_sorting, read_plexon2_event) from .spike2 import Spike2RecordingExtractor, read_spike2 from .spikegadgets import SpikeGadgetsRecordingExtractor, read_spikegadgets from .spikeglx import SpikeGLXRecordingExtractor, read_spikeglx @@ -49,12 +51,13 @@ OpenEphysBinaryRecordingExtractor, OpenEphysLegacyRecordingExtractor, PlexonRecordingExtractor, + Plexon2RecordingExtractor, Spike2RecordingExtractor, SpikeGadgetsRecordingExtractor, SpikeGLXRecordingExtractor, TdtRecordingExtractor, ] -neo_sorting_extractors_list = [BlackrockSortingExtractor, MEArecSortingExtractor, NeuralynxSortingExtractor] +neo_sorting_extractors_list = [BlackrockSortingExtractor, MEArecSortingExtractor, NeuralynxSortingExtractor, Plexon2SortingExtractor] -neo_event_extractors_list = [AlphaOmegaEventExtractor, OpenEphysBinaryEventExtractor] +neo_event_extractors_list = [AlphaOmegaEventExtractor, OpenEphysBinaryEventExtractor, Plexon2EventExtractor] diff --git a/src/spikeinterface/extractors/neoextractors/plexon2.py b/src/spikeinterface/extractors/neoextractors/plexon2.py new file mode 100644 index 0000000000..5ccceac875 --- /dev/null +++ b/src/spikeinterface/extractors/neoextractors/plexon2.py @@ -0,0 +1,102 @@ +from spikeinterface.core.core_tools import define_function_from_class + +from .neobaseextractor import (NeoBaseRecordingExtractor, NeoBaseSortingExtractor, + NeoBaseEventExtractor) + + +class Plexon2RecordingExtractor(NeoBaseRecordingExtractor): + """ + Class for reading plexon pl2 files. + + Based on :py:class:`neo.rawio.Plexon2RawIO` + + Parameters + ---------- + file_path: str + The file path to load the recordings from. + stream_id: str, optional + If there are several streams, specify the stream id you want to load. + stream_name: str, optional + If there are several streams, specify the stream name you want to load. + all_annotations: bool, default: False + Load exhaustively all annotations from neo. + """ + + mode = "file" + NeoRawIOClass = "Plexon2RawIO" + name = "plexon2" + + def __init__(self, file_path, stream_id=None, stream_name=None, all_annotations=False): + neo_kwargs = self.map_to_neo_kwargs(file_path) + NeoBaseRecordingExtractor.__init__( + self, stream_id=stream_id, stream_name=stream_name, all_annotations=all_annotations, **neo_kwargs + ) + self._kwargs.update({"file_path": str(file_path)}) + + @classmethod + def map_to_neo_kwargs(cls, file_path): + neo_kwargs = {"filename": str(file_path)} + return neo_kwargs + + +class Plexon2SortingExtractor(NeoBaseSortingExtractor): + """ + Class for reading plexon spiking data from .pl2 files. + + Based on :py:class:`neo.rawio.Plexon2RawIO` + + Parameters + ---------- + file_path: str + The file path to load the recordings from. + """ + + mode = "file" + NeoRawIOClass = "Plexon2RawIO" + handle_spike_frame_directly = True + name = "plexon2" + + def __init__(self, file_path): + from neo.rawio import Plexon2RawIO + + neo_kwargs = self.map_to_neo_kwargs(file_path) + neo_reader = Plexon2RawIO(**neo_kwargs) + neo_reader.parse_header() + NeoBaseSortingExtractor.__init__(self, **neo_kwargs) + self._kwargs.update({"file_path": str(file_path)}) + + @classmethod + def map_to_neo_kwargs(cls, file_path): + neo_kwargs = {"filename": str(file_path)} + return neo_kwargs + + +class Plexon2EventExtractor(NeoBaseEventExtractor): + """ + Class for reading plexon spiking data from .pl2 files. + + Based on :py:class:`neo.rawio.Plexon2RawIO` + + Parameters + ---------- + folder_path: str + + """ + + mode = "file" + NeoRawIOClass = "Plexon2RawIO" + name = "plexon2" + + def __init__(self, folder_path, block_index=None): + neo_kwargs = self.map_to_neo_kwargs(folder_path) + NeoBaseEventExtractor.__init__(self, block_index=block_index, **neo_kwargs) + + @classmethod + def map_to_neo_kwargs(cls, folder_path): + neo_kwargs = {"filename": str(folder_path)} + return neo_kwargs + + +read_plexon2 = define_function_from_class(source_class=Plexon2RecordingExtractor, name="read_plexon2") +read_plexon2_sorting = define_function_from_class(source_class=Plexon2SortingExtractor, name="read_plexon2_sorting") +read_plexon2_event = define_function_from_class(source_class=Plexon2EventExtractor, name='read_plexon2_event') diff --git a/src/spikeinterface/extractors/tests/test_neoextractors.py b/src/spikeinterface/extractors/tests/test_neoextractors.py index d19574e094..a28752acdd 100644 --- a/src/spikeinterface/extractors/tests/test_neoextractors.py +++ b/src/spikeinterface/extractors/tests/test_neoextractors.py @@ -120,6 +120,12 @@ class NeuroScopeSortingTest(SortingCommonTestSuite, unittest.TestCase): }, ] +class Plexon2EventTest(EventCommonTestSuite, unittest.TestCase): + ExtractorClass = Plexon2EventExtractor + downloads = ["plexon"] + entities = [ + ("plexon/4chDemoPL2.pl2"), + ] class PlexonRecordingTest(RecordingCommonTestSuite, unittest.TestCase): ExtractorClass = PlexonRecordingExtractor @@ -128,6 +134,12 @@ class PlexonRecordingTest(RecordingCommonTestSuite, unittest.TestCase): "plexon/File_plexon_3.plx", ] +class Plexon2RecordingTest(RecordingCommonTestSuite, unittest.TestCase): + ExtractorClass = Plexon2RecordingExtractor + downloads = ["plexon"] + entities = [ + ("plexon/4chDemoPL2.pl2", {"stream_id": "3"}), + ] class PlexonSortingTest(SortingCommonTestSuite, unittest.TestCase): ExtractorClass = PlexonSortingExtractor @@ -136,6 +148,12 @@ class PlexonSortingTest(SortingCommonTestSuite, unittest.TestCase): ("plexon/File_plexon_1.plx"), ] +class Plexon2SortingTest(SortingCommonTestSuite, unittest.TestCase): + ExtractorClass = Plexon2SortingExtractor + downloads = ["plexon"] + entities = [ + ("plexon/4chDemoPL2.pl2"), + ] class NeuralynxRecordingTest(RecordingCommonTestSuite, unittest.TestCase): ExtractorClass = NeuralynxRecordingExtractor From 9f42895213a47ddb9158e4cccb48dcec1dea9549 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 22 Jun 2023 08:31:48 +0000 Subject: [PATCH 02/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../extractors/neoextractors/__init__.py | 17 ++++++++++++++--- .../extractors/neoextractors/plexon2.py | 5 ++--- .../extractors/tests/test_neoextractors.py | 6 ++++++ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/__init__.py b/src/spikeinterface/extractors/neoextractors/__init__.py index 4c12017328..3360b76147 100644 --- a/src/spikeinterface/extractors/neoextractors/__init__.py +++ b/src/spikeinterface/extractors/neoextractors/__init__.py @@ -25,8 +25,14 @@ read_openephys_event, ) from .plexon import PlexonRecordingExtractor, PlexonSortingExtractor, read_plexon, read_plexon_sorting -from .plexon2 import (Plexon2SortingExtractor, Plexon2RecordingExtractor, Plexon2EventExtractor, - read_plexon2, read_plexon2_sorting, read_plexon2_event) +from .plexon2 import ( + Plexon2SortingExtractor, + Plexon2RecordingExtractor, + Plexon2EventExtractor, + read_plexon2, + read_plexon2_sorting, + read_plexon2_event, +) from .spike2 import Spike2RecordingExtractor, read_spike2 from .spikegadgets import SpikeGadgetsRecordingExtractor, read_spikegadgets from .spikeglx import SpikeGLXRecordingExtractor, read_spikeglx @@ -58,6 +64,11 @@ TdtRecordingExtractor, ] -neo_sorting_extractors_list = [BlackrockSortingExtractor, MEArecSortingExtractor, NeuralynxSortingExtractor, Plexon2SortingExtractor] +neo_sorting_extractors_list = [ + BlackrockSortingExtractor, + MEArecSortingExtractor, + NeuralynxSortingExtractor, + Plexon2SortingExtractor, +] neo_event_extractors_list = [AlphaOmegaEventExtractor, OpenEphysBinaryEventExtractor, Plexon2EventExtractor] diff --git a/src/spikeinterface/extractors/neoextractors/plexon2.py b/src/spikeinterface/extractors/neoextractors/plexon2.py index 5ccceac875..c3869dbadc 100644 --- a/src/spikeinterface/extractors/neoextractors/plexon2.py +++ b/src/spikeinterface/extractors/neoextractors/plexon2.py @@ -1,7 +1,6 @@ from spikeinterface.core.core_tools import define_function_from_class -from .neobaseextractor import (NeoBaseRecordingExtractor, NeoBaseSortingExtractor, - NeoBaseEventExtractor) +from .neobaseextractor import NeoBaseRecordingExtractor, NeoBaseSortingExtractor, NeoBaseEventExtractor class Plexon2RecordingExtractor(NeoBaseRecordingExtractor): @@ -99,4 +98,4 @@ def map_to_neo_kwargs(cls, folder_path): read_plexon2 = define_function_from_class(source_class=Plexon2RecordingExtractor, name="read_plexon2") read_plexon2_sorting = define_function_from_class(source_class=Plexon2SortingExtractor, name="read_plexon2_sorting") -read_plexon2_event = define_function_from_class(source_class=Plexon2EventExtractor, name='read_plexon2_event') +read_plexon2_event = define_function_from_class(source_class=Plexon2EventExtractor, name="read_plexon2_event") diff --git a/src/spikeinterface/extractors/tests/test_neoextractors.py b/src/spikeinterface/extractors/tests/test_neoextractors.py index a28752acdd..b14bcc9cf8 100644 --- a/src/spikeinterface/extractors/tests/test_neoextractors.py +++ b/src/spikeinterface/extractors/tests/test_neoextractors.py @@ -120,6 +120,7 @@ class NeuroScopeSortingTest(SortingCommonTestSuite, unittest.TestCase): }, ] + class Plexon2EventTest(EventCommonTestSuite, unittest.TestCase): ExtractorClass = Plexon2EventExtractor downloads = ["plexon"] @@ -127,6 +128,7 @@ class Plexon2EventTest(EventCommonTestSuite, unittest.TestCase): ("plexon/4chDemoPL2.pl2"), ] + class PlexonRecordingTest(RecordingCommonTestSuite, unittest.TestCase): ExtractorClass = PlexonRecordingExtractor downloads = ["plexon"] @@ -134,6 +136,7 @@ class PlexonRecordingTest(RecordingCommonTestSuite, unittest.TestCase): "plexon/File_plexon_3.plx", ] + class Plexon2RecordingTest(RecordingCommonTestSuite, unittest.TestCase): ExtractorClass = Plexon2RecordingExtractor downloads = ["plexon"] @@ -141,6 +144,7 @@ class Plexon2RecordingTest(RecordingCommonTestSuite, unittest.TestCase): ("plexon/4chDemoPL2.pl2", {"stream_id": "3"}), ] + class PlexonSortingTest(SortingCommonTestSuite, unittest.TestCase): ExtractorClass = PlexonSortingExtractor downloads = ["plexon"] @@ -148,6 +152,7 @@ class PlexonSortingTest(SortingCommonTestSuite, unittest.TestCase): ("plexon/File_plexon_1.plx"), ] + class Plexon2SortingTest(SortingCommonTestSuite, unittest.TestCase): ExtractorClass = Plexon2SortingExtractor downloads = ["plexon"] @@ -155,6 +160,7 @@ class Plexon2SortingTest(SortingCommonTestSuite, unittest.TestCase): ("plexon/4chDemoPL2.pl2"), ] + class NeuralynxRecordingTest(RecordingCommonTestSuite, unittest.TestCase): ExtractorClass = NeuralynxRecordingExtractor downloads = ["neuralynx"] From 2d7b08f2744c550dd630add451e85c28f4f7336d Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Tue, 29 Aug 2023 09:55:09 +0200 Subject: [PATCH 03/13] Add zugbruecke in extractors install for plexon2 --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 3ecfbe2718..ddb0de4893 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ extractors = [ "ONE-api>=1.19.1", "ibllib>=2.21.0", "pymatreader>=0.0.32", # For cell explorer matlab files + "zugbruecke", # For plexon2 ] streaming_extractors = [ From 4a9f429f2ad3b18057db6c6432960c401f0ff14c Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Tue, 29 Aug 2023 09:57:16 +0200 Subject: [PATCH 04/13] Update naming following #1626 --- src/spikeinterface/extractors/neoextractors/plexon2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spikeinterface/extractors/neoextractors/plexon2.py b/src/spikeinterface/extractors/neoextractors/plexon2.py index c3869dbadc..148deb48e9 100644 --- a/src/spikeinterface/extractors/neoextractors/plexon2.py +++ b/src/spikeinterface/extractors/neoextractors/plexon2.py @@ -52,7 +52,7 @@ class Plexon2SortingExtractor(NeoBaseSortingExtractor): mode = "file" NeoRawIOClass = "Plexon2RawIO" - handle_spike_frame_directly = True + neo_returns_frames = True name = "plexon2" def __init__(self, file_path): From aae39c6d1a2f4c3f952e19a81e031eb7abb909ae Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Tue, 29 Aug 2023 10:34:17 +0200 Subject: [PATCH 05/13] Install wine for plexon2 --- .github/actions/build-test-environment/action.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/actions/build-test-environment/action.yml b/.github/actions/build-test-environment/action.yml index 004fe31203..7b5debdd51 100644 --- a/.github/actions/build-test-environment/action.yml +++ b/.github/actions/build-test-environment/action.yml @@ -40,3 +40,9 @@ runs: tar xvzf git-annex-standalone-amd64.tar.gz echo "$(pwd)/git-annex.linux" >> $GITHUB_PATH shell: bash + - name: Install wine (needed for Plexon2) + run: | + sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list + sudo dpkg --add-architecture i386 + sudo apt-get update -qq + sudo apt-get install -yqq --allow-downgrades libc6:i386 libgcc-s1:i386 libstdc++6:i386 wine From 26fdba2b0d85f5c174f60462d6edb6128876c14a Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Tue, 29 Aug 2023 10:36:23 +0200 Subject: [PATCH 06/13] Add shell --- .github/actions/build-test-environment/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/build-test-environment/action.yml b/.github/actions/build-test-environment/action.yml index 7b5debdd51..b056bd3353 100644 --- a/.github/actions/build-test-environment/action.yml +++ b/.github/actions/build-test-environment/action.yml @@ -46,3 +46,4 @@ runs: sudo dpkg --add-architecture i386 sudo apt-get update -qq sudo apt-get install -yqq --allow-downgrades libc6:i386 libgcc-s1:i386 libstdc++6:i386 wine + shell: bash From 8bcec5f9df9d2205b0ecd222aac5df135492d730 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Tue, 29 Aug 2023 11:39:12 +0200 Subject: [PATCH 07/13] Expose sampling_frequency in pl2 sorting (needed for multi-stream) --- src/spikeinterface/extractors/neoextractors/plexon2.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/plexon2.py b/src/spikeinterface/extractors/neoextractors/plexon2.py index 148deb48e9..966fc253ad 100644 --- a/src/spikeinterface/extractors/neoextractors/plexon2.py +++ b/src/spikeinterface/extractors/neoextractors/plexon2.py @@ -48,6 +48,8 @@ class Plexon2SortingExtractor(NeoBaseSortingExtractor): ---------- file_path: str The file path to load the recordings from. + sampling_frequency: float, default: None + The sampling frequency of the sorting (required for multiple streams with different sampling frequencies). """ mode = "file" @@ -55,13 +57,13 @@ class Plexon2SortingExtractor(NeoBaseSortingExtractor): neo_returns_frames = True name = "plexon2" - def __init__(self, file_path): + def __init__(self, file_path, sampling_frequency=None): from neo.rawio import Plexon2RawIO neo_kwargs = self.map_to_neo_kwargs(file_path) neo_reader = Plexon2RawIO(**neo_kwargs) neo_reader.parse_header() - NeoBaseSortingExtractor.__init__(self, **neo_kwargs) + NeoBaseSortingExtractor.__init__(self, sampling_frequency=sampling_frequency, **neo_kwargs) self._kwargs.update({"file_path": str(file_path)}) @classmethod From 3ffb76c444e2556fd62efbfab677d6dcd1cd7706 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Tue, 29 Aug 2023 11:59:55 +0200 Subject: [PATCH 08/13] Add sampling_frequency kwargs in tests --- src/spikeinterface/extractors/tests/test_neoextractors.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/spikeinterface/extractors/tests/test_neoextractors.py b/src/spikeinterface/extractors/tests/test_neoextractors.py index 0405d7b129..e8f565bede 100644 --- a/src/spikeinterface/extractors/tests/test_neoextractors.py +++ b/src/spikeinterface/extractors/tests/test_neoextractors.py @@ -156,9 +156,7 @@ class PlexonSortingTest(SortingCommonTestSuite, unittest.TestCase): class Plexon2SortingTest(SortingCommonTestSuite, unittest.TestCase): ExtractorClass = Plexon2SortingExtractor downloads = ["plexon"] - entities = [ - ("plexon/4chDemoPL2.pl2"), - ] + entities = [("plexon/4chDemoPL2.pl2", {"sampling_frequency": 40000})] class NeuralynxRecordingTest(RecordingCommonTestSuite, unittest.TestCase): @@ -328,7 +326,7 @@ def test_pickling(self): # test = PlexonRecordingTest() # test = PlexonSortingTest() # test = NeuralynxRecordingTest() - test = BlackrockRecordingTest() + test = Plexon2RecordingTest() # test = MCSRawRecordingTest() # test = KiloSortSortingTest() # test = Spike2RecordingTest() From 6247dc090b5604dca5dd73fb151f55f781bca8d3 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Tue, 29 Aug 2023 12:55:33 +0200 Subject: [PATCH 09/13] Update self._kwargs --- src/spikeinterface/extractors/neoextractors/plexon2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spikeinterface/extractors/neoextractors/plexon2.py b/src/spikeinterface/extractors/neoextractors/plexon2.py index 966fc253ad..8dbfc67e90 100644 --- a/src/spikeinterface/extractors/neoextractors/plexon2.py +++ b/src/spikeinterface/extractors/neoextractors/plexon2.py @@ -64,7 +64,7 @@ def __init__(self, file_path, sampling_frequency=None): neo_reader = Plexon2RawIO(**neo_kwargs) neo_reader.parse_header() NeoBaseSortingExtractor.__init__(self, sampling_frequency=sampling_frequency, **neo_kwargs) - self._kwargs.update({"file_path": str(file_path)}) + self._kwargs.update({"file_path": str(file_path), "sampling_frequency": sampling_frequency}) @classmethod def map_to_neo_kwargs(cls, file_path): From dfcd3caf8a18168ae05b564d62e7ce15c3ac185d Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Tue, 29 Aug 2023 16:46:55 +0200 Subject: [PATCH 10/13] Mark plexon2 tests and install Wine only if needed --- .../actions/build-test-environment/action.yml | 7 --- .github/actions/install-wine/action.yml | 21 ++++++++ .github/workflows/full-test-with-codecov.yml | 2 +- .github/workflows/full-test.yml | 12 ++++- .../extractors/tests/test_neoextractors.py | 48 ++++++++++--------- 5 files changed, 59 insertions(+), 31 deletions(-) create mode 100644 .github/actions/install-wine/action.yml diff --git a/.github/actions/build-test-environment/action.yml b/.github/actions/build-test-environment/action.yml index b056bd3353..004fe31203 100644 --- a/.github/actions/build-test-environment/action.yml +++ b/.github/actions/build-test-environment/action.yml @@ -40,10 +40,3 @@ runs: tar xvzf git-annex-standalone-amd64.tar.gz echo "$(pwd)/git-annex.linux" >> $GITHUB_PATH shell: bash - - name: Install wine (needed for Plexon2) - run: | - sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - sudo dpkg --add-architecture i386 - sudo apt-get update -qq - sudo apt-get install -yqq --allow-downgrades libc6:i386 libgcc-s1:i386 libstdc++6:i386 wine - shell: bash diff --git a/.github/actions/install-wine/action.yml b/.github/actions/install-wine/action.yml new file mode 100644 index 0000000000..3ae08ecd34 --- /dev/null +++ b/.github/actions/install-wine/action.yml @@ -0,0 +1,21 @@ +name: Install packages +description: This action installs the package and its dependencies for testing + +inputs: + python-version: + description: 'Python version to set up' + required: false + os: + description: 'Operating system to set up' + required: false + +runs: + using: "composite" + steps: + - name: Install wine (needed for Plexon2) + run: | + sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list + sudo dpkg --add-architecture i386 + sudo apt-get update -qq + sudo apt-get install -yqq --allow-downgrades libc6:i386 libgcc-s1:i386 libstdc++6:i386 wine + shell: bash diff --git a/.github/workflows/full-test-with-codecov.yml b/.github/workflows/full-test-with-codecov.yml index a5561c2ffc..d0bf109a00 100644 --- a/.github/workflows/full-test-with-codecov.yml +++ b/.github/workflows/full-test-with-codecov.yml @@ -54,7 +54,7 @@ jobs: - name: run tests run: | source ${{ github.workspace }}/test_env/bin/activate - pytest -m "not sorters_external" --cov=./ --cov-report xml:./coverage.xml -vv -ra --durations=0 | tee report_full.txt; test ${PIPESTATUS[0]} -eq 0 || exit 1 + pytest -m "not sorters_external and not plexon2" --cov=./ --cov-report xml:./coverage.xml -vv -ra --durations=0 | tee report_full.txt; test ${PIPESTATUS[0]} -eq 0 || exit 1 echo "# Timing profile of full tests" >> $GITHUB_STEP_SUMMARY python ./.github/build_job_summary.py report_full.txt >> $GITHUB_STEP_SUMMARY cat $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/full-test.yml b/.github/workflows/full-test.yml index ac5130bade..a343500c08 100644 --- a/.github/workflows/full-test.yml +++ b/.github/workflows/full-test.yml @@ -75,6 +75,10 @@ jobs: echo "Extractors changed" echo "EXTRACTORS_CHANGED=true" >> $GITHUB_OUTPUT fi + if [[ $file == *"plexon2"* ]]; then + echo "Plexon2 changed" + echo "PLEXON2_CHANGED=true" >> $GITHUB_OUTPUT + fi if [[ $file == *"/preprocessing/"* ]]; then echo "Preprocessing changed" echo "PREPROCESSING_CHANGED=true" >> $GITHUB_OUTPUT @@ -122,11 +126,14 @@ jobs: done - name: Set execute permissions on run_tests.sh run: chmod +x .github/run_tests.sh + - name: Install Wine (Plexon2) + if: ${{ steps.modules-changed.outputs.PLEXON2_CHANGED == 'true' }} + uses: ./.github/actions/install-wine - name: Test core run: ./.github/run_tests.sh core - name: Test extractors if: ${{ steps.modules-changed.outputs.EXTRACTORS_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }} - run: ./.github/run_tests.sh "extractors and not streaming_extractors" + run: ./.github/run_tests.sh "extractors and not streaming_extractors and not plexon2" - name: Test preprocessing if: ${{ steps.modules-changed.outputs.PREPROCESSING_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }} run: ./.github/run_tests.sh preprocessing @@ -157,3 +164,6 @@ jobs: - name: Test internal sorters if: ${{ steps.modules-changed.outputs.SORTERS_INTERNAL_CHANGED == 'true' || steps.modules-changed.outputs.SORTINGCOMPONENTS_CHANGED || steps.modules-changed.outputs.CORE_CHANGED == 'true' }} run: ./.github/run_tests.sh sorters_internal + - name: Test plexon2 + if: ${{ steps.modules-changed.outputs.PLEXON2_CHANGED == 'true' }} + run: ./.github/run_tests.sh plexon2 diff --git a/src/spikeinterface/extractors/tests/test_neoextractors.py b/src/spikeinterface/extractors/tests/test_neoextractors.py index e8f565bede..da162eccf1 100644 --- a/src/spikeinterface/extractors/tests/test_neoextractors.py +++ b/src/spikeinterface/extractors/tests/test_neoextractors.py @@ -121,14 +121,6 @@ class NeuroScopeSortingTest(SortingCommonTestSuite, unittest.TestCase): ] -class Plexon2EventTest(EventCommonTestSuite, unittest.TestCase): - ExtractorClass = Plexon2EventExtractor - downloads = ["plexon"] - entities = [ - ("plexon/4chDemoPL2.pl2"), - ] - - class PlexonRecordingTest(RecordingCommonTestSuite, unittest.TestCase): ExtractorClass = PlexonRecordingExtractor downloads = ["plexon"] @@ -137,14 +129,6 @@ class PlexonRecordingTest(RecordingCommonTestSuite, unittest.TestCase): ] -class Plexon2RecordingTest(RecordingCommonTestSuite, unittest.TestCase): - ExtractorClass = Plexon2RecordingExtractor - downloads = ["plexon"] - entities = [ - ("plexon/4chDemoPL2.pl2", {"stream_id": "3"}), - ] - - class PlexonSortingTest(SortingCommonTestSuite, unittest.TestCase): ExtractorClass = PlexonSortingExtractor downloads = ["plexon"] @@ -153,12 +137,6 @@ class PlexonSortingTest(SortingCommonTestSuite, unittest.TestCase): ] -class Plexon2SortingTest(SortingCommonTestSuite, unittest.TestCase): - ExtractorClass = Plexon2SortingExtractor - downloads = ["plexon"] - entities = [("plexon/4chDemoPL2.pl2", {"sampling_frequency": 40000})] - - class NeuralynxRecordingTest(RecordingCommonTestSuite, unittest.TestCase): ExtractorClass = NeuralynxRecordingExtractor downloads = ["neuralynx"] @@ -312,6 +290,32 @@ def test_pickling(self): pass +# We mark plexon2 tests as they require additional dependencies (wine) +@pytest.mark.plexon2 +class Plexon2RecordingTest(RecordingCommonTestSuite, unittest.TestCase): + ExtractorClass = Plexon2RecordingExtractor + downloads = ["plexon"] + entities = [ + ("plexon/4chDemoPL2.pl2", {"stream_id": "3"}), + ] + + +@pytest.mark.plexon2 +class Plexon2EventTest(EventCommonTestSuite, unittest.TestCase): + ExtractorClass = Plexon2EventExtractor + downloads = ["plexon"] + entities = [ + ("plexon/4chDemoPL2.pl2"), + ] + + +@pytest.mark.plexon2 +class Plexon2SortingTest(SortingCommonTestSuite, unittest.TestCase): + ExtractorClass = Plexon2SortingExtractor + downloads = ["plexon"] + entities = [("plexon/4chDemoPL2.pl2", {"sampling_frequency": 40000})] + + if __name__ == "__main__": # test = MearecSortingTest() # test = SpikeGLXRecordingTest() From 23f8677f148a819a57c1f859aa327ca40d124f25 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Tue, 29 Aug 2023 17:33:15 +0200 Subject: [PATCH 11/13] Install zugbruecke not on win --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ddb0de4893..b0bf4cdcf5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,7 @@ extractors = [ "ONE-api>=1.19.1", "ibllib>=2.21.0", "pymatreader>=0.0.32", # For cell explorer matlab files - "zugbruecke", # For plexon2 + "zugbruecke>=0.2; sys_platform!='win32'", # For plexon2 ] streaming_extractors = [ From 99602f17ed4793bbf21b577cd8f87860cc3c3c2b Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Mon, 11 Sep 2023 10:46:51 +0200 Subject: [PATCH 12/13] Make plexon2 tests conditional on Wine dependency (on Linux) --- .../extractors/tests/test_neoextractors.py | 40 ++++++++++++++++++- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/src/spikeinterface/extractors/tests/test_neoextractors.py b/src/spikeinterface/extractors/tests/test_neoextractors.py index da162eccf1..5fe42b0c4e 100644 --- a/src/spikeinterface/extractors/tests/test_neoextractors.py +++ b/src/spikeinterface/extractors/tests/test_neoextractors.py @@ -1,5 +1,6 @@ import unittest -from platform import python_version +import platform +import subprocess from packaging import version import pytest @@ -18,6 +19,38 @@ local_folder = get_global_dataset_folder() / "ephy_testing_data" +def has_plexon2_dependencies(): + """ + Check if required Plexon2 dependencies are installed on different OS. + """ + + os_type = platform.system() + + if os_type == "Windows": + # On Windows, no need for additional dependencies + return True + + elif os_type == "Linux": + # Check for 'wine' using dpkg + try: + result_wine = subprocess.run( + ["dpkg", "-l", "wine"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True + ) + except subprocess.CalledProcessError: + return False + + # Check for 'zugbruecke' using pip + try: + import zugbruecke + + return True + except ImportError: + return False + else: + # Not sure about MacOS + raise ValueError(f"Unsupported OS: {os_type}") + + class MearecRecordingTest(RecordingCommonTestSuite, unittest.TestCase): ExtractorClass = MEArecRecordingExtractor downloads = ["mearec"] @@ -218,7 +251,7 @@ class Spike2RecordingTest(RecordingCommonTestSuite, unittest.TestCase): @pytest.mark.skipif( - version.parse(python_version()) >= version.parse("3.10"), + version.parse(platform.python_version()) >= version.parse("3.10"), reason="Sonpy only testing with Python < 3.10!", ) class CedRecordingTest(RecordingCommonTestSuite, unittest.TestCase): @@ -291,6 +324,7 @@ def test_pickling(self): # We mark plexon2 tests as they require additional dependencies (wine) +@pytest.mark.skipif(not has_plexon2_dependencies(), reason="Required dependencies not installed") @pytest.mark.plexon2 class Plexon2RecordingTest(RecordingCommonTestSuite, unittest.TestCase): ExtractorClass = Plexon2RecordingExtractor @@ -300,6 +334,7 @@ class Plexon2RecordingTest(RecordingCommonTestSuite, unittest.TestCase): ] +@pytest.mark.skipif(not has_plexon2_dependencies(), reason="Required dependencies not installed") @pytest.mark.plexon2 class Plexon2EventTest(EventCommonTestSuite, unittest.TestCase): ExtractorClass = Plexon2EventExtractor @@ -309,6 +344,7 @@ class Plexon2EventTest(EventCommonTestSuite, unittest.TestCase): ] +@pytest.mark.skipif(not has_plexon2_dependencies(), reason="Required dependencies not installed") @pytest.mark.plexon2 class Plexon2SortingTest(SortingCommonTestSuite, unittest.TestCase): ExtractorClass = Plexon2SortingExtractor From e73cf7e107026d80b176e9fb420b31cea964b730 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Wed, 13 Sep 2023 10:27:39 +0200 Subject: [PATCH 13/13] Simplify plexon2 tests (only run when dependencies are installed) --- .github/workflows/full-test-with-codecov.yml | 2 +- .github/workflows/full-test.yml | 5 +---- src/spikeinterface/extractors/tests/test_neoextractors.py | 5 +---- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/full-test-with-codecov.yml b/.github/workflows/full-test-with-codecov.yml index d0bf109a00..a5561c2ffc 100644 --- a/.github/workflows/full-test-with-codecov.yml +++ b/.github/workflows/full-test-with-codecov.yml @@ -54,7 +54,7 @@ jobs: - name: run tests run: | source ${{ github.workspace }}/test_env/bin/activate - pytest -m "not sorters_external and not plexon2" --cov=./ --cov-report xml:./coverage.xml -vv -ra --durations=0 | tee report_full.txt; test ${PIPESTATUS[0]} -eq 0 || exit 1 + pytest -m "not sorters_external" --cov=./ --cov-report xml:./coverage.xml -vv -ra --durations=0 | tee report_full.txt; test ${PIPESTATUS[0]} -eq 0 || exit 1 echo "# Timing profile of full tests" >> $GITHUB_STEP_SUMMARY python ./.github/build_job_summary.py report_full.txt >> $GITHUB_STEP_SUMMARY cat $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/full-test.yml b/.github/workflows/full-test.yml index a343500c08..8f88e84039 100644 --- a/.github/workflows/full-test.yml +++ b/.github/workflows/full-test.yml @@ -133,7 +133,7 @@ jobs: run: ./.github/run_tests.sh core - name: Test extractors if: ${{ steps.modules-changed.outputs.EXTRACTORS_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }} - run: ./.github/run_tests.sh "extractors and not streaming_extractors and not plexon2" + run: ./.github/run_tests.sh "extractors and not streaming_extractors" - name: Test preprocessing if: ${{ steps.modules-changed.outputs.PREPROCESSING_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }} run: ./.github/run_tests.sh preprocessing @@ -164,6 +164,3 @@ jobs: - name: Test internal sorters if: ${{ steps.modules-changed.outputs.SORTERS_INTERNAL_CHANGED == 'true' || steps.modules-changed.outputs.SORTINGCOMPONENTS_CHANGED || steps.modules-changed.outputs.CORE_CHANGED == 'true' }} run: ./.github/run_tests.sh sorters_internal - - name: Test plexon2 - if: ${{ steps.modules-changed.outputs.PLEXON2_CHANGED == 'true' }} - run: ./.github/run_tests.sh plexon2 diff --git a/src/spikeinterface/extractors/tests/test_neoextractors.py b/src/spikeinterface/extractors/tests/test_neoextractors.py index 5fe42b0c4e..ce2703d382 100644 --- a/src/spikeinterface/extractors/tests/test_neoextractors.py +++ b/src/spikeinterface/extractors/tests/test_neoextractors.py @@ -323,9 +323,8 @@ def test_pickling(self): pass -# We mark plexon2 tests as they require additional dependencies (wine) +# We run plexon2 tests only if we have dependencies (wine) @pytest.mark.skipif(not has_plexon2_dependencies(), reason="Required dependencies not installed") -@pytest.mark.plexon2 class Plexon2RecordingTest(RecordingCommonTestSuite, unittest.TestCase): ExtractorClass = Plexon2RecordingExtractor downloads = ["plexon"] @@ -335,7 +334,6 @@ class Plexon2RecordingTest(RecordingCommonTestSuite, unittest.TestCase): @pytest.mark.skipif(not has_plexon2_dependencies(), reason="Required dependencies not installed") -@pytest.mark.plexon2 class Plexon2EventTest(EventCommonTestSuite, unittest.TestCase): ExtractorClass = Plexon2EventExtractor downloads = ["plexon"] @@ -345,7 +343,6 @@ class Plexon2EventTest(EventCommonTestSuite, unittest.TestCase): @pytest.mark.skipif(not has_plexon2_dependencies(), reason="Required dependencies not installed") -@pytest.mark.plexon2 class Plexon2SortingTest(SortingCommonTestSuite, unittest.TestCase): ExtractorClass = Plexon2SortingExtractor downloads = ["plexon"]