From 884be4fc553cb3e56cee1e3c8742aab8834a20bd Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Wed, 1 May 2024 16:27:29 -0400 Subject: [PATCH 1/6] add errors to `ensure` functions to help end user --- src/spikeinterface/widgets/base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/spikeinterface/widgets/base.py b/src/spikeinterface/widgets/base.py index 7440c240ce..b94167d2b7 100644 --- a/src/spikeinterface/widgets/base.py +++ b/src/spikeinterface/widgets/base.py @@ -109,17 +109,17 @@ def do_plot(self): @classmethod def ensure_sorting_analyzer(cls, input): - # internal help to accept both SortingAnalyzer or MockWaveformExtractor for a ploter + # internal help to accept both SortingAnalyzer or MockWaveformExtractor for a plotter if isinstance(input, SortingAnalyzer): return input elif isinstance(input, MockWaveformExtractor): return input.sorting_analyzer else: - return input + raise TypeError("input must be a SortingAnalyzer or MockWaveformExtractor") @classmethod def ensure_sorting(cls, input): - # internal help to accept both Sorting or SortingAnalyzer or MockWaveformExtractor for a ploter + # internal help to accept both Sorting or SortingAnalyzer or MockWaveformExtractor for a plotter if isinstance(input, BaseSorting): return input elif isinstance(input, SortingAnalyzer): @@ -127,7 +127,7 @@ def ensure_sorting(cls, input): elif isinstance(input, MockWaveformExtractor): return input.sorting_analyzer.sorting else: - return input + raise TypeError("input must be a SortingAnalyzer, MockWaveformExtractor, or of type BaseSorting") @staticmethod def check_extensions(sorting_analyzer, extensions): From 0c96661505cf354b520eeb3934ad3605b862dafc Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Wed, 1 May 2024 16:50:07 -0400 Subject: [PATCH 2/6] allow for basesorting --- src/spikeinterface/widgets/crosscorrelograms.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/spikeinterface/widgets/crosscorrelograms.py b/src/spikeinterface/widgets/crosscorrelograms.py index 6eb565d56a..e70a5775e6 100644 --- a/src/spikeinterface/widgets/crosscorrelograms.py +++ b/src/spikeinterface/widgets/crosscorrelograms.py @@ -46,7 +46,9 @@ def __init__( backend=None, **backend_kwargs, ): - sorting_analyzer_or_sorting = self.ensure_sorting_analyzer(sorting_analyzer_or_sorting) + + if not isinstance(sorting_analyzer_or_sorting, BaseSorting): + sorting_analyzer_or_sorting = self.ensure_sorting_analyzer(sorting_analyzer_or_sorting) if min_similarity_for_correlograms is None: min_similarity_for_correlograms = 0 From b12c7555cbee793a32f2b6deffce618dd685485e Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Wed, 1 May 2024 17:26:07 -0400 Subject: [PATCH 3/6] add write settings.json for back compatability --- src/spikeinterface/core/sortinganalyzer.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/spikeinterface/core/sortinganalyzer.py b/src/spikeinterface/core/sortinganalyzer.py index 9edb527879..bbc428f2eb 100644 --- a/src/spikeinterface/core/sortinganalyzer.py +++ b/src/spikeinterface/core/sortinganalyzer.py @@ -416,9 +416,21 @@ def load_from_binary_folder(cls, folder, recording=None): else: sparsity = None + # PATCH: Because SortingAnalyzer added this json during the development of 0.101.0 we need to save + # this as a bridge for early adopters. The else branch can be removed in version 0.102.0/0.103.0 + # so that this can be simplified in the future + # See https://github.com/SpikeInterface/spikeinterface/issues/2788 + settings_file = folder / f"settings.json" - with open(settings_file, "r") as f: - settings = json.load(f) + if settings_file.exists(): + with open(settings_file, "r") as f: + settings = json.load(f) + else: + warnings.warn("settings.json not found for this folder writing one with return_scaled=True") + settings = dict(return_scaled=True) + with open(settings_file, "w") as f: + json.dump(check_json(settings), f, indent=4) + return_scaled = settings["return_scaled"] sorting_analyzer = SortingAnalyzer( From fea6cb4eb4d8c7c77a59bb48d3042c79ad615075 Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Fri, 3 May 2024 11:36:08 +0200 Subject: [PATCH 4/6] Remove SharedMemory from SortingAnalyzer and use NumpySorting instead. --- src/spikeinterface/core/numpyextractors.py | 2 +- src/spikeinterface/core/sortinganalyzer.py | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/spikeinterface/core/numpyextractors.py b/src/spikeinterface/core/numpyextractors.py index 7572c78adb..0e7aba3080 100644 --- a/src/spikeinterface/core/numpyextractors.py +++ b/src/spikeinterface/core/numpyextractors.py @@ -287,7 +287,7 @@ def from_sorting(source_sorting: BaseSorting, with_metadata=False, copy_spike_ve spike_vector = source_sorting.to_spike_vector() if copy_spike_vector: spike_vector = spike_vector.copy() - sorting = NumpySorting(spike_vector, source_sorting.get_sampling_frequency(), source_sorting.unit_ids) + sorting = NumpySorting(spike_vector, source_sorting.get_sampling_frequency(), source_sorting.unit_ids.copy()) if with_metadata: sorting.copy_metadata(source_sorting) return sorting diff --git a/src/spikeinterface/core/sortinganalyzer.py b/src/spikeinterface/core/sortinganalyzer.py index bbc428f2eb..0a3870c9aa 100644 --- a/src/spikeinterface/core/sortinganalyzer.py +++ b/src/spikeinterface/core/sortinganalyzer.py @@ -23,7 +23,7 @@ from .recording_tools import check_probe_do_not_overlap, get_rec_attributes from .core_tools import check_json, retrieve_importing_provenance from .job_tools import split_job_kwargs -from .numpyextractors import SharedMemorySorting +from .numpyextractors import NumpySorting from .sparsity import ChannelSparsity, estimate_sparsity from .sortingfolder import NumpyFolderSorting from .zarrextractors import get_default_zarr_compressor, ZarrSortingExtractor @@ -296,8 +296,9 @@ def create_memory(cls, sorting, recording, sparsity, return_scaled, rec_attribut # a copy is done to avoid shared dict between instances (which can block garbage collector) rec_attributes = rec_attributes.copy() - # a copy of sorting is created directly in shared memory format to avoid further duplication of spikes. - sorting_copy = SharedMemorySorting.from_sorting(sorting, with_metadata=True) + # a copy of sorting is copied in memory for fast access + sorting_copy = NumpySorting.from_sorting(sorting, with_metadata=True, copy_spike_vector=True) + sorting_analyzer = SortingAnalyzer( sorting=sorting_copy, recording=recording, @@ -375,8 +376,8 @@ def load_from_binary_folder(cls, folder, recording=None): folder = Path(folder) assert folder.is_dir(), f"This folder does not exists {folder}" - # load internal sorting copy and make it sharedmem - sorting = SharedMemorySorting.from_sorting(NumpyFolderSorting(folder / "sorting"), with_metadata=True) + # load internal sorting copy in memory + sorting = NumpySorting.from_sorting(NumpyFolderSorting(folder / "sorting"), with_metadata=True, copy_spike_vector=True) # load recording if possible if recording is None: @@ -537,11 +538,9 @@ def load_from_zarr(cls, folder, recording=None): zarr_root = zarr.open(folder, mode="r") - # load internal sorting and make it sharedmem + # load internal sorting in memory # TODO propagate storage_options - sorting = SharedMemorySorting.from_sorting( - ZarrSortingExtractor(folder, zarr_group="sorting"), with_metadata=True - ) + sorting = NumpySorting.from_sorting(ZarrSortingExtractor(folder, zarr_group="sorting"), with_metadata=True, copy_spike_vector=True) # load recording if possible if recording is None: From ba1cf461e286b47a4db4d083e014fb352511e333 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 3 May 2024 09:38:42 +0000 Subject: [PATCH 5/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/spikeinterface/core/sortinganalyzer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/spikeinterface/core/sortinganalyzer.py b/src/spikeinterface/core/sortinganalyzer.py index 0a3870c9aa..dff941eaef 100644 --- a/src/spikeinterface/core/sortinganalyzer.py +++ b/src/spikeinterface/core/sortinganalyzer.py @@ -377,7 +377,9 @@ def load_from_binary_folder(cls, folder, recording=None): assert folder.is_dir(), f"This folder does not exists {folder}" # load internal sorting copy in memory - sorting = NumpySorting.from_sorting(NumpyFolderSorting(folder / "sorting"), with_metadata=True, copy_spike_vector=True) + sorting = NumpySorting.from_sorting( + NumpyFolderSorting(folder / "sorting"), with_metadata=True, copy_spike_vector=True + ) # load recording if possible if recording is None: @@ -540,7 +542,9 @@ def load_from_zarr(cls, folder, recording=None): # load internal sorting in memory # TODO propagate storage_options - sorting = NumpySorting.from_sorting(ZarrSortingExtractor(folder, zarr_group="sorting"), with_metadata=True, copy_spike_vector=True) + sorting = NumpySorting.from_sorting( + ZarrSortingExtractor(folder, zarr_group="sorting"), with_metadata=True, copy_spike_vector=True + ) # load recording if possible if recording is None: From 25dd6d3687d548ee7882715212affee2a7db9f4b Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Fri, 3 May 2024 15:01:54 +0200 Subject: [PATCH 6/6] Fix copy metadata in numpy.form_sorting --- src/spikeinterface/core/numpyextractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spikeinterface/core/numpyextractors.py b/src/spikeinterface/core/numpyextractors.py index 0e7aba3080..00b4d9efff 100644 --- a/src/spikeinterface/core/numpyextractors.py +++ b/src/spikeinterface/core/numpyextractors.py @@ -289,7 +289,7 @@ def from_sorting(source_sorting: BaseSorting, with_metadata=False, copy_spike_ve spike_vector = spike_vector.copy() sorting = NumpySorting(spike_vector, source_sorting.get_sampling_frequency(), source_sorting.unit_ids.copy()) if with_metadata: - sorting.copy_metadata(source_sorting) + source_sorting.copy_metadata(sorting) return sorting @staticmethod