From 796e145f8d93a721a3757d36c004e9cd1d8fd094 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Mon, 29 Aug 2022 17:29:43 +0000 Subject: [PATCH 1/8] wip --- .../basesegmentationextractorinterface.py | 4 +++ .../tools/roiextractors/roiextractors.py | 26 ++++++++++++++----- tests/test_ophys/test_tools_roiextractors.py | 17 +++++++++--- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py b/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py index 6a56cd79f..7c43530b5 100644 --- a/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py +++ b/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py @@ -65,6 +65,8 @@ def run_conversion( metadata: Optional[dict] = None, overwrite: bool = False, include_roi_centroids: bool = True, + iterator_options: Optional[dict] = None, + compression_options: Optional[dict] = None, save_path: OptionalFilePathType = None, ): from ...tools.roiextractors import write_segmentation @@ -77,5 +79,7 @@ def run_conversion( overwrite=overwrite, verbose=self.verbose, include_roi_centroids=include_roi_centroids, + iterator_options=iterator_options, + compression_options=compression_options, save_path=save_path, ) diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index 62706e840..6a66fa8f3 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -28,6 +28,7 @@ from hdmf.backends.hdf5.h5_utils import H5DataIO from .imagingextractordatachunkiterator import ImagingExtractorDataChunkIterator +from ..hdmf import SliceableDataChunkIterator from ..nwb_helpers import get_default_nwbfile_metadata, make_or_load_nwbfile, get_module from ...utils import OptionalFilePathType, dict_deep_update, calculate_regular_series_rate @@ -606,7 +607,7 @@ def add_plane_segmentation( if iterator_options is None: iterator_options = dict() if compression_options is None: - compression_options = dict() + compression_options = dict(compression="gzip") def image_mask_iterator(): for roi_id in segmentation_extractor.get_roi_ids(): @@ -690,6 +691,8 @@ def add_fluorescence_traces( nwbfile: NWBFile, metadata: Optional[dict], plane_index: int = 0, + iterator_options: Optional[dict] = None, + compression_options: Optional[dict] = None, ) -> NWBFile: """ Adds the fluorescence traces specified by the metadata to the nwb file. @@ -712,6 +715,10 @@ def add_fluorescence_traces( NWBFile The nwbfile passed as an input with the fluorescence traces added. """ + if iterator_options is None: + iterator_options = dict() + if compression_options is None: + compression_options = dict(compression="gzip") # Set the defaults and required infrastructure metadata_copy = deepcopy(metadata) @@ -797,7 +804,7 @@ def add_fluorescence_traces( # Build the roi response series roi_response_series_kwargs.update( - data=np.array(trace).T, + data=H5DataIO(SliceableDataChunkIterator(trace, **iterator_options), **compression_options), rois=roi_table_region, **trace_metadata, ) @@ -905,6 +912,8 @@ def write_segmentation( buffer_size: int = 10, plane_num: int = 0, include_roi_centroids: bool = True, + iterator_options: Optional[dict] = None, + compression_options: Optional[dict] = None, save_path: OptionalFilePathType = None, # TODO: to be removed ): """ @@ -943,6 +952,10 @@ def write_segmentation( Defaults to True. """ assert save_path is None or nwbfile is None, "Either pass a save_path location, or nwbfile object, but not both!" + if iterator_options is None: + iterator_options = dict() + if compression_options is None: + compression_options = dict(compression="gzip") # parse metadata correctly considering the MultiSegmentationExtractor function: if isinstance(segext_obj, MultiSegmentationExtractor): @@ -1017,11 +1030,8 @@ def write_segmentation( nwbfile=nwbfile_out, metadata=metadata, include_roi_centroids=include_roi_centroids, - iterator_options=dict(buffer_size=buffer_size), - compression_options=dict( - compression=True, - compression_opts=9, - ), + iterator_options=iterator_options, + compression_options=compression_options, ) # Add fluorescence traces: @@ -1029,6 +1039,8 @@ def write_segmentation( segmentation_extractor=segext_obj, nwbfile=nwbfile_out, metadata=metadata, + iterator_options=iterator_options, + compression_options=compression_options, ) # Adding summary images (mean and correlation) diff --git a/tests/test_ophys/test_tools_roiextractors.py b/tests/test_ophys/test_tools_roiextractors.py index a7ea8023f..807ebc1f6 100644 --- a/tests/test_ophys/test_tools_roiextractors.py +++ b/tests/test_ophys/test_tools_roiextractors.py @@ -10,7 +10,7 @@ from hdmf.testing import TestCase from numpy.testing import assert_array_equal, assert_raises from parameterized import parameterized, param -from pynwb import NWBFile, NWBHDF5IO +from pynwb import NWBFile, NWBHDF5IO, H5DataIO from pynwb.device import Device from roiextractors.testing import ( generate_dummy_imaging_extractor, @@ -616,9 +616,18 @@ def test_add_fluorescence_traces(self): traces = self.segmentation_extractor.get_traces_dict() - assert_array_equal(fluorescence["RoiResponseSeries"].data, traces["raw"].T) - assert_array_equal(fluorescence["Deconvolved"].data, traces["deconvolved"].T) - assert_array_equal(fluorescence["Neuropil"].data, traces["neuropil"].T) + for nwb_series_name, roiextractors_name in zip( + ["RoiResponseSeries", "Deconvolved", "Neuropil"], ["raw", "deconvolved", "neuropil"] + ): + series_outer_data = fluorescence[nwb_series_name].data + assert_array_equal(series_outer_data.data.data, traces[roiextractors_name].T) + + # Check compression options are set + assert isinstance(series_outer_data, H5DataIO) + + compression_parameters = series_outer_data.get_io_params() + assert compression_parameters["compression"] == "gzip" + # Check that df/F trace data is not being written to the Fluorescence container df_over_f = ophys.get(self.df_over_f_name) assert_raises( From 6f930ad302fdc573f0c93de2123a01fafa30de15 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 29 Aug 2022 14:56:22 -0400 Subject: [PATCH 2/8] Update roiextractors.py --- src/neuroconv/tools/roiextractors/roiextractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index 6a66fa8f3..cedd88ee5 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -804,7 +804,7 @@ def add_fluorescence_traces( # Build the roi response series roi_response_series_kwargs.update( - data=H5DataIO(SliceableDataChunkIterator(trace, **iterator_options), **compression_options), + data=H5DataIO(SliceableDataChunkIterator(trace.T, **iterator_options), **compression_options), rois=roi_table_region, **trace_metadata, ) From bba075d748aeb7b7a9d211922e44e0005f27a7be Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 29 Aug 2022 15:22:09 -0400 Subject: [PATCH 3/8] debug other test --- tests/test_ophys/test_tools_roiextractors.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_ophys/test_tools_roiextractors.py b/tests/test_ophys/test_tools_roiextractors.py index efdd0277a..ffbc97f65 100644 --- a/tests/test_ophys/test_tools_roiextractors.py +++ b/tests/test_ophys/test_tools_roiextractors.py @@ -679,7 +679,14 @@ def test_add_df_over_f_trace(self): traces = segmentation_extractor.get_traces_dict() - assert_array_equal(df_over_f[trace_name].data, traces["dff"].T) + series_outer_data = df_over_f[trace_name].data + assert_array_equal(series_outer_data.data.data, traces["dff"].T) + + # Check compression options are set + assert isinstance(series_outer_data, H5DataIO) + + compression_parameters = series_outer_data.get_io_params() + assert compression_parameters["compression"] == "gzip" def test_add_fluorescence_one_of_the_traces_is_none(self): """Test that roi response series with None values are not added to the From 288aeaac3c25f7f55a056103ce2fb0911a7c4870 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 29 Aug 2022 19:22:38 +0000 Subject: [PATCH 4/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_ophys/test_tools_roiextractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ophys/test_tools_roiextractors.py b/tests/test_ophys/test_tools_roiextractors.py index ffbc97f65..666ed046e 100644 --- a/tests/test_ophys/test_tools_roiextractors.py +++ b/tests/test_ophys/test_tools_roiextractors.py @@ -681,7 +681,7 @@ def test_add_df_over_f_trace(self): series_outer_data = df_over_f[trace_name].data assert_array_equal(series_outer_data.data.data, traces["dff"].T) - + # Check compression options are set assert isinstance(series_outer_data, H5DataIO) From 09866cc0b98e84cdb1d809681ab32fc3047469b9 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Wed, 31 Aug 2022 15:37:43 +0000 Subject: [PATCH 5/8] debug --- src/neuroconv/datainterfaces/ophys/requirements.txt | 2 +- src/neuroconv/tools/roiextractors/roiextractors.py | 2 +- tests/test_ophys/test_tools_roiextractors.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/neuroconv/datainterfaces/ophys/requirements.txt b/src/neuroconv/datainterfaces/ophys/requirements.txt index f60576484..ab13aadbc 100644 --- a/src/neuroconv/datainterfaces/ophys/requirements.txt +++ b/src/neuroconv/datainterfaces/ophys/requirements.txt @@ -1 +1 @@ -roiextractors @ git+https://github.com/catalystneuro/roiextractors.git@9e955248d2532c74522cf896b7c27367304041d9 +roiextractors @ git+https://github.com/catalystneuro/roiextractors.git@f8c51f07df37fabe0c8b6fa009b5efc89f9956e0 diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index 34a5acedf..fec5464a4 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -783,7 +783,7 @@ def add_fluorescence_traces( # Build the roi response series roi_response_series_kwargs.update( - data=H5DataIO(SliceableDataChunkIterator(trace.T, **iterator_options), **compression_options), + data=H5DataIO(SliceableDataChunkIterator(trace, **iterator_options), **compression_options), rois=roi_table_region, **trace_metadata, ) diff --git a/tests/test_ophys/test_tools_roiextractors.py b/tests/test_ophys/test_tools_roiextractors.py index 666ed046e..59614fc02 100644 --- a/tests/test_ophys/test_tools_roiextractors.py +++ b/tests/test_ophys/test_tools_roiextractors.py @@ -620,7 +620,7 @@ def test_add_fluorescence_traces(self): ["RoiResponseSeries", "Deconvolved", "Neuropil"], ["raw", "deconvolved", "neuropil"] ): series_outer_data = fluorescence[nwb_series_name].data - assert_array_equal(series_outer_data.data.data, traces[roiextractors_name].T) + assert_array_equal(series_outer_data.data.data, traces[roiextractors_name]) # Check compression options are set assert isinstance(series_outer_data, H5DataIO) @@ -680,7 +680,7 @@ def test_add_df_over_f_trace(self): traces = segmentation_extractor.get_traces_dict() series_outer_data = df_over_f[trace_name].data - assert_array_equal(series_outer_data.data.data, traces["dff"].T) + assert_array_equal(series_outer_data.data.data, traces["dff"]) # Check compression options are set assert isinstance(series_outer_data, H5DataIO) From d9889bee32f4c56b600450c1f5fe0fa68bc3f8a6 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Wed, 31 Aug 2022 17:22:25 +0000 Subject: [PATCH 6/8] pin to essential fixes in ROIExtractors --- src/neuroconv/datainterfaces/ophys/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neuroconv/datainterfaces/ophys/requirements.txt b/src/neuroconv/datainterfaces/ophys/requirements.txt index ab13aadbc..0d13afeb2 100644 --- a/src/neuroconv/datainterfaces/ophys/requirements.txt +++ b/src/neuroconv/datainterfaces/ophys/requirements.txt @@ -1 +1 @@ -roiextractors @ git+https://github.com/catalystneuro/roiextractors.git@f8c51f07df37fabe0c8b6fa009b5efc89f9956e0 +roiextractors @ git+https://github.com/catalystneuro/roiextractors.git@e668d4bb3f378a308abc128fa3f4a000f0109ef1 From 9467354bbda8a9a47276dfce07261f0553c18100 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Thu, 1 Sep 2022 13:22:16 +0000 Subject: [PATCH 7/8] PR suggestions --- CHANGELOG.md | 1 + .../tools/roiextractors/roiextractors.py | 18 ++++++------------ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c42f77a74..7c6da32f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ * Implemented format-wise and modality-wise extra installation requirements. If there are any requirements to use a module or data interface, these are defined in individual requirements files at the corresponding level of the package. These are in turn easily accessible from the commands `pip install neuroconv[format_name]`. `pip install neuroconv[modality_name]` will also install all dependencies necessary to make full use of any interfaces from that modality. [PR #100](https://github.com/catalystneuro/neuroconv/pull/100) * Added frame stubbing to the `BaseSegmentationExtractorInterface`. [PR #116](https://github.com/catalystneuro/neuroconv/pull/116) * Added `mask_type: str` and `include_roi_centroids: bool` to the `add_plane_segmentation` helper and `write_segmentation` functions for the `tools.roiextractors` submodule. [PR #117](https://github.com/catalystneuro/neuroconv/pull/117) +* Added compression and iteration (with options control) to all Fluorescence traces in `write_segmentation`. [PR #120](https://github.com/catalystneuro/neuroconv/pull/120) ### Documentation and tutorial enhancements: * Unified the documentation of NeuroConv structure in the User Guide readthedocs. [PR #39](https://github.com/catalystneuro/neuroconv/pull/39) diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index fec5464a4..789915e85 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -582,10 +582,8 @@ def add_plane_segmentation( NWBFile The nwbfile passed as an input with the plane segmentation added. """ - if iterator_options is None: - iterator_options = dict() - if compression_options is None: - compression_options = dict(compression="gzip") + iterator_options = iterator_options or dict() + compression_options = compression_options or dict(compression="gzip") def image_mask_iterator(): for roi_id in segmentation_extractor.get_roi_ids(): @@ -694,10 +692,8 @@ def add_fluorescence_traces( NWBFile The nwbfile passed as an input with the fluorescence traces added. """ - if iterator_options is None: - iterator_options = dict() - if compression_options is None: - compression_options = dict(compression="gzip") + iterator_options = iterator_options or dict() + compression_options = compression_options or dict(compression="gzip") # Set the defaults and required infrastructure metadata_copy = deepcopy(metadata) @@ -933,10 +929,8 @@ def write_segmentation( nwbfile_path is None or nwbfile is None ), "Either pass a nwbfile_path location, or nwbfile object, but not both!" - if iterator_options is None: - iterator_options = dict() - if compression_options is None: - compression_options = dict(compression="gzip") + iterator_options = iterator_options or dict() + compression_options = compression_options or dict(compression="gzip") # parse metadata correctly considering the MultiSegmentationExtractor function: if isinstance(segmentation_extractor, MultiSegmentationExtractor): From a0b3665ed3066e1fc49736becdc1e1553f181a7f Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 1 Sep 2022 09:33:29 -0400 Subject: [PATCH 8/8] Update requirements-minimal.txt --- requirements-minimal.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-minimal.txt b/requirements-minimal.txt index 695b16d97..74c8e85db 100644 --- a/requirements-minimal.txt +++ b/requirements-minimal.txt @@ -9,3 +9,4 @@ pynwb>=1.4.0 psutil>=5.8.0 tqdm>=4.60.0 dandi>=0.39.6 +keyring<=23.8.2 # temporary fix around DANDI: https://github.com/jaraco/keyring/issues/593