diff --git a/CHANGELOG.md b/CHANGELOG.md index c42f77a74..7c6da32f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ * Implemented format-wise and modality-wise extra installation requirements. If there are any requirements to use a module or data interface, these are defined in individual requirements files at the corresponding level of the package. These are in turn easily accessible from the commands `pip install neuroconv[format_name]`. `pip install neuroconv[modality_name]` will also install all dependencies necessary to make full use of any interfaces from that modality. [PR #100](https://github.com/catalystneuro/neuroconv/pull/100) * Added frame stubbing to the `BaseSegmentationExtractorInterface`. [PR #116](https://github.com/catalystneuro/neuroconv/pull/116) * Added `mask_type: str` and `include_roi_centroids: bool` to the `add_plane_segmentation` helper and `write_segmentation` functions for the `tools.roiextractors` submodule. [PR #117](https://github.com/catalystneuro/neuroconv/pull/117) +* Added compression and iteration (with options control) to all Fluorescence traces in `write_segmentation`. [PR #120](https://github.com/catalystneuro/neuroconv/pull/120) ### Documentation and tutorial enhancements: * Unified the documentation of NeuroConv structure in the User Guide readthedocs. [PR #39](https://github.com/catalystneuro/neuroconv/pull/39) diff --git a/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py b/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py index 9c01334f2..fb7cc9947 100644 --- a/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py +++ b/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py @@ -67,6 +67,8 @@ def run_conversion( stub_test: bool = False, stub_frames: int = 100, include_roi_centroids: bool = True, + iterator_options: Optional[dict] = None, + compression_options: Optional[dict] = None, ): from ...tools.roiextractors import write_segmentation @@ -84,4 +86,6 @@ def run_conversion( overwrite=overwrite, verbose=self.verbose, include_roi_centroids=include_roi_centroids, + iterator_options=iterator_options, + compression_options=compression_options, ) diff --git a/src/neuroconv/datainterfaces/ophys/requirements.txt b/src/neuroconv/datainterfaces/ophys/requirements.txt index f60576484..0d13afeb2 100644 --- a/src/neuroconv/datainterfaces/ophys/requirements.txt +++ b/src/neuroconv/datainterfaces/ophys/requirements.txt @@ -1 +1 @@ -roiextractors @ git+https://github.com/catalystneuro/roiextractors.git@9e955248d2532c74522cf896b7c27367304041d9 +roiextractors @ git+https://github.com/catalystneuro/roiextractors.git@e668d4bb3f378a308abc128fa3f4a000f0109ef1 diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index dbd03fa2f..789915e85 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -28,6 +28,7 @@ from hdmf.backends.hdf5.h5_utils import H5DataIO from .imagingextractordatachunkiterator import ImagingExtractorDataChunkIterator +from ..hdmf import SliceableDataChunkIterator from ..nwb_helpers import get_default_nwbfile_metadata, make_or_load_nwbfile, get_module from ...utils import OptionalFilePathType, dict_deep_update, calculate_regular_series_rate @@ -581,10 +582,8 @@ def add_plane_segmentation( NWBFile The nwbfile passed as an input with the plane segmentation added. """ - if iterator_options is None: - iterator_options = dict() - if compression_options is None: - compression_options = dict() + iterator_options = iterator_options or dict() + compression_options = compression_options or dict(compression="gzip") def image_mask_iterator(): for roi_id in segmentation_extractor.get_roi_ids(): @@ -669,6 +668,8 @@ def add_fluorescence_traces( nwbfile: NWBFile, metadata: Optional[dict], plane_index: int = 0, + iterator_options: Optional[dict] = None, + compression_options: Optional[dict] = None, ) -> NWBFile: """ Adds the fluorescence traces specified by the metadata to the nwb file. @@ -691,6 +692,8 @@ def add_fluorescence_traces( NWBFile The nwbfile passed as an input with the fluorescence traces added. """ + iterator_options = iterator_options or dict() + compression_options = compression_options or dict(compression="gzip") # Set the defaults and required infrastructure metadata_copy = deepcopy(metadata) @@ -776,7 +779,7 @@ def add_fluorescence_traces( # Build the roi response series roi_response_series_kwargs.update( - data=np.array(trace).T, + data=H5DataIO(SliceableDataChunkIterator(trace, **iterator_options), **compression_options), rois=roi_table_region, **trace_metadata, ) @@ -884,6 +887,8 @@ def write_segmentation( buffer_size: int = 10, plane_num: int = 0, include_roi_centroids: bool = True, + iterator_options: Optional[dict] = None, + compression_options: Optional[dict] = None, ): """ Primary method for writing an SegmentationExtractor object to an NWBFile. @@ -924,6 +929,9 @@ def write_segmentation( nwbfile_path is None or nwbfile is None ), "Either pass a nwbfile_path location, or nwbfile object, but not both!" + iterator_options = iterator_options or dict() + compression_options = compression_options or dict(compression="gzip") + # parse metadata correctly considering the MultiSegmentationExtractor function: if isinstance(segmentation_extractor, MultiSegmentationExtractor): segmentation_extractors = segmentation_extractor.segmentations @@ -978,11 +986,8 @@ def write_segmentation( nwbfile=nwbfile_out, metadata=metadata, include_roi_centroids=include_roi_centroids, - iterator_options=dict(buffer_size=buffer_size), - compression_options=dict( - compression=True, - compression_opts=9, - ), + iterator_options=iterator_options, + compression_options=compression_options, ) # Add fluorescence traces: @@ -990,6 +995,8 @@ def write_segmentation( segmentation_extractor=segmentation_extractor, nwbfile=nwbfile_out, metadata=metadata, + iterator_options=iterator_options, + compression_options=compression_options, ) # Adding summary images (mean and correlation) diff --git a/tests/test_ophys/test_tools_roiextractors.py b/tests/test_ophys/test_tools_roiextractors.py index 35e0afa6b..59614fc02 100644 --- a/tests/test_ophys/test_tools_roiextractors.py +++ b/tests/test_ophys/test_tools_roiextractors.py @@ -10,7 +10,7 @@ from hdmf.testing import TestCase from numpy.testing import assert_array_equal, assert_raises from parameterized import parameterized, param -from pynwb import NWBFile, NWBHDF5IO +from pynwb import NWBFile, NWBHDF5IO, H5DataIO from pynwb.device import Device from roiextractors.testing import ( generate_dummy_imaging_extractor, @@ -616,9 +616,18 @@ def test_add_fluorescence_traces(self): traces = self.segmentation_extractor.get_traces_dict() - assert_array_equal(fluorescence["RoiResponseSeries"].data, traces["raw"].T) - assert_array_equal(fluorescence["Deconvolved"].data, traces["deconvolved"].T) - assert_array_equal(fluorescence["Neuropil"].data, traces["neuropil"].T) + for nwb_series_name, roiextractors_name in zip( + ["RoiResponseSeries", "Deconvolved", "Neuropil"], ["raw", "deconvolved", "neuropil"] + ): + series_outer_data = fluorescence[nwb_series_name].data + assert_array_equal(series_outer_data.data.data, traces[roiextractors_name]) + + # Check compression options are set + assert isinstance(series_outer_data, H5DataIO) + + compression_parameters = series_outer_data.get_io_params() + assert compression_parameters["compression"] == "gzip" + # Check that df/F trace data is not being written to the Fluorescence container df_over_f = ophys.get(self.df_over_f_name) assert_raises( @@ -670,7 +679,14 @@ def test_add_df_over_f_trace(self): traces = segmentation_extractor.get_traces_dict() - assert_array_equal(df_over_f[trace_name].data, traces["dff"].T) + series_outer_data = df_over_f[trace_name].data + assert_array_equal(series_outer_data.data.data, traces["dff"]) + + # Check compression options are set + assert isinstance(series_outer_data, H5DataIO) + + compression_parameters = series_outer_data.get_io_params() + assert compression_parameters["compression"] == "gzip" def test_add_fluorescence_one_of_the_traces_is_none(self): """Test that roi response series with None values are not added to the