Merge branch 'main' into add_depth_slice

.github/workflows/update-version.yml

@@ -0,0 +1,41 @@
+name: Update Dev Version on Release
+
+on:
+    workflow_dispatch:
+    workflow_run:
+        workflows: [auto-publish]
+        types: [completed]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Checkout New Branch
+        run: |
+          git checkout -b update_dev_version
+      - name: Update dev version
+        run: |
+          git fetch --prune --unshallow --tags
+          tags="$(git tag --list)"
+          latest_tag=${tags: -6 : 6}
+          old_version=${latest_tag:1:5}
+          echo "Old Version: $old_version"
+          old_major_version=${old_version:0:1}
+          old_minor_version=${old_version:2:1}
+          old_patch_version=${old_version:4:1}
+          new_patch_version=`expr $old_patch_version + 1`
+          new_version="$old_major_version.$old_minor_version.$new_patch_version"
+          echo "New Version: $new_version"
+          sed -i -e "s/version=\"$old_version\"/version=\"$new_version\"/g" setup.py
+          echo "old_version=$old_version" >> "$GITHUB_ENV"
+          echo "new_version=$new_version" >> "$GITHUB_ENV"
+      - name: Commit Changes and Create Pull Request
+        run: |
+          git config --global user.email 41898282+github-actions[bot]@users.noreply.github.com
+          git config --global user.name github-actions[bot]
+          git commit . -m "Update dev version from $old_version to $new_version"
+          git push origin update_dev_version
+          gh pr create --title "[Github.CI] Update dev version from $old_version to $new_version" --body "version: $old_version --> $new_version"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

CHANGELOG.md

@@ -30,6 +30,11 @@
 
 * Added support for Miniscope AVI files with the `MiniscopeImagingExtractor`. [PR #225](https://github.com/catalystneuro/roiextractors/pull/225)
 
+* Added support for incomplete file ingestion for the `Suite2pSegmentationExtractor`. [PR #227](https://github.com/catalystneuro/roiextractors/pull/227)
+
+* Bug fix for the `CaimanSegmentationExtractor`: Change reshaping from 'C' to 'F' (Fortran). [PR #227](https://github.com/catalystneuro/roiextractors/pull/227)
+
+* Bug fix for the `CaimanSegmentationExtractor`: Added importing of `self._image_correlation` and changed how `self._image_mean` to import the background component image. [PR #227](https://github.com/catalystneuro/roiextractors/pull/227)
 
 
 # v0.5.2

setup.py

@@ -22,8 +22,8 @@
 
 setup(
     name="roiextractors",
-    version="0.5.6",
-    author="Heberto Mayorquin, Szonja Weigl, Cody Baker, Ben Dichter, Alessio Buccino",
+    version="0.5.7",
+    author="Heberto Mayorquin, Szonja Weigl, Cody Baker, Ben Dichter, Alessio Buccino, Paul Adkisson",
     author_email="[email protected]",
     description="Python module for extracting optical physiology ROIs and traces for various file types and formats",
     url="https://github.com/catalystneuro/roiextractors",

src/roiextractors/__init__.py

@@ -1,7 +1,7 @@
 """Python-based module for extracting from, converting between, and handling recorded and optical imaging data from several file formats."""
 
 # Keeping __version__ accessible only to maintain backcompatability.
-# Modern appraoch (Python >= 3.8) is to use importlib
+# Modern approach (Python >= 3.8) is to use importlib
 try:
     from importlib.metadata import version
 

src/roiextractors/extraction_tools.py

@@ -149,7 +149,7 @@ def _validate_video_structure(self) -> None:
             "each property axis should be unique value between 0 and 3 (inclusive)"
         )
 
-        axis_values = set((self.rows_axis, self.columns_axis, self.channels_axis, self.frame_axis))
+        axis_values = {self.rows_axis, self.columns_axis, self.channels_axis, self.frame_axis}
         axis_values_are_not_unique = len(axis_values) != 4
         if axis_values_are_not_unique:
             raise ValueError(exception_message)
@@ -273,7 +273,7 @@ def read_numpy_memmap_video(
     return video_memap
 
 
-def _pixel_mask_extractor(image_mask_, _roi_ids):
+def _pixel_mask_extractor(image_mask_, _roi_ids) -> list:
     """Convert image mask to pixel mask.
 
     Pixel masks are an alternative data format for storage of image masks which relies on the sparsity of the images.
@@ -302,7 +302,7 @@ def _pixel_mask_extractor(image_mask_, _roi_ids):
     return pixel_mask_list
 
 
-def _image_mask_extractor(pixel_mask, _roi_ids, image_shape):
+def _image_mask_extractor(pixel_mask, _roi_ids, image_shape) -> np.ndarray:
     """Convert a pixel mask to image mask.
 
     Parameters
@@ -326,28 +326,6 @@ def _image_mask_extractor(pixel_mask, _roi_ids, image_shape):
     return image_mask
 
 
-def get_video_shape(video):
-    """Get the shape of a video (num_channels, num_frames, size_x, size_y).
-
-    Parameters
-    ----------
-    video: numpy.ndarray
-        The video to get the shape of.
-
-    Returns
-    -------
-    video_shape: tuple
-        The shape of the video (num_channels, num_frames, size_x, size_y).
-    """
-    if len(video.shape) == 3:
-        # 1 channel
-        num_channels = 1
-        num_frames, size_x, size_y = video.shape
-    else:
-        num_channels, num_frames, size_x, size_y = video.shape
-    return num_channels, num_frames, size_x, size_y
-
-
 def check_get_frames_args(func):
     """Check the arguments of the get_frames function.
 

src/roiextractors/extractors/caiman/caimansegmentationextractor.py

@@ -58,7 +58,8 @@ def __init__(self, file_path: PathType):
         self._roi_response_dff = self._trace_extractor_read("F_dff")
         self._roi_response_neuropil = self._trace_extractor_read("C")
         self._roi_response_deconvolved = self._trace_extractor_read("S")
-        self._image_correlation = self._summary_image_read()
+        self._image_correlation = self._correlation_image_read()
+        self._image_mean = self._summary_image_read()
         self._sampling_frequency = self._dataset_file["params"]["data"]["fr"][()]
         self._image_masks = self._image_mask_sparse_read()
 
@@ -112,11 +113,18 @@ def _trace_extractor_read(self, field):
         if field in self._dataset_file["estimates"]:
             return lazy_ops.DatasetView(self._dataset_file["estimates"][field]).lazy_transpose()
 
-    def _summary_image_read(self):
-        """Read the summary image (Cn) from the estimates dataset of the h5py file."""
+    def _correlation_image_read(self):
+        """Read correlation image Cn."""
         if self._dataset_file["estimates"].get("Cn"):
             return np.array(self._dataset_file["estimates"]["Cn"])
 
+    def _summary_image_read(self):
+        """Read summary image mean."""
+        if self._dataset_file["estimates"].get("b"):
+            FOV_shape = self._dataset_file["params"]["data"]["dims"][()]
+            b_sum = self._dataset_file["estimates"]["b"][:].sum(axis=1)
+            return np.array(b_sum).reshape(FOV_shape, order="F")
+
     def get_accepted_list(self):
         accepted = self._dataset_file["estimates"]["idx_components"]
         if len(accepted.shape) == 0:

src/roiextractors/extractors/hdf5imagingextractor/hdf5imagingextractor.py

@@ -13,10 +13,7 @@
 import numpy as np
 
 from ...extraction_tools import PathType, FloatType, ArrayType
-from ...extraction_tools import (
-    get_video_shape,
-    write_to_h5_dataset_format,
-)
+from ...extraction_tools import write_to_h5_dataset_format
 from ...imagingextractor import ImagingExtractor
 from lazy_ops import DatasetView
 

src/roiextractors/extractors/memmapextractors/memmapextractors.py

@@ -13,13 +13,9 @@
 from tqdm import tqdm
 
 from ...imagingextractor import ImagingExtractor
-from typing import Tuple, Dict, Optional
+from typing import Tuple, Optional
 
-from ...extraction_tools import (
-    PathType,
-    DtypeType,
-    NumpyArray,
-)
+from ...extraction_tools import PathType, DtypeType
 
 
 class MemmapImagingExtractor(ImagingExtractor):

src/roiextractors/extractors/memmapextractors/numpymemampextractor.py

@@ -8,14 +8,9 @@
 
 import os
 from pathlib import Path
-from typing import Tuple, Dict
 
-import numpy as np
-from tqdm import tqdm
-
-from ...imagingextractor import ImagingExtractor
-from typing import Tuple, Dict
 from roiextractors.extraction_tools import read_numpy_memmap_video, VideoStructure, DtypeType, PathType
+
 from .memmapextractors import MemmapImagingExtractor
 
 

src/roiextractors/extractors/numpyextractors/numpyextractors.py

@@ -14,7 +14,6 @@
 import numpy as np
 
 from ...extraction_tools import PathType, FloatType, ArrayType
-from ...extraction_tools import get_video_shape
 from ...imagingextractor import ImagingExtractor
 from ...segmentationextractor import SegmentationExtractor
 
@@ -78,7 +77,7 @@ def __init__(
             self._num_rows,
             self._num_columns,
             self._num_channels,
-        ) = get_video_shape(self._video)
+        ) = self.get_video_shape(self._video)
 
         if len(self._video.shape) == 3:
             # check if this converts to np.ndarray
@@ -91,6 +90,28 @@ def __init__(
         else:
             self._channel_names = [f"channel_{ch}" for ch in range(self._num_channels)]
 
+    @staticmethod
+    def get_video_shape(video) -> Tuple[int, int, int, int]:
+        """Get the shape of a video (num_frames, num_rows, num_columns, num_channels).
+
+        Parameters
+        ----------
+        video: numpy.ndarray
+            The video to get the shape of.
+
+        Returns
+        -------
+        video_shape: tuple
+            The shape of the video (num_frames, num_rows, num_columns, num_channels).
+        """
+        if len(video.shape) == 3:
+            # 1 channel
+            num_channels = 1
+            num_frames, num_rows, num_columns = video.shape
+        else:
+            num_frames, num_rows, num_columns, num_channels = video.shape
+        return num_frames, num_rows, num_columns, num_channels
+
     def get_frames(self, frame_idxs=None, channel: Optional[int] = 0) -> np.ndarray:
         if frame_idxs is None:
             frame_idxs = [frame for frame in range(self.get_num_frames())]

src/roiextractors/extractors/nwbextractors/nwbextractors.py

@@ -26,8 +26,6 @@
     FloatType,
     IntType,
     ArrayType,
-    check_get_frames_args,
-    check_get_videos_args,
     raise_multi_channel_or_depth_not_implemented,
 )
 from ...imagingextractor import ImagingExtractor

src/roiextractors/extractors/sbximagingextractor/sbximagingextractor.py

@@ -9,7 +9,6 @@
 from multiprocessing.sharedctypes import Value
 import os
 from pathlib import Path
-from warnings import warn
 from typing import Tuple, Optional
 
 import numpy as np

src/roiextractors/extractors/suite2p/suite2psegmentationextractor.py

@@ -10,7 +10,6 @@
 from pathlib import Path
 from typing import Optional
 from warnings import warn
-import os
 import numpy as np
 
 from ...extraction_tools import PathType
@@ -34,7 +33,7 @@ def get_available_channels(cls, folder_path: PathType):
 
         Parameters
         ----------
-        file_path : PathType
+        folder_path : PathType
             Path to Suite2p output path.
 
         Returns

src/roiextractors/multiimagingextractor.py

@@ -11,7 +11,7 @@
 
 import numpy as np
 
-from .extraction_tools import ArrayType, NumpyArray, check_get_frames_args
+from .extraction_tools import ArrayType, NumpyArray
 from .imagingextractor import ImagingExtractor
 
 
@@ -81,7 +81,7 @@ def _check_consistency_between_imaging_extractors(self):
                 len(unique_values) == 1
             ), f"{property_message} is not consistent over the files (found {unique_values})."
 
-    def _get_times(self):
+    def _get_times(self) -> np.ndarray:
         """Get all the times from the imaging extractors and combine them into a single array.
 
         Returns
@@ -200,7 +200,7 @@ def get_video(
 
         return video
 
-    def get_image_size(self) -> Tuple:
+    def get_image_size(self) -> Tuple[int, int]:
         return self._imaging_extractors[0].get_image_size()
 
     def get_num_frames(self) -> int:

src/roiextractors/segmentationextractor.py

@@ -181,7 +181,13 @@ def frame_slice(self, start_frame: Optional[int] = None, end_frame: Optional[int
         """
         return FrameSliceSegmentationExtractor(parent_segmentation=self, start_frame=start_frame, end_frame=end_frame)
 
-    def get_traces(self, roi_ids=None, start_frame=None, end_frame=None, name="raw"):
+    def get_traces(
+        self,
+        roi_ids: ArrayType = None,
+        start_frame: Optional[int] = None,
+        end_frame: Optional[int] = None,
+        name: str = "raw",
+    ):
         """Get the traces of each ROI specified by roi_ids.
 
         Parameters
@@ -237,7 +243,7 @@ def get_images_dict(self):
         """
         return dict(mean=self._image_mean, correlation=self._image_correlation)
 
-    def get_image(self, name="correlation"):
+    def get_image(self, name: str = "correlation") -> ArrayType:
         """Get specific images: mean or correlation.
 
         Parameters
@@ -253,7 +259,7 @@ def get_image(self, name="correlation"):
             raise ValueError(f"could not find {name} image, enter one of {list(self.get_images_dict().keys())}")
         return self.get_images_dict().get(name)
 
-    def get_sampling_frequency(self):
+    def get_sampling_frequency(self) -> float:
         """Get the sampling frequency in Hz.
 
         Returns
@@ -266,7 +272,7 @@ def get_sampling_frequency(self):
 
         return self._sampling_frequency
 
-    def get_num_rois(self):
+    def get_num_rois(self) -> int:
         """Get total number of Regions of Interest (ROIs) in the acquired images.
 
         Returns
@@ -278,7 +284,7 @@ def get_num_rois(self):
             if trace is not None and len(trace.shape) > 0:
                 return trace.shape[1]
 
-    def get_channel_names(self):
+    def get_channel_names(self) -> List[str]:
         """Get names of channels in the pipeline.
 
         Returns
@@ -288,7 +294,7 @@ def get_channel_names(self):
         """
         return self._channel_names
 
-    def get_num_channels(self):
+    def get_num_channels(self) -> int:
         """Get number of channels in the pipeline.
 
         Returns
@@ -342,7 +348,7 @@ def frame_to_time(self, frames: Union[IntType, ArrayType]) -> Union[FloatType, A
 
         Parameters
         ----------
-        frame_indices: int or array-like
+        frames: int or array-like
             The frame or frames to be converted to times
 
         Returns

src/roiextractors/testing.py

@@ -76,6 +76,8 @@ def generate_dummy_imaging_extractor(
         sampling frequency of the video, by default 30.
     dtype : DtypeType, optional
         dtype of the video, by default "uint16".
+    channel_names : list, optional
+        list of channel names.
 
     Returns
     -------

tests/test_internals/test_extraction_tools.py

@@ -95,7 +95,7 @@ def test_invalid_structure_with_repeated_axis(self):
             "^Invalid structure: (.*?) each property axis should be unique value between 0 and 3 (inclusive)?"
         )
         with self.assertRaisesRegex(ValueError, reg_expression):
-            video_structure = VideoStructure(
+            VideoStructure(
                 num_rows=self.num_rows,
                 num_columns=self.num_columns,
                 num_channels=self.num_channels,