From c4904c8fceee33ec962d856bb56226e16d13f868 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Fri, 10 May 2024 13:20:39 -0500 Subject: [PATCH 01/27] upload draft of multi-tiff multi-page imaging extractor. This could be used on its own or used as a base type for more specialized extractors --- .../multitiffmultipageimagingextractor.py | 84 +++++++++++++++++++ src/roiextractors/utils.py | 19 +++++ 2 files changed, 103 insertions(+) create mode 100644 src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py create mode 100644 src/roiextractors/utils.py diff --git a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py new file mode 100644 index 00000000..f3ac2797 --- /dev/null +++ b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py @@ -0,0 +1,84 @@ +from typing import Optional, Tuple + +import numpy as np +from roiextractors import ImagingExtractor +from roiextractors.extraction_tools import PathType +from tifffile import TiffFile +from tqdm import tqdm + +from ...utils import match_paths + + +class MultiTiffMultiPageImagingExtractor(ImagingExtractor): + """A ImagingExtractor for multiple TIFF files that each have multiple pages.""" + + extractor_name = "multi-tiff multi-page Imaging Extractor" + is_writable = False + + def __init__(self, folder_path: PathType, pattern: str, sampling_frequency: float): + """Create a MultiTiffMultiPageImagingExtractor instance. + + Parameters + ---------- + folder_path : str + List of path to each TIFF file. + pattern : str + F-string-style pattern to match the TIFF files. + sampling_frequency : float + The frequency at which the frames were sampled, in Hz. + """ + + super().__init__() + self.folder_path = folder_path + + self.tif_paths = match_paths(folder_path, pattern) + + self.page_tracker = [] + page_counter = 0 + for file_path in tqdm(self.tif_paths, "extracting page lengths"): + with TiffFile(file_path) as tif: + self.page_tracker.append(page_counter) + page_counter += len(tif.pages) + + page = next(tif.pages) + + self._num_frames = page_counter + self._num_columns = page.imagewidth + self._num_rows = page.imagelength + + self._sampling_frequency = sampling_frequency + + self._kwargs = {"folder_path": folder_path} + + def get_video(self, start_frame: int = None, end_frame: int = None, channel: Optional[int] = 0) -> np.ndarray: + + frame_idxs = np.arange(start_frame or 0, end_frame or self._num_frames) + file_idxs = np.searchsorted(self.page_tracker, frame_idxs) # index of the file that contains the frame + file_start_idxs = self.page_tracker[file_idxs] # index of the first frame in the file + frame_offset_idxs = frame_idxs - file_start_idxs # index of the frame in the file + # dict of file_idx: frame_offset_idxs + index_dict = {x: frame_offset_idxs[file_idxs == x] for x in np.unique(file_idxs)} + + data = [] + for file_idx, frame_offset_idxs in index_dict.items(): + with TiffFile(self.tif_paths[file_idx]) as tif: + for frame_offset_idx in frame_offset_idxs: + page = tif.pages[frame_offset_idx] + data.append(page.asarray()) + + return np.array(data) + + def get_image_size(self) -> Tuple[int, int]: + return self._num_rows, self._num_columns + + def get_num_frames(self): + return self._num_frames + + def get_sampling_frequency(self): + return self._sampling_frequency + + def get_num_channels(self): + return 1 + + def get_channel_names(self): + return ["channel_0"] \ No newline at end of file diff --git a/src/roiextractors/utils.py b/src/roiextractors/utils.py new file mode 100644 index 00000000..51835551 --- /dev/null +++ b/src/roiextractors/utils.py @@ -0,0 +1,19 @@ +import glob +import os + +from parse import parse + + +def match_paths(base, pattern, sort=True): + full_pattern = os.path.join(base, pattern) + paths = glob.glob(os.path.join(base, "*")) + out = {} + for path in paths: + parsed = parse(full_pattern, path) + if parsed is not None: + out[path] = parsed.named + + if sort: + out = dict(sorted(out.items(), key=lambda item: tuple(item[1].values()))) + + return out From fddf198079f256f06b9fb6ad6368ed75011b7eb5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 10 May 2024 18:21:13 +0000 Subject: [PATCH 02/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../tiffimagingextractors/multitiffmultipageimagingextractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py index f3ac2797..47fcc58e 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py @@ -81,4 +81,4 @@ def get_num_channels(self): return 1 def get_channel_names(self): - return ["channel_0"] \ No newline at end of file + return ["channel_0"] From ccf4d90c11290f7e55c6462eb946cc729e32f13b Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 15 May 2024 17:41:48 -0500 Subject: [PATCH 03/27] add tests --- src/roiextractors/__init__.py | 5 +++-- src/roiextractors/extractorlist.py | 2 ++ .../extractors/tiffimagingextractors/__init__.py | 1 + .../multitiffmultipageimagingextractor.py | 6 ++---- tests/test_multitiff_multipage_imaging_extractor.py | 13 +++++++++++++ 5 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 tests/test_multitiff_multipage_imaging_extractor.py diff --git a/src/roiextractors/__init__.py b/src/roiextractors/__init__.py index 1f7ec5be..14c06231 100644 --- a/src/roiextractors/__init__.py +++ b/src/roiextractors/__init__.py @@ -2,10 +2,11 @@ from importlib.metadata import version -__version__ = version("roiextractors") - from .example_datasets import toy_example from .extraction_tools import show_video from .extractorlist import * from .imagingextractor import ImagingExtractor from .segmentationextractor import SegmentationExtractor + +__version__ = version("roiextractors") + diff --git a/src/roiextractors/extractorlist.py b/src/roiextractors/extractorlist.py index f93c6c29..cf7f6525 100644 --- a/src/roiextractors/extractorlist.py +++ b/src/roiextractors/extractorlist.py @@ -23,6 +23,7 @@ BrukerTiffMultiPlaneImagingExtractor, BrukerTiffSinglePlaneImagingExtractor, MicroManagerTiffImagingExtractor, + MultiTiffMultiPageImagingExtractor, ) from .extractors.sbximagingextractor import SbxImagingExtractor from .extractors.inscopixextractors import InscopixImagingExtractor @@ -51,6 +52,7 @@ NumpyMemmapImagingExtractor, MemmapImagingExtractor, VolumetricImagingExtractor, + MultiTiffMultiPageImagingExtractor, InscopixImagingExtractor, ] diff --git a/src/roiextractors/extractors/tiffimagingextractors/__init__.py b/src/roiextractors/extractors/tiffimagingextractors/__init__.py index 5d37e2bc..a0221111 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/__init__.py +++ b/src/roiextractors/extractors/tiffimagingextractors/__init__.py @@ -43,3 +43,4 @@ ) from .brukertiffimagingextractor import BrukerTiffMultiPlaneImagingExtractor, BrukerTiffSinglePlaneImagingExtractor from .micromanagertiffimagingextractor import MicroManagerTiffImagingExtractor +from .multitiffmultipageimagingextractor import MultiTiffMultiPageImagingExtractor diff --git a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py index 47fcc58e..000c625d 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py @@ -1,11 +1,10 @@ from typing import Optional, Tuple import numpy as np -from roiextractors import ImagingExtractor -from roiextractors.extraction_tools import PathType from tifffile import TiffFile from tqdm import tqdm +from ... import ImagingExtractor from ...utils import match_paths @@ -15,7 +14,7 @@ class MultiTiffMultiPageImagingExtractor(ImagingExtractor): extractor_name = "multi-tiff multi-page Imaging Extractor" is_writable = False - def __init__(self, folder_path: PathType, pattern: str, sampling_frequency: float): + def __init__(self, folder_path: str, pattern: str, sampling_frequency: float): """Create a MultiTiffMultiPageImagingExtractor instance. Parameters @@ -51,7 +50,6 @@ def __init__(self, folder_path: PathType, pattern: str, sampling_frequency: floa self._kwargs = {"folder_path": folder_path} def get_video(self, start_frame: int = None, end_frame: int = None, channel: Optional[int] = 0) -> np.ndarray: - frame_idxs = np.arange(start_frame or 0, end_frame or self._num_frames) file_idxs = np.searchsorted(self.page_tracker, frame_idxs) # index of the file that contains the frame file_start_idxs = self.page_tracker[file_idxs] # index of the first frame in the file diff --git a/tests/test_multitiff_multipage_imaging_extractor.py b/tests/test_multitiff_multipage_imaging_extractor.py new file mode 100644 index 00000000..29253646 --- /dev/null +++ b/tests/test_multitiff_multipage_imaging_extractor.py @@ -0,0 +1,13 @@ +from roiextractors import MultiTiffMultiPageImagingExtractor + + +def test_init_multitiff_multipage_imaging_extractor(): + extractor = MultiTiffMultiPageImagingExtractor(folder_path='tests/test_files', pattern='split_{:d}.tif', sampling_frequency=1.0) + + assert extractor.get_num_channels() == 1 + assert extractor.get_num_frames() == 2000 + assert extractor.get_sampling_frequency() == 1.0 + assert extractor.get_channel_names() == ['channel_0'] + assert extractor.get_dtype() == 'uint16' + assert extractor.get_image_size() == (512, 512) + assert extractor.get_video().shape == (2000, 512, 512) \ No newline at end of file From 6de2ca4e7249531d0127b7f5ce9c7b179a648ac5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 May 2024 22:42:04 +0000 Subject: [PATCH 04/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/roiextractors/__init__.py | 1 - tests/test_multitiff_multipage_imaging_extractor.py | 10 ++++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/roiextractors/__init__.py b/src/roiextractors/__init__.py index 14c06231..839f335e 100644 --- a/src/roiextractors/__init__.py +++ b/src/roiextractors/__init__.py @@ -9,4 +9,3 @@ from .segmentationextractor import SegmentationExtractor __version__ = version("roiextractors") - diff --git a/tests/test_multitiff_multipage_imaging_extractor.py b/tests/test_multitiff_multipage_imaging_extractor.py index 29253646..cdb0aa9d 100644 --- a/tests/test_multitiff_multipage_imaging_extractor.py +++ b/tests/test_multitiff_multipage_imaging_extractor.py @@ -2,12 +2,14 @@ def test_init_multitiff_multipage_imaging_extractor(): - extractor = MultiTiffMultiPageImagingExtractor(folder_path='tests/test_files', pattern='split_{:d}.tif', sampling_frequency=1.0) + extractor = MultiTiffMultiPageImagingExtractor( + folder_path="tests/test_files", pattern="split_{:d}.tif", sampling_frequency=1.0 + ) assert extractor.get_num_channels() == 1 assert extractor.get_num_frames() == 2000 assert extractor.get_sampling_frequency() == 1.0 - assert extractor.get_channel_names() == ['channel_0'] - assert extractor.get_dtype() == 'uint16' + assert extractor.get_channel_names() == ["channel_0"] + assert extractor.get_dtype() == "uint16" assert extractor.get_image_size() == (512, 512) - assert extractor.get_video().shape == (2000, 512, 512) \ No newline at end of file + assert extractor.get_video().shape == (2000, 512, 512) From 85e90db35a15172b2bfa4e1cfb8b23cae26f3e4a Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 15 May 2024 17:48:57 -0500 Subject: [PATCH 05/27] dynamic import of tifffile --- .../multitiffmultipageimagingextractor.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py index 000c625d..959d5639 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py @@ -1,10 +1,10 @@ from typing import Optional, Tuple import numpy as np -from tifffile import TiffFile from tqdm import tqdm from ... import ImagingExtractor +from ...extraction_tools import get_package from ...utils import match_paths @@ -31,11 +31,12 @@ def __init__(self, folder_path: str, pattern: str, sampling_frequency: float): self.folder_path = folder_path self.tif_paths = match_paths(folder_path, pattern) + self._tifffile = get_package(package_name="tifffile", installation_instructions="pip install tifffile") self.page_tracker = [] page_counter = 0 for file_path in tqdm(self.tif_paths, "extracting page lengths"): - with TiffFile(file_path) as tif: + with self._tifffile.TiffFile(file_path) as tif: self.page_tracker.append(page_counter) page_counter += len(tif.pages) @@ -59,7 +60,7 @@ def get_video(self, start_frame: int = None, end_frame: int = None, channel: Opt data = [] for file_idx, frame_offset_idxs in index_dict.items(): - with TiffFile(self.tif_paths[file_idx]) as tif: + with self._tifffile.TiffFile(self.tif_paths[file_idx]) as tif: for frame_offset_idx in frame_offset_idxs: page = tif.pages[frame_offset_idx] data.append(page.asarray()) From fb772d6926b72291ce10a042338610ba59275748 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 15 May 2024 18:02:08 -0500 Subject: [PATCH 06/27] fix tests --- .../multitiffmultipageimagingextractor.py | 14 ++++++++------ .../test_multitiff_multipage_imaging_extractor.py | 9 ++++++--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py index 959d5639..4a45ff1c 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py @@ -3,7 +3,7 @@ import numpy as np from tqdm import tqdm -from ... import ImagingExtractor +from ...imagingextractor import ImagingExtractor from ...extraction_tools import get_package from ...utils import match_paths @@ -33,14 +33,15 @@ def __init__(self, folder_path: str, pattern: str, sampling_frequency: float): self.tif_paths = match_paths(folder_path, pattern) self._tifffile = get_package(package_name="tifffile", installation_instructions="pip install tifffile") - self.page_tracker = [] + page_tracker = [] page_counter = 0 for file_path in tqdm(self.tif_paths, "extracting page lengths"): with self._tifffile.TiffFile(file_path) as tif: - self.page_tracker.append(page_counter) + page_tracker.append(page_counter) page_counter += len(tif.pages) + self.page_tracker = np.array(page_tracker) - page = next(tif.pages) + page = tif.pages[0] self._num_frames = page_counter self._num_columns = page.imagewidth @@ -52,7 +53,8 @@ def __init__(self, folder_path: str, pattern: str, sampling_frequency: float): def get_video(self, start_frame: int = None, end_frame: int = None, channel: Optional[int] = 0) -> np.ndarray: frame_idxs = np.arange(start_frame or 0, end_frame or self._num_frames) - file_idxs = np.searchsorted(self.page_tracker, frame_idxs) # index of the file that contains the frame + file_idxs = np.searchsorted(self.page_tracker, frame_idxs, side="right") - 1 # index of the file that contains the frame + print(f"{file_idxs=}") file_start_idxs = self.page_tracker[file_idxs] # index of the first frame in the file frame_offset_idxs = frame_idxs - file_start_idxs # index of the frame in the file # dict of file_idx: frame_offset_idxs @@ -60,7 +62,7 @@ def get_video(self, start_frame: int = None, end_frame: int = None, channel: Opt data = [] for file_idx, frame_offset_idxs in index_dict.items(): - with self._tifffile.TiffFile(self.tif_paths[file_idx]) as tif: + with self._tifffile.TiffFile(list(self.tif_paths)[file_idx]) as tif: for frame_offset_idx in frame_offset_idxs: page = tif.pages[frame_offset_idx] data.append(page.asarray()) diff --git a/tests/test_multitiff_multipage_imaging_extractor.py b/tests/test_multitiff_multipage_imaging_extractor.py index cdb0aa9d..940bcac0 100644 --- a/tests/test_multitiff_multipage_imaging_extractor.py +++ b/tests/test_multitiff_multipage_imaging_extractor.py @@ -1,9 +1,12 @@ from roiextractors import MultiTiffMultiPageImagingExtractor +from tests.setup_paths import OPHYS_DATA_PATH + def test_init_multitiff_multipage_imaging_extractor(): extractor = MultiTiffMultiPageImagingExtractor( - folder_path="tests/test_files", pattern="split_{:d}.tif", sampling_frequency=1.0 + folder_path=OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "splits", pattern="split_{:d}.tif", + sampling_frequency=1.0 ) assert extractor.get_num_channels() == 1 @@ -11,5 +14,5 @@ def test_init_multitiff_multipage_imaging_extractor(): assert extractor.get_sampling_frequency() == 1.0 assert extractor.get_channel_names() == ["channel_0"] assert extractor.get_dtype() == "uint16" - assert extractor.get_image_size() == (512, 512) - assert extractor.get_video().shape == (2000, 512, 512) + assert extractor.get_image_size() == (60, 80) + assert extractor.get_video().shape == (2000, 60, 80) From 31c9df483f803d9484213a456c42c395f52b1db7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 May 2024 23:02:19 +0000 Subject: [PATCH 07/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../multitiffmultipageimagingextractor.py | 4 +++- tests/test_multitiff_multipage_imaging_extractor.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py index 4a45ff1c..a8dec5b2 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py @@ -53,7 +53,9 @@ def __init__(self, folder_path: str, pattern: str, sampling_frequency: float): def get_video(self, start_frame: int = None, end_frame: int = None, channel: Optional[int] = 0) -> np.ndarray: frame_idxs = np.arange(start_frame or 0, end_frame or self._num_frames) - file_idxs = np.searchsorted(self.page_tracker, frame_idxs, side="right") - 1 # index of the file that contains the frame + file_idxs = ( + np.searchsorted(self.page_tracker, frame_idxs, side="right") - 1 + ) # index of the file that contains the frame print(f"{file_idxs=}") file_start_idxs = self.page_tracker[file_idxs] # index of the first frame in the file frame_offset_idxs = frame_idxs - file_start_idxs # index of the frame in the file diff --git a/tests/test_multitiff_multipage_imaging_extractor.py b/tests/test_multitiff_multipage_imaging_extractor.py index 940bcac0..73d84c72 100644 --- a/tests/test_multitiff_multipage_imaging_extractor.py +++ b/tests/test_multitiff_multipage_imaging_extractor.py @@ -5,8 +5,9 @@ def test_init_multitiff_multipage_imaging_extractor(): extractor = MultiTiffMultiPageImagingExtractor( - folder_path=OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "splits", pattern="split_{:d}.tif", - sampling_frequency=1.0 + folder_path=OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "splits", + pattern="split_{:d}.tif", + sampling_frequency=1.0, ) assert extractor.get_num_channels() == 1 From 7263bd93879e5c0ffc8b8a9340a795e37883ed4b Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 15 May 2024 18:28:28 -0500 Subject: [PATCH 08/27] fix test --- ...est_multitiff_multipage_imaging_extractor.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/test_multitiff_multipage_imaging_extractor.py b/tests/test_multitiff_multipage_imaging_extractor.py index 940bcac0..17d9c246 100644 --- a/tests/test_multitiff_multipage_imaging_extractor.py +++ b/tests/test_multitiff_multipage_imaging_extractor.py @@ -5,7 +5,7 @@ def test_init_multitiff_multipage_imaging_extractor(): extractor = MultiTiffMultiPageImagingExtractor( - folder_path=OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "splits", pattern="split_{:d}.tif", + folder_path=OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "splits", pattern="split_{split:d}.tif", sampling_frequency=1.0 ) @@ -16,3 +16,18 @@ def test_init_multitiff_multipage_imaging_extractor(): assert extractor.get_dtype() == "uint16" assert extractor.get_image_size() == (60, 80) assert extractor.get_video().shape == (2000, 60, 80) + assert list(extractor.tif_paths.keys()) == [ + str(OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "splits" / x) for x in + ( + "split_1.tif", + "split_2.tif", + "split_3.tif", + "split_4.tif", + "split_5.tif", + "split_6.tif", + "split_7.tif", + "split_8.tif", + "split_9.tif", + "split_10.tif", + ) + ] From f760ba630332ea00c95b80e28dffc573fcf47a38 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 May 2024 23:29:07 +0000 Subject: [PATCH 09/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_multitiff_multipage_imaging_extractor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_multitiff_multipage_imaging_extractor.py b/tests/test_multitiff_multipage_imaging_extractor.py index 3c09811e..fc7ed4b0 100644 --- a/tests/test_multitiff_multipage_imaging_extractor.py +++ b/tests/test_multitiff_multipage_imaging_extractor.py @@ -18,8 +18,8 @@ def test_init_multitiff_multipage_imaging_extractor(): assert extractor.get_image_size() == (60, 80) assert extractor.get_video().shape == (2000, 60, 80) assert list(extractor.tif_paths.keys()) == [ - str(OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "splits" / x) for x in - ( + str(OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "splits" / x) + for x in ( "split_1.tif", "split_2.tif", "split_3.tif", @@ -30,5 +30,5 @@ def test_init_multitiff_multipage_imaging_extractor(): "split_8.tif", "split_9.tif", "split_10.tif", - ) + ) ] From 7892f8d4700a06d7c35d980bb507b1f0604c7da6 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 15 May 2024 18:30:11 -0500 Subject: [PATCH 10/27] rmv print --- .../tiffimagingextractors/multitiffmultipageimagingextractor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py index a8dec5b2..d659bdf5 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py @@ -56,7 +56,6 @@ def get_video(self, start_frame: int = None, end_frame: int = None, channel: Opt file_idxs = ( np.searchsorted(self.page_tracker, frame_idxs, side="right") - 1 ) # index of the file that contains the frame - print(f"{file_idxs=}") file_start_idxs = self.page_tracker[file_idxs] # index of the first frame in the file frame_offset_idxs = frame_idxs - file_start_idxs # index of the frame in the file # dict of file_idx: frame_offset_idxs From fda7b9c040294557f352d2f70c0f93b7cecaa64e Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 15 May 2024 19:23:13 -0500 Subject: [PATCH 11/27] use multiimaginginterface --- .../multitiffmultipageimagingextractor.py | 65 ++----------------- ...t_multitiff_multipage_imaging_extractor.py | 2 +- 2 files changed, 7 insertions(+), 60 deletions(-) diff --git a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py index d659bdf5..6a98b175 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py @@ -1,14 +1,13 @@ from typing import Optional, Tuple import numpy as np -from tqdm import tqdm -from ...imagingextractor import ImagingExtractor -from ...extraction_tools import get_package +from ...multiimagingextractor import MultiImagingExtractor +from .tiffimagingextractor import TiffImagingExtractor from ...utils import match_paths -class MultiTiffMultiPageImagingExtractor(ImagingExtractor): +class MultiTiffMultiPageImagingExtractor(MultiImagingExtractor): """A ImagingExtractor for multiple TIFF files that each have multiple pages.""" extractor_name = "multi-tiff multi-page Imaging Extractor" @@ -27,60 +26,8 @@ def __init__(self, folder_path: str, pattern: str, sampling_frequency: float): The frequency at which the frames were sampled, in Hz. """ - super().__init__() self.folder_path = folder_path - self.tif_paths = match_paths(folder_path, pattern) - self._tifffile = get_package(package_name="tifffile", installation_instructions="pip install tifffile") - - page_tracker = [] - page_counter = 0 - for file_path in tqdm(self.tif_paths, "extracting page lengths"): - with self._tifffile.TiffFile(file_path) as tif: - page_tracker.append(page_counter) - page_counter += len(tif.pages) - self.page_tracker = np.array(page_tracker) - - page = tif.pages[0] - - self._num_frames = page_counter - self._num_columns = page.imagewidth - self._num_rows = page.imagelength - - self._sampling_frequency = sampling_frequency - - self._kwargs = {"folder_path": folder_path} - - def get_video(self, start_frame: int = None, end_frame: int = None, channel: Optional[int] = 0) -> np.ndarray: - frame_idxs = np.arange(start_frame or 0, end_frame or self._num_frames) - file_idxs = ( - np.searchsorted(self.page_tracker, frame_idxs, side="right") - 1 - ) # index of the file that contains the frame - file_start_idxs = self.page_tracker[file_idxs] # index of the first frame in the file - frame_offset_idxs = frame_idxs - file_start_idxs # index of the frame in the file - # dict of file_idx: frame_offset_idxs - index_dict = {x: frame_offset_idxs[file_idxs == x] for x in np.unique(file_idxs)} - - data = [] - for file_idx, frame_offset_idxs in index_dict.items(): - with self._tifffile.TiffFile(list(self.tif_paths)[file_idx]) as tif: - for frame_offset_idx in frame_offset_idxs: - page = tif.pages[frame_offset_idx] - data.append(page.asarray()) - - return np.array(data) - - def get_image_size(self) -> Tuple[int, int]: - return self._num_rows, self._num_columns - - def get_num_frames(self): - return self._num_frames - - def get_sampling_frequency(self): - return self._sampling_frequency - - def get_num_channels(self): - return 1 - - def get_channel_names(self): - return ["channel_0"] + imaging_extractors = [TiffImagingExtractor(x, sampling_frequency) for x in self.tif_paths] + super().__init__(imaging_extractors=imaging_extractors) + self._kwargs.update({"folder_path": folder_path}) diff --git a/tests/test_multitiff_multipage_imaging_extractor.py b/tests/test_multitiff_multipage_imaging_extractor.py index fc7ed4b0..493c64a9 100644 --- a/tests/test_multitiff_multipage_imaging_extractor.py +++ b/tests/test_multitiff_multipage_imaging_extractor.py @@ -13,7 +13,7 @@ def test_init_multitiff_multipage_imaging_extractor(): assert extractor.get_num_channels() == 1 assert extractor.get_num_frames() == 2000 assert extractor.get_sampling_frequency() == 1.0 - assert extractor.get_channel_names() == ["channel_0"] + assert extractor.get_channel_names() is None assert extractor.get_dtype() == "uint16" assert extractor.get_image_size() == (60, 80) assert extractor.get_video().shape == (2000, 60, 80) From 96152ac9ed9008e48182445a07ed6c053a0e310e Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 15 May 2024 19:37:17 -0500 Subject: [PATCH 12/27] add parse as a dependency --- requirements-full.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements-full.txt b/requirements-full.txt index 2586b930..02e43298 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -3,3 +3,5 @@ scanimage-tiff-reader==1.4.1.4 neuroconv[video]>=0.4.6 # Uses the VideoCaptureContext class natsort>=8.3.1 isx>=1.0.4 +parse>=1.20.0 + From dbdf11e2efd138ffd7ca96d01d57dfe4cc7e599b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 May 2024 00:37:27 +0000 Subject: [PATCH 13/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- requirements-full.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-full.txt b/requirements-full.txt index 02e43298..e4f3cfe1 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -4,4 +4,3 @@ neuroconv[video]>=0.4.6 # Uses the VideoCaptureContext class natsort>=8.3.1 isx>=1.0.4 parse>=1.20.0 - From 4925b40602d88e40efba48f9172fbcc05d91bafe Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 15 May 2024 19:43:38 -0500 Subject: [PATCH 14/27] move parse to minimal requirements --- requirements-full.txt | 1 - requirements-minimal.txt | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-full.txt b/requirements-full.txt index 02e43298..1500b227 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -3,5 +3,4 @@ scanimage-tiff-reader==1.4.1.4 neuroconv[video]>=0.4.6 # Uses the VideoCaptureContext class natsort>=8.3.1 isx>=1.0.4 -parse>=1.20.0 diff --git a/requirements-minimal.txt b/requirements-minimal.txt index 59234aae..e834fd08 100644 --- a/requirements-minimal.txt +++ b/requirements-minimal.txt @@ -6,3 +6,4 @@ dill>=0.3.2 scipy>=1.5.2 psutil>=5.8.0 PyYAML +parse>=1.20.0 From 435f60046047b5d00dfd7931eafcbe8a1577aaae Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 May 2024 00:44:26 +0000 Subject: [PATCH 15/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- requirements-full.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-full.txt b/requirements-full.txt index 1500b227..2586b930 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -3,4 +3,3 @@ scanimage-tiff-reader==1.4.1.4 neuroconv[video]>=0.4.6 # Uses the VideoCaptureContext class natsort>=8.3.1 isx>=1.0.4 - From c1ae185dad16311d412f66b88a9528c39961a5e5 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 15 May 2024 19:48:16 -0500 Subject: [PATCH 16/27] Add docstring for match_paths --- src/roiextractors/utils.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/roiextractors/utils.py b/src/roiextractors/utils.py index 51835551..e373112f 100644 --- a/src/roiextractors/utils.py +++ b/src/roiextractors/utils.py @@ -1,10 +1,27 @@ import glob import os +from typing import Dict, Any from parse import parse -def match_paths(base, pattern, sort=True): +def match_paths(base: str, pattern: str, sort=True) -> Dict[str, Dict[str, Any]]: + """ + Match paths in a directory to a pattern. + + Parameters + ---------- + base: str + The base directory to search in. + pattern: str + The f-string pattern to match the paths to. + sort: bool, default=True + Whether to sort the output by the values of the named groups in the pattern. + + Returns + ------- + dict + """ full_pattern = os.path.join(base, pattern) paths = glob.glob(os.path.join(base, "*")) out = {} From a798a7cfca4aa58b27f0dccd24a51d9d4ce1e20d Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 15 May 2024 19:53:45 -0500 Subject: [PATCH 17/27] pass docstring tests --- .../multitiffmultipageimagingextractor.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py index 6a98b175..c358d5e6 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py @@ -1,9 +1,7 @@ -from typing import Optional, Tuple +"""MultiTiffMultiPageImagingExtractor class.""" -import numpy as np - -from ...multiimagingextractor import MultiImagingExtractor from .tiffimagingextractor import TiffImagingExtractor +from ...multiimagingextractor import MultiImagingExtractor from ...utils import match_paths From a904272ab574569f7319de2a69063ee29d0ba950 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 15 May 2024 21:26:07 -0500 Subject: [PATCH 18/27] use named vars? --- .../tiffimagingextractors/multitiffmultipageimagingextractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py index c358d5e6..4f51e94b 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py @@ -26,6 +26,6 @@ def __init__(self, folder_path: str, pattern: str, sampling_frequency: float): self.folder_path = folder_path self.tif_paths = match_paths(folder_path, pattern) - imaging_extractors = [TiffImagingExtractor(x, sampling_frequency) for x in self.tif_paths] + imaging_extractors = [TiffImagingExtractor(file_path=x, sampling_frequency=sampling_frequency) for x in self.tif_paths] super().__init__(imaging_extractors=imaging_extractors) self._kwargs.update({"folder_path": folder_path}) From 66289fb298fcbafefe80268447fde1b27235b538 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 May 2024 02:26:19 +0000 Subject: [PATCH 19/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../multitiffmultipageimagingextractor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py index 4f51e94b..f2137cd8 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py @@ -26,6 +26,8 @@ def __init__(self, folder_path: str, pattern: str, sampling_frequency: float): self.folder_path = folder_path self.tif_paths = match_paths(folder_path, pattern) - imaging_extractors = [TiffImagingExtractor(file_path=x, sampling_frequency=sampling_frequency) for x in self.tif_paths] + imaging_extractors = [ + TiffImagingExtractor(file_path=x, sampling_frequency=sampling_frequency) for x in self.tif_paths + ] super().__init__(imaging_extractors=imaging_extractors) self._kwargs.update({"folder_path": folder_path}) From 293dc1ce6fc38fadfeb79db935c8dddfce8a5602 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 15 May 2024 21:28:04 -0500 Subject: [PATCH 20/27] add docstring for module --- src/roiextractors/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/roiextractors/utils.py b/src/roiextractors/utils.py index e373112f..5e214172 100644 --- a/src/roiextractors/utils.py +++ b/src/roiextractors/utils.py @@ -1,3 +1,5 @@ +"""Utility functions for the ROIExtractors package.""" + import glob import os from typing import Dict, Any From b47ad48148381dbbbda6c6ae04baf2f7d04ceafa Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 15 May 2024 23:13:21 -0500 Subject: [PATCH 21/27] add check for if no TIFF files were found --- .../tiffimagingextractors/multitiffmultipageimagingextractor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py index f2137cd8..59013ace 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py @@ -26,6 +26,8 @@ def __init__(self, folder_path: str, pattern: str, sampling_frequency: float): self.folder_path = folder_path self.tif_paths = match_paths(folder_path, pattern) + if len(self.tif_paths) == 0: + raise ValueError("No TIFF files found in the folder_path with the given pattern.") imaging_extractors = [ TiffImagingExtractor(file_path=x, sampling_frequency=sampling_frequency) for x in self.tif_paths ] From c54c0521893f94d1e00fa978f988f2d733e88bf4 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 15 May 2024 23:15:14 -0500 Subject: [PATCH 22/27] move multitiffmultipage extractor over to existing tiffimagingextractor.py --- .../tiffimagingextractors/__init__.py | 3 +- .../multitiffmultipageimagingextractor.py | 35 ------------------- .../tiffimagingextractor.py | 31 ++++++++++++++++ 3 files changed, 32 insertions(+), 37 deletions(-) delete mode 100644 src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py diff --git a/src/roiextractors/extractors/tiffimagingextractors/__init__.py b/src/roiextractors/extractors/tiffimagingextractors/__init__.py index a0221111..bf972007 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/__init__.py +++ b/src/roiextractors/extractors/tiffimagingextractors/__init__.py @@ -33,7 +33,7 @@ Specialized extractor for reading TIFF files produced via Micro-Manager. """ -from .tiffimagingextractor import TiffImagingExtractor +from .tiffimagingextractor import TiffImagingExtractor, MultiTiffMultiPageImagingExtractor from .scanimagetiffimagingextractor import ( ScanImageTiffImagingExtractor, ScanImageTiffMultiPlaneImagingExtractor, @@ -43,4 +43,3 @@ ) from .brukertiffimagingextractor import BrukerTiffMultiPlaneImagingExtractor, BrukerTiffSinglePlaneImagingExtractor from .micromanagertiffimagingextractor import MicroManagerTiffImagingExtractor -from .multitiffmultipageimagingextractor import MultiTiffMultiPageImagingExtractor diff --git a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py deleted file mode 100644 index 59013ace..00000000 --- a/src/roiextractors/extractors/tiffimagingextractors/multitiffmultipageimagingextractor.py +++ /dev/null @@ -1,35 +0,0 @@ -"""MultiTiffMultiPageImagingExtractor class.""" - -from .tiffimagingextractor import TiffImagingExtractor -from ...multiimagingextractor import MultiImagingExtractor -from ...utils import match_paths - - -class MultiTiffMultiPageImagingExtractor(MultiImagingExtractor): - """A ImagingExtractor for multiple TIFF files that each have multiple pages.""" - - extractor_name = "multi-tiff multi-page Imaging Extractor" - is_writable = False - - def __init__(self, folder_path: str, pattern: str, sampling_frequency: float): - """Create a MultiTiffMultiPageImagingExtractor instance. - - Parameters - ---------- - folder_path : str - List of path to each TIFF file. - pattern : str - F-string-style pattern to match the TIFF files. - sampling_frequency : float - The frequency at which the frames were sampled, in Hz. - """ - - self.folder_path = folder_path - self.tif_paths = match_paths(folder_path, pattern) - if len(self.tif_paths) == 0: - raise ValueError("No TIFF files found in the folder_path with the given pattern.") - imaging_extractors = [ - TiffImagingExtractor(file_path=x, sampling_frequency=sampling_frequency) for x in self.tif_paths - ] - super().__init__(imaging_extractors=imaging_extractors) - self._kwargs.update({"folder_path": folder_path}) diff --git a/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py index 9ae0a09d..7a4a7865 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py @@ -14,6 +14,7 @@ import numpy as np from tqdm import tqdm +from ... import MultiImagingExtractor from ...imagingextractor import ImagingExtractor from ...extraction_tools import ( PathType, @@ -151,3 +152,33 @@ def write_imaging(imaging, save_path, overwrite: bool = False, chunk_size=None, ) chunk_frames = np.squeeze(video) tif.save(chunk_frames, contiguous=True, metadata=None) + + +class MultiTiffMultiPageImagingExtractor(MultiImagingExtractor): + """A ImagingExtractor for multiple TIFF files that each have multiple pages.""" + + extractor_name = "multi-tiff multi-page Imaging Extractor" + is_writable = False + + def __init__(self, folder_path: str, pattern: str, sampling_frequency: float): + """Create a MultiTiffMultiPageImagingExtractor instance. + + Parameters + ---------- + folder_path : str + List of path to each TIFF file. + pattern : str + F-string-style pattern to match the TIFF files. + sampling_frequency : float + The frequency at which the frames were sampled, in Hz. + """ + + self.folder_path = folder_path + self.tif_paths = match_paths(folder_path, pattern) + if len(self.tif_paths) == 0: + raise ValueError("No TIFF files found in the folder_path with the given pattern.") + imaging_extractors = [ + TiffImagingExtractor(file_path=x, sampling_frequency=sampling_frequency) for x in self.tif_paths + ] + super().__init__(imaging_extractors=imaging_extractors) + self._kwargs.update({"folder_path": folder_path}) \ No newline at end of file From a9153f1c1944b67d60b2f0f3752395858bbf11ee Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 May 2024 04:15:26 +0000 Subject: [PATCH 23/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../extractors/tiffimagingextractors/tiffimagingextractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py index 7a4a7865..7cb35e09 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py @@ -181,4 +181,4 @@ def __init__(self, folder_path: str, pattern: str, sampling_frequency: float): TiffImagingExtractor(file_path=x, sampling_frequency=sampling_frequency) for x in self.tif_paths ] super().__init__(imaging_extractors=imaging_extractors) - self._kwargs.update({"folder_path": folder_path}) \ No newline at end of file + self._kwargs.update({"folder_path": folder_path}) From 5c7b751fbde59be0e2fda2f49256ae24cd4e0805 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Thu, 16 May 2024 11:45:35 -0500 Subject: [PATCH 24/27] make sure match_paths can sub-select --- tests/test_internals/test_utils.py | 56 ++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 tests/test_internals/test_utils.py diff --git a/tests/test_internals/test_utils.py b/tests/test_internals/test_utils.py new file mode 100644 index 00000000..15d8f9dc --- /dev/null +++ b/tests/test_internals/test_utils.py @@ -0,0 +1,56 @@ +import os + +from roiextractors.utils import match_paths +from tempfile import TemporaryDirectory + + +def test_match_paths(): + # create temporary directory + with TemporaryDirectory() as tmpdir: + # create temporary files + files = [ + "split_1.tif", + "split_2.tif", + "split_3.tif", + "split_4.tif", + "split_5.tif", + "split_6.tif", + "split_7.tif", + "split_8.tif", + "split_9.tif", + "split_10.tif", + ] + for file in files: + with open(os.path.join(tmpdir, file), "w") as f: + f.write("") + + # test match_paths + out = match_paths(tmpdir, "split_{split:d}.tif") + assert list(out.keys()) == [os.path.join(tmpdir, x) for x in files] + assert list([x["split"] for x in out.values()]) == list(range(1, 11)) + + +def test_match_paths_sub_select(): + # create temporary directory + with TemporaryDirectory() as tmpdir: + # create temporary files + files = [ + "chanA_split_1.tif", + "chanA_split_2.tif", + "chanA_split_3.tif", + "chanA_split_4.tif", + "chanA_split_5.tif", + "chanB_split_1.tif", + "chanB_split_2.tif", + "chanB_split_3.tif", + "chanB_split_4.tif", + "chanB_split_5.tif", + ] + for file in files: + with open(os.path.join(tmpdir, file), "w") as f: + f.write("") + + # test match_paths + out = match_paths(tmpdir, "chanA_split_{split:d}.tif") + assert list(out.keys()) == [os.path.join(tmpdir, x) for x in files[:5]] + assert list([x["split"] for x in out.values()]) == list(range(1, 6)) From a383ecd6a7c7b6c44ccb305edf988f4eab351b4e Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Fri, 17 May 2024 18:41:14 -0500 Subject: [PATCH 25/27] refactor --- src/roiextractors/extractorlist.py | 6 ++- .../tiffimagingextractors/__init__.py | 2 +- .../tiffimagingextractor.py | 51 ++++++++++++++----- 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/src/roiextractors/extractorlist.py b/src/roiextractors/extractorlist.py index cf7f6525..cc4265c1 100644 --- a/src/roiextractors/extractorlist.py +++ b/src/roiextractors/extractorlist.py @@ -23,7 +23,8 @@ BrukerTiffMultiPlaneImagingExtractor, BrukerTiffSinglePlaneImagingExtractor, MicroManagerTiffImagingExtractor, - MultiTiffMultiPageImagingExtractor, + MultiTiffImagingExtractor, + FolderTiffImagingExtractor, ) from .extractors.sbximagingextractor import SbxImagingExtractor from .extractors.inscopixextractors import InscopixImagingExtractor @@ -52,7 +53,8 @@ NumpyMemmapImagingExtractor, MemmapImagingExtractor, VolumetricImagingExtractor, - MultiTiffMultiPageImagingExtractor, + MultiTiffImagingExtractor, + FolderTiffImagingExtractor, InscopixImagingExtractor, ] diff --git a/src/roiextractors/extractors/tiffimagingextractors/__init__.py b/src/roiextractors/extractors/tiffimagingextractors/__init__.py index bf972007..e16a9eac 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/__init__.py +++ b/src/roiextractors/extractors/tiffimagingextractors/__init__.py @@ -33,7 +33,7 @@ Specialized extractor for reading TIFF files produced via Micro-Manager. """ -from .tiffimagingextractor import TiffImagingExtractor, MultiTiffMultiPageImagingExtractor +from .tiffimagingextractor import TiffImagingExtractor, MultiTiffImagingExtractor, FolderTiffImagingExtractor from .scanimagetiffimagingextractor import ( ScanImageTiffImagingExtractor, ScanImageTiffMultiPlaneImagingExtractor, diff --git a/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py index 7cb35e09..c5ead623 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py @@ -22,6 +22,7 @@ raise_multi_channel_or_depth_not_implemented, get_package, ) +from ...utils import match_paths class TiffImagingExtractor(ImagingExtractor): @@ -154,31 +155,53 @@ def write_imaging(imaging, save_path, overwrite: bool = False, chunk_size=None, tif.save(chunk_frames, contiguous=True, metadata=None) -class MultiTiffMultiPageImagingExtractor(MultiImagingExtractor): +class MultiTiffImagingExtractor(MultiImagingExtractor): """A ImagingExtractor for multiple TIFF files that each have multiple pages.""" extractor_name = "multi-tiff multi-page Imaging Extractor" is_writable = False - def __init__(self, folder_path: str, pattern: str, sampling_frequency: float): - """Create a MultiTiffMultiPageImagingExtractor instance. + def __init__(self, file_paths: list[str], sampling_frequency: float): + """Create a MultiTiffImagingExtractor instance. Parameters ---------- - folder_path : str - List of path to each TIFF file. - pattern : str - F-string-style pattern to match the TIFF files. + file_paths: list of str + List of paths to the TIFF files. sampling_frequency : float The frequency at which the frames were sampled, in Hz. """ - - self.folder_path = folder_path - self.tif_paths = match_paths(folder_path, pattern) - if len(self.tif_paths) == 0: - raise ValueError("No TIFF files found in the folder_path with the given pattern.") + self.file_paths = file_paths imaging_extractors = [ - TiffImagingExtractor(file_path=x, sampling_frequency=sampling_frequency) for x in self.tif_paths + TiffImagingExtractor(file_path=x, sampling_frequency=sampling_frequency) for x in self.file_paths ] super().__init__(imaging_extractors=imaging_extractors) - self._kwargs.update({"folder_path": folder_path}) + self._kwargs.update({"file_paths": file_paths}) + + +class FolderTiffImagingExtractor(MultiTiffImagingExtractor): + """A ImagingExtractor for multiple TIFF files in a folder that each have multiple pages.""" + + extractor_name = "folder-tiff multi-page Imaging Extractor" + is_writable = False + + def __init__(self, folder_path: PathType, pattern: str, sampling_frequency: float): + """Create a FolderTiffImagingExtractor instance. + + Parameters + ---------- + folder_path: PathType + Path to the folder containing the TIFF files. + pattern : str + The f-string pattern to match the TIFF files in the folder. + sampling_frequency : float + The frequency at which the frames were sampled, in Hz. + """ + folder_path = Path(folder_path) + file_paths = match_paths(str(folder_path), pattern) + super().__init__(file_paths=file_paths, sampling_frequency=sampling_frequency) + self._kwargs.update({ + "folder_path": str(folder_path.absolute()), + "pattern": pattern, + }) + From fb936062cc5f964d4a9f24108b2c35d883746f7e Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Fri, 17 May 2024 18:45:57 -0500 Subject: [PATCH 26/27] update tests --- .../tiffimagingextractor.py | 2 +- ...t_multitiff_multipage_imaging_extractor.py | 41 +++++++++++++++++-- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py index c5ead623..5f0043cb 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py @@ -14,7 +14,7 @@ import numpy as np from tqdm import tqdm -from ... import MultiImagingExtractor +from ...multiimagingextractor import MultiImagingExtractor from ...imagingextractor import ImagingExtractor from ...extraction_tools import ( PathType, diff --git a/tests/test_multitiff_multipage_imaging_extractor.py b/tests/test_multitiff_multipage_imaging_extractor.py index 493c64a9..c2554bd3 100644 --- a/tests/test_multitiff_multipage_imaging_extractor.py +++ b/tests/test_multitiff_multipage_imaging_extractor.py @@ -1,10 +1,10 @@ -from roiextractors import MultiTiffMultiPageImagingExtractor +from roiextractors import MultiTiffImagingExtractor, FolderTiffImagingExtractor from tests.setup_paths import OPHYS_DATA_PATH -def test_init_multitiff_multipage_imaging_extractor(): - extractor = MultiTiffMultiPageImagingExtractor( +def test_init_folder_tiff_imaging_extractor_multi_page(): + extractor = FolderTiffImagingExtractor( folder_path=OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "splits", pattern="split_{split:d}.tif", sampling_frequency=1.0, @@ -17,7 +17,7 @@ def test_init_multitiff_multipage_imaging_extractor(): assert extractor.get_dtype() == "uint16" assert extractor.get_image_size() == (60, 80) assert extractor.get_video().shape == (2000, 60, 80) - assert list(extractor.tif_paths.keys()) == [ + assert list(extractor.file_paths) == [ str(OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "splits" / x) for x in ( "split_1.tif", @@ -32,3 +32,36 @@ def test_init_multitiff_multipage_imaging_extractor(): "split_10.tif", ) ] + + +def test_init_multitiff_imaging_extractor_multi_page(): + extractor = MultiTiffImagingExtractor( + file_paths=[ + OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "splits" / f"split_{i}.tif" + for i in range(1, 11) + ], + sampling_frequency=1.0, + ) + + assert extractor.get_num_channels() == 1 + assert extractor.get_num_frames() == 2000 + assert extractor.get_sampling_frequency() == 1.0 + assert extractor.get_channel_names() is None + assert extractor.get_dtype() == "uint16" + assert extractor.get_image_size() == (60, 80) + assert extractor.get_video().shape == (2000, 60, 80) + assert list(extractor.file_paths) == [ + OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "splits" / x + for x in ( + "split_1.tif", + "split_2.tif", + "split_3.tif", + "split_4.tif", + "split_5.tif", + "split_6.tif", + "split_7.tif", + "split_8.tif", + "split_9.tif", + "split_10.tif", + ) + ] From eb5cc8e5af8d3ed913f9f520b0809559b5a571ea Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 May 2024 23:46:10 +0000 Subject: [PATCH 27/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../tiffimagingextractors/tiffimagingextractor.py | 11 ++++++----- tests/test_multitiff_multipage_imaging_extractor.py | 5 +---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py index 5f0043cb..d3e4b120 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py @@ -200,8 +200,9 @@ def __init__(self, folder_path: PathType, pattern: str, sampling_frequency: floa folder_path = Path(folder_path) file_paths = match_paths(str(folder_path), pattern) super().__init__(file_paths=file_paths, sampling_frequency=sampling_frequency) - self._kwargs.update({ - "folder_path": str(folder_path.absolute()), - "pattern": pattern, - }) - + self._kwargs.update( + { + "folder_path": str(folder_path.absolute()), + "pattern": pattern, + } + ) diff --git a/tests/test_multitiff_multipage_imaging_extractor.py b/tests/test_multitiff_multipage_imaging_extractor.py index c2554bd3..51f3e6a5 100644 --- a/tests/test_multitiff_multipage_imaging_extractor.py +++ b/tests/test_multitiff_multipage_imaging_extractor.py @@ -36,10 +36,7 @@ def test_init_folder_tiff_imaging_extractor_multi_page(): def test_init_multitiff_imaging_extractor_multi_page(): extractor = MultiTiffImagingExtractor( - file_paths=[ - OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "splits" / f"split_{i}.tif" - for i in range(1, 11) - ], + file_paths=[OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "splits" / f"split_{i}.tif" for i in range(1, 11)], sampling_frequency=1.0, )