From ab0451b91bc68d8920579ad5a722e90196bdd365 Mon Sep 17 00:00:00 2001 From: eftychios pnevmatikakis Date: Fri, 9 Aug 2019 09:28:23 -0400 Subject: [PATCH 01/16] Update params.py --- caiman/source_extraction/cnmf/params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caiman/source_extraction/cnmf/params.py b/caiman/source_extraction/cnmf/params.py index 9b89134cb..416fda761 100644 --- a/caiman/source_extraction/cnmf/params.py +++ b/caiman/source_extraction/cnmf/params.py @@ -560,7 +560,7 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), 'decay_time': decay_time, 'dxy': dxy, 'var_name_hdf5': var_name_hdf5, - 'caiman_version': '1.5.2', + 'caiman_version': '1.5.3', 'last_commit': None, } From f513bc3ba32d3c7f32a4cd73a34cb7cbf884e53d Mon Sep 17 00:00:00 2001 From: HagaiHargil Date: Tue, 13 Aug 2019 10:48:57 +0300 Subject: [PATCH 02/16] Remove mutable argument from func def --- caiman/source_extraction/cnmf/cnmf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/caiman/source_extraction/cnmf/cnmf.py b/caiman/source_extraction/cnmf/cnmf.py index d42d7622e..f36cc0bca 100644 --- a/caiman/source_extraction/cnmf/cnmf.py +++ b/caiman/source_extraction/cnmf/cnmf.py @@ -302,7 +302,7 @@ def __init__(self, n_processes, k=5, gSig=[4, 4], gSiz=None, merge_thresh=0.8, p self.estimates = Estimates(A=Ain, C=Cin, b=b_in, f=f_in, dims=self.params.data['dims']) - def fit_file(self, motion_correct=False, indices=[slice(None)]*2): + def fit_file(self, motion_correct=False, indices=None): """ This method packages the analysis pipeline (motion correction, memory mapping, patch based CNMF processing) in a single method that can be @@ -320,6 +320,8 @@ def fit_file(self, motion_correct=False, indices=[slice(None)]*2): Returns: cnmf object with the current estimates """ + if indices is None: + indices = [slice(None)] * 2 fnames = self.params.get('data', 'fnames') if os.path.exists(fnames[0]): _, extension = os.path.splitext(fnames[0])[:2] From 27969adeb60a676d0589d3ce4576978f9cc63d66 Mon Sep 17 00:00:00 2001 From: HagaiHargil Date: Tue, 13 Aug 2019 11:16:53 +0300 Subject: [PATCH 03/16] Convert list slices to tuples --- caiman/source_extraction/cnmf/cnmf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/caiman/source_extraction/cnmf/cnmf.py b/caiman/source_extraction/cnmf/cnmf.py index f36cc0bca..3cfd5dd02 100644 --- a/caiman/source_extraction/cnmf/cnmf.py +++ b/caiman/source_extraction/cnmf/cnmf.py @@ -321,7 +321,7 @@ def fit_file(self, motion_correct=False, indices=None): cnmf object with the current estimates """ if indices is None: - indices = [slice(None)] * 2 + indices = (slice(None), slice(None)) fnames = self.params.get('data', 'fnames') if os.path.exists(fnames[0]): _, extension = os.path.splitext(fnames[0])[:2] @@ -380,7 +380,7 @@ def refit(self, images, dview=None): cnm.mmap_file = self.mmap_file return cnm.fit(images) - def fit(self, images, indices=[slice(None), slice(None)]): + def fit(self, images, indices=(slice(None), slice(None))): """ This method uses the cnmf algorithm to find sources in data. it is calling every function from the cnmf folder From 628338164ec7ea6dacb0da1e41ff59658cd958f0 Mon Sep 17 00:00:00 2001 From: HagaiHargil Date: Tue, 13 Aug 2019 11:18:13 +0300 Subject: [PATCH 04/16] Ignore vscode's base folder --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 73ea7a621..db8d64aa3 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,7 @@ Icon ._* *.log *.err -*.*.out +*.*.out # Files that might appear in the root of a volume .DocumentRevisions-V100 @@ -75,6 +75,7 @@ wheels/ *.egg-info/ .installed.cfg *.egg +.vscode/ # PyInstaller # Usually these files are written by a python script from a template From 4e6c869cccf8f19a700651dba5d3e73ff0004efd Mon Sep 17 00:00:00 2001 From: HagaiHargil Date: Tue, 13 Aug 2019 16:34:14 +0300 Subject: [PATCH 05/16] Added mmapping tests --- caiman/mmapping.py | 44 +++++++++++++++++---------------- caiman/tests/test_memmapping.py | 22 +++++++++++++++++ 2 files changed, 45 insertions(+), 21 deletions(-) create mode 100644 caiman/tests/test_memmapping.py diff --git a/caiman/mmapping.py b/caiman/mmapping.py index d85f3c3df..0564b74a3 100644 --- a/caiman/mmapping.py +++ b/caiman/mmapping.py @@ -22,6 +22,7 @@ import sys import tifffile from typing import Any, Dict, List, Optional, Tuple, Union +import pathlib import caiman as cm from caiman.paths import memmap_frames_filename @@ -55,26 +56,27 @@ def load_memmap(filename:str, mode:str='r') -> Tuple[Any,Tuple,int]: Raises: - Exception "Unknown file extension" + ValueError "Unknown file extension" """ - if ('.mmap' in filename): - # Strip path components and use CAIMAN_DATA/example_movies - # TODO: Eventually get the code to save these in a different dir - file_to_load = filename - filename = os.path.split(filename)[-1] - fpart = filename.split('_')[1:-1] # The filename encodes the structure of the map - d1, d2, d3, T, order = int(fpart[-9]), int(fpart[-7] - ), int(fpart[-5]), int(fpart[-1]), fpart[-3] - Yr = np.memmap(file_to_load, mode=mode, shape=prepare_shape(( - d1 * d2 * d3, T)), dtype=np.float32, order=order) - if d3 == 1: - return (Yr, (d1, d2), T) - else: - return (Yr, (d1, d2, d3), T) - else: + if pathlib.Path(filename).suffix != '.mmap': logging.error("Unknown extension for file " + str(filename)) - raise Exception('Unknown file extension (should be .mmap)') + raise ValueError('Unknown file extension (should be .mmap)') + + # Strip path components and use CAIMAN_DATA/example_movies + # TODO: Eventually get the code to save these in a different dir + file_to_load = filename + filename = os.path.split(filename)[-1] + fpart = filename.split('_')[1:-1] # The filename encodes the structure of the map + d1, d2, d3, T, order = int(fpart[-9]), int(fpart[-7] + ), int(fpart[-5]), int(fpart[-1]), fpart[-3] + Yr = np.memmap(file_to_load, mode=mode, shape=prepare_shape(( + d1 * d2 * d3, T)), dtype=np.float32, order=order) + if d3 == 1: + return (Yr, (d1, d2), T) + else: + return (Yr, (d1, d2, d3), T) + #%% def save_memmap_each(fnames:List[str], dview=None, base_name:str=None, resize_fact=(1, 1, 1), remove_init:int=0, @@ -353,9 +355,9 @@ def save_memmap(filenames:List[str], base_name:str='Yr', resize_fact:Tuple=(1, 1 slices: slice object or list of slice objects slice can be used to select portion of the movies in time and x,y - directions. For instance - slices = [slice(0,200),slice(0,100),slice(0,100)] will take - the first 200 frames and the 100 pixels along x and y dimensions. + directions. For instance + slices = [slice(0,200),slice(0,100),slice(0,100)] will take + the first 200 frames and the 100 pixels along x and y dimensions. Returns: fname_new: the name of the mapped file, the format is such that the name will contain the frame dimensions and the number of frames @@ -441,7 +443,7 @@ def save_memmap(filenames:List[str], base_name:str='Yr', resize_fact:Tuple=(1, 1 Yr = np.array(Yr)[remove_init:, idx_xy[0], idx_xy[1], idx_xy[2]] if border_to_0 > 0: - if slices is not None: + if slices is not None: if type(slices) is list: raise Exception('You cannot slice in x and y and then use add_to_movie: if you only want to slice in time do not pass in a list but just a slice object') diff --git a/caiman/tests/test_memmapping.py b/caiman/tests/test_memmapping.py new file mode 100644 index 000000000..418577bc0 --- /dev/null +++ b/caiman/tests/test_memmapping.py @@ -0,0 +1,22 @@ +import pathlib + +from caiman import mmapping +from caiman.paths import caiman_datadir + +def test_load_raises_wrong_ext(): + fname = "a.mmapp" + try: + mmapping.load_memmap(fname) + except ValueError: + assert True + else: + assert False + +def test_load_raises_multiple_ext(): + fname = "a.mmap.mma" + try: + mmapping.load_memmap(fname) + except ValueError: + assert True + else: + assert False From 26438a662eec3bb0cf7400f12631989c0e16784d Mon Sep 17 00:00:00 2001 From: HagaiHargil Date: Tue, 13 Aug 2019 16:34:36 +0300 Subject: [PATCH 06/16] Test modification --- caiman/tests/test_memmapping.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/caiman/tests/test_memmapping.py b/caiman/tests/test_memmapping.py index 418577bc0..c2097a986 100644 --- a/caiman/tests/test_memmapping.py +++ b/caiman/tests/test_memmapping.py @@ -1,5 +1,3 @@ -import pathlib - from caiman import mmapping from caiman.paths import caiman_datadir From 21e7b65919f8d3784c84272bd101e3d992f8f813 Mon Sep 17 00:00:00 2001 From: HagaiHargil Date: Tue, 13 Aug 2019 16:35:02 +0300 Subject: [PATCH 07/16] Removed unnecessary import --- caiman/tests/test_memmapping.py | 1 - 1 file changed, 1 deletion(-) diff --git a/caiman/tests/test_memmapping.py b/caiman/tests/test_memmapping.py index c2097a986..f4c0e1408 100644 --- a/caiman/tests/test_memmapping.py +++ b/caiman/tests/test_memmapping.py @@ -1,5 +1,4 @@ from caiman import mmapping -from caiman.paths import caiman_datadir def test_load_raises_wrong_ext(): fname = "a.mmapp" From 5828581782c3859ad75098fa7f61fdea2b12d507 Mon Sep 17 00:00:00 2001 From: HagaiHargil Date: Tue, 13 Aug 2019 18:12:09 +0300 Subject: [PATCH 08/16] Commented out a possible bug --- caiman/source_extraction/cnmf/cnmf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/caiman/source_extraction/cnmf/cnmf.py b/caiman/source_extraction/cnmf/cnmf.py index 3cfd5dd02..562384503 100644 --- a/caiman/source_extraction/cnmf/cnmf.py +++ b/caiman/source_extraction/cnmf/cnmf.py @@ -347,7 +347,9 @@ def fit_file(self, motion_correct=False, indices=None): else: b0 = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int) self.estimates.shifts = mc.shifts_rig - b0 = 0 if self.params.get('motion', 'border_nan') is 'copy' else 0 + # FIXME Huh? + # b0 = 0 if self.params.get('motion', 'border_nan') is 'copy' else 0 + b0 = 0 fname_new = mmapping.save_memmap(fname_mc, base_name='memmap_', order='C', border_to_0=b0) else: From 1c036a1ff41affcdff062431e448825104e9e78a Mon Sep 17 00:00:00 2001 From: HagaiHargil Date: Wed, 14 Aug 2019 09:12:21 +0300 Subject: [PATCH 09/16] Fixed merge artifact --- caiman/mmapping.py | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/caiman/mmapping.py b/caiman/mmapping.py index d11067b03..c052f2778 100644 --- a/caiman/mmapping.py +++ b/caiman/mmapping.py @@ -35,6 +35,7 @@ def prepare_shape(mytuple: Tuple) -> Tuple: return tuple(map(lambda x: np.uint64(x), mytuple)) + #%% def load_memmap(filename: str, mode: str = 'r') -> Tuple[Any, Tuple, int]: """ Load a memory mapped file created by the function save_memmap @@ -60,38 +61,22 @@ def load_memmap(filename: str, mode: str = 'r') -> Tuple[Any, Tuple, int]: ValueError "Unknown file extension" """ - if ('.mmap' in filename): - # Strip path components and use CAIMAN_DATA/example_movies - # TODO: Eventually get the code to save these in a different dir - file_to_load = filename - filename = os.path.split(filename)[-1] - fpart = filename.split('_')[1:-1] # The filename encodes the structure of the map - d1, d2, d3, T, order = int(fpart[-9]), int(fpart[-7]), int(fpart[-5]), int(fpart[-1]), fpart[-3] - Yr = np.memmap(file_to_load, mode=mode, shape=prepare_shape((d1 * d2 * d3, T)), dtype=np.float32, order=order) - if d3 == 1: - return (Yr, (d1, d2), T) - else: - return (Yr, (d1, d2, d3), T) - else: + if pathlib.Path(filename).suffix != '.mmap': logging.error("Unknown extension for file " + str(filename)) raise ValueError('Unknown file extension (should be .mmap)') - # Strip path components and use CAIMAN_DATA/example_movies # TODO: Eventually get the code to save these in a different dir file_to_load = filename filename = os.path.split(filename)[-1] - fpart = filename.split('_')[1:-1] # The filename encodes the structure of the map - d1, d2, d3, T, order = int(fpart[-9]), int(fpart[-7] - ), int(fpart[-5]), int(fpart[-1]), fpart[-3] - Yr = np.memmap(file_to_load, mode=mode, shape=prepare_shape(( - d1 * d2 * d3, T)), dtype=np.float32, order=order) + fpart = filename.split('_')[1:-1] # The filename encodes the structure of the map + d1, d2, d3, T, order = int(fpart[-9]), int(fpart[-7]), int(fpart[-5]), int(fpart[-1]), fpart[-3] + Yr = np.memmap(file_to_load, mode=mode, shape=prepare_shape((d1 * d2 * d3, T)), dtype=np.float32, order=order) if d3 == 1: return (Yr, (d1, d2), T) else: return (Yr, (d1, d2, d3), T) - #%% def save_memmap_each(fnames: List[str], dview=None, From 0a832a933e0c2a2ab97f87c939f051f5977d166d Mon Sep 17 00:00:00 2001 From: HagaiHargil Date: Wed, 14 Aug 2019 09:43:59 +0300 Subject: [PATCH 10/16] Fix tupled indices --- caiman/source_extraction/cnmf/cnmf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/caiman/source_extraction/cnmf/cnmf.py b/caiman/source_extraction/cnmf/cnmf.py index 085da36a8..af8dd39cb 100644 --- a/caiman/source_extraction/cnmf/cnmf.py +++ b/caiman/source_extraction/cnmf/cnmf.py @@ -408,6 +408,8 @@ def fit(self, images, indices=(slice(None), slice(None))): # Todo : to compartment if isinstance(indices, slice): indices = [indices] + if isinstance(indices, tuple): + indices = list(indices) indices = [slice(None)] + indices if len(indices) < len(images.shape): indices = indices + [slice(None)]*(len(images.shape) - len(indices)) From 6e5579b7f57cb807796d8c12bc7da6421148f439 Mon Sep 17 00:00:00 2001 From: HagaiHargil Date: Wed, 14 Aug 2019 10:38:53 +0300 Subject: [PATCH 11/16] Bug fixes and flag for comp eval --- caiman/source_extraction/cnmf/cnmf.py | 46 +++++++++++++++++++++------ 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/caiman/source_extraction/cnmf/cnmf.py b/caiman/source_extraction/cnmf/cnmf.py index af8dd39cb..7c27ce586 100644 --- a/caiman/source_extraction/cnmf/cnmf.py +++ b/caiman/source_extraction/cnmf/cnmf.py @@ -32,6 +32,7 @@ import psutil import scipy import sys +import glob from .estimates import Estimates from .initialization import initialize_components, compute_W @@ -46,6 +47,8 @@ from ...components_evaluation import estimate_components_quality from ...motion_correction import MotionCorrect from ...utils.utils import save_dict_to_hdf5, load_dict_from_hdf5 +from caiman import summary_images +from caiman import cluster try: cv2.setNumThreads(0) @@ -302,21 +305,23 @@ def __init__(self, n_processes, k=5, gSig=[4, 4], gSiz=None, merge_thresh=0.8, p self.estimates = Estimates(A=Ain, C=Cin, b=b_in, f=f_in, dims=self.params.data['dims']) - def fit_file(self, motion_correct=False, indices=None): + def fit_file(self, motion_correct=False, indices=None, include_eval=False): """ This method packages the analysis pipeline (motion correction, memory - mapping, patch based CNMF processing) in a single method that can be - called on a specific (sequence of) file(s). It is assumed that the CNMF - object already contains a params object where the location of the files - and all the relevant parameters have been specified. The method does - not perform the quality evaluation step. Consult demo_pipeline for an - example. + mapping, patch based CNMF processing and component evaluation) in a + single method that can be called on a specific (sequence of) file(s). + It is assumed that the CNMF object already contains a params object + where the location of the files and all the relevant parameters have + been specified. The method will perform the last step, i.e. component + evaluation, if the flag "include_eval" is set to `True`. Args: motion_correct (bool) flag for performing motion correction indices (list of slice objects) perform analysis only on a part of the FOV + include_eval (bool) + flag for performing component evaluation Returns: cnmf object with the current estimates """ @@ -347,7 +352,6 @@ def fit_file(self, motion_correct=False, indices=None): else: b0 = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int) self.estimates.shifts = mc.shifts_rig - # FIXME Huh? # b0 = 0 if self.params.get('motion', 'border_nan') is 'copy' else 0 b0 = 0 fname_new = mmapping.save_memmap(fname_mc, base_name='memmap_', order='C', @@ -359,7 +363,31 @@ def fit_file(self, motion_correct=False, indices=None): images = np.reshape(Yr.T, [T] + list(dims), order='F') self.mmap_file = fname_new - return self.fit(images, indices=indices) + if not include_eval: + return self.fit(images, indices=indices) + + fit_cnm = self.fit(images, indices=indices) + Cn = summary_images.local_correlations(images, swap_dim=False) + Cn[np.isnan(Cn)] = 0 + fit_cnm.save(fname_new[:-5]+'_init.hdf5') + fit_cnm.params.change_params({'p': self.params.get('preprocess', 'p')}) + # %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution + cnm2 = fit_cnm.refit(images, dview=self.dview) + cnm2.estimates.evaluate_components(images, cnm2.params, dview=self.dview) + #%% update object with selected components + cnm2.estimates.select_components(use_object=True) + #%% Extract DF/F values + cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250) + cnm2.estimates.Cn = Cn + cnm2.save(cnm2.mmap_file[:-4] + 'hdf5') + + cluster.stop_server(dview=self.dview) + log_files = glob.glob('*_LOG_*') + for log_file in log_files: + os.remove(log_file) + + return cnm2 + def refit(self, images, dview=None): """ From 673bc61b4114188cb92c650be77640d3386209b7 Mon Sep 17 00:00:00 2001 From: HagaiHargil Date: Wed, 14 Aug 2019 11:39:35 +0300 Subject: [PATCH 12/16] More memmapping tests --- caiman/tests/test_memmapping.py | 24 ++++++++++++++++++ ..._d1_10_d2_11_d3_13_order_F_frames_12_.mmap | Bin 0 -> 68640 bytes ...__d1_10_d2_11_d3_1_order_F_frames_12_.mmap | Bin 0 -> 5280 bytes 3 files changed, 24 insertions(+) create mode 100644 testdata/memmap__d1_10_d2_11_d3_13_order_F_frames_12_.mmap create mode 100644 testdata/memmap__d1_10_d2_11_d3_1_order_F_frames_12_.mmap diff --git a/caiman/tests/test_memmapping.py b/caiman/tests/test_memmapping.py index f4c0e1408..5a57a3a1e 100644 --- a/caiman/tests/test_memmapping.py +++ b/caiman/tests/test_memmapping.py @@ -1,4 +1,10 @@ +import pathlib + +import numpy as np + from caiman import mmapping +from caiman.paths import caiman_datadir + def test_load_raises_wrong_ext(): fname = "a.mmapp" @@ -9,6 +15,7 @@ def test_load_raises_wrong_ext(): else: assert False + def test_load_raises_multiple_ext(): fname = "a.mmap.mma" try: @@ -17,3 +24,20 @@ def test_load_raises_multiple_ext(): assert True else: assert False + + +def test_load_successful_2d(): + fname = pathlib.Path(caiman_datadir()) / 'testdata' / 'memmap__d1_10_d2_11_d3_1_order_F_frames_12_.mmap' + Yr, (d1, d2), T = mmapping.load_memmap(str(fname)) + assert (d1, d2) == (10, 11) + assert T == 12 + + +def test_load_successful_3d(): + fname = pathlib.Path(caiman_datadir()) / 'testdata' / 'memmap__d1_10_d2_11_d3_13_order_F_frames_12_.mmap' + Yr, (d1, d2, d3), T = mmapping.load_memmap(str(fname)) + assert (d1, d2, d3) == (10, 11, 13) + assert T == 12 + assert isinstance(Yr, np.memmap) + + diff --git a/testdata/memmap__d1_10_d2_11_d3_13_order_F_frames_12_.mmap b/testdata/memmap__d1_10_d2_11_d3_13_order_F_frames_12_.mmap new file mode 100644 index 0000000000000000000000000000000000000000..7a6d92370ceb0ea96989dfc4c38efd8c0bb9b0fa GIT binary patch literal 68640 zcmeIuF#!Mo0K%a4Pwi<5h(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r K3>YwA-~$5;F8~1m literal 0 HcmV?d00001 diff --git a/testdata/memmap__d1_10_d2_11_d3_1_order_F_frames_12_.mmap b/testdata/memmap__d1_10_d2_11_d3_1_order_F_frames_12_.mmap new file mode 100644 index 0000000000000000000000000000000000000000..e14df0c135783a7a3553d2b2c34aee87fe3589ae GIT binary patch literal 5280 ucmeIuF#!Mo0K%a4Pi+hzh(KY$fB^#r3>YwAz<>b*1`HT5V8DQZ&%gi_pa1~? literal 0 HcmV?d00001 From a2caa3556dc16aa4bf5113a756b48694b35c7eaa Mon Sep 17 00:00:00 2001 From: HagaiHargil Date: Wed, 14 Aug 2019 12:46:02 +0300 Subject: [PATCH 13/16] fix mmap test files not showing --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index db8d64aa3..7cf6d9ddc 100644 --- a/.gitignore +++ b/.gitignore @@ -97,6 +97,7 @@ nosetests.xml coverage.xml *,cover .hypothesis/ +!testdata/*.mmap # Translations *.mo From eb47becad1d4b83ddb49db63504190709f605ffb Mon Sep 17 00:00:00 2001 From: HagaiHargil Date: Thu, 15 Aug 2019 10:24:10 +0300 Subject: [PATCH 14/16] Returned b0 behavior and added better basename --- caiman/source_extraction/cnmf/cnmf.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/caiman/source_extraction/cnmf/cnmf.py b/caiman/source_extraction/cnmf/cnmf.py index 7c27ce586..7ed8dd51c 100644 --- a/caiman/source_extraction/cnmf/cnmf.py +++ b/caiman/source_extraction/cnmf/cnmf.py @@ -33,6 +33,7 @@ import scipy import sys import glob +import pathlib from .estimates import Estimates from .initialization import initialize_components, compute_W @@ -335,6 +336,7 @@ def fit_file(self, motion_correct=False, indices=None, include_eval=False): logging.warning("Error: File not found, with file list:\n" + fnames[0]) raise Exception('File not found!') + base_name = pathlib.Path(fnames[0]).stem + "_memmap_" if extension == '.mmap': fname_new = fnames[0] Yr, dims, T = mmapping.load_memmap(fnames[0]) @@ -352,13 +354,16 @@ def fit_file(self, motion_correct=False, indices=None, include_eval=False): else: b0 = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int) self.estimates.shifts = mc.shifts_rig + # TODO - b0 is currently direction inspecific, which can cause + # sub-optimal behavior. See + # https://github.com/flatironinstitute/CaImAn/pull/618#discussion_r313960370 + # for further details. # b0 = 0 if self.params.get('motion', 'border_nan') is 'copy' else 0 b0 = 0 - fname_new = mmapping.save_memmap(fname_mc, base_name='memmap_', order='C', + fname_new = mmapping.save_memmap(fname_mc, base_name=base_name, order='C', border_to_0=b0) else: - fname_new = mmapping.save_memmap(fnames, base_name='memmap_', order='C') - # now load the file + fname_new = mmapping.save_memmap(fnames, base_name=base_name, order='C') Yr, dims, T = mmapping.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') From 31b3359f66790cfc49dfd7e71523a138697e2be8 Mon Sep 17 00:00:00 2001 From: HagaiHargil Date: Thu, 15 Aug 2019 11:03:58 +0300 Subject: [PATCH 15/16] Create memmapped files on the fly --- .gitignore | 1 - caiman/tests/test_memmapping.py | 24 +++++++++++++++--- ..._d1_10_d2_11_d3_13_order_F_frames_12_.mmap | Bin 68640 -> 0 bytes ...__d1_10_d2_11_d3_1_order_F_frames_12_.mmap | Bin 5280 -> 0 bytes 4 files changed, 21 insertions(+), 4 deletions(-) delete mode 100644 testdata/memmap__d1_10_d2_11_d3_13_order_F_frames_12_.mmap delete mode 100644 testdata/memmap__d1_10_d2_11_d3_1_order_F_frames_12_.mmap diff --git a/.gitignore b/.gitignore index 7cf6d9ddc..db8d64aa3 100644 --- a/.gitignore +++ b/.gitignore @@ -97,7 +97,6 @@ nosetests.xml coverage.xml *,cover .hypothesis/ -!testdata/*.mmap # Translations *.mo diff --git a/caiman/tests/test_memmapping.py b/caiman/tests/test_memmapping.py index 5a57a3a1e..4207319aa 100644 --- a/caiman/tests/test_memmapping.py +++ b/caiman/tests/test_memmapping.py @@ -1,10 +1,13 @@ import pathlib import numpy as np +import nose from caiman import mmapping from caiman.paths import caiman_datadir +twod_fname = pathlib.Path(caiman_datadir()) / 'testdata/memmap__d1_10_d2_11_d3_1_order_F_frames_12_.mmap' +threed_fname = pathlib.Path(caiman_datadir()) / 'testdata/memmap__d1_10_d2_11_d3_13_order_F_frames_12_.mmap' def test_load_raises_wrong_ext(): fname = "a.mmapp" @@ -26,15 +29,30 @@ def test_load_raises_multiple_ext(): assert False +def setup_2d_mmap(): + np.memmap(twod_fname, mode='w+', dtype=np.float32, shape=(12, 10, 11, 13), order='F') + +def teardown_2d_mmap(): + twod_fname.unlink() + +def setup_3d_mmap(): + np.memmap(threed_fname, mode='w+', dtype=np.float32, shape=(12, 10, 11, 13), order='F') + +def teardown_3d_mmap(): + threed_fname.unlink() + + +@nose.with_setup(setup_2d_mmap, teardown_2d_mmap) def test_load_successful_2d(): - fname = pathlib.Path(caiman_datadir()) / 'testdata' / 'memmap__d1_10_d2_11_d3_1_order_F_frames_12_.mmap' + fname = pathlib.Path(caiman_datadir()) / 'testdata' / twod_fname Yr, (d1, d2), T = mmapping.load_memmap(str(fname)) assert (d1, d2) == (10, 11) assert T == 12 + assert isinstance(Yr, np.memmap) - +@nose.with_setup(setup_3d_mmap, teardown_3d_mmap) def test_load_successful_3d(): - fname = pathlib.Path(caiman_datadir()) / 'testdata' / 'memmap__d1_10_d2_11_d3_13_order_F_frames_12_.mmap' + fname = pathlib.Path(caiman_datadir()) / 'testdata' / threed_fname Yr, (d1, d2, d3), T = mmapping.load_memmap(str(fname)) assert (d1, d2, d3) == (10, 11, 13) assert T == 12 diff --git a/testdata/memmap__d1_10_d2_11_d3_13_order_F_frames_12_.mmap b/testdata/memmap__d1_10_d2_11_d3_13_order_F_frames_12_.mmap deleted file mode 100644 index 7a6d92370ceb0ea96989dfc4c38efd8c0bb9b0fa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 68640 zcmeIuF#!Mo0K%a4Pwi<5h(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r K3>YwA-~$5;F8~1m diff --git a/testdata/memmap__d1_10_d2_11_d3_1_order_F_frames_12_.mmap b/testdata/memmap__d1_10_d2_11_d3_1_order_F_frames_12_.mmap deleted file mode 100644 index e14df0c135783a7a3553d2b2c34aee87fe3589ae..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5280 ucmeIuF#!Mo0K%a4Pi+hzh(KY$fB^#r3>YwAz<>b*1`HT5V8DQZ&%gi_pa1~? From e3df6a10d6d1079236ec916dfb49666f10de207d Mon Sep 17 00:00:00 2001 From: HagaiHargil Date: Thu, 15 Aug 2019 11:10:51 +0300 Subject: [PATCH 16/16] improved test readability --- caiman/tests/test_memmapping.py | 34 +++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/caiman/tests/test_memmapping.py b/caiman/tests/test_memmapping.py index 4207319aa..6a3e6df9f 100644 --- a/caiman/tests/test_memmapping.py +++ b/caiman/tests/test_memmapping.py @@ -6,8 +6,16 @@ from caiman import mmapping from caiman.paths import caiman_datadir -twod_fname = pathlib.Path(caiman_datadir()) / 'testdata/memmap__d1_10_d2_11_d3_1_order_F_frames_12_.mmap' -threed_fname = pathlib.Path(caiman_datadir()) / 'testdata/memmap__d1_10_d2_11_d3_13_order_F_frames_12_.mmap' + +TWO_D_FNAME = ( + pathlib.Path(caiman_datadir()) + / "testdata/memmap__d1_10_d2_11_d3_1_order_F_frames_12_.mmap" +) +THREE_D_FNAME = ( + pathlib.Path(caiman_datadir()) + / "testdata/memmap__d1_10_d2_11_d3_13_order_F_frames_12_.mmap" +) + def test_load_raises_wrong_ext(): fname = "a.mmapp" @@ -30,32 +38,38 @@ def test_load_raises_multiple_ext(): def setup_2d_mmap(): - np.memmap(twod_fname, mode='w+', dtype=np.float32, shape=(12, 10, 11, 13), order='F') + np.memmap( + TWO_D_FNAME, mode="w+", dtype=np.float32, shape=(12, 10, 11, 13), order="F" + ) + def teardown_2d_mmap(): - twod_fname.unlink() + TWO_D_FNAME.unlink() + def setup_3d_mmap(): - np.memmap(threed_fname, mode='w+', dtype=np.float32, shape=(12, 10, 11, 13), order='F') + np.memmap( + THREE_D_FNAME, mode="w+", dtype=np.float32, shape=(12, 10, 11, 13), order="F" + ) + def teardown_3d_mmap(): - threed_fname.unlink() + THREE_D_FNAME.unlink() @nose.with_setup(setup_2d_mmap, teardown_2d_mmap) def test_load_successful_2d(): - fname = pathlib.Path(caiman_datadir()) / 'testdata' / twod_fname + fname = pathlib.Path(caiman_datadir()) / "testdata" / TWO_D_FNAME Yr, (d1, d2), T = mmapping.load_memmap(str(fname)) assert (d1, d2) == (10, 11) assert T == 12 assert isinstance(Yr, np.memmap) + @nose.with_setup(setup_3d_mmap, teardown_3d_mmap) def test_load_successful_3d(): - fname = pathlib.Path(caiman_datadir()) / 'testdata' / threed_fname + fname = pathlib.Path(caiman_datadir()) / "testdata" / THREE_D_FNAME Yr, (d1, d2, d3), T = mmapping.load_memmap(str(fname)) assert (d1, d2, d3) == (10, 11, 13) assert T == 12 assert isinstance(Yr, np.memmap) - -