From 05498fe535e3c1d3b9274183e903c65e63c4f5f1 Mon Sep 17 00:00:00 2001 From: Ali Khan Date: Tue, 1 Oct 2024 21:02:44 -0400 Subject: [PATCH] back to reading whole zstack tif at once --- workflow/rules/import.smk | 2 +- workflow/scripts/tif_to_zarr_gcs.py | 13 ++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/workflow/rules/import.smk b/workflow/rules/import.smk index 8a3f320..3906b36 100644 --- a/workflow/rules/import.smk +++ b/workflow/rules/import.smk @@ -280,7 +280,7 @@ rule tif_to_zarr_gcs: ), group: "preproc" - threads: 16 #config["cores_per_rule"] + threads: config["cores_per_rule"] container: config["containers"]["spimprep"] script: diff --git a/workflow/scripts/tif_to_zarr_gcs.py b/workflow/scripts/tif_to_zarr_gcs.py index ef454d2..9e69ee9 100644 --- a/workflow/scripts/tif_to_zarr_gcs.py +++ b/workflow/scripts/tif_to_zarr_gcs.py @@ -33,8 +33,8 @@ def read_tiff_slice(fs,gcs_uri, key=0): with fs.open(gcs_uri, 'rb') as file: return tifffile.imread(file, key=key) -def read_page_as_numpy(tif_file_uri, page, fs): - """Gets a single page (i.e., 2D image) from a tif file z-stack stored in a cloud URI.""" +def read_stack_as_numpy(tif_file_uri, fs): + """Gets the full stack (i.e., 3D image) from a tif file z-stack stored in a cloud URI.""" # Open the file from the cloud storage using fsspec with fs.open(tif_file_uri, 'rb') as f: @@ -42,7 +42,7 @@ def read_page_as_numpy(tif_file_uri, page, fs): file_buffer = f.read() # Use pyvips to read from the buffer - return pyvips.Image.new_from_buffer(file_buffer, "", page=page).numpy() + return pyvips.Image.new_from_buffer(file_buffer, "").numpy() @@ -122,12 +122,7 @@ def build_zstack_from_single(gcs_uri,zstack_metadata,fs): tif_file = in_tif_pattern.format(tilex=tilex,tiley=tiley,prefix=metadata['prefixes'][0],channel=channel) - pages=[] - #read each page - for i_z in range(size_z): - pages.append(da.from_delayed(delayed(read_page_as_numpy)('gcs://'+tif_file,i_z,fs),shape=(size_y,size_x),dtype='uint16')) - - zstacks.append(da.stack(pages)) + zstacks.append(da.from_delayed(delayed(read_page_as_numpy)('gcs://'+tif_file,fs),shape=(size_z,size_y,size_x),dtype='uint16')) else: zstacks.append(build_zstack(fs.glob('gcs://'+in_tif_glob.format(tilex=tilex,tiley=tiley,prefix=metadata['prefixes'][0],channel=channel,zslice='*')),fs=fs))