Skip to content

Commit

Permalink
set chunking in bdv n5 (default 32x32x32)
Browse files Browse the repository at this point in the history
To hopefully address performance issues in the bigstitcher step when
dealing with large datasets.

Also sets dtype to int16 instead of uint16 for this only -- some imagesc
posts describing the black bars had this as a fix..
  • Loading branch information
akhanf committed Sep 3, 2024
1 parent 8c9ec37 commit 1b1afc4
Show file tree
Hide file tree
Showing 3 changed files with 6 additions and 4 deletions.
2 changes: 1 addition & 1 deletion workflow/rules/bigstitcher.smk
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ rule zarr_to_bdv:
)
/ "dataset.xml"
),
chunks=(1,1,32,32,32)
chunks=(32,32,32) #the previous default was 1 x Nx x Ny (Nx Ny were full tile size!)
output:
bdv_n5=temp(
directory(
Expand Down
7 changes: 4 additions & 3 deletions workflow/scripts/zarr_to_n5_bdv.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def update_xml_h5_to_n5(in_xml,out_xml,in_n5):
max_downsampling_layers=snakemake.params.max_downsampling_layers

#load data (tiles,chans,zslices,x,y)
darr = da.from_zarr(in_zarr,chunks=snakemake.params.chunks).astype(np.int16)
darr = da.from_zarr(in_zarr)

(n_tiles,n_chans,n_z,n_x,n_y) = darr.shape

Expand All @@ -52,7 +52,7 @@ def update_xml_h5_to_n5(in_xml,out_xml,in_n5):
overwrite=True,
nchannels=len(metadata['channels']),
ntiles=len(metadata['tiles_x'])*len(metadata['tiles_y']),
blockdim=(snakemake.params.chunks[2:]))
blockdim=((1,256,256),))

bdv_writer.set_attribute_labels('channel', metadata['channels'])

Expand Down Expand Up @@ -90,6 +90,7 @@ def update_xml_h5_to_n5(in_xml,out_xml,in_n5):
print('removing empty bdv h5/xml')
rmtree(temp_bdv_dir)


print('writing data to n5')
n5_store = zarr.n5.N5Store(snakemake.output.bdv_n5)

Expand All @@ -98,7 +99,7 @@ def update_xml_h5_to_n5(in_xml,out_xml,in_n5):
ds_list=[] #for setup-level attrs
for ds in range(max_downsampling_layers):
step=2**ds #1,2,4,8..
zstack = da.squeeze(darr[tile_i,chan_i,:,::step,::step])
zstack = da.squeeze(darr[tile_i,chan_i,:,::step,::step]).rechunk(snakemake.params.chunks).astype(np.int16)
print(f'writing to setup{setup_i}/timepoint0/s{ds}')
with ProgressBar():
zstack.to_zarr(n5_store,component=f'setup{setup_i}/timepoint0/s{ds}',overwrite=True,compute=True)
Expand Down
1 change: 1 addition & 0 deletions workflow/scripts/zarr_to_ome_zarr.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@

darr_list.append(da.from_zarr(in_zarr,component=f'{group_name}/s0',chunks=rechunk_size))


#append to omero metadata
channel_metadata={key:val for key,val in snakemake.config['ome_zarr']['omero_metadata']['channels']['defaults'].items()}
channel_name=stains[zarr_i]
Expand Down

0 comments on commit 1b1afc4

Please sign in to comment.