updated chunking sizes

and moves rechunking from zarr2bdv to apply_flatfield_corr
should hopefully improve performance

workflow/rules/bigstitcher.smk

@@ -40,7 +40,7 @@ rule zarr_to_bdv:
             )
             / "dataset.xml"
         ),
-        chunks=(32,32,32) #the previous default was 1 x Nx x Ny (Nx Ny were full tile size!)
+        chunks=(128,256,256) #the previous default was 1 x Nx x Ny (Nx Ny were full tile size!)
     output:
         bdv_n5=temp(
             directory(

workflow/rules/flatfield_corr.smk

@@ -78,6 +78,8 @@ rule apply_basic_flatfield_corr:
             stain=get_stains(wildcards),
             allow_missing=True,
         ),
+    params:
+        out_chunks=(1,1,128,256,256) #make this a config option -- setting it here instead of rechunking in zarr2bdv
     output:
         zarr=temp(
             directory(

workflow/scripts/apply_basic_flatfield_corr_zarr.py

@@ -47,7 +47,7 @@ def apply_basic_parallel(x):
     chan_arr_list.append(arr_corr)
 
 #stack along chans
-arr_stacked = da.stack(chan_arr_list,axis=1)
+arr_stacked = da.stack(chan_arr_list,axis=1).rechunk(snakemake.params.out_chunks)
 
 with ProgressBar():
     da.to_zarr(arr_stacked,snakemake.output.zarr,overwrite=True,dimension_separator='/')

workflow/scripts/zarr_to_n5_bdv.py

@@ -99,7 +99,7 @@ def update_xml_h5_to_n5(in_xml,out_xml,in_n5):
     ds_list=[] #for setup-level attrs
     for ds in range(max_downsampling_layers):
         step=2**ds  #1,2,4,8.. 
-        zstack = da.squeeze(darr[tile_i,chan_i,:,::step,::step]).rechunk(snakemake.params.chunks).astype(np.int16)
+        zstack = da.squeeze(darr[tile_i,chan_i,:,::step,::step]).astype(np.int16)
         print(f'writing to setup{setup_i}/timepoint0/s{ds}')
         with ProgressBar():
             zstack.to_zarr(n5_store,component=f'setup{setup_i}/timepoint0/s{ds}',overwrite=True,compute=True)