update resources

- cores_per_rule -> total_cores
- added total_mem_mb
- better calculation of resources for tif_to_zarr (mem-based calc of
threads)
- snakefmt

config/config.yml

@@ -7,7 +7,9 @@ remote_creds: '~/.config/gcloud/application_default_credentials.json' #this is n
 
 write_ome_zarr_direct: True #use this to skip writing the final zarr output to work first and copying afterwards -- useful when work is not a fast local disk
 
-cores_per_rule: 32 
+#total resources available, used to set rule resources
+total_cores: 32 
+total_mem_mb: 128000 
 
 #import wildcards:  tilex, tiley, channel, zslice (and prefix - unused)
 import_blaze:

workflow/Snakefile

@@ -33,7 +33,7 @@ rule all:
     input:
         get_all_targets(),
         get_bids_toplevel_targets(),
-#        get_qc_targets(),  #need to skip this if using prestitched
+    #        get_qc_targets(),  #need to skip this if using prestitched
     localrule: True
 
 

workflow/rules/bigstitcher.smk

@@ -85,7 +85,7 @@ rule zarr_to_bdv:
             desc="{desc}",
             suffix="log.txt",
         ),
-    threads: config["cores_per_rule"]
+    threads: config["total_cores"]
     group:
         "preproc"
     container:
@@ -152,8 +152,8 @@ rule bigstitcher_stitching:
         config["containers"]["spimprep"]
     resources:
         runtime=30,
-        mem_mb=40000,
-    threads: config["cores_per_rule"]
+        mem_mb=int(config["total_mem_mb"] * 0.9),
+    threads: config["total_cores"]
     group:
         "preproc"
     shell:
@@ -216,8 +216,8 @@ rule bigstitcher_solver:
         config["containers"]["spimprep"]
     resources:
         runtime=30,
-        mem_mb=40000,
-    threads: config["cores_per_rule"]
+        mem_mb=int(config["total_mem_mb"] * 0.9),
+    threads: config["total_cores"]
     group:
         "preproc"
     shell:
@@ -226,7 +226,7 @@ rule bigstitcher_solver:
         " -s STITCHING --lambda 0.1 "
         " {params.method} && "
         "{params.rm_old_xml}"
-        #lambda 0.1 is default (can expose this if needed)
+
 
 rule bigstitcher_fusion:
     input:
@@ -310,8 +310,8 @@ rule bigstitcher_fusion:
         config["containers"]["spimprep"]
     resources:
         runtime=30,
-        mem_mb=30000,
-    threads: config["cores_per_rule"]
+        mem_mb=int(config["total_mem_mb"] * 0.9),
+    threads: config["total_cores"]
     group:
         "preproc"
     shell:

workflow/rules/common.smk

@@ -128,7 +128,6 @@ def get_all_subj_html(wildcards):
     htmls = []
 
     for i in range(len(datasets)):
-
         html = "{root}/qc/sub-{subject}_sample-{sample}_acq-{acq}/subject.html".format(
             root=root,
             subject=datasets.loc[i, "subject"],
@@ -257,7 +256,6 @@ def get_stains(wildcards):
     return df.iloc[0][stain_columns].dropna().tolist()
 
 
-
 def get_output_ome_zarr_uri():
     if is_remote(config["root"]):
         return _bids(

workflow/rules/flatfield_corr.smk

@@ -79,7 +79,11 @@ rule apply_basic_flatfield_corr:
             allow_missing=True,
         ),
     params:
-        out_chunks=[128, 128, 128],
+        out_chunks=[
+            config["bigstitcher"]["fuse_dataset"]["block_size_z"],
+            config["bigstitcher"]["fuse_dataset"]["block_size_y"],
+            config["bigstitcher"]["fuse_dataset"]["block_size_x"],
+        ],
     output:
         zarr=temp(
             directory(
@@ -115,7 +119,7 @@ rule apply_basic_flatfield_corr:
     resources:
         runtime=60,
         mem_mb=32000,
-    threads: config["cores_per_rule"]
+    threads: config["total_cores"]
     group:
         "preproc"
     script:

workflow/rules/import.smk

@@ -227,7 +227,9 @@ rule tif_to_zarr:
         ),
     group:
         "preproc"
-    threads: config["cores_per_rule"]
+    resources:
+        mem_mb=config["total_mem_mb"],
+    threads: int(config["total_mem_mb"] / 8000)  #this is memory-limited -- seems to need ~8000mb for each thread, so threads=total_mem_mb / 8000 
     container:
         config["containers"]["spimprep"]
     script:
@@ -280,7 +282,9 @@ rule tif_to_zarr_gcs:
         ),
     group:
         "preproc"
-    threads: 16
+    resources:
+        mem_mb=config["total_mem_mb"],
+    threads: int(config["total_mem_mb"] / 8000)  #this is memory-limited -- seems to need ~8000mb for each thread, so threads=total_mem_mb / 8000 
     container:
         config["containers"]["spimprep"]
     script:

workflow/rules/ome_zarr.smk

@@ -1,3 +1,5 @@
+
+
 rule zarr_to_ome_zarr:
     input:
         **get_storage_creds(),
@@ -27,7 +29,9 @@ rule zarr_to_ome_zarr:
         storage_provider_settings=workflow.storage_provider_settings,
     output:
         **get_output_ome_zarr("blaze"),
-    threads: config["cores_per_rule"]
+    resources:
+        mem_mb=config["total_mem_mb"],
+    threads: config["total_cores"]
     log:
         bids(
             root="logs",
@@ -77,10 +81,10 @@ rule tif_stacks_to_ome_zarr:
         config["containers"]["spimprep"]
     group:
         "preproc"
-    threads: config["cores_per_rule"]
+    threads: config["total_cores"]
     resources:
         runtime=360,
-        mem_mb=32000,
+        mem_mb=config["total_mem_mb"],
     script:
         "../scripts/tif_stacks_to_ome_zarr.py"
 
@@ -165,7 +169,7 @@ rule ome_zarr_to_nii:
         ),
     group:
         "preproc"
-    threads: config["cores_per_rule"]
+    threads: config["total_cores"]
     container:
         config["containers"]["spimprep"]
     script: