diff --git a/config/config.yml b/config/config.yml index 9858eb7..71974f2 100644 --- a/config/config.yml +++ b/config/config.yml @@ -7,7 +7,9 @@ remote_creds: '~/.config/gcloud/application_default_credentials.json' #this is n write_ome_zarr_direct: True #use this to skip writing the final zarr output to work first and copying afterwards -- useful when work is not a fast local disk -cores_per_rule: 32 +#total resources available, used to set rule resources +total_cores: 32 +total_mem_mb: 128000 #import wildcards: tilex, tiley, channel, zslice (and prefix - unused) import_blaze: diff --git a/workflow/Snakefile b/workflow/Snakefile index c997f26..70b5ced 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -33,7 +33,7 @@ rule all: input: get_all_targets(), get_bids_toplevel_targets(), -# get_qc_targets(), #need to skip this if using prestitched + # get_qc_targets(), #need to skip this if using prestitched localrule: True diff --git a/workflow/rules/bigstitcher.smk b/workflow/rules/bigstitcher.smk index 24a4a3e..f280d9a 100644 --- a/workflow/rules/bigstitcher.smk +++ b/workflow/rules/bigstitcher.smk @@ -85,7 +85,7 @@ rule zarr_to_bdv: desc="{desc}", suffix="log.txt", ), - threads: config["cores_per_rule"] + threads: config["total_cores"] group: "preproc" container: @@ -152,8 +152,8 @@ rule bigstitcher_stitching: config["containers"]["spimprep"] resources: runtime=30, - mem_mb=40000, - threads: config["cores_per_rule"] + mem_mb=int(config["total_mem_mb"] * 0.9), + threads: config["total_cores"] group: "preproc" shell: @@ -216,8 +216,8 @@ rule bigstitcher_solver: config["containers"]["spimprep"] resources: runtime=30, - mem_mb=40000, - threads: config["cores_per_rule"] + mem_mb=int(config["total_mem_mb"] * 0.9), + threads: config["total_cores"] group: "preproc" shell: @@ -226,7 +226,7 @@ rule bigstitcher_solver: " -s STITCHING --lambda 0.1 " " {params.method} && " "{params.rm_old_xml}" - #lambda 0.1 is default (can expose this if needed) + rule bigstitcher_fusion: input: @@ -310,8 +310,8 @@ rule bigstitcher_fusion: config["containers"]["spimprep"] resources: runtime=30, - mem_mb=30000, - threads: config["cores_per_rule"] + mem_mb=int(config["total_mem_mb"] * 0.9), + threads: config["total_cores"] group: "preproc" shell: diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index 80a773a..cb882fc 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -128,7 +128,6 @@ def get_all_subj_html(wildcards): htmls = [] for i in range(len(datasets)): - html = "{root}/qc/sub-{subject}_sample-{sample}_acq-{acq}/subject.html".format( root=root, subject=datasets.loc[i, "subject"], @@ -257,7 +256,6 @@ def get_stains(wildcards): return df.iloc[0][stain_columns].dropna().tolist() - def get_output_ome_zarr_uri(): if is_remote(config["root"]): return _bids( diff --git a/workflow/rules/flatfield_corr.smk b/workflow/rules/flatfield_corr.smk index 82470ec..e167c2d 100644 --- a/workflow/rules/flatfield_corr.smk +++ b/workflow/rules/flatfield_corr.smk @@ -79,7 +79,11 @@ rule apply_basic_flatfield_corr: allow_missing=True, ), params: - out_chunks=[128, 128, 128], + out_chunks=[ + config["bigstitcher"]["fuse_dataset"]["block_size_z"], + config["bigstitcher"]["fuse_dataset"]["block_size_y"], + config["bigstitcher"]["fuse_dataset"]["block_size_x"], + ], output: zarr=temp( directory( @@ -115,7 +119,7 @@ rule apply_basic_flatfield_corr: resources: runtime=60, mem_mb=32000, - threads: config["cores_per_rule"] + threads: config["total_cores"] group: "preproc" script: diff --git a/workflow/rules/import.smk b/workflow/rules/import.smk index 421ff80..e8ac38c 100644 --- a/workflow/rules/import.smk +++ b/workflow/rules/import.smk @@ -227,7 +227,9 @@ rule tif_to_zarr: ), group: "preproc" - threads: config["cores_per_rule"] + resources: + mem_mb=config["total_mem_mb"], + threads: int(config["total_mem_mb"] / 8000) #this is memory-limited -- seems to need ~8000mb for each thread, so threads=total_mem_mb / 8000 container: config["containers"]["spimprep"] script: @@ -280,7 +282,9 @@ rule tif_to_zarr_gcs: ), group: "preproc" - threads: 16 + resources: + mem_mb=config["total_mem_mb"], + threads: int(config["total_mem_mb"] / 8000) #this is memory-limited -- seems to need ~8000mb for each thread, so threads=total_mem_mb / 8000 container: config["containers"]["spimprep"] script: diff --git a/workflow/rules/ome_zarr.smk b/workflow/rules/ome_zarr.smk index 0ea5854..95e647d 100644 --- a/workflow/rules/ome_zarr.smk +++ b/workflow/rules/ome_zarr.smk @@ -1,3 +1,5 @@ + + rule zarr_to_ome_zarr: input: **get_storage_creds(), @@ -27,7 +29,9 @@ rule zarr_to_ome_zarr: storage_provider_settings=workflow.storage_provider_settings, output: **get_output_ome_zarr("blaze"), - threads: config["cores_per_rule"] + resources: + mem_mb=config["total_mem_mb"], + threads: config["total_cores"] log: bids( root="logs", @@ -77,10 +81,10 @@ rule tif_stacks_to_ome_zarr: config["containers"]["spimprep"] group: "preproc" - threads: config["cores_per_rule"] + threads: config["total_cores"] resources: runtime=360, - mem_mb=32000, + mem_mb=config["total_mem_mb"], script: "../scripts/tif_stacks_to_ome_zarr.py" @@ -165,7 +169,7 @@ rule ome_zarr_to_nii: ), group: "preproc" - threads: config["cores_per_rule"] + threads: config["total_cores"] container: config["containers"]["spimprep"] script: