updates for compatibility with input zstacks (#48)

Blaze microscope now outputs data with one tif file per zstack. This adds compatibility for it.

- Includes a number of other fixes and cleaning up too (removes unused rules/scripts). Tested locally and on gcs.
- Some refactoring of global config params for resources (adds total_cores, total_mem_mb)
- currently disables qc reports (was taking long for gcs runs - need to revisit that in another PR)

README.md

@@ -18,6 +18,7 @@ Takes TIF images (tiled or prestitched) and outputs a validated BIDS Microscopy
  - Python >= 3.11
  - Lightsheet data:
     - Raw Ultramicroscope Blaze OME TIFF files (include `blaze` in the acquisition tag)
+        - can be 2D or 3D TIFF files
     - Prestitched TIFF files (include `prestitched` in the acquisition tag)
 
 
@@ -63,10 +64,8 @@ or for snakemake<8.0, use:
 snakemake -c all --use-singularity 
 ```
 
-Note: if you run the workflow on a system with large memory, you will need to set the heap size for the stitching and fusion rules. This can be done with e.g.: `--set-resources bigstitcher_spark_stitching:mem_mb=60000 bigstitcher_spark_fusion:mem_mb=100000`
+Note: if you run the workflow on a system with large memory, you will need to set the heap size for the stitching and fusion rules. This can be done with e.g.: `--set-resources bigstitcher_stitching:mem_mb=60000 bigstitcher_fusion:mem_mb=100000`
 
 7. If you want to run the workflow using a batch job submission server, please see the executor plugins here: https://snakemake.github.io/snakemake-plugin-catalog/
 
-
-Alternate usage of this workflow (making use of conda) is described in the [Snakemake Workflow Catalog](https://snakemake.github.io/snakemake-workflow-catalog?repo=khanlab/SPIMprep).
 <!--intro-end-->

config/config.yml

@@ -1,18 +1,20 @@
 datasets: 'config/datasets.tsv'
 
-
 root: 'bids' # can use a s3:// or gcs:// prefix to write output to cloud storage
 work: 'work' 
 
 remote_creds: '~/.config/gcloud/application_default_credentials.json' #this is needed so we can pass creds to container
 
-write_ome_zarr_direct: True #use this to skip writing the final zarr output to work first and copying afterwards -- useful when work is not a fast local disk
+use_zipstore: False #if True, produce SPIM.ome.zarr.zip instead of SPIM.ome.zarr
 
-cores_per_rule: 32 
+#total resources available, used to set rule resources
+total_cores: 32 
+total_mem_mb: 128000 
 
 #import wildcards:  tilex, tiley, channel, zslice (and prefix - unused)
 import_blaze:
   raw_tif_pattern: "{prefix}_Blaze[{tilex} x {tiley}]_C{channel}_xyz-Table Z{zslice}.ome.tif"
+  raw_tif_pattern_zstack:  "{prefix}_Blaze[{tilex} x {tiley}]_C{channel}.ome.tif"
   intensity_rescaling: 0.5 #raw images seem to be at the upper end of uint16 (over-saturated) -- causes wrapping issues when adjusting with flatfield correction etc. this rescales the raw data as it imports it..
 
 import_prestitched:
@@ -36,16 +38,14 @@ bigstitcher:
     downsample_in_x: 4
     downsample_in_y: 4
     downsample_in_z: 1
-    method: "phase_corr" #unused
-    methods: #unused
+    methods: #unused, only for reference
       phase_corr: "Phase Correlation" 
       optical_flow: "Lucas-Kanade"  
   filter_pairwise_shifts:
-    enabled: 1 #unused
     min_r: 0.7
     max_shift_total: 50
   global_optimization:
-    enabled: 1 
+    enabled: 1
     method: TWO_ROUND_ITERATIVE
     methods: #unused, only for reference
       ONE_ROUND_SIMPLE: "One-Round"
@@ -64,7 +64,7 @@ bigstitcher:
     block_size_factor_z: 32
 
 ome_zarr:
-  desc: sparkstitchedflatcorr
+  desc: stitchedflatcorr
   max_downsampling_layers: 5 # e.g. 4 levels: { 0: orig, 1: ds2, 2: ds4, 3: ds8, 4: ds16}
   rechunk_size: #z, y, x 
     - 1 
@@ -100,7 +100,6 @@ ome_zarr:
       id: 0
       name: spim
       version: "0.4"  
-  use_zipstore: False #if True, produce SPIM.ome.zarr.zip instead of SPIM.ome.zarr
 
 nifti:
   levels: #cannot be higher than max_downsampling_layers in ome_zarr
@@ -155,5 +154,5 @@ report:
 
 
 containers:
-  spimprep: 'docker://khanlab/spimprep-deps:main'
+  spimprep: 'docker://khanlab/spimprep-deps:v0.1.0'
 

pyproject.toml

@@ -20,13 +20,22 @@ universal-pathlib = "^0.2.2"
 snakefmt = "^0.10.0"
 mkdocs-material = "^9.5.20"
 mkdocs-include-markdown-plugin = "^6.0.6"
+poethepoet = "^0.29.0"
 
 
 [tool.poetry.group.dataset_creation.dependencies]
 typer = "^0.12.3"
 xmltodict = "^0.13.0"
 tifffile = "^2024.5.10"
 
+[tool.poe.tasks]
+test_localin_gcsout = "snakemake --dry-run --config datasets=testing/dryrun_tests/datasets_local.tsv root='gcs://khanlab-lightsheet/data/test_bids'"
+test_localin_localout = "snakemake --dry-run --config datasets=testing/dryrun_tests/datasets_local.tsv root=bids"
+test_gcsin_gcsout = "snakemake --dry-run --config datasets=testing/dryrun_tests/datasets_gcs.tsv root='gcs://khanlab-lightsheet/data/test_bids'"
+test_gcsin_localout = "snakemake --dry-run --config datasets=testing/dryrun_tests/datasets_gcs.tsv root=bids"
+test_localin_localout_zipstore = "snakemake --dry-run --config datasets=testing/dryrun_tests/datasets_local.tsv root=bids use_zipstore=True"
+test_localin_gcsout_zipstore = "snakemake --dry-run --config datasets=testing/dryrun_tests/datasets_local.tsv root='gcs://khanlab-lightsheet/data/test_bids' use_zipstore=True"
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

resources/qc/ff_html_temp.html

@@ -43,13 +43,13 @@ <h2>Chunk - {{ chunk_num }} Channel - {{ loop.index0 }}</h2>
         <tr>
     {%- for image in channel %}
             <td>
-                <img src="{{ image.img_corr }}"></img>
-                <h3>Corrected</h3>
+                <img src="{{ image.img_uncorr }}"></img>
+                <h3>Uncorrected</h3>
                 <p>Slice-{{ image.slice }}</p>
             </td>
             <td>
-                <img src="{{ image.img_uncorr }}"></img>
-                <h3>Uncorrected</h3>
+                <img src="{{ image.img_corr }}"></img>
+                <h3>Corrected</h3>
                 <p>Slice-{{ image.slice }}</p>
             </td>
     {%- endfor %}
@@ -117,4 +117,4 @@ <h3>Uncorrected</h3>
             }
         })
     </script>
-</body>
+</body>

testing/dryrun_tests/datasets_gcs.tsv

@@ -0,0 +1,2 @@
+subject	sample	acq	stain_0	stain_1	stain_2	dataset_path
+mouse1	brain	blaze	Lectin	PI	Abeta	gcs://my-bucket/test_data

testing/dryrun_tests/datasets_local.tsv

@@ -0,0 +1,2 @@
+subject	sample	acq	stain_0	stain_1	stain_2	dataset_path
+mouse1	brain	blaze	Lectin	PI	Abeta	.test/dryrun/data

workflow/Snakefile

@@ -33,7 +33,7 @@ rule all:
     input:
         get_all_targets(),
         get_bids_toplevel_targets(),
-        get_qc_targets(),  #need to skip this if using prestitched
+    #        get_qc_targets(),  #need to skip this if using prestitched
     localrule: True