Skip to content

Commit

Permalink
refactoring for zipstore and ome zarr outputs
Browse files Browse the repository at this point in the history
- fixes a bug (was writing to zarr to .snakemake_touch)
- hopefully cleans things up and makes more readable
  • Loading branch information
akhanf committed Oct 8, 2024
1 parent f1f4ba6 commit 7c190d8
Show file tree
Hide file tree
Showing 5 changed files with 212 additions and 124 deletions.
3 changes: 2 additions & 1 deletion config/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ remote_creds: '~/.config/gcloud/application_default_credentials.json' #this is n

write_ome_zarr_direct: True #use this to skip writing the final zarr output to work first and copying afterwards -- useful when work is not a fast local disk

use_zipstore: False #if True, produce SPIM.ome.zarr.zip instead of SPIM.ome.zarr

#total resources available, used to set rule resources
total_cores: 32
total_mem_mb: 128000
Expand Down Expand Up @@ -100,7 +102,6 @@ ome_zarr:
id: 0
name: spim
version: "0.4"
use_zipstore: False #if True, produce SPIM.ome.zarr.zip instead of SPIM.ome.zarr

nifti:
levels: #cannot be higher than max_downsampling_layers in ome_zarr
Expand Down
141 changes: 87 additions & 54 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,22 @@ universal-pathlib = "^0.2.2"
snakefmt = "^0.10.0"
mkdocs-material = "^9.5.20"
mkdocs-include-markdown-plugin = "^6.0.6"
poethepoet = "^0.29.0"


[tool.poetry.group.dataset_creation.dependencies]
typer = "^0.12.3"
xmltodict = "^0.13.0"
tifffile = "^2024.5.10"

[tool.poe.tasks]
test_localin_gcsout = "snakemake --dry-run --config datasets=testing/dryrun_tests/datasets_local.tsv root='gcs://khanlab-lightsheet/data/test_bids'"
test_localin_localout = "snakemake --dry-run --config datasets=testing/dryrun_tests/datasets_gcs.tsv root=bids"
test_gcsin_gcsout = "snakemake --dry-run --config datasets=testing/dryrun_tests/datasets_gcs.tsv root='gcs://khanlab-lightsheet/data/test_bids'"
test_gcsin_localout = "snakemake --dry-run --config datasets=testing/dryrun_tests/datasets_gcs.tsv root=bids"
test_localin_localout_zipstore = "snakemake --dry-run --config datasets=testing/dryrun_tests/datasets_gcs.tsv root=bids use_zipstore=True"
test_localin_gcsout_zipstore = "snakemake --dry-run --config datasets=testing/dryrun_tests/datasets_gcs.tsv root='gcs://khanlab-lightsheet/data/test_bids' use_zipstore=True"

[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
Loading

0 comments on commit 7c190d8

Please sign in to comment.