From 0209e91494e29b960964e1ef2448b199781c4614 Mon Sep 17 00:00:00 2001 From: Ali Khan Date: Wed, 14 Feb 2024 10:08:53 -0500 Subject: [PATCH] update readme, and version of snakebids now uses latest spec so entities (e.g. sample) are ordered correctly in the output filenames --- README.md | 32 +++-- poetry.lock | 291 ++++++++++++++++++++------------------------- pyproject.toml | 5 +- workflow/Snakefile | 5 +- 4 files changed, 155 insertions(+), 178 deletions(-) diff --git a/README.md b/README.md index f5806fa..8dbe66a 100644 --- a/README.md +++ b/README.md @@ -11,34 +11,46 @@ Also performs simple affine registration to Allen Brain Atlas v3 (mouse atlas), ## Requirements - - Linux system with Singularity/Apptainer installed. + - Linux system with Singularity/Apptainer installed + - (Note: container will be automatically pulled when you run the workflow) - Python >= 3.11 - - Lightsheet data in a folder, or in a tar file (see tif filename pattern in `config.yml`) + - Lightsheet data as OME tif files in a folder, or in a tar file (see tif filename pattern in `config.yml`) ## Usage -Clone this repository to the folder you want to run the workflow in, update the `config/datasets.tsv` to point to your dataset(s), and optionally update the `config/config.yml` with customized options. +1. Clone this repository to the folder you want to run the workflow in +``` +git clone https://github.com/khanlab/spimprep +``` -Install dependencies with: +2. Create and activate a virtual environment, then install dependencies with: ``` pip install . ``` -Then, to do a dry-run use: + +3. Update the `config/datasets.tsv` spreadsheet to point to your dataset(s). Each dataset's tif files should be in it's own folder or tar file, with no other tif files. Enter the path to each dataset in the `dataset_path` column. The first three columns identify the subject, sample, acquisition, which become part of the resulting filenames (BIDS naming). The `stain_0` and `stain_1` identify what stains were used for each channel. Use `autof` to indicate the autofluorescence channel, as this is used for registration. + +4. The `config/config.yml` can be edited to customize any workflow parameters. The most important ones are the `root` and `work` variables. The `root` path is where the results will end up, by default this is a subfolder called results. The `work` path is where any intermediate scratch files are produced. By default the files in `work` are deleted after they are no longer needed in the workflow, unless you use the `--notemp` command-line option. The workflow writes a large number of small files in parallel to the `work` folder, so for optimum performance this should be a fast local disk, and not a networked file system (i.e. shared disk). + +Note: you can use environment variables when specifying `root` or `work`, e.g. so `work: '$SLURM_TMPDIR` can be used on HPC servers. +5. Perform a dry-run to make sure the workflow is configured properly. This will only print what the workflow will run, and will not run anything. ``` snakemake -np ``` -To run on all cores use: +6. To run the workflow, parallelizing on all cores, using Singularity (aka Apptainer) for dependencies, use: ``` -snakemake -c all --sdm apptainer +snakemake -c all --sdm apptainer ``` -### WIP: The usage of this workflow is described in the [Snakemake Workflow Catalog](https://snakemake.github.io/snakemake-workflow-catalog/?usage=khanlab%2Fspimprep). +Note: if you run the workflow on a system with large memory, you will need to set the heap size for the stitching and fusion rules. This can be done with e.g.: `--set-resources bigstitcher:mem_mb=60000 fuse_dataset:mem_mb=100000` + +7. If you want to run the workflow using a batch job submission server, please see the executor plugins here: https://snakemake.github.io/snakemake-plugin-catalog/ -## TODO: +### WIP: The usage of this workflow is described in the [Snakemake Workflow Catalog](https://snakemake.github.io/snakemake-workflow-catalog/?usage=khanlab%2Fspimprep). + - - [ ] any changes to post to Snakemake Workflow Catalog, and verify usage there diff --git a/poetry.lock b/poetry.lock index ffd69cc..2538d55 100644 --- a/poetry.lock +++ b/poetry.lock @@ -24,18 +24,6 @@ files = [ {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, ] -[[package]] -name = "argparse-dataclass" -version = "2.0.0" -description = "Declarative CLIs with argparse and dataclasses" -category = "main" -optional = false -python-versions = ">=3.8" -files = [ - {file = "argparse_dataclass-2.0.0-py3-none-any.whl", hash = "sha256:3ffc8852a88d9d98d1364b4441a712491320afb91fb56049afd8a51d74bb52d2"}, - {file = "argparse_dataclass-2.0.0.tar.gz", hash = "sha256:09ab641c914a2f12882337b9c3e5086196dbf2ee6bf0ef67895c74002cc9297f"}, -] - [[package]] name = "arrow" version = "1.3.0" @@ -259,21 +247,6 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -[[package]] -name = "conda-inject" -version = "1.3.1" -description = "Helper functions for injecting a conda environment into the current python environment (by modifying sys.path, without actually changing the current python environment)." -category = "main" -optional = false -python-versions = ">=3.9,<4.0" -files = [ - {file = "conda_inject-1.3.1-py3-none-any.whl", hash = "sha256:0a106bb0ef3553e82b6e7ef343162305c44dad7789c1909eed1abe83548c7fc6"}, - {file = "conda_inject-1.3.1.tar.gz", hash = "sha256:9e8d902230261beba74083aae12c2c5a395e29b408469fefadc8aaf51ee441e5"}, -] - -[package.dependencies] -pyyaml = ">=6.0,<7.0" - [[package]] name = "configargparse" version = "1.7" @@ -610,60 +583,6 @@ files = [ {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] -[[package]] -name = "immutables" -version = "0.20" -description = "Immutable Collections" -category = "main" -optional = false -python-versions = ">=3.8.0" -files = [ - {file = "immutables-0.20-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dea0ae4d7f31b145c18c16badeebc2f039d09411be4a8febb86e1244cf7f1ce0"}, - {file = "immutables-0.20-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2dd0dcef2f8d4523d34dbe1d2b7804b3d2a51fddbd104aad13f506a838a2ea15"}, - {file = "immutables-0.20-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:393dde58ffd6b4c089ffdf4cef5fe73dad37ce4681acffade5f5d5935ec23c93"}, - {file = "immutables-0.20-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1214b5a175df783662b7de94b4a82db55cc0ee206dd072fa9e279fb8895d8df"}, - {file = "immutables-0.20-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2761e3dc2a6406943ce77b3505e9b3c1187846de65d7247548dc7edaa202fcba"}, - {file = "immutables-0.20-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2bcea81e7516bd823b4ed16f4f794531097888675be13e833b1cc946370d5237"}, - {file = "immutables-0.20-cp310-cp310-win32.whl", hash = "sha256:d828e7580f1fa203ddeab0b5e91f44bf95706e7f283ca9fbbcf0ae08f63d3084"}, - {file = "immutables-0.20-cp310-cp310-win_amd64.whl", hash = "sha256:380e2957ba3d63422b2f3fbbff0547c7bbe6479d611d3635c6411005a4264525"}, - {file = "immutables-0.20-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:532be32c7a25dae6cade28825c76d3004cf4d166a0bfacf04bda16056d59ba26"}, - {file = "immutables-0.20-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5302ce9c7827f8300f3dc34a695abb71e4a32bab09e65e5ad6e454785383347f"}, - {file = "immutables-0.20-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b51aec54b571ae466113509d4dc79a2808dc2ae9263b71fd6b37778cb49eb292"}, - {file = "immutables-0.20-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47f56aea56e597ecf6631f24a4e26007b6a5f4fe30278b96eb90bc1f60506164"}, - {file = "immutables-0.20-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:085ac48ee3eef7baf070f181cae574489bbf65930a83ec5bbd65c9940d625db3"}, - {file = "immutables-0.20-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f063f53b5c0e8f541ae381f1d828f3d05bbed766a2d6c817f9218b8b37a4cb66"}, - {file = "immutables-0.20-cp311-cp311-win32.whl", hash = "sha256:b0436cc831b47e26bef637bcf143cf0273e49946cfb7c28c44486d70513a3080"}, - {file = "immutables-0.20-cp311-cp311-win_amd64.whl", hash = "sha256:5bb32aee1ea16fbb90f58f8bd96016bca87aba0a8e574e5fa218d0d83b142851"}, - {file = "immutables-0.20-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4ba726b7a3a696b9d4b122fa2c956bc68e866f3df1b92765060c88c64410ff82"}, - {file = "immutables-0.20-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5a88adf1dcc9d8ab07dba5e74deefcd5b5e38bc677815cbf9365dc43b69f1f08"}, - {file = "immutables-0.20-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1009a4e00e2e69a9b40c2f1272795f5a06ad72c9bf4638594d518e9cbd7a721a"}, - {file = "immutables-0.20-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96899994842c37cf4b9d6d2bedf685aae7810bd73f1538f8cba5426e2d65cb85"}, - {file = "immutables-0.20-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a606410b2ccb6ae339c3f26cccc9a92bcb16dc06f935d51edfd8ca68cf687e50"}, - {file = "immutables-0.20-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e8e82754f72823085643a2c0e6a4c489b806613e94af205825fa81df2ba147a0"}, - {file = "immutables-0.20-cp312-cp312-win32.whl", hash = "sha256:525fb361bd7edc8a891633928d549713af8090c79c25af5cc06eb90b48cb3c64"}, - {file = "immutables-0.20-cp312-cp312-win_amd64.whl", hash = "sha256:a82afc3945e9ceb9bcd416dc4ed9b72f92760c42787e26de50610a8b81d48120"}, - {file = "immutables-0.20-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f17f25f21e82a1c349a61191cfb13e442a348b880b74cb01b00e0d1e848b63f4"}, - {file = "immutables-0.20-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:65954eb861c61af48debb1507518d45ae7d594b4fba7282785a70b48c5f51f9b"}, - {file = "immutables-0.20-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62f8a7a22939278127b7a206d05679b268b9cf665437125625348e902617cbad"}, - {file = "immutables-0.20-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac86f4372f4cfaa00206c12472fd3a78753092279e0552b7e1880944d71b04fe"}, - {file = "immutables-0.20-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e771198edc11a9e02ffa693911b3918c6cde0b64ad2e6672b076dbe005557ad8"}, - {file = "immutables-0.20-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fc739fc07cff5df2e4f31addbd48660b5ac0da56e9f719f8bb45da8ddd632c63"}, - {file = "immutables-0.20-cp38-cp38-win32.whl", hash = "sha256:c086ccb44d9d3824b9bf816365d10b1b82837efc7119f8bab56bd7a27ed805a9"}, - {file = "immutables-0.20-cp38-cp38-win_amd64.whl", hash = "sha256:9cd2ee9c10bf00be3c94eb51854bc0b761326bd0a7ea0dad4272a3f182269ae6"}, - {file = "immutables-0.20-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d4f78cb748261f852953620ed991de74972446fd484ec69377a41e2f1a1beb75"}, - {file = "immutables-0.20-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d6449186ea91b7c17ec8e7bd9bf059858298b1db5c053f5d27de8eba077578ce"}, - {file = "immutables-0.20-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85dd9765b068f7beb297553fddfcf7f904bd58a184c520830a106a58f0c9bfb4"}, - {file = "immutables-0.20-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f349a7e0327b92dcefb863e49ace086f2f26e6689a4e022c98720c6e9696e763"}, - {file = "immutables-0.20-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e3a5462f6d3549bbf7d02ce929fb0cb6df9539445f0589105de4e8b99b906e69"}, - {file = "immutables-0.20-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cc51a01a64a6d2cd7db210a49ad010c2ac2e9e026745f23fd31e0784096dcfff"}, - {file = "immutables-0.20-cp39-cp39-win32.whl", hash = "sha256:83794712f0507416f2818edc63f84305358b8656a93e5b9e2ab056d9803c7507"}, - {file = "immutables-0.20-cp39-cp39-win_amd64.whl", hash = "sha256:2837b1078abc66d9f009bee9085cf62515d5516af9a5c9ea2751847e16efd236"}, - {file = "immutables-0.20.tar.gz", hash = "sha256:1d2f83e6a6a8455466cd97b9a90e2b4f7864648616dfa6b19d18f49badac3876"}, -] - -[package.extras] -test = ["flake8 (>=5.0,<6.0)", "mypy (>=1.4,<2.0)", "pycodestyle (>=2.9,<3.0)", "pytest (>=7.4,<8.0)"] - [[package]] name = "importlib-resources" version = "6.1.1" @@ -803,6 +722,22 @@ traitlets = ">=5.3" docs = ["myst-parser", "pydata-sphinx-theme", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling", "traitlets"] test = ["ipykernel", "pre-commit", "pytest", "pytest-cov", "pytest-timeout"] +[[package]] +name = "lazy-loader" +version = "0.3" +description = "lazy_loader" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "lazy_loader-0.3-py3-none-any.whl", hash = "sha256:1e9e76ee8631e264c62ce10006718e80b2cfc74340d17d1031e0f84af7478554"}, + {file = "lazy_loader-0.3.tar.gz", hash = "sha256:3b68898e34f5b2a29daaaac172c6555512d0f32074f147e2254e4a6d9d838f37"}, +] + +[package.extras] +lint = ["pre-commit (>=3.3)"] +test = ["pytest (>=7.4)", "pytest-cov (>=4.1)"] + [[package]] name = "markupsafe" version = "2.1.5" @@ -1050,10 +985,7 @@ files = [ ] [package.dependencies] -numpy = [ - {version = ">=1.23.2,<2", markers = "python_version == \"3.11\""}, - {version = ">=1.26.0,<2", markers = "python_version >= \"3.12\""}, -] +numpy = {version = ">=1.26.0,<2", markers = "python_version >= \"3.12\""} python-dateutil = ">=2.8.2" pytz = ">=2020.1" tzdata = ">=2022.7" @@ -1188,14 +1120,14 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] [[package]] name = "pulp" -version = "2.8.0" +version = "2.7.0" description = "PuLP is an LP modeler written in python. PuLP can generate MPS or LP files and call GLPK, COIN CLP/CBC, CPLEX, and GUROBI to solve linear problems." category = "main" optional = false -python-versions = ">=3.7" +python-versions = "*" files = [ - {file = "PuLP-2.8.0-py3-none-any.whl", hash = "sha256:4a19814a5b0a4392d788ac2315263435293579b0583c3469943fe0c6a586f263"}, - {file = "PuLP-2.8.0.tar.gz", hash = "sha256:4903bf96110bbab8ed2c68533f90565ebb76aa367d9e4df38e51bf727927c125"}, + {file = "PuLP-2.7.0-py3-none-any.whl", hash = "sha256:b6de42c929e80325bf44cc7a2997f02535440800c376b9eb8cb7b4670ed53769"}, + {file = "PuLP-2.7.0.tar.gz", hash = "sha256:e73ee6b32d639c9b8cf4b4aded334ba158be5f8313544e056f796ace0a10ae63"}, ] [[package]] @@ -1692,6 +1624,85 @@ files = [ {file = "rpds_py-0.17.1.tar.gz", hash = "sha256:0210b2668f24c078307260bf88bdac9d6f1093635df5123789bfee4d8d7fc8e7"}, ] +[[package]] +name = "ruamel-yaml" +version = "0.18.6" +description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruamel.yaml-0.18.6-py3-none-any.whl", hash = "sha256:57b53ba33def16c4f3d807c0ccbc00f8a6081827e81ba2491691b76882d0c636"}, + {file = "ruamel.yaml-0.18.6.tar.gz", hash = "sha256:8b27e6a217e786c6fbe5634d8f3f11bc63e0f80f6a5890f28863d9c45aac311b"}, +] + +[package.dependencies] +"ruamel.yaml.clib" = {version = ">=0.2.7", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.13\""} + +[package.extras] +docs = ["mercurial (>5.7)", "ryd"] +jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] + +[[package]] +name = "ruamel-yaml-clib" +version = "0.2.8" +description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b42169467c42b692c19cf539c38d4602069d8c1505e97b86387fcf7afb766e1d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:07238db9cbdf8fc1e9de2489a4f68474e70dffcb32232db7c08fa61ca0c7c462"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:fff3573c2db359f091e1589c3d7c5fc2f86f5bdb6f24252c2d8e539d4e45f412"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:aa2267c6a303eb483de8d02db2871afb5c5fc15618d894300b88958f729ad74f"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:840f0c7f194986a63d2c2465ca63af8ccbbc90ab1c6001b1978f05119b5e7334"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:024cfe1fc7c7f4e1aff4a81e718109e13409767e4f871443cbff3dba3578203d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win32.whl", hash = "sha256:c69212f63169ec1cfc9bb44723bf2917cbbd8f6191a00ef3410f5a7fe300722d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win_amd64.whl", hash = "sha256:cabddb8d8ead485e255fe80429f833172b4cadf99274db39abc080e068cbcc31"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bef08cd86169d9eafb3ccb0a39edb11d8e25f3dae2b28f5c52fd997521133069"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:b16420e621d26fdfa949a8b4b47ade8810c56002f5389970db4ddda51dbff248"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:25c515e350e5b739842fc3228d662413ef28f295791af5e5110b543cf0b57d9b"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_24_aarch64.whl", hash = "sha256:1707814f0d9791df063f8c19bb51b0d1278b8e9a2353abbb676c2f685dee6afe"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:46d378daaac94f454b3a0e3d8d78cafd78a026b1d71443f4966c696b48a6d899"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:09b055c05697b38ecacb7ac50bdab2240bfca1a0c4872b0fd309bb07dc9aa3a9"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win32.whl", hash = "sha256:53a300ed9cea38cf5a2a9b069058137c2ca1ce658a874b79baceb8f892f915a7"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win_amd64.whl", hash = "sha256:c2a72e9109ea74e511e29032f3b670835f8a59bbdc9ce692c5b4ed91ccf1eedb"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ebc06178e8821efc9692ea7544aa5644217358490145629914d8020042c24aa1"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:edaef1c1200c4b4cb914583150dcaa3bc30e592e907c01117c08b13a07255ec2"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d176b57452ab5b7028ac47e7b3cf644bcfdc8cacfecf7e71759f7f51a59e5c92"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_24_aarch64.whl", hash = "sha256:1dc67314e7e1086c9fdf2680b7b6c2be1c0d8e3a8279f2e993ca2a7545fecf62"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3213ece08ea033eb159ac52ae052a4899b56ecc124bb80020d9bbceeb50258e9"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aab7fd643f71d7946f2ee58cc88c9b7bfc97debd71dcc93e03e2d174628e7e2d"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-win32.whl", hash = "sha256:5c365d91c88390c8d0a8545df0b5857172824b1c604e867161e6b3d59a827eaa"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-win_amd64.whl", hash = "sha256:1758ce7d8e1a29d23de54a16ae867abd370f01b5a69e1a3ba75223eaa3ca1a1b"}, + {file = "ruamel.yaml.clib-0.2.8-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a5aa27bad2bb83670b71683aae140a1f52b0857a2deff56ad3f6c13a017a26ed"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c58ecd827313af6864893e7af0a3bb85fd529f862b6adbefe14643947cfe2942"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f481f16baec5290e45aebdc2a5168ebc6d35189ae6fea7a58787613a25f6e875"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_24_aarch64.whl", hash = "sha256:77159f5d5b5c14f7c34073862a6b7d34944075d9f93e681638f6d753606c6ce6"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:7f67a1ee819dc4562d444bbafb135832b0b909f81cc90f7aa00260968c9ca1b3"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4ecbf9c3e19f9562c7fdd462e8d18dd902a47ca046a2e64dba80699f0b6c09b7"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:87ea5ff66d8064301a154b3933ae406b0863402a799b16e4a1d24d9fbbcbe0d3"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win32.whl", hash = "sha256:75e1ed13e1f9de23c5607fe6bd1aeaae21e523b32d83bb33918245361e9cc51b"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win_amd64.whl", hash = "sha256:3f215c5daf6a9d7bbed4a0a4f760f3113b10e82ff4c5c44bec20a68c8014f675"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1b617618914cb00bf5c34d4357c37aa15183fa229b24767259657746c9077615"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:a6a9ffd280b71ad062eae53ac1659ad86a17f59a0fdc7699fd9be40525153337"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_24_aarch64.whl", hash = "sha256:305889baa4043a09e5b76f8e2a51d4ffba44259f6b4c72dec8ca56207d9c6fe1"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:700e4ebb569e59e16a976857c8798aee258dceac7c7d6b50cab63e080058df91"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e2b4c44b60eadec492926a7270abb100ef9f72798e18743939bdbf037aab8c28"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e79e5db08739731b0ce4850bed599235d601701d5694c36570a99a0c5ca41a9d"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win32.whl", hash = "sha256:955eae71ac26c1ab35924203fda6220f84dce57d6d7884f189743e2abe3a9fbe"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win_amd64.whl", hash = "sha256:56f4252222c067b4ce51ae12cbac231bce32aee1d33fbfc9d17e5b8d6966c312"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:03d1162b6d1df1caa3a4bd27aa51ce17c9afc2046c31b0ad60a0a96ec22f8001"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:bba64af9fa9cebe325a62fa398760f5c7206b215201b0ec825005f1b18b9bccf"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_24_aarch64.whl", hash = "sha256:a1a45e0bb052edf6a1d3a93baef85319733a888363938e1fc9924cb00c8df24c"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:da09ad1c359a728e112d60116f626cc9f29730ff3e0e7db72b9a2dbc2e4beed5"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:184565012b60405d93838167f425713180b949e9d8dd0bbc7b49f074407c5a8b"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a75879bacf2c987c003368cf14bed0ffe99e8e85acfa6c0bfffc21a090f16880"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win32.whl", hash = "sha256:84b554931e932c46f94ab306913ad7e11bba988104c5cff26d90d03f68258cd5"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win_amd64.whl", hash = "sha256:25ac8c08322002b06fa1d49d1646181f0b2c72f5cbc15a85e80b4c30a544bb15"}, + {file = "ruamel.yaml.clib-0.2.8.tar.gz", hash = "sha256:beb2e0404003de9a4cab9753a8805a8fe9320ee6673136ed7f04255fe60bb512"}, +] + [[package]] name = "scipy" version = "1.12.0" @@ -1891,71 +1902,70 @@ files = [ [[package]] name = "snakebids" -version = "0.10.0" +version = "0.0.0" description = "BIDS integration into snakemake workflows" category = "main" optional = false -python-versions = ">=3.8,<3.13" -files = [ - {file = "snakebids-0.10.0-py3-none-any.whl", hash = "sha256:32395bdcb610387a15977c5c7e148a6a441df01fb06bb76406c28653f7412e14"}, - {file = "snakebids-0.10.0.tar.gz", hash = "sha256:ebb1e7fdc0d68d8ad8373b595bf8d868f41886f04661d37e9a0376df0d2aeff2"}, -] +python-versions = ">=3.8,<4.0" +files = [] +develop = false [package.dependencies] attrs = ">=21.2.0" -boutiques = ">=0.5.25,<0.6.0" +boutiques = "^0.5.25" copier = ">=8.2.0" importlib-resources = ">=5.12.0" jinja2-time = ">=0.2.0" +lazy-loader = ">=0.3" more-itertools = ">=8" numpy = {version = ">=1.23.2", markers = "python_version >= \"3.11\""} +pandas = {version = ">=2.1.1", markers = "python_version >= \"3.12\""} pvandyken-deprecated = "0.0.4" pybids = ">=0.16.0,<0.17" -PyYAML = ">=6" requests = ">=2.31.0" +ruamel-yaml = ">=0.17.2" scipy = {version = ">=1.10.0", markers = "python_version >= \"3.9\""} snakemake = [ - {version = ">=5.28.0", markers = "python_version >= \"3.8\""}, - {version = ">=7.18.2", markers = "python_version >= \"3.11\""}, + {version = ">=5.28.0,<8", markers = "python_version >= \"3.8\""}, + {version = ">=7.18.2,<8", markers = "python_version >= \"3.11\""}, ] typing-extensions = ">=3.10.0" +[package.source] +type = "git" +url = "https://github.com/khanlab/snakebids" +reference = "main" +resolved_reference = "8acbd9f773b71a0c43ddf692968e4c45b1b7707f" + [[package]] name = "snakemake" -version = "8.4.8" +version = "7.32.4" description = "Workflow management system to create reproducible and scalable data analyses" category = "main" optional = false -python-versions = ">=3.11" +python-versions = ">=3.7" files = [ - {file = "snakemake-8.4.8-py3-none-any.whl", hash = "sha256:f04b62ed5177820a523d0db7e342b12a180572fbf0679b0538613d8dbc57e330"}, - {file = "snakemake-8.4.8.tar.gz", hash = "sha256:ab64a4f402f78ec252f211d9cc900344fdc2a000dadcec91833efb5fd0f275e7"}, + {file = "snakemake-7.32.4.tar.gz", hash = "sha256:fdc3f15dd7b06fabb7da30d460e0a3b1fba08e4ea91f9c32c47a83705cdc7b6e"}, ] [package.dependencies] appdirs = "*" -conda-inject = ">=1.3.1,<2.0" configargparse = "*" -connection-pool = ">=0.0.3" +connection_pool = ">=0.0.3" datrie = "*" docutils = "*" -dpath = ">=2.1.6,<3.0.0" gitpython = "*" humanfriendly = "*" -immutables = "*" jinja2 = ">=3.0,<4.0" jsonschema = "*" nbformat = "*" packaging = "*" psutil = "*" -pulp = ">=2.3.1,<2.9" +pulp = ">=2.0" pyyaml = "*" requests = "*" reretry = "*" -smart-open = ">=3.0" -snakemake-interface-common = ">=1.15.0,<2.0" -snakemake-interface-executor-plugins = ">=8.1.3,<9.0" -snakemake-interface-storage-plugins = ">=3.0.0,<4.0" +smart_open = ">=3.0" stopit = "*" tabulate = "*" throttler = "*" @@ -1964,61 +1974,12 @@ wrapt = "*" yte = ">=1.5.1,<2.0" [package.extras] -messaging = ["slack-sdk"] +azure = ["azure-batch", "azure-core", "azure-identity", "azure-mgmt-batch", "azure-storage-blob"] +google-cloud = ["google-api-python-client", "google-cloud-storage", "google-crc32c", "oauth2client"] +messaging = ["slacker"] pep = ["eido", "peppy"] reports = ["pygments"] -[[package]] -name = "snakemake-interface-common" -version = "1.15.3" -description = "Common functions and classes for Snakemake and its plugins" -category = "main" -optional = false -python-versions = ">=3.8,<4.0" -files = [ - {file = "snakemake_interface_common-1.15.3-py3-none-any.whl", hash = "sha256:78241fbbce2b8cc33a3024ddd9f0fcc697f79c6b61e84677d6478fbc6cfa0c0f"}, - {file = "snakemake_interface_common-1.15.3.tar.gz", hash = "sha256:7587c5e553c6ddaf0e2542bad1ffda55c3adf5b8ca651f4bc359a17d3f747ceb"}, -] - -[package.dependencies] -argparse-dataclass = ">=2.0.0,<3.0.0" -ConfigArgParse = ">=1.7,<2.0" - -[[package]] -name = "snakemake-interface-executor-plugins" -version = "8.2.0" -description = "This package provides a stable interface for interactions between Snakemake and its executor plugins." -category = "main" -optional = false -python-versions = ">=3.11,<4.0" -files = [ - {file = "snakemake_interface_executor_plugins-8.2.0-py3-none-any.whl", hash = "sha256:b5d7f83b699492648bd44fc601736f15c86a2aa7ba8e5e011282e83a38b0e05f"}, - {file = "snakemake_interface_executor_plugins-8.2.0.tar.gz", hash = "sha256:4c74e3e1751bab6b266baf8688e854b8b4c5c5e10f5e34c581f42d69af4ff13b"}, -] - -[package.dependencies] -argparse-dataclass = ">=2.0.0,<3.0.0" -snakemake-interface-common = ">=1.12.0,<2.0.0" -throttler = ">=1.2.2,<2.0.0" - -[[package]] -name = "snakemake-interface-storage-plugins" -version = "3.0.0" -description = "This package provides a stable interface for interactions between Snakemake and its storage plugins." -category = "main" -optional = false -python-versions = ">=3.11,<4.0" -files = [ - {file = "snakemake_interface_storage_plugins-3.0.0-py3-none-any.whl", hash = "sha256:a2bff701cc23930143d2699520080234f8cc8ce59774be938d46fb86b82e848c"}, - {file = "snakemake_interface_storage_plugins-3.0.0.tar.gz", hash = "sha256:f20d85ee7e86a1e2ffa3f72e2385dd5abb17fa7b58a26cba8ba59096872fe169"}, -] - -[package.dependencies] -reretry = ">=0.11.8,<0.12.0" -snakemake-interface-common = ">=1.12.0,<2.0.0" -throttler = ">=1.2.2,<2.0.0" -wrapt = ">=1.15.0,<2.0.0" - [[package]] name = "sqlalchemy" version = "2.0.27" @@ -2357,4 +2318,4 @@ pyyaml = ">=6.0,<7.0" [metadata] lock-version = "2.0" python-versions = ">=3.11,<3.13" -content-hash = "e4baedd9e4b8f11caed8c6824e3fb88061e1e6eb6e2a3d6feb20fda53cec08df" +content-hash = "b32cf7377ac448d7f19b28f247dcf05496cc37a00d28ce2a4a7e58abd22ae219" diff --git a/pyproject.toml b/pyproject.toml index b620cff..9ffc562 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,8 +8,9 @@ packages = [] [tool.poetry.dependencies] python = ">=3.11,<3.13" -snakemake = "^8.4.2" -snakebids = "^0.10.0" +snakemake = "<8.0.0" +pulp = "2.7.0" +snakebids = {git = "https://github.com/khanlab/snakebids", rev = "main"} [build-system] requires = ["poetry-core"] diff --git a/workflow/Snakefile b/workflow/Snakefile index fdcd44b..d7d0f8c 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -1,5 +1,5 @@ import json -from snakebids import bids +from snakebids import bids, set_bids_spec from pathlib import Path import pandas as pd import os @@ -15,6 +15,9 @@ conda: config["conda_envs"]["global"] root = os.path.expandvars(config["root"]) work = os.path.expandvars(config["work"]) +#this is needed to use the latest bids spec with the pre-release snakebids +set_bids_spec("v0_10_1") + # read datasets tsv datasets = pd.read_csv( config["datasets"],