diff --git a/.github/workflows/docker_base.yml b/.github/workflows/docker_base.yml index 0bf10f2..8b8a30f 100644 --- a/.github/workflows/docker_base.yml +++ b/.github/workflows/docker_base.yml @@ -21,7 +21,7 @@ jobs: run: podman login -u ${QUAY_USER} -p ${QUAY_PASSWORD} quay.io - name: Build container - run: podman build --tag beckerlab/aradeepopsis-base containers/base + run: podman build --format docker --tag beckerlab/aradeepopsis-base containers/base - name: Push container run: podman push beckerlab/aradeepopsis-base docker://quay.io/beckerlab/aradeepopsis-base:${GITHUB_REF##*/} \ No newline at end of file diff --git a/.github/workflows/docker_dpp.yml b/.github/workflows/docker_dpp.yml index 4f56524..ee42149 100644 --- a/.github/workflows/docker_dpp.yml +++ b/.github/workflows/docker_dpp.yml @@ -21,7 +21,7 @@ jobs: run: podman login -u ${QUAY_USER} -p ${QUAY_PASSWORD} quay.io - name: Build container - run: podman build --tag beckerlab/aradeepopsis-dpp containers/addons/dpp + run: podman build --format docker --tag beckerlab/aradeepopsis-dpp containers/addons/dpp - name: Push container run: podman push beckerlab/aradeepopsis-dpp docker://quay.io/beckerlab/aradeepopsis-dpp:${GITHUB_REF##*/} \ No newline at end of file diff --git a/.github/workflows/docker_shiny.yml b/.github/workflows/docker_shiny.yml index eca5602..9c14d43 100644 --- a/.github/workflows/docker_shiny.yml +++ b/.github/workflows/docker_shiny.yml @@ -21,7 +21,7 @@ jobs: run: podman login -u ${QUAY_USER} -p ${QUAY_PASSWORD} quay.io - name: Build container - run: podman build --tag beckerlab/aradeepopsis-shiny containers/shiny + run: podman build --format docker --tag beckerlab/aradeepopsis-shiny containers/shiny - name: Push container run: podman push beckerlab/aradeepopsis-shiny docker://quay.io/beckerlab/aradeepopsis-shiny:${GITHUB_REF##*/} \ No newline at end of file diff --git a/.github/workflows/minimal_test.yml b/.github/workflows/minimal_test.yml index fc1c5fe..88f5e90 100644 --- a/.github/workflows/minimal_test.yml +++ b/.github/workflows/minimal_test.yml @@ -11,18 +11,54 @@ on: jobs: CI: env: + PROFILE: ${{ matrix.engine }} NXF_VER: ${{ matrix.nxf_ver }} - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: - nxf_ver: ["20.07.1", ""] + engine: ["docker", "podman", "charliecloud", "conda"] + nxf_ver: ["20.07.1",""] + os: ["ubuntu-latest", "macos-latest"] + exclude: + - engine: "charliecloud" + nxf_ver: "20.07.1" # charliecloud needs 21.04.0 or later + - os: "macos-latest" + engine: "docker" + - os: "macos-latest" + engine: "podman" + - os: "macos-latest" + engine: "charliecloud" steps: - uses: actions/checkout@v2 + - name: Set up Python + if: ${{ matrix.engine == 'charliecloud' }} + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install Charliecloud + if: ${{ matrix.engine == 'charliecloud' }} + run: | + python -m pip install lark-parser requests + wget -qO- https://github.com/hpc/charliecloud/releases/download/v0.22/charliecloud-0.22.tar.gz | tar -xvz + cd charliecloud-0.22 + ./configure + make + sudo make install + - name: Install Nextflow run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - - name: Run aradeepopsis on test images + + - name: Run ARADEEPOPSIS on ubuntu + if: ${{ matrix.os == 'ubuntu-latest' }} + run: | + nextflow run ${GITHUB_WORKSPACE} --save_rosette --save_histogram --save_hull --summary_diagnostics --shiny false -profile test,${PROFILE} + + - name: Run ARADEEPOPSIS on macOS + if: ${{ matrix.os == 'macos-latest' }} run: | - nextflow run ${GITHUB_WORKSPACE} --save_rosette --save_histogram --save_hull --summary_diagnostics --shiny false -profile test,docker + sudo chown -R ${UID} ${CONDA} + nextflow run ${GITHUB_WORKSPACE} --save_rosette --save_histogram --save_hull --summary_diagnostics --shiny false -profile test,conda diff --git a/CHANGELOG.md b/CHANGELOG.md index 5adfb0d..414c0ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [v1.3.1](https://github.com/Gregor-Mendel-Institute/aradeepopsis/releases/tag/v1.3.1) - 2021-05-14 + +* fixed illegible text in Markdown documents when browsing Github in Dark Mode (thanks @greymonroe for pointing this out) +* added config profile for BioHPC Genomics cluster at LRZ +* removed unneeded `tensorflow-estimator` dependency +* updated `tensorflow-base` `2.0.0` > `2.4.1` +* updated `imagemagick` dependency `7.0.10_28` > `7.0.11_12` +* updated `shiny` dependency `1.5.0` > `1.6.0` +* updated `tidyverse` dependency `1.3.0` > `1.3.1` +* updated `shinythemes` dependency `1.1.2` > `1.2.0` +* updated `corrplot` dependency `0.84` > `0.88` +* changed container base to micromamba for smaller container images and faster builds +* added CI tests for `conda` profile on macOS and additional container engines on linux (`charliecloud` and `podman`) + ## [v1.3](https://github.com/Gregor-Mendel-Institute/aradeepopsis/releases/tag/v1.3) - 2021-01-11 * changed container registry from docker.io to quay.io diff --git a/README.md b/README.md index cd6975b..3e61c4c 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,12 @@ ![Latest Release](https://img.shields.io/github/v/release/Gregor-Mendel-Institute/aradeepopsis?sort=semver) ![License](https://img.shields.io/github/license/Gregor-Mendel-Institute/aradeepopsis) [![DOI](https://zenodo.org/badge/240287274.svg)](https://zenodo.org/badge/latestdoi/240287274) -[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A520.07.1-important.svg)](https://www.nextflow.io/) -[![conda](https://img.shields.io/badge/install%20with-conda-brightgreen.svg)](https://conda.io/) +[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A520.07.1-important.svg)](https://www.nextflow.io/) +[![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) +[![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) +[![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) +[![run with podman](https://img.shields.io/badge/run%20with-podman-1d355c.svg?labelColor=000000&logo=podman)](https://docs.podman.io/) +[![run with charliecloud](https://img.shields.io/badge/run%20with-charliecloud-1d355c.svg?labelColor=000000)](https://hpc.github.io/charliecloud/) # Introduction @@ -62,7 +66,7 @@ Running the pipeline with the provided models requires at least 6GB of memory fo 1. Install [`Nextflow`](https://www.nextflow.io/index.html#GetStarted) -2. Install either [`conda`](https://docs.conda.io/projects/conda/en/latest/user-guide/install/), [`Docker`](https://docs.docker.com/install/), [`podman`](https://podman.io/getting-started/installation) or [`Singularity`](https://sylabs.io/guides/3.0/user-guide/installation.html) +2. Install either [`conda`](https://docs.conda.io/projects/conda/en/latest/user-guide/install/), [`Docker`](https://docs.docker.com/install/), [`podman`](https://podman.io/getting-started/installation), [`Charliecloud`](https://hpc.github.io/charliecloud/) or [`Singularity`](https://sylabs.io/guides/3.0/user-guide/installation.html). > Note: To run the pipeline on MacOS with Docker, it is necessary to increase the allowed memory usage from the default setting of 2GB to 6GB (See the [Docker documentation](https://docs.docker.com/docker-for-mac/#resources) for instructions) @@ -71,7 +75,7 @@ Running the pipeline with the provided models requires at least 6GB of memory fo To run the pipeline you have to provide single-pot plant images: ```bash -nextflow run Gregor-Mendel-Institute/aradeepopsis --images 'path/to/images/*{png|jpg}' -profile {conda|docker|podman|singularity} +nextflow run Gregor-Mendel-Institute/aradeepopsis --images 'path/to/images/*{png|jpg}' -profile {conda|docker|podman|singularity|charliecloud} ``` ### Example to run on the CBE cluster using Singularity diff --git a/conf/base.config b/conf/base.config index eca612d..1cf3f16 100644 --- a/conf/base.config +++ b/conf/base.config @@ -1,5 +1,5 @@ /* -Copyright (C) 2019-2020 Patrick Hüther +Copyright (C) 2019-2021 Patrick Hüther This file is part of ARADEEPOPSIS. ARADEEPOPSIS is free software: you can redistribute it and/or modify @@ -23,41 +23,43 @@ along with ARADEEPOPSIS. If not, see . env { PYTHONPATH = "${PYTHONPATH}:$baseDir/bin" - TF_CPP_MIN_LOG_LEVEL = 2 + PYTHONNOUSERSITE = 1 + R_PROFILE_USER = "/.Rprofile" + R_ENVIRON_USER = "/.Renviron" } process { withName: build_records { - container = 'quay.io/beckerlab/aradeepopsis-base:1.3' + container = 'quay.io/beckerlab/aradeepopsis-base:1.3.1' cpus = 1 memory = 2.GB time = 1.h } withName: run_predictions { - container = 'quay.io/beckerlab/aradeepopsis-base:1.3' + container = 'quay.io/beckerlab/aradeepopsis-base:1.3.1' cpus = 2 memory = params.multiscale ? 12.GB : 6.GB time = 4.h } withName: run_predictions_DPP { - container = 'quay.io/beckerlab/aradeepopsis-dpp:1.3' + container = 'quay.io/beckerlab/aradeepopsis-dpp:1.3.1' cpus = 2 memory = 2.GB time = 4.h } withName: extract_traits { - container = 'quay.io/beckerlab/aradeepopsis-base:1.3' + container = 'quay.io/beckerlab/aradeepopsis-base:1.3.1' cpus = 1 memory = 2.GB time = 1.h } withName: draw_diagnostics { - container = 'quay.io/beckerlab/aradeepopsis-base:1.3' + container = 'quay.io/beckerlab/aradeepopsis-base:1.3.1' cpus = 1 memory = 2.GB time = 1.h } withName: launch_shiny { - container = 'quay.io/beckerlab/aradeepopsis-shiny:1.3' + container = 'quay.io/beckerlab/aradeepopsis-shiny:1.3.1' } } diff --git a/conf/lrz.config b/conf/biohpc_gen.config similarity index 76% rename from conf/lrz.config rename to conf/biohpc_gen.config index 56dfd4c..90ed935 100644 --- a/conf/lrz.config +++ b/conf/biohpc_gen.config @@ -1,5 +1,5 @@ /* -Copyright (C) 2019-2020 Patrick Hüther, Niklas Schandry +Copyright (C) 2019-2021 Patrick Hüther, Niklas Schandry This file is part of ARADEEPOPSIS. ARADEEPOPSIS is free software: you can redistribute it and/or modify @@ -17,17 +17,23 @@ along with ARADEEPOPSIS. If not, see . */ /* ------------------------------------------------- - * Nextflow config file for LRZ Linux Cluster (CM2) + * Nextflow config file for BioHPC Genomics * ------------------------------------------------- */ +env { + SLURM_CLUSTERS='biohpc_gen' +} + +charliecloud { + enabled = true +} + process { executor = 'slurm' - queue = 'cm2_std' - queueSize = 50 - clusterOptions = "--clusters cm2 --qos cm2_std" - errorStrategy = { task.exitStatus in [104,134,135,137,139,140,143] ? 'retry' : 'finish' } - maxRetries = 3 + queue = { task.memory <= 1536.GB ? (task.time > 2.d || task.memory > 384.GB ? 'biohpc_gen_production' : 'biohpc_gen_normal') : 'biohpc_gen_highmem' } + beforeScript = 'module use /dss/dsslegfs02/pn73se/pn73se-dss-0000/spack/modules/x86_avx2/linux*' + module = 'charliecloud/0.22:miniconda3' withName: build_records { cpus = { 1 * task.attempt } diff --git a/conf/cbe.config b/conf/cbe.config index 1a75520..b782c58 100644 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -1,5 +1,5 @@ /* -Copyright (C) 2019-2020 Patrick Hüther +Copyright (C) 2019-2021 Patrick Hüther This file is part of ARADEEPOPSIS. ARADEEPOPSIS is free software: you can redistribute it and/or modify @@ -27,7 +27,7 @@ process { executor = 'slurm' module = 'anaconda3/2019.10' queue = 'c' - clusterOptions = { task.time <= 8.h ? '--qos short': task.time <= 48.h ? '--qos medium' : '--qos long' } + clusterOptions = { task.time <= 1.h ? '--qos rapid' : task.time <= 8.h ? '--qos short': task.time <= 48.h ? '--qos medium' : '--qos long' } errorStrategy = { task.exitStatus in [104,134,135,137,139,140,143] ? 'retry' : 'finish' } maxRetries = 3 diff --git a/conf/charliecloud.config b/conf/charliecloud.config index 2f9af09..d9d3049 100644 --- a/conf/charliecloud.config +++ b/conf/charliecloud.config @@ -21,15 +21,10 @@ along with ARADEEPOPSIS. If not, see . * ------------------------------------------------- */ -env { - //for charliecloud, the PATH environment within the Docker image needs to be passed explicitly - PATH = "/opt/conda/envs/aradeepopsis-dpp-v1.3/bin:/opt/conda/envs/aradeepopsis-shiny-v1.3/bin:/opt/conda/envs/aradeepopsis-base-v1.3/bin:$PATH" -} - charliecloud { enabled = true } manifest { - nextflowVersion = '>=20.12.0-edge' -} \ No newline at end of file + nextflowVersion = '>=21.04.0' +} diff --git a/conf/conda.config b/conf/conda.config index 6b828b8..a4d1ba7 100644 --- a/conf/conda.config +++ b/conf/conda.config @@ -1,5 +1,5 @@ /* -Copyright (C) 2019-2020 Patrick Hüther +Copyright (C) 2019-2021 Patrick Hüther This file is part of ARADEEPOPSIS. ARADEEPOPSIS is free software: you can redistribute it and/or modify @@ -21,10 +21,6 @@ along with ARADEEPOPSIS. If not, see . * ------------------------------------------------- */ -env { - PYTHONPATH = "${PYTHONPATH}:$baseDir/bin" -} - process { withName: build_records { conda = "$baseDir/containers/base/environment.yml" diff --git a/containers/addons/dpp/Dockerfile b/containers/addons/dpp/Dockerfile index 0bb85cc..eee5083 100644 --- a/containers/addons/dpp/Dockerfile +++ b/containers/addons/dpp/Dockerfile @@ -1,8 +1,9 @@ -FROM continuumio/miniconda3 +FROM mambaorg/micromamba:0.13.0 LABEL authors="patrick.huether@gmi.oeaw.ac.at" \ description="Container image containing DPP dependencies for ARADEEPOPSIS" COPY environment.yml / -RUN apt-get update && apt-get install -y procps graphviz && apt-get clean -y -RUN conda env create -f /environment.yml && conda clean -afy -ENV PATH /opt/conda/envs/aradeepopsis-dpp-v1.3/bin:$PATH +RUN apt-get update \ + && apt-get install --no-install-recommends -y procps \ + && micromamba install -y -n base -f /environment.yml \ + && micromamba clean -a && rm -rf /var/lib/{apt,dpkg,cache,log} diff --git a/containers/addons/dpp/environment.yml b/containers/addons/dpp/environment.yml index 71b1279..75b4fdd 100644 --- a/containers/addons/dpp/environment.yml +++ b/containers/addons/dpp/environment.yml @@ -1,15 +1,10 @@ -name: aradeepopsis-dpp-v1.3 +name: aradeepopsis-dpp channels: - conda-forge - defaults dependencies: - - python=3.7 - - pip - - conda-forge::scipy=1.5.0 - - conda-forge::opt-einsum=3.0.0 - - conda-forge::tqdm=4.47.0 - - conda-forge::pillow=7.1.2 + - conda-forge::python=3.7 + - conda-forge::pip=21.1.1 - pip: - - tensorflow==1.15 - - opencv-python==4.2.0.34 - - https://github.com/p2irc/deepplantphenomics/archive/2.1.0.tar.gz + - opencv-python-headless==4.5.2.52 + - https://github.com/p2irc/deepplantphenomics/archive/2.1.0.tar.gz \ No newline at end of file diff --git a/containers/base/Dockerfile b/containers/base/Dockerfile index 496ba4f..eee5083 100644 --- a/containers/base/Dockerfile +++ b/containers/base/Dockerfile @@ -1,8 +1,9 @@ -FROM continuumio/miniconda3 +FROM mambaorg/micromamba:0.13.0 LABEL authors="patrick.huether@gmi.oeaw.ac.at" \ - description="Container image containing base dependencies for ARADEEPOPSIS" + description="Container image containing DPP dependencies for ARADEEPOPSIS" COPY environment.yml / -RUN apt-get update && apt-get install -y procps graphviz && apt-get clean -y -RUN conda env create -f /environment.yml && conda clean -afy -ENV PATH /opt/conda/envs/aradeepopsis-base-v1.3/bin:$PATH +RUN apt-get update \ + && apt-get install --no-install-recommends -y procps \ + && micromamba install -y -n base -f /environment.yml \ + && micromamba clean -a && rm -rf /var/lib/{apt,dpkg,cache,log} diff --git a/containers/base/environment.yml b/containers/base/environment.yml index ee0d468..6550f84 100644 --- a/containers/base/environment.yml +++ b/containers/base/environment.yml @@ -1,9 +1,8 @@ -name: aradeepopsis-base-v1.3 +name: aradeepopsis-base channels: - conda-forge - defaults dependencies: - - anaconda::tensorflow-base=2.0.0 - - anaconda::tensorflow-estimator=2.0.0 + - conda-forge::tensorflow-base=2.4.1 - conda-forge::scikit-image=0.18.1 - - conda-forge::imagemagick=7.0.10_28 + - conda-forge::imagemagick=7.0.11_12 \ No newline at end of file diff --git a/containers/shiny/Dockerfile b/containers/shiny/Dockerfile index d1e14fc..d1e6336 100644 --- a/containers/shiny/Dockerfile +++ b/containers/shiny/Dockerfile @@ -1,10 +1,11 @@ -FROM continuumio/miniconda3 +FROM mambaorg/micromamba:0.13.0 LABEL authors="patrick.huether@gmi.oeaw.ac.at" \ - description="Container image containing shiny dependencies for ARADEEPOPSIS" + description="Container image containing DPP dependencies for ARADEEPOPSIS" COPY environment.yml / -RUN apt-get update && apt-get install -y procps graphviz && apt-get clean -y -RUN conda env create -f /environment.yml && conda clean -afy -ENV PATH /opt/conda/envs/aradeepopsis-shiny-v1.3/bin:$PATH +RUN apt-get update \ + && apt-get install --no-install-recommends -y procps \ + && micromamba install -y -n base -f /environment.yml \ + && micromamba clean -a && rm -rf /var/lib/{apt,dpkg,cache,log} -EXPOSE 44333 \ No newline at end of file +EXPOSE 44333 diff --git a/containers/shiny/environment.yml b/containers/shiny/environment.yml index fc0e5fe..db31c62 100644 --- a/containers/shiny/environment.yml +++ b/containers/shiny/environment.yml @@ -1,14 +1,14 @@ -name: aradeepopsis-shiny-v1.3 +name: aradeepopsis-shiny channels: - conda-forge - r - defaults dependencies: - - conda-forge::r-tidyverse=1.3.0 - - conda-forge::r-shiny=1.5.0 - - conda-forge::r-shinythemes=1.1.2 + - conda-forge::r-tidyverse=1.3.1 + - conda-forge::r-shiny=1.6.0 + - conda-forge::r-shinythemes=1.2.0 - conda-forge::r-shinycssloaders=1.0.0 - conda-forge::r-slickr=0.5.0 - - conda-forge::r-corrplot=0.84 + - conda-forge::r-corrplot=0.88 - conda-forge::r-jpeg=0.1_8.1 - - r::r-radarchart=0.3.1 + - r::r-radarchart=0.3.1 \ No newline at end of file diff --git a/docs/img/BGR.png b/docs/img/BGR.png index 6ca8319..76aad00 100644 Binary files a/docs/img/BGR.png and b/docs/img/BGR.png differ diff --git a/docs/img/CB.png b/docs/img/CB.png index 17e1a58..781037e 100644 Binary files a/docs/img/CB.png and b/docs/img/CB.png differ diff --git a/docs/img/CD.png b/docs/img/CD.png index 3a8865c..b548bee 100644 Binary files a/docs/img/CD.png and b/docs/img/CD.png differ diff --git a/docs/img/CR.png b/docs/img/CR.png index 35b9f54..4933016 100644 Binary files a/docs/img/CR.png and b/docs/img/CR.png differ diff --git a/docs/img/S_green.png b/docs/img/S_green.png index c0a1403..f3456ff 100644 Binary files a/docs/img/S_green.png and b/docs/img/S_green.png differ diff --git a/docs/img/pipeline_graph.png b/docs/img/pipeline_graph.png index 59186af..771aed5 100644 Binary files a/docs/img/pipeline_graph.png and b/docs/img/pipeline_graph.png differ diff --git a/docs/img/traits.png b/docs/img/traits.png index 47321c9..347c089 100644 Binary files a/docs/img/traits.png and b/docs/img/traits.png differ diff --git a/docs/parameters.md b/docs/parameters.md index 037d425..fb5566a 100644 --- a/docs/parameters.md +++ b/docs/parameters.md @@ -127,11 +127,11 @@ Launch a [Shiny](https://shiny.rstudio.com/) app as the last step of the pipelin > cd /path/to/results > > # if using the conda environment -> conda create -f /path/to/cloned/repository/environment.yml -> conda activate aradeepopsis +> conda create -f /path/to/cloned/repository/containers/shiny/environment.yml +> conda activate aradeepopsis-shiny > R -e "shiny::runApp('app.R', port=44333)" > > # if using the container image -> {docker|podman} run -v $(pwd):/mnt/shiny -p 44333:44333 quay.io/beckerlab/aradeepopsis-shiny:1.3 R -e "shiny::runApp('/mnt/shiny/app.R', port=44333, host='0.0.0.0')" +> {docker|podman} run -v $(pwd):/mnt/shiny -p 44333:44333 quay.io/beckerlab/aradeepopsis-shiny:1.3.1 R -e "shiny::runApp('/mnt/shiny/app.R', port=44333, host='0.0.0.0')" > ``` > The shiny app can then be opened in a browser by typing localhost:44333 in the address bar. It will terminate when the browser window is closed. diff --git a/nextflow.config b/nextflow.config index fb2b69e..386b756 100644 --- a/nextflow.config +++ b/nextflow.config @@ -72,14 +72,14 @@ profiles { test { includeConfig 'conf/test.config' } // cluster specific profiles cbe { includeConfig 'conf/cbe.config' } - lrzcm2 { includeConfig 'conf/lrz.config' } + biohpc_gen { includeConfig 'conf/biohpc_gen.config' } } manifest { homePage = 'https://github.com/Gregor-Mendel-Institute/aradeepopsis' description = "Nextflow pipeline to run semantic segmentation on plant rosette images with DeepLab V3+" name = 'aradeepopsis' - version = '1.3' + version = '1.3.1' mainScript = 'main.nf' nextflowVersion = '>=20.07.1' doi = '10.1105/tpc.20.00318'