From 39cc0f5c4c825dcb580aec9896274fadf2485bac Mon Sep 17 00:00:00 2001 From: Daniel Park Date: Wed, 1 Nov 2017 09:47:19 -0400 Subject: [PATCH] docker build optimization (#708) This changes how the docker image is built. Changes include: - Change from installing the viral-ngs conda package (easy-deploy-viral-ngs.sh setup) to something closer to setup-git (that uses the current working copy instead of cloning fresh from github). - Divorce the conda build & deploy steps from the docker build & deploy -- put each in their own separate space in the Travis build matrix and cut end-to-end build time in half. - Separate out basic infrastructural setup in the Dockerfile to a separate repository such that this Dockerfile only installs the components specific to viral-ngs and its dependencies. This also reduces build time by caching all of the apt-get steps. - Move Dockerfile to root of git repo. This allows for a direct build of either the github URI or the working copy (e.g. docker build . or docker build https://github.com/broadinstitute/viral-ngs.git). --- .dockerignore | 4 + .travis.yml | 16 +- Dockerfile | 40 + docker/Dockerfile | 98 --- docker/easy-deploy-script/README.md | 59 -- .../easy-deploy-viral-ngs.sh | 732 ------------------ docker/install-viral-ngs.sh | 22 + easy-deploy-script/easy-deploy-viral-ngs.sh | 84 +- requirements-conda.txt | 1 - travis/before_install.sh | 7 - travis/{build-package.sh => build-conda.sh} | 11 - travis/deploy-docker.sh | 6 +- travis/install-conda-build.sh | 7 + travis/install-conda.sh | 4 - travis/upgrade-docker.sh | 8 + 15 files changed, 142 insertions(+), 957 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile delete mode 100644 docker/Dockerfile delete mode 100644 docker/easy-deploy-script/README.md delete mode 100755 docker/easy-deploy-script/easy-deploy-viral-ngs.sh create mode 100755 docker/install-viral-ngs.sh rename travis/{build-package.sh => build-conda.sh} (79%) create mode 100755 travis/install-conda-build.sh create mode 100755 travis/upgrade-docker.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..08d68e72f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +*.pyc +__pycache__ +tools/conda-cache +tools/conda-tools diff --git a/.travis.yml b/.travis.yml index b9946a828..48943e253 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,11 +14,14 @@ matrix: - os: linux python: 3.6 env: - - BUILD_PACKAGE=true + - BUILD_PACKAGE=conda # $ANACONDA_TOKEN for uploading builds to anaconda.org ("broad-viral" channel) - secure: SLPB86BpMIiNncMioxVk9cLrqaSNt8F1QDtxkrdLq9j7wXzFqGa7cipG6UJ6Om7GvoF49DpACfGPTA4ycr+T4cH3pWXpBHrBhV8TyKJb23cOmg5+7zqJQTzuwNqKOT7t9rnBkf1uzVXBcgqKaD6XW/nEvNFK00I0cvjlCp8vgxE= # $TRAVIS_ACCESS_TOKEN_FOR_OTHER_REPO (viral-ngs-deploy) - secure: ChB0K3gPr5HknxYA41xCrpgChHDmLkqc79p1NABB/tbqOEnrPzDPqE+FU4/QlmeV96jMYn4uyLVauJpzVXyBIVoOa8guqoF5VdiKlAhaUwh9UQJ75i3SKQtGBrqaTXSDVI1vJARMiGabduCrcNJxVsxV9Bm+YzTq6tuhWyqR4fs= + - os: linux + env: + - BUILD_PACKAGE=docker # DOCKER_USER - secure: hYX8492Wqpq3yqv+eHBV9c6VY8JlUSS8mUDfT1eWNEZF2vw8WrnTt8SrLIPC8etQUk1ZaSI/8XbJQKEr9LRqgvuO07AIv6BxYCg9l9BPHCj6B2YTTPo2qPkPapfvtGVd7PUZcWDUvzvPxJqMveuKVkTnCuuQSWwR68Y/Khxj8UY= # DOCKER_PASS @@ -56,24 +59,27 @@ env: - secure: KX7DwKRD85S7NgspxevgbulTtV+jHQIiM6NBus2/Ur/P0RMdpt0EQQ2wDq79qGN70bvvkw901N7EjSYd+GWCAM7StXtaxnLRrrZ3XI1gX7KMk8E3QzPf0zualLDs7cuQmL6l6WiElUAEqumLc7WGpLZZLdSPzNqFSg+CBKCmTI8= git: - depth: 60 + depth: 50 before_install: - travis/before_install.sh install: - - source travis/install-conda.sh + - if [[ "$BUILD_PACKAGE" != "docker" ]]; then source travis/install-conda.sh; fi - if [ -n "$PERFORM_TESTING" ]; then source travis/install-tools.sh; fi - if [ -n "$PERFORM_TESTING" ]; then travis/install-pip.sh; fi + - if [[ "$BUILD_PACKAGE" == "conda" ]]; then travis/install-conda-build.sh; fi + - if [[ "$BUILD_PACKAGE" == "docker" ]]; then travis/upgrade-docker.sh; fi script: - if [ -n "$PERFORM_TESTING" ]; then travis/tests-long.sh; fi - if [ -n "$PERFORM_TESTING" ]; then travis/tests-unit.sh; fi - - if [ -n "$BUILD_PACKAGE" ]; then travis/build-package.sh; fi - - if [ -n "$BUILD_PACKAGE" ]; then travis/deploy-docker.sh; fi + - if [[ "$BUILD_PACKAGE" == "conda" ]]; then travis_wait travis/build-conda.sh; fi + - if [[ "$BUILD_PACKAGE" == "docker" ]]; then travis_wait docker build --rm -t local/viral-ngs:build .; docker run --rm local/viral-ngs:build assembly.py --version; fi after_success: - if [ -n "$PERFORM_TESTING" ]; then coveralls; fi + - if [[ "$BUILD_PACKAGE" == "docker" ]]; then travis/deploy-docker.sh; fi after_script: # correct post-build failure on OSX due to non-zero exit code diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..0c847f8d6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +FROM broadinstitute/viral-baseimage:0.1.2 + +LABEL maintainer "Chris Tomkins-Tinch " + +# to build: +# docker build --rm . +# to run: +# Download licensed copies of GATK and Novoalign to the host machine (for Linux-64) +# export GATK_PATH=/path/to/gatk/ +# export NOVOALIGN_PATH=/path/to/novoalign/ +# docker run --rm -v $GATK_PATH:/gatk -v $NOVOALIGN_PATH:/novoalign -v /path/to/dir/on/host:/user-data -t -i ".py subcommand" +# if you receive a "no space on device" error: +# docker kill $(docker ps -a -q) +# docker rm $(docker ps -a -q) +# docker rmi $(docker images -q) +# docker volume rm $(docker volume ls -qf dangling=true) + +# DEBIAN_FRONTEND: Silence some warnings about Readline. Read more over here: +# https://github.com/phusion/baseimage-docker/issues/58 +ENV DEBIAN_FRONTEND=noninteractive INSTALL_PATH="/opt/viral-ngs" VIRAL_NGS_PATH="/opt/viral-ngs/source" + +# copy basic files +COPY docker/env_wrapper.sh docker/install-viral-ngs.sh easy-deploy-script/easy-deploy-viral-ngs.sh $INSTALL_PATH/ +RUN chmod a+x $INSTALL_PATH/*.sh + +# Prepare viral-ngs user and installation directory +COPY . $VIRAL_NGS_PATH/ +WORKDIR $INSTALL_PATH +RUN ./install-viral-ngs.sh + +# Volume setup: make external tools and data available within the container +VOLUME ["/gatk", "/novoalign", "/user-data"] +# DEBIAN_FRONTEND: Silence some warnings about Readline. Read more over here: +# https://github.com/phusion/baseimage-docker/issues/58 +ENV GATK_PATH="/gatk" NOVOALIGN_PATH="/novoalign" VIRAL_NGS_DOCKER_DATA_PATH="/user-data" DEBIAN_FRONTEND=teletype + +# It's a wrapper script to load the viral-ngs environment via the easy-deploy script +# and then run any commands desired +ENTRYPOINT ["/opt/viral-ngs/env_wrapper.sh"] +CMD ["/bin/bash"] diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index aa3a75a3d..000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,98 +0,0 @@ -FROM phusion/baseimage:0.9.22 - -LABEL maintainer "Chris Tomkins-Tinch " - -# to build: -# docker build --rm . -# to run: -# Download licensed copies of GATK and Novoalign to the host machine (for Linux-64) -# export GATK_PATH=/path/to/gatk/ -# export NOVOALIGN_PATH=/path/to/novoalign/ -# docker run --rm -v $GATK_PATH:/gatk -v $NOVOALIGN_PATH:/novoalign -v /path/to/dir/on/host:/user-data -t -i ".py subcommand" -# if you receive a "no space on device" error: -# docker kill $(docker ps -a -q) -# docker rm $(docker ps -a -q) -# docker rmi $(docker images -q) -# docker volume rm $(docker volume ls -qf dangling=true) - -# These are used to determine which conda package name and version to pull from anaconda.org -ARG VIRAL_NGS_PACKAGE -ARG VIRAL_NGS_VERSION - -# Silence some warnings about Readline. Checkout more over here: -# https://github.com/phusion/baseimage-docker/issues/58 -ENV DEBIAN_FRONTEND noninteractive - -############################## -# System packages, Google Cloud SDK, and locale -############################## -# ca-certificates and wget needed for gosu -# bzip2, liblz4-toolk, and pigz are useful for packaging and archival -# google-cloud-sdk needed when using this in GCE, xenial is what phusion/baseimage is based on -# removing /var/lib/apt/lists/* frees some space -# -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-xenial main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \ - && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ - && apt-get update \ - && apt-get install -y -qq --no-install-recommends ca-certificates wget rsync curl bzip2 python less nano vim locales google-cloud-sdk liblz4-tool pigz \ - && apt-get upgrade -y \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && locale-gen en_US.UTF-8 - -# Set default locale to en_US.UTF-8 -ENV LANG="en_US.UTF-8" LANGUAGE="en_US:en" LC_ALL="en_US.UTF-8" - -# grab gosu for easy step-down from root -# verify the signature -# verify that the binary works -ENV GOSU_VERSION 1.10 -RUN set -ex; \ - dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ - wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ - wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \ - export GNUPGHOME="$(mktemp -d)"; \ - gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ - gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ - rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc; \ - chmod +x /usr/local/bin/gosu; \ - gosu nobody true - -############################## -# Prepare user and installation directory -############################## -ENV INSTALL_DIR="/opt/viral-ngs" -RUN mkdir -p $INSTALL_DIR - -############################## -# Volume setup -############################## -# make external tools and data available within the container -VOLUME ["/gatk", "/novoalign", "/user-data"] -ENV GATK_PATH="/gatk" NOVOALIGN_PATH="/novoalign" VIRAL_NGS_DOCKER_DATA_PATH="/user-data" -# It's a wrapper script to load the viral-ngs environment via the easy-deploy script -# and then run any commands desired -# Put this up here to avoid long waiting time after heavier build layer below -ADD ./env_wrapper.sh $INSTALL_DIR/env_wrapper.sh -RUN chmod -R +x $INSTALL_DIR/env_wrapper.sh - -############################## -# Setup viral-ngs -############################## -# download and use the viral-ngs easy deploy script to install viral-ngs -#RUN curl -LO https://raw.githubusercontent.com/broadinstitute/viral-ngs-deploy/master/easy-deploy-script/easy-deploy-viral-ngs.sh -# Docker does not support conditional copies, but we can get around it by copying a file we know to exist and then wildcarded files -COPY Dockerfile viral-ngs*.tar.bz2 $INSTALL_DIR/ -ADD ./easy-deploy-script/easy-deploy-viral-ngs.sh $INSTALL_DIR/easy-deploy-viral-ngs.sh -RUN chmod +x $INSTALL_DIR/easy-deploy-viral-ngs.sh -WORKDIR $INSTALL_DIR - -ENV VIRAL_NGS_PACKAGE $VIRAL_NGS_PACKAGE -ENV VIRAL_NGS_VERSION $VIRAL_NGS_VERSION -RUN echo "Running easy install from conda: $VIRAL_NGS_PACKAGE=$VIRAL_NGS_VERSION" -RUN ./easy-deploy-viral-ngs.sh setup --viral-ngs-version $VIRAL_NGS_VERSION - -ENV DEBIAN_FRONTEND teletype - -ENTRYPOINT ["/opt/viral-ngs/env_wrapper.sh"] -CMD ["/bin/bash"] diff --git a/docker/easy-deploy-script/README.md b/docker/easy-deploy-script/README.md deleted file mode 100644 index 6b98eec45..000000000 --- a/docker/easy-deploy-script/README.md +++ /dev/null @@ -1,59 +0,0 @@ -## Easy deployment of viral-ngs - -**viral-ngs** can be deployed with help from the script in this directory, `easy-deploy-viral-ngs.sh`. This script will install an independent copy of viral-ngs from the latest source, install all dependencies, and make it simple to activate the viral-ngs environment and create projects. - -### One-line install command - -This one-line command will install viral-ngs on a 64-bit macOS or Linux system: - -``` -./easy-deploy-script/easy-deploy-viral-ngs.sh setup -``` - -### One-line install command for Broad Institute users - -This one-line command will download the `easy-deploy-viral-ngs.sh` script and setup viral-ngs in the current working directory. Simply ssh to one of the login nodes and paste this command: - - wget https://raw.githubusercontent.com/broadinstitute/viral-ngs-deploy/master/easy-deploy-script/easy-deploy-viral-ngs.sh && chmod a+x ./easy-deploy-viral-ngs.sh && reuse UGER && qrsh -l h_vmem=4G -cwd -N "viral-ngs_deploy" -q interactive ./easy-deploy-viral-ngs.sh setup - -**Note:** The script will run the install on a UGER interactive node, so you must have the ability to create to start a new interactive session. A project can be specified via `qrsh -P ""` - -### Usage - -* `./easy-deploy-viral-ngs.sh setup` Installs a fresh copy of viral-ngs, installs all dependencies, and creates a directory, `viral-ngs-etc`, in the current working directory. - -**Resulting directories**: - -``` -viral-ngs-etc/ - conda-env/ - viral-ngs/ - mc3/ -``` - -* `source ./easy-deploy-viral-ngs.sh load` Loads the dotkits needed by viral-ngs and activates the Python virtual environment - -* `./easy-deploy-viral-ngs.sh create-project ` Creates a directory for a new Snakemake-compatible project, with data directories and symlinked run scripts. Copies in the files `Snakefile` and `config.yaml` - -**Resulting directories**: - -``` -viral-ngs-etc/ - projects/ - / - Snakefile - bin/ (symlink) - config.yaml - data/ - log/ - reports/ - run-pipe_LSF.sh (symlink) - run-pipe_UGER.sh (symlink) - samples-assembly-failures.txt - samples-assembly.txt - samples-depletion.txt - samples-runs.txt - tmp/ - venv/ (symlink) - [...other project files...] -``` diff --git a/docker/easy-deploy-script/easy-deploy-viral-ngs.sh b/docker/easy-deploy-script/easy-deploy-viral-ngs.sh deleted file mode 100755 index ccd4204ef..000000000 --- a/docker/easy-deploy-script/easy-deploy-viral-ngs.sh +++ /dev/null @@ -1,732 +0,0 @@ -#!/bin/bash - -#set -e -o pipefail - -STARTING_DIR=$(pwd) - -# way to get the absolute path to this script that should -# work regardless of whether or not this script has been sourced -# Find original directory of bash script, resovling symlinks -# http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in/246128#246128 -function absolute_path() { - local SOURCE="$1" - while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - if [[ "$OSTYPE" == "darwin"* ]]; then - SOURCE="$(readlink "$SOURCE")" - else - SOURCE="$(readlink -f "$SOURCE")" - fi - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located - done - echo "$SOURCE" -} -SOURCE="${BASH_SOURCE[0]}" -SCRIPT=$(absolute_path "$SOURCE") -SCRIPT_DIRNAME="$(dirname "$SOURCE")" -SCRIPTPATH="$(cd -P "$(echo $SCRIPT_DIRNAME)" &> /dev/null && pwd)" -SCRIPT="$SCRIPTPATH/$(basename "$SCRIPT")" - -CONDA_PREFIX_LENGTH_LIMIT=250 - -CONTAINING_DIR="viral-ngs-etc" -VIRAL_NGS_DIR="viral-ngs" -CONDA_ENV_BASENAME="conda-env" -CONDA_ENV_CACHE="conda-cache" -PROJECTS_DIR="projects" -MINICONDA_DIR="mc3" - -INSTALL_PATH="${VIRAL_NGS_INSTALL_PATH:-$SCRIPTPATH/$CONTAINING_DIR}" -INSTALL_PATH=$(absolute_path "$INSTALL_PATH") - -VIRAL_CONDA_ENV_PATH="$INSTALL_PATH/$CONDA_ENV_BASENAME" -PROJECTS_PATH="$INSTALL_PATH/$PROJECTS_DIR" -VIRAL_NGS_PATH="$INSTALL_PATH/$VIRAL_NGS_DIR" -MINICONDA_PATH="$INSTALL_PATH/$MINICONDA_DIR" - -SELF_UPDATE_URL="https://raw.githubusercontent.com/broadinstitute/viral-ngs/master/easy-deploy-script/easy-deploy-viral-ngs.sh" - -CONDA_CHANNEL_STRING="-c broad-viral -c bioconda -c conda-forge -c defaults -c r" - -if [ -z "$VIRAL_NGS_PACKAGE" ]; then - VIRAL_NGS_PACKAGE="viral-ngs" -fi - -# determine if this script has been sourced -# via: http://stackoverflow.com/a/28776166/2328433 -([[ -n $ZSH_EVAL_CONTEXT && $ZSH_EVAL_CONTEXT =~ :file$ ]] || - [[ -n $KSH_VERSION && $(cd "$(dirname -- "$0")" && - printf '%s' "${PWD%/}/")$(basename -- "$0") != "${.sh.file}" ]] || - [[ -n $BASH_VERSION && $0 != "$BASH_SOURCE" ]]) && sourced=1 || sourced=0 - -current_prefix_length=$(echo $MINICONDA_PATH | wc -c | sed -n '1h;1!H;${;g;s/^[ \t]*//g;s/[ \t]*$//g;p;}') # sed trims whitespace -if [ $current_prefix_length -ge $CONDA_PREFIX_LENGTH_LIMIT ]; then - echo "ERROR: The conda path to be created by this script is too long to work with conda ($current_prefix_length characters):" - echo "$MINICONDA_PATH" - echo "This is a known bug in conda ($CONDA_PREFIX_LENGTH_LIMIT character limit): " - echo "https://github.com/conda/conda-build/pull/877" - echo "To prevent this error, move this script higher in the filesystem hierarchy." - exit 80 -fi - -python_check=$(hash python) -if [ $? -ne 0 ]; then - echo "It looks like Python is not installed. Exiting." - if [[ $sourced -eq 0 ]]; then - exit 1 - else - return 1 - fi -fi - -ram_check=$(python -c "bytearray(768000000)" &> /dev/null) -if [ $? -ne 0 ]; then - echo "" - echo "Unable to allocate 768MB." - echo "==============================================================" - echo "It appears your current system does not have enough free RAM." - echo "Consider logging in to a machine with more available memory." - echo "==============================================================" - echo "" - - if [[ $sourced -eq 0 ]]; then - exit 1 - else - return 1 - fi -fi - - -# Usage: puniq [] -# Remove duplicate entries from a PATH style value while retaining -# the original order. Use PATH if no is given. -# -# Example: -# $ puniq /usr/bin:/usr/local/bin:/usr/bin -# /usr/bin:/usr/local/bin -puniq() { - echo ${1%:} |tr : '\n' |nl |sort -u -k 2,2 |sort -n | - cut -f 2- |tr '\n' : |sed -e 's/:$//' -e 's/^://' -} - - -function set_locale(){ - export LANG="$1" - export LC_CTYPE="$1" - export LC_NUMERIC="$1" - export LC_TIME="$1" - export LC_COLLATE="$1" - export LC_MONETARY="$1" - export LC_MESSAGES="$1" - export LC_PAPER="$1" - export LC_NAME="$1" - export LC_ADDRESS="$1" - export LC_TELEPHONE="$1" - export LC_MEASUREMENT="$1" - export LC_IDENTIFICATION="$1" - export LC_ALL="$1" -} - - -function ask() { - while true; do - - if [ "${2:-}" = "Y" ]; then - prompt="Y/n" - default=Y - elif [ "${2:-}" = "N" ]; then - prompt="y/N" - default=N - else - prompt="y/n" - default= - fi - - # Ask the question - read -p "$1 [$prompt] " REPLY - - # Default? - if [ -z "$REPLY" ]; then - REPLY=$default - fi - - # Check if the reply is valid - case "$REPLY" in - Y*|y*) echo " "; return 0 ;; - N*|n*) echo " "; return 1 ;; - esac - - done -} - -if [[ "$OSTYPE" == "darwin"* ]]; then - set_locale "en_US.UTF-8" -else - set_locale "en_US.utf8" -fi - -function prepend_miniconda(){ - if [ -d "$MINICONDA_PATH/bin" ]; then - echo "Miniconda installed." - - echo "Prepending miniconda to PATH..." - PATH="$MINICONDA_PATH/bin:$PATH" - export PATH=$(puniq $PATH) - hash -r - - # update to the latest conda this way, since the shell script - # is often months out of date - #if [ -z "$SKIP_CONDA_UPDATE" ]; then - # echo "Updating conda..." - # conda update -y conda - #fi - else - echo "Miniconda directory not found." - if [[ $sourced -eq 0 ]]; then - exit 1 - else - return 1 - fi - fi -} - -function install_miniconda(){ - if [ -d "$MINICONDA_PATH/bin" ]; then - echo "Miniconda directory exists." - else - echo "Downloading and installing Miniconda..." - - if [[ "$(python -c 'import os; print(os.uname()[0])')" == "Darwin" ]]; then - miniconda_url=https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - else - miniconda_url=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh - fi - curl -sSL $miniconda_url > "$INSTALL_PATH/Miniconda3-latest-x86_64.sh" - chmod +x "$INSTALL_PATH/Miniconda3-latest-x86_64.sh" - "$INSTALL_PATH/Miniconda3-latest-x86_64.sh" -b -f -p "$MINICONDA_PATH" - - rm "$INSTALL_PATH/Miniconda3-latest-x86_64.sh" - fi - - if [ -d "$MINICONDA_PATH/bin" ]; then - prepend_miniconda - conda install -q -y -c defaults conda #==4.0.10 - else - echo "It looks like the Miniconda installation failed" - if [[ $sourced -eq 0 ]]; then - exit 1 - else - return 1 - fi - fi -} - -function install_viral_ngs_conda(){ - # provide an avenue to specify a package path, or to use a previously-built local package - if [ $# -eq 2 ]; then - if [ "$2" == "--use-local" ]; then - conda install -q $CONDA_CHANNEL_STRING --override-channels -y -p "$VIRAL_CONDA_ENV_PATH" --use-local $VIRAL_NGS_PACKAGE || exit 1 - echo "using local...." - exit 0 - else - conda install -q $CONDA_CHANNEL_STRING --override-channels -y -p "$VIRAL_CONDA_ENV_PATH" $2 || exit 1 - fi - - elif [ $# -eq 3 ]; then - if [ "$2" == "--viral-ngs-version" ]; then - conda install -q $CONDA_CHANNEL_STRING --override-channels -y -p "$VIRAL_CONDA_ENV_PATH" $VIRAL_NGS_PACKAGE=$3 || exit 1 - else - echo "--viral-ngs-version specified but no version given" - fi - elif [ $# -eq 1 ]; then - conda install -q $CONDA_CHANNEL_STRING --override-channels -y -p "$VIRAL_CONDA_ENV_PATH" $VIRAL_NGS_PACKAGE || exit 1 - fi -} - -function install_viral_ngs_git(){ - if [ ! -L "$VIRAL_NGS_PATH" ]; then - # First optional argument specifies non-master branch - if [[ $# -eq 1 ]]; then - git clone https://github.com/broadinstitute/viral-ngs.git --branch $1 "$VIRAL_NGS_PATH" - else - git clone https://github.com/broadinstitute/viral-ngs.git "$VIRAL_NGS_PATH" - fi - else - echo "$VIRAL_NGS_PATH/ symlink already exists. Skipping link." - fi -} - -function install_viral_ngs_conda_dependencies() { - conda install -q $CONDA_CHANNEL_STRING --override-channels -y -p $VIRAL_CONDA_ENV_PATH --file "$VIRAL_NGS_PATH/requirements-conda.txt" || exit 1 - conda install -q $CONDA_CHANNEL_STRING --override-channels -y -p $VIRAL_CONDA_ENV_PATH --file "$VIRAL_NGS_PATH/requirements-py3.txt" || exit 1 - conda install -q $CONDA_CHANNEL_STRING --override-channels -y -p $VIRAL_CONDA_ENV_PATH --file "$VIRAL_NGS_PATH/requirements-conda-tests.txt" || exit 1 -} - - -function install_tools(){ - # install tools - #pytest $VIRAL_NGS_PATH/test/unit/test_tools.py - - # get the version of gatk expected based on the installed conda package - EXPECTED_GATK_VERSION=$(conda list | grep gatk | awk -F" " '{print $2}') - if [ -z "$GATK_JAR_PATH" ]; then - # if the env var is not set, try to get the jar location using the default Broad path - if [[ "$(hash dnsdomainname &> /dev/null && dnsdomainname || echo '')" == *"broadinstitute.org" || "$HOSTNAME" == *".broadinstitute.org" || "$DOMAINNAME" == "broadinstitute.org" ]]; then - echo "This script is being run on a Broad Institute system." - echo "Trying to find GATK..." - export GATK_JAR_PATH=$(ls /humgen/gsa-hpprojects/GATK/bin &> /dev/null && sleep 5 && find /humgen/gsa-hpprojects/GATK/bin/GenomeAnalysisTK-$EXPECTED_GATK_VERSION-* -maxdepth 0 -type d)/GenomeAnalysisTK.jar - fi - fi - - # if the gatk jar file exists, call gatk-register - if [ -e "$GATK_JAR_PATH" ]; then - echo "GATK found: $GATK_JAR_PATH" - gatk-register $GATK_JAR_PATH - else - echo "GATK jar could not be found on this system for GATK version $EXPECTED_GATK_VERSION" - echo "Please activate the viral-ngs conda environment and 'gatk-register /path/to/GenomeAnalysisTK.jar'" - exit 0 - fi - - echo "" - if [ ! -z "$NOVOALIGN_PATH" ]; then - novoalign-license-register "$NOVOALIGN_PATH/novoalign.lic" - elif [ ! -z "$NOVOALIGN_LICENSE_PATH" ]; then - novoalign-license-register "$NOVOALIGN_LICENSE_PATH" - else - echo "No Novoalign license found via NOVOALIGN_PATH or NOVOALIGN_LICENSE_PATH" - echo "Please activate the viral-ngs conda environment and run 'novoalign-license-register /path/to/novoalign.lic'" - fi -} - -function create_project(){ - echo "Checking and populating project directory..." - # first arg is project folder name - starting_dir=$(pwd) - - mkdir -p $PROJECTS_PATH - PROJECT_PATH="$PROJECTS_PATH/$1" - mkdir -p "$PROJECT_PATH" - pushd "$PROJECT_PATH" > /dev/null - mkdir -p data log reports tmp - pushd data > /dev/null - mkdir -p 00_raw 01_cleaned 01_per_sample 02_align_to_self 02_assembly 03_align_to_ref 03_interhost 04_intrahost - popd > /dev/null - touch samples-metagenomics.txt - touch samples-depletion.txt - touch samples-assembly.txt - touch samples-runs.txt - touch samples-assembly-failures.txt - pushd > /dev/null - - if [ ! -e "$PROJECT_PATH/config.yaml" ]; then - cp "$VIRAL_NGS_PATH/pipes/config.yaml" "$PROJECT_PATH" - fi - if [ ! -L "$PROJECT_PATH/Snakefile" ]; then - ln -s "$VIRAL_NGS_PATH/pipes/Snakefile" "$PROJECT_PATH/Snakefile" - fi - - if [ ! -L "$PROJECT_PATH/bin" ]; then - ln -s "$VIRAL_NGS_PATH" "$PROJECT_PATH/bin" - fi - - if [ ! -L "$PROJECT_PATH/conda-env" ]; then - ln -s "$INSTALL_PATH/$CONDA_ENV_BASENAME" "$PROJECT_PATH/conda-env" - fi - - if [ ! -L "$PROJECT_PATH/mc3" ]; then - ln -s "$MINICONDA_PATH" "$PROJECT_PATH/mc3" - fi - - if [ -z "$OMIT_UGER_PROJECT_FILES" ]; then - if [ ! -L "$PROJECT_PATH/run-pipe_UGER.sh" ]; then - ln -s "$VIRAL_NGS_PATH/pipes/Broad_UGER/run-pipe.sh" "$PROJECT_PATH/run-pipe_UGER.sh" - fi - if [ ! -L "$PROJECT_PATH/run-pipe.sh" ]; then - ln -s "run-pipe_UGER.sh" "$PROJECT_PATH/run-pipe.sh" - fi - fi - - cd "$starting_dir" -} - -function activate_env(){ - if [ -d "$INSTALL_PATH" ]; then - echo "viral-ngs parent directory found" - else - echo "viral-ngs parent directory not found: $INSTALL_PATH not found." - echo "Have you run the setup?" - echo "Usage: $0 setup" - cd "$STARTING_DIR" - return 1 - fi - - # Add viral-ngs scripts to PATH for git-based installs - if [[ -d "$VIRAL_NGS_PATH/.git" ]]; then - PATH="$VIRAL_NGS_PATH:$PATH" - PATH=$(puniq $PATH) - fi - - if [ -d "$VIRAL_CONDA_ENV_PATH" ]; then - if [ -z "$CONDA_DEFAULT_ENV" ]; then - echo "Activating viral-ngs environment..." - prepend_miniconda - - source activate $VIRAL_CONDA_ENV_PATH - - # unset JAVA_HOME if set, so we can use the conda-supplied version - if [ ! -z "$JAVA_HOME" ]; then - unset JAVA_HOME - fi - - # override $PS1 to have a shorter prompt - export PS1="(\[\033[1m\]viral-ngs\[\033[0m\])\s:\h:\w \! \$ " - else - if [[ "$CONDA_DEFAULT_ENV" != "$VIRAL_CONDA_ENV_PATH" ]]; then - echo "It looks like a conda environment is already active," - echo "however it is not the viral-ngs environment." - echo "To use viral-ngs with your project, deactivate the" - echo "current environment and then source this file." - echo "Example: source deactivate && source $(basename $SCRIPT) load" - else - echo "The viral-ngs environment is already active." - fi - return 0 - fi - else - echo "$VIRAL_CONDA_ENV_PATH/ does not exist. Exiting." - cd "$STARTING_DIR" - return 1 - fi -} - -function print_usage(){ - echo "Usage: $(basename $SCRIPT) {load,create-project,setup|setup-py2|setup-git,upgrade|upgrade-deps,update-easy-deploy}" -} - -function symlink_viral_ngs(){ - # the conda-installed viral-ngs folder resides within the - # opt/ directory of the conda environment, but it contains - # a version number, so we'll ls and grep for the name - # and assume it's the first one to show up - if [ ! -L "$VIRAL_NGS_PATH" ]; then - echo "Linking to current viral-ngs install..." - EXPECTED_VIRAL_NGS_VERSION=$(conda list $VIRAL_NGS_PACKAGE | grep $VIRAL_NGS_PACKAGE | grep -v packages | awk -F" " '{print $2}') - VIRAL_NGS_CONDA_PATH="$VIRAL_CONDA_ENV_PATH/opt/"$(ls -1 "$VIRAL_CONDA_ENV_PATH/opt/" | grep "$EXPECTED_VIRAL_NGS_VERSION" | grep -m 1 "viral-ngs") - - if [ -d "$VIRAL_NGS_CONDA_PATH" ]; then - ln -s "$VIRAL_NGS_CONDA_PATH" "$VIRAL_NGS_PATH" - else - echo "Could not find viral-ngs install in conda env:" - echo "$VIRAL_NGS_CONDA_PATH" - exit 1 - fi - else - echo "$VIRAL_NGS_DIR/ symlink already exists. Skipping link." - fi -} - -function check_viral_ngs_version(){ - if [ -d "$VIRAL_NGS_PATH/.git" ]; then - pushd "$VIRAL_NGS_PATH" > /dev/null - git remote update - git status -uno - popd > /dev/null - elif [ -z "$SKIP_VERSION_CHECK" ]; then - # this must be run within an active conda environment - # so, after a call to "activate_env" - echo "Checking viral-ngs version..." - CURRENT_VER="$(conda list --no-pip viral-ngs | grep viral-ngs | grep -v packages | awk -F' ' '{print $2}')" - # perhaps a better way... - AVAILABLE_VER="$(conda search --override-channels -f $CONDA_CHANNEL_STRING --override-channels viral-ngs --json | grep version | tail -n 1 | awk -F' ' '{print $2}' | perl -lape 's/[\",]+//g')" - if [ "$CURRENT_VER" != "$AVAILABLE_VER" ]; then - echo "" - echo "============================================================================================================" - echo "Your copy of viral-ngs appears to be outdated ($CURRENT_VER). A newer version is available ($AVAILABLE_VER)." - echo "Check the release notes and consider upgrading:" - echo "https://github.com/broadinstitute/viral-ngs/releases" - echo "To upgrade: $(basename $SCRIPT) upgrade" - echo "============================================================================================================" - echo "" - else - echo "viral-ngs is up to date ($CURRENT_VER)" - fi - fi -} - -function updateSelf() { - # this function overwrites this script with one downloaded from - # the first argument passed to the funciton, $1 - - echo "Performing self-update..." - - cp "$SCRIPT" "$SCRIPT.bak" - - # Download new version - echo -n "Downloading latest version..." - if ! wget --quiet --output-document="$SCRIPT.tmp" "$1" ; then - echo "Error while trying to wget new version!" - echo "File requested: $SELF_UPDATE_URL" - exit 1 - fi - echo "done." - - # Copy permissions from old version - if [[ "$OSTYPE" == "darwin"* ]]; then - OCTAL_MODE=$(stat -f '%A' $SCRIPT) - else - OCTAL_MODE=$(stat -c '%a' $SCRIPT) - fi - if ! chmod $OCTAL_MODE "$SCRIPT.tmp" ; then - echo "Failed: Error while trying to set mode on $SCRIPT.tmp." - exit 1 - fi - - # Spawn update script - cat > update-easy-deploy-script.sh << EOF -#!/bin/bash -# Overwrite old file with new -if mv "$SCRIPT.tmp" "$SCRIPT"; then - echo "done." - echo "Self-update complete." - rm \$0 -else - echo "Failed!" -fi -EOF - - echo -n "Overwriting old script with new one..." - exec /bin/bash update-easy-deploy-script.sh -} - -if [ $# -eq 0 ]; then - print_usage - if [[ $sourced -eq 0 ]]; then - exit 1 - else - return 1 - fi -else - case "$1" in - "setup"|"setup-py2"|"setup-git") - if ! [ $# -eq 1 -o $# -eq 2 -o $# -eq 3 ]; then - - echo "Usage: $(basename $SCRIPT) setup" - if [[ $sourced -eq 0 ]]; then - exit 1 - else - return 1 - fi - fi - - if [[ $sourced -eq 1 ]]; then - echo "ABORTING. $(basename $SCRIPT) must not be sourced during setup" - echo "Usage: $(basename $SCRIPT) setup" - return 1 - elif [ ! -z "$CONDA_DEFAULT_ENV" ]; then - echo "The viral-ngs setup cannot be run while a conda environment is active." - echo "The current environment must first be disabled via 'source deactivate'" - exit 1 - fi - - mkdir -p "$INSTALL_PATH" - install_miniconda - - if [ ! -d "$VIRAL_CONDA_ENV_PATH" ]; then - # provide an option to use Python 2 in the conda environment - if [ "$1" == "setup-py2" ]; then - conda create -q $CONDA_CHANNEL_STRING --override-channels -y -p "$VIRAL_CONDA_ENV_PATH" python=2 || exit 1 - else - conda create -q $CONDA_CHANNEL_STRING --override-channels -y -p "$VIRAL_CONDA_ENV_PATH" python=3.6 || exit 1 - fi - - if [[ "$1" == "setup-git" ]]; then - install_viral_ngs_git $2 - install_viral_ngs_conda_dependencies - else - install_viral_ngs_conda $@ - fi - else - echo "$VIRAL_CONDA_ENV_PATH/ already exists. Skipping conda env setup." - fi - - conda clean -y --all - - activate_env - - if [[ "$1" != "setup-git" ]]; then - symlink_viral_ngs - fi - - install_tools - - echo "Setup complete. Do you want to start a project? Run:" - echo "$0 create-project [/containing/path]" - echo "" - ;; - "load") - if [ $# -eq 1 ]; then - if [[ $sourced -eq 0 ]]; then - echo "ABORTING. $(basename $SCRIPT) must be sourced." - echo "Usage: source $(basename $SCRIPT) load" - else - activate_env - - check_viral_ngs_version - - return 0 - fi - else - echo "Usage: source $(basename $SCRIPT) load" - if [[ $sourced -eq 0 ]]; then - exit 1 - else - return 1 - fi - fi - ;; - "upgrade-deps") - if [ ! -d "$VIRAL_NGS_PATH/.git" ]; then - echo "upgrade-deps requires a viral-ngs installation from git" - echo "Use 'upgrade' for a versioned install" - exit 1 - fi - activate_env - install_viral_ngs_conda_dependencies - ;; - "upgrade") - if [ $# -eq 1 ]; then - if [[ $sourced -eq 1 ]]; then - echo "ABORTING. $(basename $SCRIPT) must not be sourced during upgrade" - echo "Usage: $(basename $SCRIPT) upgrade" - return 1 - else - echo "Upgrading viral-ngs..." - - if [ -z "$CONDA_DEFAULT_ENV" ]; then - activate_env - else - if [ "$CONDA_DEFAULT_ENV" != "$VIRAL_CONDA_ENV_PATH" ]; then - echo "A viral-ngs upgrade cannot be run while a conda environment is active." - echo "The current environment must first be disabled via 'source deactivate'" - exit 1 - fi - fi - - if [ ! -z "$(conda list $VIRAL_NGS_PACKAGE | grep $VIRAL_NGS_PACKAGE | grep -v packages | awk -F" " '{print $2}')" ]; then - if [ -L "$VIRAL_NGS_PATH" ]; then - rm $VIRAL_NGS_PATH # remove symlink - fi - conda update -y $CONDA_CHANNEL_STRING --override-channels viral-ngs - - # recreate symlink to folder for latest viral-ngs in conda-env/opt/ - symlink_viral_ngs - - echo "" - echo "==================================================================================" - echo "Note that if viral-ngs-derived files are present in your project folders," - echo "they may need to be updated. Check the release notes and commit log for more info:" - echo "https://github.com/broadinstitute/viral-ngs/releases" - echo "==================================================================================" - echo "" - else - echo "The viral-ngs package does not appear to be installed. Have you run setup?" - echo "Usage: $(basename $SCRIPT) setup" - exit 1 - fi - #source deactivate - exit 0 - fi - else - echo "Usage: source $(basename $SCRIPT) upgrade" - if [[ $sourced -eq 0 ]]; then - exit 1 - else - return 1 - fi - fi - ;; - "update-easy-deploy") - if [ $# -eq 1 ]; then - if [[ $sourced -eq 1 ]]; then - echo "ABORTING. $(basename $SCRIPT) must not be sourced during upgrade" - echo "Usage: $(basename $SCRIPT) update-easy-deploy" - return 1 - else - if [ -z "$CONDA_DEFAULT_ENV" ]; then - if [ ! -z "$SKIP_SELF_UPDATE_CONFIRM" ] || $(ask "Are you sure you want to update the easy deploy script to the latest version?" Y); then - updateSelf "$SELF_UPDATE_URL" - fi - else - echo "It looks like a conda environment is active." - echo "To update this script, first deactivate the environment" - echo "then call update-easy-deploy. Example:" - echo " source deactivate && $(basename $SCRIPT) update-easy-deploy" - exit 1 - fi - - fi - else - echo "Usage: source $(basename $SCRIPT) update-easy-deploy" - if [[ $sourced -eq 0 ]]; then - exit 1 - else - return 1 - fi - fi - ;; - "create-project") - if [ $# -ne 2 -a $# -ne 3 ]; then - echo "Usage: $(basename $SCRIPT) create-project [/containing/path]" - if [[ $sourced -eq 0 ]]; then - exit 1 - else - return 1 - fi - else - # check for viral-ngs install; abort project creation if missing - if [ ! -d "$VIRAL_CONDA_ENV_PATH" ]; then - echo "It looks like viral-ngs has not yet been installed." - echo "Directory does not exist: $VIRAL_CONDA_ENV_PATH" - echo "First run $(basename $SCRIPT) setup" - if [[ $sourced -eq 0 ]]; then - exit 1 - else - return 1 - fi - fi - ORIGINAL_PROJECTS_PATH=$PROJECTS_PATH - PROJECT_NAME="$2" - if [ $# -eq 3 ]; then - PROJECTS_PATH="$3" - echo "Creating project in path: $PROJECTS_PATH/$PROJECT_NAME" - else - echo "Creating project: $PROJECT_NAME" - fi - - create_project $PROJECT_NAME && echo "Project created: $PROJECTS_PATH/$PROJECT_NAME" && echo "OK" - - echo "" - - if [[ "$CONDA_DEFAULT_ENV" != "$VIRAL_CONDA_ENV_PATH" ]]; then - echo "It looks like the vial-ngs environment is not active." - echo "To use viral-ngs with your project, source this file." - echo "Example: source $(basename $SCRIPT) load" - else - # if the viral-ngs environment is active and we have sourced this file - if [[ $sourced -eq 1 ]]; then - # change to the project directory - if [ -d "$PROJECTS_PATH/$PROJECT_NAME" ]; then - cd "$PROJECTS_PATH/$PROJECT_NAME" - fi - return 0 - fi - fi - PROJECTS_PATH=$ORIGINAL_PROJECTS_PATH - fi - ;; - *) - print_usage - ;; - esac -fi diff --git a/docker/install-viral-ngs.sh b/docker/install-viral-ngs.sh new file mode 100755 index 000000000..6c51c9f96 --- /dev/null +++ b/docker/install-viral-ngs.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# +# This script requires INSTALL_PATH (typically /opt/viral-ngs) +# and VIRAL_NGS_PATH (typically /opt/viral-ngs/source) to be set. + +set -e -o pipefail + +export VIRAL_CONDA_ENV_PATH=/opt/miniconda + +mkdir -p $INSTALL_PATH/viral-ngs-etc +ln -s $VIRAL_NGS_PATH $INSTALL_PATH/viral-ngs-etc/viral-ngs +ln -s $VIRAL_CONDA_ENV_PATH $INSTALL_PATH/viral-ngs-etc/conda-env + +# setup/install viral-ngs +sync +$INSTALL_PATH/easy-deploy-viral-ngs.sh setup-git-local + +# this not only prints the current version string, but it also saves it to the +# VERSION file for later use +PATH=$VIRAL_CONDA_ENV_PATH/bin:$PATH +echo -n "viral-ngs version: " +$VIRAL_NGS_PATH/assembly.py --version diff --git a/easy-deploy-script/easy-deploy-viral-ngs.sh b/easy-deploy-script/easy-deploy-viral-ngs.sh index ccd4204ef..52f889581 100755 --- a/easy-deploy-script/easy-deploy-viral-ngs.sh +++ b/easy-deploy-script/easy-deploy-viral-ngs.sh @@ -39,10 +39,14 @@ MINICONDA_DIR="mc3" INSTALL_PATH="${VIRAL_NGS_INSTALL_PATH:-$SCRIPTPATH/$CONTAINING_DIR}" INSTALL_PATH=$(absolute_path "$INSTALL_PATH") -VIRAL_CONDA_ENV_PATH="$INSTALL_PATH/$CONDA_ENV_BASENAME" +if [ -z "$VIRAL_CONDA_ENV_PATH" ]; then + VIRAL_CONDA_ENV_PATH="$INSTALL_PATH/$CONDA_ENV_BASENAME" +fi PROJECTS_PATH="$INSTALL_PATH/$PROJECTS_DIR" VIRAL_NGS_PATH="$INSTALL_PATH/$VIRAL_NGS_DIR" -MINICONDA_PATH="$INSTALL_PATH/$MINICONDA_DIR" +if [ -z "$MINICONDA_PATH" ]; then + MINICONDA_PATH="$INSTALL_PATH/$MINICONDA_DIR" +fi SELF_UPDATE_URL="https://raw.githubusercontent.com/broadinstitute/viral-ngs/master/easy-deploy-script/easy-deploy-viral-ngs.sh" @@ -244,7 +248,7 @@ function install_viral_ngs_conda(){ } function install_viral_ngs_git(){ - if [ ! -L "$VIRAL_NGS_PATH" ]; then + if [ ! -d "$VIRAL_NGS_PATH" ]; then # First optional argument specifies non-master branch if [[ $# -eq 1 ]]; then git clone https://github.com/broadinstitute/viral-ngs.git --branch $1 "$VIRAL_NGS_PATH" @@ -257,15 +261,12 @@ function install_viral_ngs_git(){ } function install_viral_ngs_conda_dependencies() { - conda install -q $CONDA_CHANNEL_STRING --override-channels -y -p $VIRAL_CONDA_ENV_PATH --file "$VIRAL_NGS_PATH/requirements-conda.txt" || exit 1 - conda install -q $CONDA_CHANNEL_STRING --override-channels -y -p $VIRAL_CONDA_ENV_PATH --file "$VIRAL_NGS_PATH/requirements-py3.txt" || exit 1 - conda install -q $CONDA_CHANNEL_STRING --override-channels -y -p $VIRAL_CONDA_ENV_PATH --file "$VIRAL_NGS_PATH/requirements-conda-tests.txt" || exit 1 + conda install -q $CONDA_CHANNEL_STRING --override-channels -y -p $VIRAL_CONDA_ENV_PATH --file "$VIRAL_NGS_PATH/requirements-conda.txt" --file "$VIRAL_NGS_PATH/requirements-py3.txt" --file "$VIRAL_NGS_PATH/requirements-conda-tests.txt" || exit 1 } function install_tools(){ # install tools - #pytest $VIRAL_NGS_PATH/test/unit/test_tools.py # get the version of gatk expected based on the installed conda package EXPECTED_GATK_VERSION=$(conda list | grep gatk | awk -F" " '{print $2}') @@ -289,9 +290,9 @@ function install_tools(){ fi echo "" - if [ ! -z "$NOVOALIGN_PATH" ]; then + if [ -n "$NOVOALIGN_PATH" ]; then novoalign-license-register "$NOVOALIGN_PATH/novoalign.lic" - elif [ ! -z "$NOVOALIGN_LICENSE_PATH" ]; then + elif [ -n "$NOVOALIGN_LICENSE_PATH" ]; then novoalign-license-register "$NOVOALIGN_LICENSE_PATH" else echo "No Novoalign license found via NOVOALIGN_PATH or NOVOALIGN_LICENSE_PATH" @@ -372,10 +373,12 @@ function activate_env(){ echo "Activating viral-ngs environment..." prepend_miniconda - source activate $VIRAL_CONDA_ENV_PATH + if [[ "$CONDA_DEFAULT_ENV" != "$VIRAL_CONDA_ENV_PATH" ]]; then + source activate $VIRAL_CONDA_ENV_PATH + fi # unset JAVA_HOME if set, so we can use the conda-supplied version - if [ ! -z "$JAVA_HOME" ]; then + if [ -n "$JAVA_HOME" ]; then unset JAVA_HOME fi @@ -401,7 +404,7 @@ function activate_env(){ } function print_usage(){ - echo "Usage: $(basename $SCRIPT) {load,create-project,setup|setup-py2|setup-git,upgrade|upgrade-deps,update-easy-deploy}" + echo "Usage: $(basename $SCRIPT) {load,create-project,setup|setup-py2|setup-git|setup-git-local,upgrade|upgrade-deps,update-easy-deploy}" } function symlink_viral_ngs(){ @@ -409,7 +412,7 @@ function symlink_viral_ngs(){ # opt/ directory of the conda environment, but it contains # a version number, so we'll ls and grep for the name # and assume it's the first one to show up - if [ ! -L "$VIRAL_NGS_PATH" ]; then + if [ ! -d "$VIRAL_NGS_PATH" ]; then echo "Linking to current viral-ngs install..." EXPECTED_VIRAL_NGS_VERSION=$(conda list $VIRAL_NGS_PACKAGE | grep $VIRAL_NGS_PACKAGE | grep -v packages | awk -F" " '{print $2}') VIRAL_NGS_CONDA_PATH="$VIRAL_CONDA_ENV_PATH/opt/"$(ls -1 "$VIRAL_CONDA_ENV_PATH/opt/" | grep "$EXPECTED_VIRAL_NGS_VERSION" | grep -m 1 "viral-ngs") @@ -427,29 +430,31 @@ function symlink_viral_ngs(){ } function check_viral_ngs_version(){ - if [ -d "$VIRAL_NGS_PATH/.git" ]; then - pushd "$VIRAL_NGS_PATH" > /dev/null - git remote update - git status -uno - popd > /dev/null - elif [ -z "$SKIP_VERSION_CHECK" ]; then - # this must be run within an active conda environment - # so, after a call to "activate_env" - echo "Checking viral-ngs version..." - CURRENT_VER="$(conda list --no-pip viral-ngs | grep viral-ngs | grep -v packages | awk -F' ' '{print $2}')" - # perhaps a better way... - AVAILABLE_VER="$(conda search --override-channels -f $CONDA_CHANNEL_STRING --override-channels viral-ngs --json | grep version | tail -n 1 | awk -F' ' '{print $2}' | perl -lape 's/[\",]+//g')" - if [ "$CURRENT_VER" != "$AVAILABLE_VER" ]; then - echo "" - echo "============================================================================================================" - echo "Your copy of viral-ngs appears to be outdated ($CURRENT_VER). A newer version is available ($AVAILABLE_VER)." - echo "Check the release notes and consider upgrading:" - echo "https://github.com/broadinstitute/viral-ngs/releases" - echo "To upgrade: $(basename $SCRIPT) upgrade" - echo "============================================================================================================" - echo "" + if [ -z "$SKIP_VERSION_CHECK" ]; then + if [ -d "$VIRAL_NGS_PATH/.git" ]; then + pushd "$VIRAL_NGS_PATH" > /dev/null + git remote update + git status -uno + popd > /dev/null else - echo "viral-ngs is up to date ($CURRENT_VER)" + # this must be run within an active conda environment + # so, after a call to "activate_env" + echo "Checking viral-ngs version..." + CURRENT_VER="$(conda list --no-pip viral-ngs | grep viral-ngs | grep -v packages | awk -F' ' '{print $2}')" + # perhaps a better way... + AVAILABLE_VER="$(conda search --override-channels -f $CONDA_CHANNEL_STRING --override-channels viral-ngs --json | grep version | tail -n 1 | awk -F' ' '{print $2}' | perl -lape 's/[\",]+//g')" + if [ "$CURRENT_VER" != "$AVAILABLE_VER" ]; then + echo "" + echo "============================================================================================================" + echo "Your copy of viral-ngs appears to be outdated ($CURRENT_VER). A newer version is available ($AVAILABLE_VER)." + echo "Check the release notes and consider upgrading:" + echo "https://github.com/broadinstitute/viral-ngs/releases" + echo "To upgrade: $(basename $SCRIPT) upgrade" + echo "============================================================================================================" + echo "" + else + echo "viral-ngs is up to date ($CURRENT_VER)" + fi fi fi } @@ -508,7 +513,7 @@ if [ $# -eq 0 ]; then fi else case "$1" in - "setup"|"setup-py2"|"setup-git") + "setup"|"setup-py2"|"setup-git"|"setup-git-local") if ! [ $# -eq 1 -o $# -eq 2 -o $# -eq 3 ]; then echo "Usage: $(basename $SCRIPT) setup" @@ -540,7 +545,7 @@ else conda create -q $CONDA_CHANNEL_STRING --override-channels -y -p "$VIRAL_CONDA_ENV_PATH" python=3.6 || exit 1 fi - if [[ "$1" == "setup-git" ]]; then + if [[ "$1" == "setup-git" || "$1" == "setup-git-local" ]]; then install_viral_ngs_git $2 install_viral_ngs_conda_dependencies else @@ -548,6 +553,9 @@ else fi else echo "$VIRAL_CONDA_ENV_PATH/ already exists. Skipping conda env setup." + if [[ "$1" == "setup-git-local" ]]; then + install_viral_ngs_conda_dependencies + fi fi conda clean -y --all @@ -558,7 +566,9 @@ else symlink_viral_ngs fi + #if [[ "$1" != "setup-git-local" ]]; then install_tools + #fi echo "Setup complete. Do you want to start a project? Run:" echo "$0 create-project [/containing/path]" diff --git a/requirements-conda.txt b/requirements-conda.txt index 6e0da87bd..ab6aa1bfd 100644 --- a/requirements-conda.txt +++ b/requirements-conda.txt @@ -32,6 +32,5 @@ future==0.16.0 matplotlib=1.5.3 pysam=0.12.0.1 pybedtools=0.7.10 -Jinja2=2.8 PyYAML=3.11 six=1.10.0 diff --git a/travis/before_install.sh b/travis/before_install.sh index 5ba636a2b..c8ce1a64c 100755 --- a/travis/before_install.sh +++ b/travis/before_install.sh @@ -26,10 +26,3 @@ if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then # See: https://github.com/travis-ci/travis-ci/issues/6307 rvm get head fi - -if [ ! -z "$BUILD_PACKAGE" ]; then - # upgrade docker (Travis's is old) - sudo apt-get update - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce - docker --version -fi diff --git a/travis/build-package.sh b/travis/build-conda.sh similarity index 79% rename from travis/build-package.sh rename to travis/build-conda.sh index 17db1a8f7..1f523e665 100755 --- a/travis/build-package.sh +++ b/travis/build-conda.sh @@ -25,8 +25,6 @@ if [ -n "$TRAVIS_TAG" ]; then python packaging/conda-recipe/render-recipe.py "$TRAVIS_TAG" --build-reqs requirements-conda.txt --run-reqs requirements-conda.txt --py3-run-reqs requirements-py3.txt --py2-run-reqs requirements-py2.txt --test-reqs requirements-conda-tests.txt CONDA_PERL=5.22.0 conda build -c broad-viral -c r -c bioconda -c conda-forge -c defaults --python "$TRAVIS_PYTHON_VERSION" --token "$ANACONDA_TOKEN" packaging/conda-recipe/viral-ngs - # Tell docker build which conda package to pull (this will change later) - DOCKER_BUILD_ARGS="--build-arg VIRAL_NGS_PACKAGE=viral-ngs --build-arg VIRAL_NGS_VERSION=$VIRAL_NGS_VERSION" else # This is a development build @@ -48,13 +46,4 @@ else python packaging/conda-recipe/render-recipe.py "$CONDA_PKG_VERSION" --package-name "viral-ngs-dev" --download-filename "$TRAVIS_COMMIT" --build-reqs requirements-conda.txt --run-reqs requirements-conda.txt --py3-run-reqs requirements-py3.txt --py2-run-reqs requirements-py2.txt --test-reqs requirements-conda-tests.txt CONDA_PERL=5.22.0 conda build -c broad-viral -c r -c bioconda -c conda-forge -c defaults --python "$TRAVIS_PYTHON_VERSION" --token "$ANACONDA_TOKEN" --output-folder "$CONDA_PACKAGE_OUTDIR" packaging/conda-recipe/viral-ngs - # Tell docker build which conda package to pull (this will change later) - DOCKER_BUILD_ARGS="--build-arg VIRAL_NGS_PACKAGE=viral-ngs-dev --build-arg VIRAL_NGS_VERSION=$CONDA_PKG_VERSION" fi - -# ----- -# eventually separate off this bottom section after converting docker build to something based on setup-git - -# build & test docker image -docker build $DOCKER_BUILD_ARGS --rm -t local/viral-ngs:build ./docker -docker run --rm local/viral-ngs:build illumina.py diff --git a/travis/deploy-docker.sh b/travis/deploy-docker.sh index 0514c36b9..e0144a39c 100755 --- a/travis/deploy-docker.sh +++ b/travis/deploy-docker.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -o pipefail -if [[ -n "$BUILD_PACKAGE" && -n "$DOCKER_PASS" && -n "$DOCKER_USER" ]]; then +if [[ -n "$DOCKER_PASS" && -n "$DOCKER_USER" ]]; then echo "Deploying docker image to DockerHub" # log in to DockerHub @@ -12,7 +12,7 @@ if [[ -n "$BUILD_PACKAGE" && -n "$DOCKER_PASS" && -n "$DOCKER_USER" ]]; then # this is an official tagged release DOCKER_REPO="broadinstitute/viral-ngs" DOCKER_SHORT_TAG="latest" - DOCKER_LONG_TAG="$TRAVIS_TAG" + DOCKER_LONG_TAG="$(echo $TRAVIS_TAG | sed 's/^v//')" else DOCKER_REPO="broadinstitute/viral-ngs-dev" if [ -n "$TRAVIS_PULL_REQUEST_BRANCH" ]; then @@ -41,6 +41,6 @@ if [[ -n "$BUILD_PACKAGE" && -n "$DOCKER_PASS" && -n "$DOCKER_USER" ]]; then done else - echo "Skipping DockerHub deploy" + echo "Skipping DockerHub deploy unless DOCKER_USER and DOCKER_PASS variables are set." fi diff --git a/travis/install-conda-build.sh b/travis/install-conda-build.sh new file mode 100755 index 000000000..1b2e1243e --- /dev/null +++ b/travis/install-conda-build.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +conda install -q -y conda-build==3.0.25 +conda install -q -y anaconda-client +conda create -m -q -y -p tools/conda-tools/default Jinja2==2.8 diff --git a/travis/install-conda.sh b/travis/install-conda.sh index 33cbd05e8..a4994021a 100755 --- a/travis/install-conda.sh +++ b/travis/install-conda.sh @@ -40,7 +40,6 @@ else # if it does not exist, we need to install miniconda fi hash -r conda config --set always_yes yes --set changeps1 no - conda config --set anaconda_upload yes # for uploading packages after successful build conda config --add channels r conda config --add channels defaults conda config --add channels conda-forge @@ -49,9 +48,6 @@ else # if it does not exist, we need to install miniconda conda install --quiet -y conda #conda=4.2 # pin to 4.2.* until this is fixed: https://github.com/conda/conda-build/issues/1666 conda config --set auto_update_conda false conda install --quiet -y java-jdk==8.0.112 - conda install --quiet -y conda-build==3.0.25 # needed to build recipe - conda install --quiet -y anaconda-client # needed to upload build package to anaconda.org - #conda install --quiet -y -c conda-forge -f curl # the bioconda curl is broken as of 21 Feb 2017 fi conda info -a # for debugging diff --git a/travis/upgrade-docker.sh b/travis/upgrade-docker.sh new file mode 100755 index 000000000..727192727 --- /dev/null +++ b/travis/upgrade-docker.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e + +# upgrade docker (Travis's is old) +sudo apt-get update +sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce +docker --version