From 9149e0559923703cc4ac7509822084da6ead1bec Mon Sep 17 00:00:00 2001 From: Chris Tomkins-Tinch Date: Fri, 13 Oct 2017 19:31:13 -0400 Subject: [PATCH] fixes related to testing+building conda recipes (#694) * pin conda-built to newer version * switch to curl for miniconda acquisition * render+build the conda package on PRs, on master, and on tags, but only upload to anaconda.org on tags * travis-trigger curl call is now silent, except on error * move build-an-deploy script to after_success deploy conditional is now in the script itself, so we can call it every time * placeholder version number for conda build test * recipe render can now specify filename override to handle downloading file for branch --- .travis.yml | 15 +++--- packaging/conda-recipe/render-recipe.py | 7 +++ travis/build-and-deploy.sh | 62 +++++++++++++++++++++++++ travis/deploy.sh | 52 --------------------- travis/install-conda.sh | 11 +++-- travis/install-tools.sh | 6 +-- travis/trigger-travis.sh | 2 +- 7 files changed, 86 insertions(+), 69 deletions(-) create mode 100755 travis/build-and-deploy.sh delete mode 100755 travis/deploy.sh diff --git a/.travis.yml b/.travis.yml index be5f04ab1..1584b58f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,15 +59,16 @@ script: after_success: - coveralls + - travis/build-and-deploy.sh $TRAVIS_TAG before_deploy: - source travis/install-conda.sh - conda install -y jinja2 # needed to render conda recipe -deploy: - provider: script - skip_cleanup: true # retain build artifacts, including dependencies - script: travis/deploy.sh $TRAVIS_TAG - on: - tags: true - #all_branches: true +# deploy: +# provider: script +# skip_cleanup: true # retain build artifacts, including dependencies +# script: travis/build-and-deploy.sh $TRAVIS_TAG +# on: +# tags: true +# all_branches: true diff --git a/packaging/conda-recipe/render-recipe.py b/packaging/conda-recipe/render-recipe.py index fc30c2340..fe34d53c3 100755 --- a/packaging/conda-recipe/render-recipe.py +++ b/packaging/conda-recipe/render-recipe.py @@ -156,6 +156,7 @@ def url_md5(url): while True: try: print("Downloading source package for hash calculation...") + print(url) response = urlopen(url) for chunk in iter(lambda: response.read(CHUNK_SIZE), b""): hash_md5.update(chunk) @@ -197,6 +198,10 @@ def url_md5(url): parser.add_argument('--test-reqs', nargs='*', dest='test_requirements', type=argparse.FileType('r'), help='test-time requirements file') + parser.add_argument('--download-filename', dest='src_download_filename', + type=str, + help='An argument to override the usual filename to download; ' + 'useful for specifying a branch name.') try: args = parser.parse_args() @@ -213,6 +218,8 @@ def url_md5(url): # store two separate version strings, one to use for the conda package and one # that should match github tagged releases recipe_variables["PKG_VERSION"] = str(args_dict.pop("version")) + if "src_download_filename" in args_dict: + recipe_variables["PKG_VERSION"] = str(args_dict.pop("src_download_filename")) # strip "v" prefix from versions that look like v1.14.0 if recipe_variables["PKG_VERSION"].startswith("v"): diff --git a/travis/build-and-deploy.sh b/travis/build-and-deploy.sh new file mode 100755 index 000000000..205610e28 --- /dev/null +++ b/travis/build-and-deploy.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# This script performs various packing and deployment operations. +# It assumes it will be caused as a deploy hook of TravisCI; ex.: +# +# deploy: +# provider: script +# script: travis/deploy.sh $TRAVIS_TAG +# on: +# tags: true +# all_branches: master + + +# way to get the absolute path to this script that should +# work regardless of whether or not this script has been sourced +# Find original directory of bash script, resovling symlinks +# http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in/246128#246128 +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + if [[ "$OSTYPE" == "darwin"* ]]; then + SOURCE="$(readlink "$SOURCE")" + else + SOURCE="$(readlink -f "$SOURCE")" + fi + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +SCRIPT=$SOURCE +SCRIPT_DIRNAME="$(dirname "$SOURCE")" +SCRIPTPATH="$(cd -P "$SCRIPT_DIRNAME" &> /dev/null && pwd)" +SCRIPT="$SCRIPTPATH/$(basename "$SCRIPT")" + +PKG_VERSION=$1 + +# === Build conda package and upload + +echo "Python binary: $(which python)" +echo "Python version: $(python --version)" + +# If this is a PR, on the master branch, or is a tag, render and build the conda package. If it is a tag, also upload to anaconda.org +if [[ ( -n $TRAVIS_PULL_REQUEST && $TRAVIS_PULL_REQUEST != "false" ) || $TRAVIS_BRANCH = "master" || -n "$TRAVIS_TAG" ]]; then + echo "Rendering and building conda package..." + # Render recipe from template and dependency files, setting the tag as the current version + # if this is a tag build+upload, otherwise just test building + if [ -n "$TRAVIS_TAG" ]; then + # if the ANACONDA_TOKEN is defined (not on an external branch) + if [ ! -z "$ANACONDA_TOKEN" ]; then + conda config --set anaconda_upload yes + python packaging/conda-recipe/render-recipe.py "$PKG_VERSION" --build-reqs requirements-conda.txt --run-reqs requirements-conda.txt --py3-run-reqs requirements-py3.txt --py2-run-reqs requirements-py2.txt --test-reqs requirements-conda-tests.txt && \ + CONDA_PERL=5.22.0 conda build -c broad-viral -c r -c bioconda -c conda-forge -c defaults --python "$TRAVIS_PYTHON_VERSION" --token "$ANACONDA_TOKEN" packaging/conda-recipe/viral-ngs && \ + ./travis/trigger-tests-in-other-repo.sh + # check the exit code of conda build, and if successful, + # trigger the viral-ngs-deploy repository to test/build the docker container + else + echo "ANACONDA_TOKEN is not defined. Conda package upload is only supported for branches on the original repository." + fi + else + python packaging/conda-recipe/render-recipe.py "0.0.0" --download-filename "$TRAVIS_BRANCH" --build-reqs requirements-conda.txt --run-reqs requirements-conda.txt --py3-run-reqs requirements-py3.txt --py2-run-reqs requirements-py2.txt --test-reqs requirements-conda-tests.txt && \ + CONDA_PERL=5.22.0 conda build -c broad-viral -c r -c bioconda -c conda-forge -c defaults --python "$TRAVIS_PYTHON_VERSION" --no-anaconda-upload packaging/conda-recipe/viral-ngs + fi +fi + diff --git a/travis/deploy.sh b/travis/deploy.sh deleted file mode 100755 index 87da374b3..000000000 --- a/travis/deploy.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -# This script performs various packing and deployment operations. -# It assumes it will be caused as a deploy hook of TravisCI; ex.: -# -# deploy: -# provider: script -# script: travis/deploy.sh $TRAVIS_TAG -# on: -# tags: true -# all_branches: master - - -# way to get the absolute path to this script that should -# work regardless of whether or not this script has been sourced -# Find original directory of bash script, resovling symlinks -# http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in/246128#246128 -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - if [[ "$OSTYPE" == "darwin"* ]]; then - SOURCE="$(readlink "$SOURCE")" - else - SOURCE="$(readlink -f "$SOURCE")" - fi - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -SCRIPT=$SOURCE -SCRIPT_DIRNAME="$(dirname "$SOURCE")" -SCRIPTPATH="$(cd -P "$SCRIPT_DIRNAME" &> /dev/null && pwd)" -SCRIPT="$SCRIPTPATH/$(basename "$SCRIPT")" - -PKG_VERSION=$1 - -# === Build conda package and upload - -echo "Python binary: $(which python)" -echo "Python version: $(python --version)" - -# if the ANACONDA_TOKEN is defined (not on an external branch) -if [ ! -z "$ANACONDA_TOKEN" ]; then - echo "Running $SCRIPTPATH/package-conda.sh" - # Render recipe from template and dependency files, setting the tag as the current version - conda config --set anaconda_upload yes - python packaging/conda-recipe/render-recipe.py "$PKG_VERSION" --build-reqs requirements-conda.txt --run-reqs requirements-conda.txt --py3-run-reqs requirements-py3.txt --py2-run-reqs requirements-py2.txt --test-reqs requirements-conda-tests.txt && \ - CONDA_PERL=5.22.0 conda build -c broad-viral -c r -c bioconda -c conda-forge -c defaults --python "$TRAVIS_PYTHON_VERSION" --token "$ANACONDA_TOKEN" packaging/conda-recipe/viral-ngs && \ - ./travis/trigger-tests-in-other-repo.sh - # check the exit code of conda build, and if successful, - # trigger the viral-ngs-deploy repository to test/build the docker container -else - echo "ANACONDA_TOKEN is not defined. Conda package upload is only supported for branches on the original repository." -fi diff --git a/travis/install-conda.sh b/travis/install-conda.sh index 1f7ab349c..33cbd05e8 100755 --- a/travis/install-conda.sh +++ b/travis/install-conda.sh @@ -17,15 +17,15 @@ else # if it does not exist, we need to install miniconda if [[ "$TRAVIS_PYTHON_VERSION" == 2* ]]; then if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - wget --quiet https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh -O miniconda.sh; + curl -S https://repo.continuum.io/miniconda/Miniconda2-latest-MacOSX-x86_64.sh > miniconda.sh; else - wget --quiet https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; + curl -S https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh > miniconda.sh; fi else if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh; + curl -S https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh > miniconda.sh; else - wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + curl -S https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh > miniconda.sh; fi fi @@ -45,10 +45,11 @@ else # if it does not exist, we need to install miniconda conda config --add channels defaults conda config --add channels conda-forge conda config --add channels bioconda + conda config --add channels broad-viral conda install --quiet -y conda #conda=4.2 # pin to 4.2.* until this is fixed: https://github.com/conda/conda-build/issues/1666 conda config --set auto_update_conda false conda install --quiet -y java-jdk==8.0.112 - conda install --quiet -y conda-build # needed to build recipe + conda install --quiet -y conda-build==3.0.25 # needed to build recipe conda install --quiet -y anaconda-client # needed to upload build package to anaconda.org #conda install --quiet -y -c conda-forge -f curl # the bioconda curl is broken as of 21 Feb 2017 fi diff --git a/travis/install-tools.sh b/travis/install-tools.sh index 0516a8654..555eedd18 100755 --- a/travis/install-tools.sh +++ b/travis/install-tools.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -e -o pipefail if [ ! -d $GATK_PATH ]; then if [ -z "$BUNDLE_SECRET" ]; then @@ -37,12 +37,10 @@ echo "Installing and validating bioinformatic tools" export CONDA_ENVS_PATH=tools/conda-cache:tools/conda-tools/default for i in $(seq 3); do - conda create --quiet -y -m -c broad-viral -c r -c bioconda -c conda-forge -c defaults -p tools/conda-tools/default --file requirements-conda.txt python="$TRAVIS_PYTHON_VERSION" && break + conda create --quiet -y -m -c broad-viral -c r -c bioconda -c conda-forge -c defaults -p tools/conda-tools/default --file requirements-conda.txt --file requirements-conda-tests.txt python="$TRAVIS_PYTHON_VERSION" && break sleep 5 done -while read requirement; do conda install --quiet -y -m -c broad-viral -c r -c bioconda -c conda-forge -c defaults -p tools/conda-tools/default $requirement; done < requirements-conda-tests.txt - conda list echo 'Sourcing default environment' diff --git a/travis/trigger-travis.sh b/travis/trigger-travis.sh index a33802856..47606d436 100755 --- a/travis/trigger-travis.sh +++ b/travis/trigger-travis.sh @@ -164,7 +164,7 @@ echo "Travis API request body:\n$body" echo "Making request to start tests in other repository..." # It does not work to put / in place of %2F in the URL below. -curl -s -X POST \ +curl -S -X POST \ -H "Content-Type: application/json" \ -H "Accept: application/json" \ -H "Travis-API-Version: 3" \