fixes related to testing+building conda recipes (#694)

* pin conda-built to newer version

* switch to curl for miniconda acquisition

* render+build the conda package on PRs, on master, and on tags, but only upload to anaconda.org on tags

* travis-trigger curl call is now silent, except on error

* move build-an-deploy script to after_success

deploy conditional is now in the script itself, so we can call it every
time

* placeholder version number for conda build test

* recipe render can now specify filename override to handle downloading file for branch

.travis.yml

@@ -59,15 +59,16 @@ script:
 
 after_success:
   - coveralls
+  - travis/build-and-deploy.sh $TRAVIS_TAG
 
 before_deploy:
   - source travis/install-conda.sh
   - conda install -y jinja2 # needed to render conda recipe
 
-deploy:
-  provider: script
-  skip_cleanup: true # retain build artifacts, including dependencies
-  script: travis/deploy.sh $TRAVIS_TAG
-  on:
-    tags: true
-    #all_branches: true
+# deploy:
+#   provider: script
+#   skip_cleanup: true # retain build artifacts, including dependencies
+#   script: travis/build-and-deploy.sh $TRAVIS_TAG
+#   on:
+#     tags: true
+#     all_branches: true

packaging/conda-recipe/render-recipe.py

@@ -156,6 +156,7 @@ def url_md5(url):
     while True:
         try:
             print("Downloading source package for hash calculation...")
+            print(url)
             response = urlopen(url)
             for chunk in iter(lambda: response.read(CHUNK_SIZE), b""):
                 hash_md5.update(chunk)
@@ -197,6 +198,10 @@ def url_md5(url):
     parser.add_argument('--test-reqs', nargs='*', dest='test_requirements',
                         type=argparse.FileType('r'),
                         help='test-time requirements file')
+    parser.add_argument('--download-filename', dest='src_download_filename',
+                        type=str,
+                        help='An argument to override the usual filename to download; '
+                        'useful for specifying a branch name.')
 
     try:
        args = parser.parse_args()
@@ -213,6 +218,8 @@ def url_md5(url):
     # store two separate version strings, one to use for the conda package and one
     # that should match github tagged releases
     recipe_variables["PKG_VERSION"] = str(args_dict.pop("version"))
+    if "src_download_filename" in args_dict:
+        recipe_variables["PKG_VERSION"] = str(args_dict.pop("src_download_filename"))
 
     # strip "v" prefix from versions that look like v1.14.0
     if recipe_variables["PKG_VERSION"].startswith("v"):

travis/build-and-deploy.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# This script performs various packing and deployment operations.
+# It assumes it will be caused as a deploy hook of TravisCI; ex.:
+#
+# deploy:
+#   provider: script
+#   script: travis/deploy.sh $TRAVIS_TAG
+#   on:
+#     tags: true
+#     all_branches: master
+
+
+# way to get the absolute path to this script that should
+# work regardless of whether or not this script has been sourced
+# Find original directory of bash script, resovling symlinks
+# http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in/246128#246128
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+    DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+    if [[ "$OSTYPE" == "darwin"* ]]; then
+        SOURCE="$(readlink "$SOURCE")"
+    else
+        SOURCE="$(readlink -f "$SOURCE")"
+    fi
+    [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+SCRIPT=$SOURCE
+SCRIPT_DIRNAME="$(dirname "$SOURCE")"
+SCRIPTPATH="$(cd -P "$SCRIPT_DIRNAME" &> /dev/null && pwd)"
+SCRIPT="$SCRIPTPATH/$(basename "$SCRIPT")"
+
+PKG_VERSION=$1
+
+# === Build conda package and upload
+
+echo "Python binary: $(which python)"
+echo "Python version: $(python --version)"
+
+# If this is a PR, on the master branch, or is a tag, render and build the conda package. If it is a tag, also upload to anaconda.org
+if [[ ( -n $TRAVIS_PULL_REQUEST && $TRAVIS_PULL_REQUEST != "false" ) || $TRAVIS_BRANCH = "master" || -n "$TRAVIS_TAG" ]]; then
+    echo "Rendering and building conda package..."
+    # Render recipe from template and dependency files, setting the tag as the current version
+    # if this is a tag build+upload, otherwise just test building
+    if [ -n "$TRAVIS_TAG" ]; then
+         # if the ANACONDA_TOKEN is defined (not on an external branch)
+        if [ ! -z "$ANACONDA_TOKEN" ]; then
+            conda config --set anaconda_upload yes
+            python packaging/conda-recipe/render-recipe.py "$PKG_VERSION" --build-reqs requirements-conda.txt --run-reqs requirements-conda.txt --py3-run-reqs requirements-py3.txt --py2-run-reqs requirements-py2.txt --test-reqs requirements-conda-tests.txt && \
+            CONDA_PERL=5.22.0 conda build -c broad-viral -c r -c bioconda -c conda-forge -c defaults --python "$TRAVIS_PYTHON_VERSION" --token "$ANACONDA_TOKEN" packaging/conda-recipe/viral-ngs && \
+            ./travis/trigger-tests-in-other-repo.sh
+            # check the exit code of conda build, and if successful,
+            # trigger the viral-ngs-deploy repository to test/build the docker container
+        else
+            echo "ANACONDA_TOKEN is not defined. Conda package upload is only supported for branches on the original repository."
+        fi
+    else
+        python packaging/conda-recipe/render-recipe.py "0.0.0" --download-filename "$TRAVIS_BRANCH" --build-reqs requirements-conda.txt --run-reqs requirements-conda.txt --py3-run-reqs requirements-py3.txt --py2-run-reqs requirements-py2.txt --test-reqs requirements-conda-tests.txt && \
+        CONDA_PERL=5.22.0 conda build -c broad-viral -c r -c bioconda -c conda-forge -c defaults --python "$TRAVIS_PYTHON_VERSION" --no-anaconda-upload packaging/conda-recipe/viral-ngs
+    fi
+fi  
+

travis/install-conda.sh

@@ -17,15 +17,15 @@ else # if it does not exist, we need to install miniconda
 
     if [[ "$TRAVIS_PYTHON_VERSION" == 2* ]]; then
         if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-            wget --quiet https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh -O miniconda.sh;
+            curl -S https://repo.continuum.io/miniconda/Miniconda2-latest-MacOSX-x86_64.sh > miniconda.sh;
         else
-            wget --quiet https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh;
+            curl -S https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh > miniconda.sh;
         fi
      else
         if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-            wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh;
+            curl -S https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh > miniconda.sh;
         else
-            wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
+            curl -S https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh > miniconda.sh;
         fi
     fi
 
@@ -45,10 +45,11 @@ else # if it does not exist, we need to install miniconda
     conda config --add channels defaults
     conda config --add channels conda-forge
     conda config --add channels bioconda
+    conda config --add channels broad-viral
     conda install --quiet -y conda #conda=4.2 # pin to 4.2.* until this is fixed: https://github.com/conda/conda-build/issues/1666
     conda config --set auto_update_conda false
     conda install --quiet -y java-jdk==8.0.112
-    conda install --quiet -y conda-build # needed to build recipe
+    conda install --quiet -y conda-build==3.0.25 # needed to build recipe
     conda install --quiet -y anaconda-client # needed to upload build package to anaconda.org
     #conda install --quiet -y -c conda-forge -f curl # the bioconda curl is broken as of 21 Feb 2017
 fi

travis/install-tools.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -e
+set -e -o pipefail
 
 if [ ! -d $GATK_PATH ]; then
   if [ -z "$BUNDLE_SECRET" ]; then
@@ -37,12 +37,10 @@ echo "Installing and validating bioinformatic tools"
 export CONDA_ENVS_PATH=tools/conda-cache:tools/conda-tools/default
 
 for i in $(seq 3); do
-  conda create --quiet -y -m -c broad-viral -c r -c bioconda -c conda-forge -c defaults -p tools/conda-tools/default --file requirements-conda.txt python="$TRAVIS_PYTHON_VERSION" && break
+  conda create --quiet -y -m -c broad-viral -c r -c bioconda -c conda-forge -c defaults -p tools/conda-tools/default --file requirements-conda.txt --file requirements-conda-tests.txt python="$TRAVIS_PYTHON_VERSION" && break
   sleep 5
 done
 
-while read requirement; do conda install --quiet -y -m -c broad-viral -c r -c bioconda -c conda-forge -c defaults -p tools/conda-tools/default $requirement; done < requirements-conda-tests.txt
-
 conda list
 
 echo 'Sourcing default environment'

travis/trigger-travis.sh

@@ -164,7 +164,7 @@ echo "Travis API request body:\n$body"
 echo "Making request to start tests in other repository..."
 
 # It does not work to put / in place of %2F in the URL below. 
-curl -s -X POST \
+curl -S -X POST \
   -H "Content-Type: application/json" \
   -H "Accept: application/json" \
   -H "Travis-API-Version: 3" \