docker build optimization (#708)

This changes how the docker image is built. Changes include:

- Change from installing the viral-ngs conda package (easy-deploy-viral-ngs.sh setup) to something closer to setup-git (that uses the current working copy instead of cloning fresh from github).
- Divorce the conda build & deploy steps from the docker build & deploy -- put each in their own separate space in the Travis build matrix and cut end-to-end build time in half.
- Separate out basic infrastructural setup in the Dockerfile to a separate repository such that this Dockerfile only installs the components specific to viral-ngs and its dependencies. This also reduces build time by caching all of the apt-get steps.
- Move Dockerfile to root of git repo. This allows for a direct build of either the github URI or the working copy (e.g. docker build . or docker build https://github.com/broadinstitute/viral-ngs.git).

.dockerignore

@@ -0,0 +1,4 @@
+*.pyc
+__pycache__
+tools/conda-cache
+tools/conda-tools

.travis.yml

@@ -14,11 +14,14 @@ matrix:
         - os: linux
           python: 3.6
           env:
-            - BUILD_PACKAGE=true
+            - BUILD_PACKAGE=conda
             # $ANACONDA_TOKEN for uploading builds to anaconda.org ("broad-viral" channel)
             - secure: SLPB86BpMIiNncMioxVk9cLrqaSNt8F1QDtxkrdLq9j7wXzFqGa7cipG6UJ6Om7GvoF49DpACfGPTA4ycr+T4cH3pWXpBHrBhV8TyKJb23cOmg5+7zqJQTzuwNqKOT7t9rnBkf1uzVXBcgqKaD6XW/nEvNFK00I0cvjlCp8vgxE=
             # $TRAVIS_ACCESS_TOKEN_FOR_OTHER_REPO (viral-ngs-deploy)
             - secure: ChB0K3gPr5HknxYA41xCrpgChHDmLkqc79p1NABB/tbqOEnrPzDPqE+FU4/QlmeV96jMYn4uyLVauJpzVXyBIVoOa8guqoF5VdiKlAhaUwh9UQJ75i3SKQtGBrqaTXSDVI1vJARMiGabduCrcNJxVsxV9Bm+YzTq6tuhWyqR4fs=
+        - os: linux
+          env:
+            - BUILD_PACKAGE=docker
             # DOCKER_USER
             - secure: hYX8492Wqpq3yqv+eHBV9c6VY8JlUSS8mUDfT1eWNEZF2vw8WrnTt8SrLIPC8etQUk1ZaSI/8XbJQKEr9LRqgvuO07AIv6BxYCg9l9BPHCj6B2YTTPo2qPkPapfvtGVd7PUZcWDUvzvPxJqMveuKVkTnCuuQSWwR68Y/Khxj8UY=
             # DOCKER_PASS
@@ -56,24 +59,27 @@ env:
   - secure: KX7DwKRD85S7NgspxevgbulTtV+jHQIiM6NBus2/Ur/P0RMdpt0EQQ2wDq79qGN70bvvkw901N7EjSYd+GWCAM7StXtaxnLRrrZ3XI1gX7KMk8E3QzPf0zualLDs7cuQmL6l6WiElUAEqumLc7WGpLZZLdSPzNqFSg+CBKCmTI8=
 
 git:
-  depth: 60
+  depth: 50
 
 before_install:
   - travis/before_install.sh
 
 install:
-  - source travis/install-conda.sh
+  - if [[ "$BUILD_PACKAGE" != "docker" ]]; then source travis/install-conda.sh; fi
   - if [ -n "$PERFORM_TESTING" ]; then source travis/install-tools.sh; fi
   - if [ -n "$PERFORM_TESTING" ]; then travis/install-pip.sh; fi
+  - if [[ "$BUILD_PACKAGE" == "conda" ]]; then travis/install-conda-build.sh; fi
+  - if [[ "$BUILD_PACKAGE" == "docker" ]]; then travis/upgrade-docker.sh; fi
 
 script:
   - if [ -n "$PERFORM_TESTING" ]; then travis/tests-long.sh; fi
   - if [ -n "$PERFORM_TESTING" ]; then travis/tests-unit.sh; fi
-  - if [ -n "$BUILD_PACKAGE" ]; then travis/build-package.sh; fi
-  - if [ -n "$BUILD_PACKAGE" ]; then travis/deploy-docker.sh; fi
+  - if [[ "$BUILD_PACKAGE" == "conda" ]]; then travis_wait travis/build-conda.sh; fi
+  - if [[ "$BUILD_PACKAGE" == "docker" ]]; then travis_wait docker build --rm -t local/viral-ngs:build .; docker run --rm local/viral-ngs:build assembly.py --version; fi
 
 after_success:
   - if [ -n "$PERFORM_TESTING" ]; then coveralls; fi
+  - if [[ "$BUILD_PACKAGE" == "docker" ]]; then travis/deploy-docker.sh; fi
 
 after_script:
   # correct post-build failure on OSX due to non-zero exit code

Dockerfile

@@ -0,0 +1,40 @@
+FROM broadinstitute/viral-baseimage:0.1.2
+
+LABEL maintainer "Chris Tomkins-Tinch <[email protected]>"
+
+# to build:
+#   docker build --rm .
+# to run:
+#   Download licensed copies of GATK and Novoalign to the host machine (for Linux-64)
+#   export GATK_PATH=/path/to/gatk/
+#   export NOVOALIGN_PATH=/path/to/novoalign/
+#   docker run --rm -v $GATK_PATH:/gatk -v $NOVOALIGN_PATH:/novoalign -v /path/to/dir/on/host:/user-data -t -i <image_ID> "<command>.py subcommand"
+# if you receive a "no space on device" error:
+#   docker kill $(docker ps -a -q)
+#   docker rm $(docker ps -a -q)
+#   docker rmi $(docker images -q)
+#   docker volume rm $(docker volume ls -qf dangling=true)
+
+# DEBIAN_FRONTEND: Silence some warnings about Readline. Read more over here:
+# https://github.com/phusion/baseimage-docker/issues/58
+ENV DEBIAN_FRONTEND=noninteractive INSTALL_PATH="/opt/viral-ngs" VIRAL_NGS_PATH="/opt/viral-ngs/source"
+
+# copy basic files
+COPY docker/env_wrapper.sh docker/install-viral-ngs.sh easy-deploy-script/easy-deploy-viral-ngs.sh $INSTALL_PATH/
+RUN chmod a+x $INSTALL_PATH/*.sh
+
+# Prepare viral-ngs user and installation directory
+COPY . $VIRAL_NGS_PATH/
+WORKDIR $INSTALL_PATH
+RUN ./install-viral-ngs.sh
+
+# Volume setup: make external tools and data available within the container
+VOLUME ["/gatk", "/novoalign", "/user-data"]
+# DEBIAN_FRONTEND: Silence some warnings about Readline. Read more over here:
+# https://github.com/phusion/baseimage-docker/issues/58
+ENV GATK_PATH="/gatk" NOVOALIGN_PATH="/novoalign" VIRAL_NGS_DOCKER_DATA_PATH="/user-data" DEBIAN_FRONTEND=teletype
+
+# It's a wrapper script to load the viral-ngs environment via the easy-deploy script
+# and then run any commands desired
+ENTRYPOINT ["/opt/viral-ngs/env_wrapper.sh"]
+CMD ["/bin/bash"]