diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..aa69bb0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +/LICENCE +/prerelease.sh +/README.md +/INSTALL +/LICENCE +/CHANGES.md +/.gitignore +/.git +/perl/docs +/perl/docs.tar.gz diff --git a/.travis.yml b/.travis.yml index f84a890..a6e402c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,58 +2,30 @@ notifications: slack: wtsi-cgpit:ptUMR1tkNyZJYd9TpGoss8WR email: false -env: - - CC=gcc +sudo: false -addons: - apt: - update: true - packages: - - build-essential - - autoconf - - bsdtar - - time - - curl - - libcurl4-openssl-dev - - nettle-dev - - zlib1g-dev - - libncurses5-dev - - libpstreams-dev - - unzip - - libpng12-dev - - libexpat1-dev - - libgoogle-perftools-dev - - lsof - - libbz2-dev - - liblzma-dev - - libgnutls-dev - - libtasn1-6-dev - - p11-kit - - libxml2-dev - - libgd-dev - - psmisc - - libdb-dev - -install: true - -language: perl - -perl: - - "5.22-shrplib" +services: + - docker script: - set -e - - export PATH=$HOME/install/bin:$HOME/install/biobambam2/bin:$PATH - - git clone --depth 1 --single-branch --branch develop https://github.com/cancerit/cgpBigWig.git - - cd cgpBigWig - - ./setup.sh $HOME/PCAP-opt - - cd ../ - - git clone --depth 1 --single-branch --branch develop https://github.com/cancerit/PCAP-core.git - - cd PCAP-core - - ./setup.sh $HOME/PCAP-opt - - cd ../ - - git clone --depth 1 --single-branch --branch dev https://github.com/cancerit/cgpVcf.git - - cd cgpVcf - - ./setup.sh $HOME/PCAP-opt - - cd ../ - - ./setup.sh $HOME/PCAP-opt + - echo 'Build and check docker image' + - docker build -t cgppindel . + - docker images | grep -c cgppindel + - echo 'Verify program(s) from each inherited base image - dockstore-cgpbigwig' + - docker run -t --rm cgppindel bwjoin --version + - echo 'Verify program(s) from each inherited base image - dockstore-cgpmap' + - docker run -t --rm cgppindel bwa_mem.pl -version + - docker run -t --rm cgppindel bammarkduplicates2 --version + - docker run -t --rm cgppindel samtools --version + - docker run -t --rm cgppindel bash -c 'bwa 2>&1 | grep Version' + - echo 'Verify program(s) from this repo' + - docker run -t --rm cgppindel pindel.pl --version + - docker run -t --rm cgppindel pindel_input_gen.pl --version + - docker run -t --rm cgppindel pindel_2_combined_vcf.pl --version + - docker run -t --rm cgppindel FlagVcf.pl --version + - docker run -t --rm cgppindel pindel_merge_vcf_bam.pl --version + - docker run -t --rm cgppindel pindel_np_from_vcf.pl --version + - docker run -t --rm cgppindel pindel_germ_bed.pl --version + - docker run -t --rm cgppindel which pindel + - docker run -t --rm cgppindel which filter_pindel_reads diff --git a/CHANGES.md b/CHANGES.md index 8347e96..8d812b2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ # CHANGES +## NEXT + +* Added Dockerfile and docker documentation + ## 3.2.0 * Tabix search for high depth/excluded regions now performed in memory using IntervalTrees @@ -153,15 +157,4 @@ Comparing sites in VCF files... Found 15321 SNPs common to both files. Found 0 SNPs only in main file. Found 0 SNPs only in second file. -After filtering, kept 16309 out of a possible 16309 Sites -Run Time = 6.00 seconds -``` - -## 2.0.4 - -* Permits empty results files - -## 2.0.0 - -* Migrates all Tabix and Bio::DB::Sam to Bio::DB::HTS::Tabix and Bio::DB::Sam -* Cleans up install +After diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..efffa68 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,81 @@ +FROM quay.io/wtsicgp/dockstore-cgpmap:3.1.4 as builder + +USER root + +# ALL tool versions used by opt-build.sh +ENV VER_CGPVCF="v2.2.1" +ENV VER_VCFTOOLS="0.1.16" + +RUN apt-get -yq update +RUN apt-get install -yq --no-install-recommends \ +locales \ +g++ \ +make \ +gcc \ +pkg-config \ +zlib1g-dev + +RUN locale-gen en_US.UTF-8 +RUN update-locale LANG=en_US.UTF-8 + +ENV OPT /opt/wtsi-cgp +ENV PATH $OPT/bin:$OPT/biobambam2/bin:$PATH +ENV PERL5LIB $OPT/lib/perl5 +ENV LD_LIBRARY_PATH $OPT/lib +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 + +# build tools from other repos +ADD build/opt-build.sh build/ +RUN bash build/opt-build.sh $OPT + +# build the tools in this repo, separate to reduce build time on errors +COPY . . +RUN bash build/opt-build-local.sh $OPT + +FROM ubuntu:16.04 + +LABEL maintainer="cgphelp@sanger.ac.uk" \ + uk.ac.sanger.cgp="Cancer, Ageing and Somatic Mutation, Wellcome Trust Sanger Institute" \ + version="1.0.0" \ + description="cgpPindel docker" + +RUN apt-get -yq update +RUN apt-get install -yq --no-install-recommends \ +apt-transport-https \ +locales \ +curl \ +ca-certificates \ +libperlio-gzip-perl \ +bzip2 \ +psmisc \ +time \ +zlib1g \ +liblzma5 \ +libncurses5 \ +p11-kit \ +unattended-upgrades && \ +unattended-upgrade -d -v && \ +apt-get remove -yq unattended-upgrades && \ +apt-get autoremove -yq + +RUN locale-gen en_US.UTF-8 +RUN update-locale LANG=en_US.UTF-8 + +ENV OPT /opt/wtsi-cgp +ENV PATH $OPT/bin:$OPT/biobambam2/bin:$PATH +ENV PERL5LIB $OPT/lib/perl5 +ENV LD_LIBRARY_PATH $OPT/lib +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 + +RUN mkdir -p $OPT +COPY --from=builder $OPT $OPT + +## USER CONFIGURATION +RUN adduser --disabled-password --gecos '' ubuntu && chsh -s /bin/bash && mkdir -p /home/ubuntu + +USER ubuntu +WORKDIR /home/ubuntu + +CMD ["/bin/bash"] diff --git a/README.md b/README.md index 521fa64..4ec8b56 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ cgpPindel contains the Cancer Genome Projects workflow for [Pindel][pindel-core]. +[![Quay Badge][quay-status]][quay-repo] + | Master | Develop | | --------------------------------------------- | ----------------------------------------------- | | [![Master Badge][travis-master]][travis-base] | [![Develop Badge][travis-develop]][travis-base] | @@ -18,12 +20,16 @@ Contents: -- [Docker, Singularity and Dockstore](#docker-singularity-and-dockstore) -- [Dependencies/Install](#dependenciesinstall) -- [Creating a release](#creating-a-release) - - [Preparation](#preparation) - - [Cutting the release](#cutting-the-release) -- [LICENCE](#licence) +- [cgpPindel](#cgppindel) + - [Docker, Singularity and Dockstore](#docker-singularity-and-dockstore) + - [Dependencies/Install](#dependenciesinstall) + - [Creating a release](#creating-a-release) + - [Preparation](#preparation) + - [Release process](#release-process) + - [Code changes](#code-changes) + - [Docker image](#docker-image) + - [Cutting the release](#cutting-the-release) + - [LICENCE](#licence) @@ -31,12 +37,13 @@ Contents: There are pre-built images containing this codebase on quay.io. +* [cgpPindel][cgpPindel-git]: Contained within this repository - contains the cgpPindel package * [dockstore-cgpwxs][ds-cgpwxs-git]: Contains tools specific to WXS analysis. * [dockstore-cgpwgs][ds-cgpwgs-git]: Contains additional tools for WGS analysis. These were primarily designed for use with dockstore.org but can be used as normal containers. -The docker images are know to work correctly after import into a singularity image. +The docker images are known to work correctly after import into a singularity image. ## Dependencies/Install @@ -66,15 +73,34 @@ Please be aware that this expects basic C compilation libraries and tools to be * Commit/push all relevant changes. * Pull a clean version of the repo and use this for the following steps. -### Cutting the release +### Release process + +This project is maintained using HubFlow. + +#### Code changes + +1. Make appropriate changes +2. Update `perl/lib/Sanger/CGP/Pindel.pm` to the correct version (adding rc/beta to end if applicable). +3. Update `CHANGES.md` to show major items. +4. Run `./prerelease.sh` +5. Check all tests and coverage reports are acceptable. +6. Commit the updated docs and updated module/version. +7. Push commits. + +#### Docker image -1. Update `perl/lib/Sanger/CGP/Pindel.pm` to the correct version (adding rc/beta to end if applicable). -1. Update `CHANGES.md` to show major items. -1. Run `./prerelease.sh` -1. Check all tests and coverage reports are acceptable. -1. Commit the updated docs and updated module/version. -1. Push commits. 1. Use the GitHub tools to draft a release. +2. Build image locally +3. Run example inputs and verify any changes are acceptable +4. Bump version in `Dockerfile` +5. Push changes + +#### Cutting the release + +1. Check state on Travis +2. Generate the release (add notes to GitHub) +3. Confirm that image has been built on [quay.io][quay-builds] +4. Update the [dockstore][dockstore-cgpPindel] entry, see [their docs][dockstore-get-started]. ## LICENCE @@ -114,9 +140,18 @@ identical to a statement that reads ‘Copyright (c) 2005, 2006, 2007, 2008, [pcap-core-rel]: https://github.com/cancerit/PCAP-core/releases [ds-cgpwxs-git]: https://github.com/cancerit/dockstore-cgpwxs [ds-cgpwgs-git]: https://github.com/cancerit/dockstore-cgpwgs +[cgpPindel-git]: https://github.com/cancerit/cgpPindel [pindel-core]: http://gmt.genome.wustl.edu/pindel/current [travis-base]: https://travis-ci.org/cancerit/cgpPindel [travis-master]: https://travis-ci.org/cancerit/cgpPindel.svg?branch=master [travis-develop]: https://travis-ci.org/cancerit/cgpPindel.svg?branch=dev + + +[quay-status]: https://quay.io/repository/wtsicgp/cgpPindel/status +[quay-repo]: https://quay.io/repository/wtsicgp/cgpPindel +[quay-builds]: https://quay.io/repository/wtsicgp/cgpPindel?tab=builds + + +[dockstore-cgpPindel]: https://dockstore.org/containers/quay.io/wtsicgp/cgpPindel diff --git a/build/opt-build-local.sh b/build/opt-build-local.sh new file mode 100755 index 0000000..9cfb57b --- /dev/null +++ b/build/opt-build-local.sh @@ -0,0 +1,65 @@ +#! /bin/bash + +set -xe + +if [[ -z "${TMPDIR}" ]]; then + TMPDIR=/tmp +fi + +set -u + +if [ "$#" -lt "1" ] ; then + echo "Please provide an installation path such as /opt/ICGC" + exit 1 +fi + +# get path to this script +SCRIPT_PATH=`dirname $0`; +SCRIPT_PATH=`(cd $SCRIPT_PATH && pwd)` + +# get the location to install to +INST_PATH=$1 +mkdir -p $1 +INST_PATH=`(cd $1 && pwd)` +echo $INST_PATH + +# get current directory +INIT_DIR=`pwd` + +CPU=`grep -c ^processor /proc/cpuinfo` +if [ $? -eq 0 ]; then + if [ "$CPU" -gt "6" ]; then + CPU=6 + fi +else + CPU=1 +fi +echo "Max compilation CPUs set to $CPU" + +SETUP_DIR=$INIT_DIR/install_tmp +mkdir -p $SETUP_DIR/distro # don't delete the actual distro directory until the very end +mkdir -p $INST_PATH/bin +cd $SETUP_DIR + +# make sure tools installed can see the install loc of libraries +set +u +export LD_LIBRARY_PATH=`echo $INST_PATH/lib:$LD_LIBRARY_PATH | perl -pe 's/:\$//;'` +export PATH=`echo $INST_PATH/bin:$PATH | perl -pe 's/:\$//;'` +export MANPATH=`echo $INST_PATH/man:$INST_PATH/share/man:$MANPATH | perl -pe 's/:\$//;'` +export PERL5LIB=`echo $INST_PATH/lib/perl5:$PERL5LIB | perl -pe 's/:\$//;'` +set -u + +## cgpPindel - should be the build root +if [ ! -e $SETUP_DIR/cgpPindel.success ]; then + cd $INIT_DIR + if [ ! -e $SETUP_DIR/cgpPindel_c.success ]; then + g++ -O3 -o $INST_PATH/bin/pindel c++/pindel.cpp + g++ -O3 -o $INST_PATH/bin/filter_pindel_reads c++/filter_pindel_reads.cpp + touch $SETUP_DIR/cgpPindel_c.success + fi + cd perl + cpanm --no-interactive --notest --mirror http://cpan.metacpan.org --notest -l $INST_PATH --installdeps . + cpanm -v --no-interactive --mirror http://cpan.metacpan.org -l $INST_PATH . + cd $SETUP_DIR + touch $SETUP_DIR/cgpPindel.success +fi diff --git a/build/opt-build.sh b/build/opt-build.sh new file mode 100644 index 0000000..21b0ff4 --- /dev/null +++ b/build/opt-build.sh @@ -0,0 +1,77 @@ +#! /bin/bash + +set -xe + +if [[ -z "${TMPDIR}" ]]; then + TMPDIR=/tmp +fi + +set -u + +if [ "$#" -lt "1" ] ; then + echo "Please provide an installation path such as /opt/ICGC" + exit 1 +fi + +# get path to this script +SCRIPT_PATH=`dirname $0`; +SCRIPT_PATH=`(cd $SCRIPT_PATH && pwd)` + +# get the location to install to +INST_PATH=$1 +mkdir -p $1 +INST_PATH=`(cd $1 && pwd)` +echo $INST_PATH + +# get current directory +INIT_DIR=`pwd` + +CPU=`grep -c ^processor /proc/cpuinfo` +if [ $? -eq 0 ]; then + if [ "$CPU" -gt "6" ]; then + CPU=6 + fi +else + CPU=1 +fi +echo "Max compilation CPUs set to $CPU" + +SETUP_DIR=$INIT_DIR/install_tmp +mkdir -p $SETUP_DIR/distro # don't delete the actual distro directory until the very end +mkdir -p $INST_PATH/bin +cd $SETUP_DIR + +# make sure tools installed can see the install loc of libraries +set +u +export LD_LIBRARY_PATH=`echo $INST_PATH/lib:$LD_LIBRARY_PATH | perl -pe 's/:\$//;'` +export PATH=`echo $INST_PATH/bin:$PATH | perl -pe 's/:\$//;'` +export MANPATH=`echo $INST_PATH/man:$INST_PATH/share/man:$MANPATH | perl -pe 's/:\$//;'` +export PERL5LIB=`echo $INST_PATH/lib/perl5:$PERL5LIB | perl -pe 's/:\$//;'` +set -u + +## vcftools +if [ ! -e $SETUP_DIR/vcftools.success ]; then + curl -sSL --retry 10 https://github.com/vcftools/vcftools/releases/download/v${VER_VCFTOOLS}/vcftools-${VER_VCFTOOLS}.tar.gz > distro.tar.gz + rm -rf distro/* + tar --strip-components 2 -C distro -xzf distro.tar.gz + cd distro + ./configure --prefix=$INST_PATH --with-pmdir=lib/perl5 + make -j$CPU + make install + cd $SETUP_DIR + rm -rf distro.* distro/* + touch $SETUP_DIR/vcftools.success +fi + +### cgpVcf +if [ ! -e $SETUP_DIR/cgpVcf.success ]; then + curl -sSL --retry 10 https://github.com/cancerit/cgpVcf/archive/${VER_CGPVCF}.tar.gz > distro.tar.gz + rm -rf distro/* + tar --strip-components 1 -C distro -xzf distro.tar.gz + cd distro + cpanm --no-interactive --notest --mirror http://cpan.metacpan.org --notest -l $INST_PATH --installdeps . + cpanm -v --no-interactive --mirror http://cpan.metacpan.org -l $INST_PATH . + cd $SETUP_DIR + rm -rf distro.* distro/* + touch $SETUP_DIR/cgpVcf.success +fi