Skip to content

Commit

Permalink
Merge pull request #69 from cancerit/feature/docker
Browse files Browse the repository at this point in the history
Feature/docker
  • Loading branch information
keiranmraine authored May 15, 2019
2 parents 1f83f42 + 07a3bcd commit ac49f01
Show file tree
Hide file tree
Showing 7 changed files with 310 additions and 77 deletions.
10 changes: 10 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
/LICENCE
/prerelease.sh
/README.md
/INSTALL
/LICENCE
/CHANGES.md
/.gitignore
/.git
/perl/docs
/perl/docs.tar.gz
74 changes: 23 additions & 51 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,58 +2,30 @@ notifications:
slack: wtsi-cgpit:ptUMR1tkNyZJYd9TpGoss8WR
email: false

env:
- CC=gcc
sudo: false

addons:
apt:
update: true
packages:
- build-essential
- autoconf
- bsdtar
- time
- curl
- libcurl4-openssl-dev
- nettle-dev
- zlib1g-dev
- libncurses5-dev
- libpstreams-dev
- unzip
- libpng12-dev
- libexpat1-dev
- libgoogle-perftools-dev
- lsof
- libbz2-dev
- liblzma-dev
- libgnutls-dev
- libtasn1-6-dev
- p11-kit
- libxml2-dev
- libgd-dev
- psmisc
- libdb-dev

install: true

language: perl

perl:
- "5.22-shrplib"
services:
- docker

script:
- set -e
- export PATH=$HOME/install/bin:$HOME/install/biobambam2/bin:$PATH
- git clone --depth 1 --single-branch --branch develop https://github.com/cancerit/cgpBigWig.git
- cd cgpBigWig
- ./setup.sh $HOME/PCAP-opt
- cd ../
- git clone --depth 1 --single-branch --branch develop https://github.com/cancerit/PCAP-core.git
- cd PCAP-core
- ./setup.sh $HOME/PCAP-opt
- cd ../
- git clone --depth 1 --single-branch --branch dev https://github.com/cancerit/cgpVcf.git
- cd cgpVcf
- ./setup.sh $HOME/PCAP-opt
- cd ../
- ./setup.sh $HOME/PCAP-opt
- echo 'Build and check docker image'
- docker build -t cgppindel .
- docker images | grep -c cgppindel
- echo 'Verify program(s) from each inherited base image - dockstore-cgpbigwig'
- docker run -t --rm cgppindel bwjoin --version
- echo 'Verify program(s) from each inherited base image - dockstore-cgpmap'
- docker run -t --rm cgppindel bwa_mem.pl -version
- docker run -t --rm cgppindel bammarkduplicates2 --version
- docker run -t --rm cgppindel samtools --version
- docker run -t --rm cgppindel bash -c 'bwa 2>&1 | grep Version'
- echo 'Verify program(s) from this repo'
- docker run -t --rm cgppindel pindel.pl --version
- docker run -t --rm cgppindel pindel_input_gen.pl --version
- docker run -t --rm cgppindel pindel_2_combined_vcf.pl --version
- docker run -t --rm cgppindel FlagVcf.pl --version
- docker run -t --rm cgppindel pindel_merge_vcf_bam.pl --version
- docker run -t --rm cgppindel pindel_np_from_vcf.pl --version
- docker run -t --rm cgppindel pindel_germ_bed.pl --version
- docker run -t --rm cgppindel which pindel
- docker run -t --rm cgppindel which filter_pindel_reads
17 changes: 5 additions & 12 deletions CHANGES.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# CHANGES

## NEXT

* Added Dockerfile and docker documentation

## 3.2.0

* Tabix search for high depth/excluded regions now performed in memory using IntervalTrees
Expand Down Expand Up @@ -153,15 +157,4 @@ Comparing sites in VCF files...
Found 15321 SNPs common to both files.
Found 0 SNPs only in main file.
Found 0 SNPs only in second file.
After filtering, kept 16309 out of a possible 16309 Sites
Run Time = 6.00 seconds
```

## 2.0.4

* Permits empty results files

## 2.0.0

* Migrates all Tabix and Bio::DB::Sam to Bio::DB::HTS::Tabix and Bio::DB::Sam
* Cleans up install
After
81 changes: 81 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
FROM quay.io/wtsicgp/dockstore-cgpmap:3.1.4 as builder

USER root

# ALL tool versions used by opt-build.sh
ENV VER_CGPVCF="v2.2.1"
ENV VER_VCFTOOLS="0.1.16"

RUN apt-get -yq update
RUN apt-get install -yq --no-install-recommends \
locales \
g++ \
make \
gcc \
pkg-config \
zlib1g-dev

RUN locale-gen en_US.UTF-8
RUN update-locale LANG=en_US.UTF-8

ENV OPT /opt/wtsi-cgp
ENV PATH $OPT/bin:$OPT/biobambam2/bin:$PATH
ENV PERL5LIB $OPT/lib/perl5
ENV LD_LIBRARY_PATH $OPT/lib
ENV LC_ALL en_US.UTF-8
ENV LANG en_US.UTF-8

# build tools from other repos
ADD build/opt-build.sh build/
RUN bash build/opt-build.sh $OPT

# build the tools in this repo, separate to reduce build time on errors
COPY . .
RUN bash build/opt-build-local.sh $OPT

FROM ubuntu:16.04

LABEL maintainer="[email protected]" \
uk.ac.sanger.cgp="Cancer, Ageing and Somatic Mutation, Wellcome Trust Sanger Institute" \
version="1.0.0" \
description="cgpPindel docker"

RUN apt-get -yq update
RUN apt-get install -yq --no-install-recommends \
apt-transport-https \
locales \
curl \
ca-certificates \
libperlio-gzip-perl \
bzip2 \
psmisc \
time \
zlib1g \
liblzma5 \
libncurses5 \
p11-kit \
unattended-upgrades && \
unattended-upgrade -d -v && \
apt-get remove -yq unattended-upgrades && \
apt-get autoremove -yq

RUN locale-gen en_US.UTF-8
RUN update-locale LANG=en_US.UTF-8

ENV OPT /opt/wtsi-cgp
ENV PATH $OPT/bin:$OPT/biobambam2/bin:$PATH
ENV PERL5LIB $OPT/lib/perl5
ENV LD_LIBRARY_PATH $OPT/lib
ENV LC_ALL en_US.UTF-8
ENV LANG en_US.UTF-8

RUN mkdir -p $OPT
COPY --from=builder $OPT $OPT

## USER CONFIGURATION
RUN adduser --disabled-password --gecos '' ubuntu && chsh -s /bin/bash && mkdir -p /home/ubuntu

USER ubuntu
WORKDIR /home/ubuntu

CMD ["/bin/bash"]
63 changes: 49 additions & 14 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

cgpPindel contains the Cancer Genome Projects workflow for [Pindel][pindel-core].

[![Quay Badge][quay-status]][quay-repo]

| Master | Develop |
| --------------------------------------------- | ----------------------------------------------- |
| [![Master Badge][travis-master]][travis-base] | [![Develop Badge][travis-develop]][travis-base] |
Expand All @@ -18,25 +20,30 @@ Contents:

<!-- TOC depthFrom:2 depthTo:6 withLinks:1 updateOnSave:1 orderedList:0 -->

- [Docker, Singularity and Dockstore](#docker-singularity-and-dockstore)
- [Dependencies/Install](#dependenciesinstall)
- [Creating a release](#creating-a-release)
- [Preparation](#preparation)
- [Cutting the release](#cutting-the-release)
- [LICENCE](#licence)
- [cgpPindel](#cgppindel)
- [Docker, Singularity and Dockstore](#docker-singularity-and-dockstore)
- [Dependencies/Install](#dependenciesinstall)
- [Creating a release](#creating-a-release)
- [Preparation](#preparation)
- [Release process](#release-process)
- [Code changes](#code-changes)
- [Docker image](#docker-image)
- [Cutting the release](#cutting-the-release)
- [LICENCE](#licence)

<!-- /TOC -->

## Docker, Singularity and Dockstore

There are pre-built images containing this codebase on quay.io.

* [cgpPindel][cgpPindel-git]: Contained within this repository - contains the cgpPindel package
* [dockstore-cgpwxs][ds-cgpwxs-git]: Contains tools specific to WXS analysis.
* [dockstore-cgpwgs][ds-cgpwgs-git]: Contains additional tools for WGS analysis.

These were primarily designed for use with dockstore.org but can be used as normal containers.

The docker images are know to work correctly after import into a singularity image.
The docker images are known to work correctly after import into a singularity image.

## Dependencies/Install

Expand Down Expand Up @@ -66,15 +73,34 @@ Please be aware that this expects basic C compilation libraries and tools to be
* Commit/push all relevant changes.
* Pull a clean version of the repo and use this for the following steps.

### Cutting the release
### Release process

This project is maintained using HubFlow.

#### Code changes

1. Make appropriate changes
2. Update `perl/lib/Sanger/CGP/Pindel.pm` to the correct version (adding rc/beta to end if applicable).
3. Update `CHANGES.md` to show major items.
4. Run `./prerelease.sh`
5. Check all tests and coverage reports are acceptable.
6. Commit the updated docs and updated module/version.
7. Push commits.

#### Docker image

1. Update `perl/lib/Sanger/CGP/Pindel.pm` to the correct version (adding rc/beta to end if applicable).
1. Update `CHANGES.md` to show major items.
1. Run `./prerelease.sh`
1. Check all tests and coverage reports are acceptable.
1. Commit the updated docs and updated module/version.
1. Push commits.
1. Use the GitHub tools to draft a release.
2. Build image locally
3. Run example inputs and verify any changes are acceptable
4. Bump version in `Dockerfile`
5. Push changes

#### Cutting the release

1. Check state on Travis
2. Generate the release (add notes to GitHub)
3. Confirm that image has been built on [quay.io][quay-builds]
4. Update the [dockstore][dockstore-cgpPindel] entry, see [their docs][dockstore-get-started].

## LICENCE

Expand Down Expand Up @@ -114,9 +140,18 @@ identical to a statement that reads ‘Copyright (c) 2005, 2006, 2007, 2008,
[pcap-core-rel]: https://github.com/cancerit/PCAP-core/releases
[ds-cgpwxs-git]: https://github.com/cancerit/dockstore-cgpwxs
[ds-cgpwgs-git]: https://github.com/cancerit/dockstore-cgpwgs
[cgpPindel-git]: https://github.com/cancerit/cgpPindel
[pindel-core]: http://gmt.genome.wustl.edu/pindel/current

<!-- Travis -->
[travis-base]: https://travis-ci.org/cancerit/cgpPindel
[travis-master]: https://travis-ci.org/cancerit/cgpPindel.svg?branch=master
[travis-develop]: https://travis-ci.org/cancerit/cgpPindel.svg?branch=dev

<!-- Quay.io -->
[quay-status]: https://quay.io/repository/wtsicgp/cgpPindel/status
[quay-repo]: https://quay.io/repository/wtsicgp/cgpPindel
[quay-builds]: https://quay.io/repository/wtsicgp/cgpPindel?tab=builds

<!-- Dockstore -->
[dockstore-cgpPindel]: https://dockstore.org/containers/quay.io/wtsicgp/cgpPindel
65 changes: 65 additions & 0 deletions build/opt-build-local.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
#! /bin/bash

set -xe

if [[ -z "${TMPDIR}" ]]; then
TMPDIR=/tmp
fi

set -u

if [ "$#" -lt "1" ] ; then
echo "Please provide an installation path such as /opt/ICGC"
exit 1
fi

# get path to this script
SCRIPT_PATH=`dirname $0`;
SCRIPT_PATH=`(cd $SCRIPT_PATH && pwd)`

# get the location to install to
INST_PATH=$1
mkdir -p $1
INST_PATH=`(cd $1 && pwd)`
echo $INST_PATH

# get current directory
INIT_DIR=`pwd`

CPU=`grep -c ^processor /proc/cpuinfo`
if [ $? -eq 0 ]; then
if [ "$CPU" -gt "6" ]; then
CPU=6
fi
else
CPU=1
fi
echo "Max compilation CPUs set to $CPU"

SETUP_DIR=$INIT_DIR/install_tmp
mkdir -p $SETUP_DIR/distro # don't delete the actual distro directory until the very end
mkdir -p $INST_PATH/bin
cd $SETUP_DIR

# make sure tools installed can see the install loc of libraries
set +u
export LD_LIBRARY_PATH=`echo $INST_PATH/lib:$LD_LIBRARY_PATH | perl -pe 's/:\$//;'`
export PATH=`echo $INST_PATH/bin:$PATH | perl -pe 's/:\$//;'`
export MANPATH=`echo $INST_PATH/man:$INST_PATH/share/man:$MANPATH | perl -pe 's/:\$//;'`
export PERL5LIB=`echo $INST_PATH/lib/perl5:$PERL5LIB | perl -pe 's/:\$//;'`
set -u

## cgpPindel - should be the build root
if [ ! -e $SETUP_DIR/cgpPindel.success ]; then
cd $INIT_DIR
if [ ! -e $SETUP_DIR/cgpPindel_c.success ]; then
g++ -O3 -o $INST_PATH/bin/pindel c++/pindel.cpp
g++ -O3 -o $INST_PATH/bin/filter_pindel_reads c++/filter_pindel_reads.cpp
touch $SETUP_DIR/cgpPindel_c.success
fi
cd perl
cpanm --no-interactive --notest --mirror http://cpan.metacpan.org --notest -l $INST_PATH --installdeps .
cpanm -v --no-interactive --mirror http://cpan.metacpan.org -l $INST_PATH .
cd $SETUP_DIR
touch $SETUP_DIR/cgpPindel.success
fi
Loading

0 comments on commit ac49f01

Please sign in to comment.