Skip to content

Commit

Permalink
Refactor all configs. Use singularity for UPPMAX.
Browse files Browse the repository at this point in the history
  • Loading branch information
ewels committed Aug 31, 2017
1 parent f5dee4e commit 29331ea
Show file tree
Hide file tree
Showing 10 changed files with 255 additions and 563 deletions.
3 changes: 3 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -79,3 +79,6 @@ RUN mkdir /opt/Qualimap && \

# Install MultiQC
RUN pip install git+https://github.com/ewels/MultiQC.git

# Create UPPMAX directories for Singularity
RUN mkdir /pica /proj /sw
6 changes: 5 additions & 1 deletion Dockerfile-bwa
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
FROM openjdk:8
# openjdk:8 moved from debian jessie to stretch after u131, which breaks everything (bowtie)
FROM openjdk:8u121

LABEL authors="[email protected],[email protected]" \
description="Docker image containing all requirements for NGI-MethylSeq pipeline"
Expand Down Expand Up @@ -89,3 +90,6 @@ RUN mkdir /opt/MethylDackel && \

# Install MultiQC
RUN pip install multiqc

# Create UPPMAX directories for Singularity
RUN mkdir /pica /proj /sw
152 changes: 7 additions & 145 deletions conf/aws.config
Original file line number Diff line number Diff line change
Expand Up @@ -16,153 +16,15 @@ docker {
process {
container = 'scilifelab/ngi-methylseq'
executor = 'ignite'
time = { 2.h * task.attempt }
errorStrategy = { task.exitStatus == 143 ? 'retry' : 'finish' }
maxRetries = 3
maxErrors = '-1'


errorStrategy = { ( task.exitStatus == 143 || task.exitStatus == 137 ) ? 'retry' : 'finish' }
maxRetries = 3
maxErrors = '-1'

// Process resource requirements
$makeBismarkIndex {
time = { 36.h * task.attempt }
}
$fastqc {
errorStrategy = { ( task.exitStatus == 143 || task.exitStatus == 137 ) ? 'retry' : 'ignore' }
}
$trim_galore {
time = { 12.h * task.attempt }
}
$bismark_align {
time = { 36.h * task.attempt }
}
$bismark_deduplicate {
time = { 12.h * task.attempt }
}
$bismark_methXtract {
time = { 8.h * task.attempt }
}
$bismark_report {
errorStrategy = { ( task.exitStatus == 143 || task.exitStatus == 137 ) ? 'retry' : 'ignore' }
}
$bismark_summary {
errorStrategy = { ( task.exitStatus == 143 || task.exitStatus == 137 ) ? 'retry' : 'ignore' }
}
$qualimap {
time = { 6.h * task.attempt }
errorStrategy = { ( task.exitStatus == 143 || task.exitStatus == 137 ) ? 'retry' : 'ignore' }
}
$get_software_versions {
errorStrategy = 'ignore'
}
$multiqc {
errorStrategy = { ( task.exitStatus == 143 || task.exitStatus == 137 ) ? 'retry' : 'ignore' }
}

$bwamem_align {
time = { 24.h * task.attempt }
}
$samtools_sort {
time = { 8.h * task.attempt }
}

}

params {
saveReference = true
// illumina iGenomes reference file paths on AWS
genomes {
'GRCh37' {
bismark = 's3://ngi-igenomes/igenomes/Homo_sapiens/Ensembl/GRCh37/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Homo_sapiens/Ensembl/GRCh37/Sequence/WholeGenomeFasta'
}
'GRCm38' {
bismark = 's3://ngi-igenomes/igenomes/Mus_musculus/Ensembl/GRCm38/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Mus_musculus/Ensembl/GRCm38/Sequence/WholeGenomeFasta'
}
'TAIR10' {
bismark = 's3://ngi-igenomes/igenomes/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/WholeGenomeFasta'
}
'EB2' {
bismark = 's3://ngi-igenomes/igenomes/Bacillus_subtilis_168/Ensembl/EB2/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Bacillus_subtilis_168/Ensembl/EB2/Sequence/WholeGenomeFasta'
}
'UMD3.1' {
bismark = 's3://ngi-igenomes/igenomes/Bos_taurus/Ensembl/UMD3.1/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Bos_taurus/Ensembl/UMD3.1/Sequence/WholeGenomeFasta'
}
'WBcel235' {
bismark = 's3://ngi-igenomes/igenomes/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/WholeGenomeFasta'
}
'CanFam3.1' {
bismark = 's3://ngi-igenomes/igenomes/Canis_familiaris/Ensembl/CanFam3.1/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Canis_familiaris/Ensembl/CanFam3.1/Sequence/WholeGenomeFasta'
}
'GRCz10' {
bismark = 's3://ngi-igenomes/igenomes/Danio_rerio/Ensembl/GRCz10/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Danio_rerio/Ensembl/GRCz10/Sequence/WholeGenomeFasta'
}
'BDGP6' {
bismark = 's3://ngi-igenomes/igenomes/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/WholeGenomeFasta'
}
'EquCab2' {
bismark = 's3://ngi-igenomes/igenomes/Equus_caballus/Ensembl/EquCab2/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Equus_caballus/Ensembl/EquCab2/Sequence/WholeGenomeFasta'
}
'EB1' {
bismark = 's3://ngi-igenomes/igenomes/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/WholeGenomeFasta'
}
'Galgal4' {
bismark = 's3://ngi-igenomes/igenomes/Gallus_gallus/Ensembl/Galgal4/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Gallus_gallus/Ensembl/Galgal4/Sequence/WholeGenomeFasta'
}
'Gm01' {
bismark = 's3://ngi-igenomes/igenomes/Glycine_max/Ensembl/Gm01/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Glycine_max/Ensembl/Gm01/Sequence/WholeGenomeFasta'
}
'Mmul_1' {
bismark = 's3://ngi-igenomes/igenomes/Macaca_mulatta/Ensembl/Mmul_1/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Macaca_mulatta/Ensembl/Mmul_1/Sequence/WholeGenomeFasta'
}
'IRGSP-1.0' {
bismark = 's3://ngi-igenomes/igenomes/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/WholeGenomeFasta'
}
'CHIMP2.1.4' {
bismark = 's3://ngi-igenomes/igenomes/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/WholeGenomeFasta'
}
'Rnor_6.0' {
bismark = 's3://ngi-igenomes/igenomes/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/WholeGenomeFasta'
}
'R64-1-1' {
bismark = 's3://ngi-igenomes/igenomes/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/WholeGenomeFasta'
}
'EF2' {
bismark = 's3://ngi-igenomes/igenomes/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/WholeGenomeFasta'
}
'Sbi1' {
bismark = 's3://ngi-igenomes/igenomes/Sorghum_bicolor/Ensembl/Sbi1/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Sorghum_bicolor/Ensembl/Sbi1/Sequence/WholeGenomeFasta'
}
'Sscrofa10.2' {
bismark = 's3://ngi-igenomes/igenomes/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/WholeGenomeFasta'
}
'AGPv3' {
bismark = 's3://ngi-igenomes/igenomes/Zea_mays/Ensembl/AGPv3/Sequence/BismarkIndex'
fasta = 's3://ngi-igenomes/igenomes/Zea_mays/Ensembl/AGPv3/Sequence/WholeGenomeFasta'
}
}

igenomes_base = 's3://ngi-igenomes/igenomes/'
// Max resources based on a typical AWS m4.4xlarge EC2 instance
// Customise with --max_memory, --max_cpus and --max_time (or overwrite with another config file)
// For example, for m4.2xlarge, use `--max_cpus 8 --max_memory 32`
max_memory = 64.GB
max_cpus = 16
max_time = 240.h
}
92 changes: 92 additions & 0 deletions conf/base.config
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
/*
vim: syntax=groovy
-*- mode: groovy;-*-
* -------------------------------------------------
* Base Nextflow config file
* -------------------------------------------------
* This file contains basic process requirement setup
* It DOES NOT contain any config for cluster, so will run
* in default mode by itself. It should be used with
* the max_memory, max_cpus and max_time params for
* customising hardware limits
*/

process {
cpus = { check_max(1 * task.attempt, 'cpus') }
memory = { check_max( 8.GB * task.attempt, 'memory') }
time = { check_max(2.h * task.attempt, 'time') }

errorStrategy = { ( task.exitStatus == 143 || task.exitStatus == 137 ) ? 'retry' : 'finish' }
maxRetries = 3
maxErrors = '-1'

// Resource requirements
$makeBismarkIndex {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max( 64.GB * task.attempt, 'memory') }
time = { check_max(36.h * task.attempt, 'time') }
}
$fastqc {
errorStrategy = { ( task.exitStatus == 143 || task.exitStatus == 137 ) ? 'retry' : 'ignore' }
}
$trim_galore {
cpus = { check_max(2 * task.attempt, 'cpus') }
memory = { check_max( 16.GB * task.attempt, 'memory') }
time = { check_max(12.h * task.attempt, 'time') }
}
$bismark_align {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max( 64.GB * task.attempt, 'memory') }
time = { check_max(36.h * task.attempt, 'time') }
}
$bismark_deduplicate {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max( 64.GB * task.attempt, 'memory') }
time = { check_max(12.h * task.attempt, 'time') }
}
$bismark_methXtract {
cpus = { check_max(4 * task.attempt, 'cpus') }
memory = { check_max( 32.GB * task.attempt, 'memory') }
time = { check_max(8.h * task.attempt, 'time') }
}
$bismark_report {
errorStrategy = { ( task.exitStatus == 143 || task.exitStatus == 137 ) ? 'retry' : 'ignore' }
}
$bismark_summary {
errorStrategy = { ( task.exitStatus == 143 || task.exitStatus == 137 ) ? 'retry' : 'ignore' }
}
$qualimap {
cpus = { check_max(4 * task.attempt, 'cpus') }
memory = { check_max( 32.GB * task.attempt, 'memory') }
time = { check_max(6.h * task.attempt, 'time') }
errorStrategy = { ( task.exitStatus == 143 || task.exitStatus == 137 ) ? 'retry' : 'ignore' }
}
$get_software_versions {
executor = 'local'
errorStrategy = 'ignore'
}
$multiqc {
executor = 'local'
errorStrategy = { ( task.exitStatus == 143 || task.exitStatus == 137 ) ? 'retry' : 'ignore' }
}

$bwamem_align {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max( 64.GB * task.attempt, 'memory') }
time = { check_max(24.h * task.attempt, 'time') }
}
$samtools_sort {
cpus = { check_max(4 * task.attempt, 'cpus') }
memory = { check_max( 32.GB * task.attempt, 'memory') }
time = { check_max(8.h * task.attempt, 'time') }
}
$markDuplicates {
cpus = { check_max(4 * task.attempt, 'cpus') }
memory = { check_max( 32.GB * task.attempt, 'memory') }
}
$methyldackel {
cpus = { check_max(6 * task.attempt, 'cpus') }
memory = { check_max( 48.GB * task.attempt, 'memory') }
}

}
23 changes: 0 additions & 23 deletions conf/docker.config

This file was deleted.

Loading

0 comments on commit 29331ea

Please sign in to comment.