Merge pull request #69 from CCBR/iss-67

Fix docker containers & bind mounts

.github/workflows/build.yml

@@ -32,4 +32,9 @@ jobs:
           pip install .[dev,test]
       - name: Test stub run
         run: |
-          champagne run -profile ci_stub -stub
+          champagne run -profile ci_stub,docker -stub
+      - name: "Upload Artifact"
+        uses: actions/upload-artifact@v3
+        with:
+          name: nextflow-log
+          path: .nextflow.log

CHANGELOG.md

@@ -3,11 +3,13 @@
 ### New features
 
 - Implemented peak calling with sicer2, macs2, and gem. (#52)
-- New GitHub Actions workflow to launch a stub run from the champagne CLI.
 
 ### Bug fixes
 
 - CLI error when biowulf-specific environment variables are not defined. (#54)
+- Using `--mount type=bind` instead of `--bind` for Docker compatibility. (#69)
+- Specify containers in process definitions instead of `withName`/`withLabel` for better control. (#69)
+  - Shared containers are specified as parameters in the config file.
 
 ## CHAMPAGNE 0.1.0
 

conf/ci_stub.config

@@ -18,20 +18,20 @@ params {
     }
     align {
         index_dir = "${baseDir}/tests/data/"         // dummy value, all blank files here
-        blacklist = "${params.genome}.blacklist"
-        blacklist_files = "${params.align.index_dir}${params.align.blacklist}*"
-        reference_files = "${params.align.index_dir}${params.genome}*"
+        blacklist_files = "${baseDir}/tests/data/test.blacklist*"
+        reference_files = "${baseDir}/tests/data/test/*"
         min_quality = 6
         effective_genome_size = 2700000000
-        chrom_sizes = "${params.align.index_dir}${params.genome}.fa.sizes"
+        chrom_sizes = "${baseDir}/tests/data/test.fa.sizes"
     }
-    gene_info = "${params.align.index_dir}${params.genome}.geneinfo.bed"
+    gene_info = "${baseDir}/tests/data/test.geneinfo.bed"
     deeptools {
         bin_size = 10000 // this value is only to make bamCoverage run faster. use smaller value for real data.
         smooth_length = 75
         normalize_using = "RPGC"
         excluded_chroms = "chrM chrX chrY"
     }
+    chromosomes_dir = "${baseDir}/tests/data/chroms/*"
 }
 
 process {

conf/modules.config

@@ -16,63 +16,11 @@ process {
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
-}
-
-// Docker/Singularity containers
-// Do not specify cpus, memory, or time here -- use process labels from conf/base.config
-process {
-
-    // catch any process without a container.
-    // this must be first before any other withName selectors.
-    // oddly, it will override any withLabel selectors even though
-    // label should have a higher priority than name.
-    // https://www.nextflow.io/docs/latest/config.html#selector-priority
-    withName: '.*' {
-        container = 'nciccbr/ccbr_ubuntu_base_20.04:latest'
-    }
-
-    // custom process containers
-
-    withName: 'TRIM.*' {
-        container = 'nciccbr/ncigb_cutadapt_v1.18:latest'
-    }
-    withName: 'FASTQC.*' {
-        container = 'nciccbr/ccrgb_qctools:latest'
-    }
-    withName: 'FASTQ_SCREEN' {
-        container = 'nciccbr/ccbr_fastq_screen_0.14.1'
-        // TODO bind flag not recognized by Docker on gh actions?
-        containerOptions = "--bind ${params.fastq_screen.db_dir}"
-    }
-    withName: 'PRESEQ' {
-        container = 'nciccbr/ccbr_preseq_v2.0:v1'
+    withName: FASTQ_SCREEN {
+        // use bind for singularity and volume for docker
+        containerOptions = { workflow.containerEngine == 'singularity' ?
+            "--mount type=bind,src=$params.fastq_screen.db_dir,dst=$params.fastq_screen.db_dir" :
+            "--volume $params.fastq_screen.db_dir:$params.fastq_screen.db_dir "}
     }
-    withName: 'PHANTOM_PEAKS' {
-        container = 'quay.io/biocontainers/phantompeakqualtools:1.2.2--hdfd78af_1'
-    }
-    withName: 'DEDUPLICATE' {
-        container = 'nciccbr/ccbr_macs2_2.2.9.1:v1'
-    }
-    withName: 'NGSQC_GEN' {
-        container = 'nciccbr/ccbr_ngsqc_0.31:v1'
-    }
-    // deeptools
-    // Previously tried using withLabel selector, but it gets overridden by `withName: '.*'`
-    withName: 'BAM_COVERAGE|BIGWIG_SUM|PLOT_CORRELATION|PLOT_PCA|PLOT_FINGERPRINT|COMPUTE_MATRIX|PLOT_HEATMAP|PLOT_PROFILE|NORMALIZE_INPUT' {
-        container = 'nciccbr/ccbr_deeptools_3.5.3:v1'
-    }
-    withName: 'MULTIQC' {
-        container = 'nciccbr/ccbr_multiqc_1.15:v1'
-    }
-    withName: 'SICER.*' {
-        container = 'nciccbr/ccbr_sicer2_1.0.3:v1'
-    }
-    withName: 'MACS.*' {
-        container = 'nciccbr/ccbr_macs2_2.2.9.1'
-    }
-    withName: 'GEM.*' {
-        container = 'nciccbr/ccbr_gem_3.4:v1'
-        containerOptions = "--bind ${params.chromosomes_dir}"
-    }
-
 }
+// Do not specify cpus, memory, or time here -- use process labels from conf/base.config

conf/test.config

@@ -32,7 +32,7 @@ params {
     }
     gene_info = "/data/CCBR_Pipeliner/db/PipeDB/Indices/${params.genome}_basic/geneinfo.bed"
     gem_read_dist = 'https://groups.csail.mit.edu/cgs/gem/download/Read_Distribution_default.txt'
-    chromosomes_dir = "/data/CCBR_Pipeliner/db/PipeDB/Indices/${params.genome}_basic/Chromsomes/"
+    chromosomes_dir = "/data/CCBR_Pipeliner/db/PipeDB/Indices/${params.genome}_basic/Chromsomes/*"
 }
 
 dag {

conf/test_mm10.config

@@ -17,5 +17,5 @@ params {
         chrom_sizes = "${params.align.index_dir}${params.genome}.fa.sizes"       // source: https://github.com/CCBR/Pipeliner/blob/86c6ccaa3d58381a0ffd696bbf9c047e4f991f9e/hg38.json#L359
     }
     gene_info = "/data/CCBR_Pipeliner/db/PipeDB/Indices/${params.genome}_basic/geneinfo.bed"
-    chromosomes_dir = "/data/CCBR_Pipeliner/db/PipeDB/Indices/${params.genome}_basic/Chromsomes/"
+    chromosomes_dir = "/data/CCBR_Pipeliner/db/PipeDB/Indices/${params.genome}_basic/Chromsomes/*"
 }

main.nf

@@ -103,7 +103,7 @@ workflow {
   PPQT_PROCESS(PHANTOM_PEAKS.out.fraglen)
   frag_lengths = PPQT_PROCESS.out.fraglen
 
-  ///* optional qc
+  ///* start qc
   QC_STATS(
     raw_fastqs,
     ALIGN_GENOME.out.flagstat,
@@ -167,7 +167,7 @@ workflow {
   )
 
   NORMALIZE_INPUT(ch_ip_ctrl_bigwig)
-  //*/
+  //*/ // end of qc
   // peak calling
 
   genome_frac = CALC_GENOME_FRAC(chrom_sizes)
@@ -191,9 +191,10 @@ workflow {
     .combine(Channel.fromPath(params.gem_read_dists))
     .combine(chrom_sizes)
     .set { ch_gem }
+  chrom_files = Channel.fromPath(params.chromosomes_dir).collect()
 
   ch_tagalign  | SICER
   ch_tagalign  | MACS_BROAD
   ch_tagalign  | MACS_NARROW
-  ch_gem       | GEM
+  GEM(ch_gem, chrom_files)
 }

modules/local/align.nf

@@ -4,6 +4,8 @@ process ALIGN_BLACKLIST {
     label 'align'
     label 'process_higher'
 
+    container = "${params.containers.base}"
+
     input:
         tuple val(meta), path(fastq)
         path(blacklist_files)
@@ -35,6 +37,8 @@ process ALIGN_GENOME {
     label 'align'
     label 'process_higher'
 
+    container = "${params.containers.base}"
+
     input:
         tuple val(meta), path(fastq)
         path(reference_files)
@@ -76,6 +80,8 @@ process INDEX_BAM {
     label 'align'
     label 'process_higher'
 
+    container = "${params.containers.base}"
+
     input:
         tuple val(meta), path(bam)
 

modules/local/deeptools.nf

@@ -5,6 +5,8 @@ process BAM_COVERAGE {
     label 'deeptools'
     label 'process_higher'
 
+    container = "${params.containers.deeptools}"
+
     input:
         tuple val(meta), path(bam), path(bai)
         tuple val(meta), val(fraglen)
@@ -38,6 +40,8 @@ process BIGWIG_SUM {
     label 'deeptools'
     label 'process_high'
 
+    container = "${params.containers.deeptools}"
+
     input:
         path(bigwigs)
 
@@ -62,6 +66,8 @@ process PLOT_CORRELATION {
     label 'qc'
     label 'deeptools'
 
+    container = "${params.containers.deeptools}"
+
     input:
         tuple path(array), val(plottype)
 
@@ -93,6 +99,8 @@ process PLOT_PCA {
     label 'qc'
     label 'deeptools'
 
+    container = "${params.containers.deeptools}"
+
     input:
         path(array)
 
@@ -119,6 +127,8 @@ process PLOT_FINGERPRINT {
   label 'deeptools'
   label 'process_higher'
 
+  container = "${params.containers.deeptools}"
+
   input:
     tuple val(meta), path(bams), path(bais)
 
@@ -154,6 +164,8 @@ process PLOT_FINGERPRINT {
 process BED_PROTEIN_CODING {
     label 'qc'
 
+    container = "${params.containers.base}"
+
     input:
       path(bed)
 
@@ -176,6 +188,8 @@ process COMPUTE_MATRIX {
   label 'deeptools'
   label 'process_higher'
 
+  container = "${params.containers.deeptools}"
+
   input:
     path(bigwigs)
     tuple path(bed), val(mattype)
@@ -224,6 +238,8 @@ process PLOT_HEATMAP {
   label 'qc'
   label 'deeptools'
 
+  container = "${params.containers.deeptools}"
+
   input:
     path(mat)
 
@@ -253,6 +269,8 @@ process PLOT_PROFILE {
   label 'qc'
   label 'deeptools'
 
+  container = "${params.containers.deeptools}"
+
   input:
     path(mat)
 
@@ -286,6 +304,8 @@ process NORMALIZE_INPUT {
   label 'deeptools'
   label 'process_higher'
 
+  container = "${params.containers.deeptools}"
+
   input:
     tuple val(meta), path(chip), path(input)
 

modules/local/peaks.nf

@@ -2,6 +2,8 @@
 process CALC_GENOME_FRAC {
     label 'peaks'
 
+    container = "${params.containers.base}"
+
     input:
         path(chrom_sizes)
 
@@ -26,6 +28,8 @@ process SICER {
     label 'peaks'
     label 'process_high'
 
+    container = "${params.containers.sicer}"
+
     input:
         tuple val(meta), path(chip), path(input), val(fraglen), val(genome_frac)
 
@@ -76,6 +80,8 @@ process MACS_BROAD {
     tag { meta.id }
     label 'peaks'
 
+    container = "${params.containers.macs2}"
+
     input:
         tuple val(meta), path(chip), path(input), val(fraglen), val(genome_frac)
 
@@ -112,6 +118,8 @@ process MACS_NARROW {
     tag { meta.id }
     label 'peaks'
 
+    container = "${params.containers.macs2}"
+
     input:
         tuple val(meta), path(chip), path(input), val(fraglen), val(genome_frac)
 
@@ -146,8 +154,11 @@ process GEM {
     label 'peaks'
     label 'process_high'
 
+    container = "${params.containers.gem}"
+
     input:
         tuple val(meta), path(chip), path(input), path(read_dists), path(chrom_sizes)
+        path(chrom_files)
 
     output:
         path("${meta.id}/*.GEM_events.txt")
@@ -159,7 +170,7 @@ process GEM {
       --t ${task.cpus} \\
       --d ${read_dists} \\
       --g ${chrom_sizes} \\
-      --genome ${params.chromosomes_dir} \\
+      --genome ${chrom_files} \\
       --expt ${chip} \\
       --ctrl ${input} \\
       --out ${meta.id} \\
@@ -171,6 +182,7 @@ process GEM {
 
     stub:
     """
-    touch ${meta.id}.GEM_events.txt
+    mkdir ${meta.id}
+    touch ${meta.id}/${meta.id}.GEM_events.txt
     """
 }