Merge pull request #79 from CCBR/iss-78

Use 'dir' type and checkIfExists for Channel.fromPath

CHANGELOG.md

@@ -11,9 +11,14 @@
   - Error when biowulf-specific environment variables are not defined. (#54)
   - The host is now correctly detected as biowulf via `scontrol`. (#75)
 - Containers:
-  - When mount binding paths to containers, we use `--mount type=bind` for Singularity and `--volume` for Docker for compatibility. (#69)
   - Containers are now specified in process definitions instead of `withName`/`withLabel` for better control. (#69)
-    - Shared containers are specified as parameters in the config file.
+    - Shared containers are specified as parameters in the config file `conf/containers.config`.
+  - No longer use `--mount type=bind` or `--volume` for making directories available to processes in containers. Instead, use Nextflow's `Channel.fromPath` constructor with `type: 'dir'`. (#71)
+
+### API-breaking changes
+
+- An error is thrown when a required input file doesn't exist. (#71)
+  - Previously, the workflow quietly didn't run the process(es) that required the missing file.
 
 ## CHAMPAGNE 0.1.0
 

MANIFEST.in

@@ -3,6 +3,7 @@ recursive-include bin
 recursive-include conf
 recursive-include modules
 recursive-include submodules
+recursive-include tests
 include CITATION.cff
 include LICENSE
 include VERSION

conf/ci_stub.config

@@ -13,10 +13,22 @@ params {
     publish_dir_mode = "symlink"
 
     // CCBR shared resource paths
-    index_dir = "${baseDir}/tests/data/"
+    index_dir = "tests/data"
     fastq_screen {
-        conf = "${baseDir}/assets/fastq_screen_biowulf.conf"
-        db_dir = "${baseDir}/tests/data/"
+        conf = "assets/fastq_screen_biowulf.conf"
+        db_dir = "tests/data/"
+    }
+    multiqc_config = "assets/multiqc_config.yaml"
+    genomes {
+        'test' { // blank files for testing stubs on GitHub Actions
+            blacklist = 'test.blacklist'
+            blacklist_files = "tests/data/test.blacklist"
+            reference_files = "tests/data/test/*"
+            effective_genome_size = 2700000000
+            chrom_sizes = "tests/data/test.fa.sizes"
+            gene_info = "tests/data/geneinfo.bed"
+            chromosomes_dir = "tests/data/chroms/"
+        }
     }
 
 }

conf/genomes.config

@@ -7,7 +7,7 @@ params {
             effective_genome_size = 2700000000
             chrom_sizes = "${params.index_dir}/hg38_basic/indexes/hg38.fa.sizes"
             gene_info = "${params.index_dir}/hg38_basic/geneinfo.bed"
-            chromosomes_dir = "${params.index_dir}/hg38_basic/Chromsomes/*"
+            chromosomes_dir = "${params.index_dir}/hg38_basic/Chromsomes/"
         }
         'mm10' {
             blacklist = 'mm10.blacklist'
@@ -16,16 +16,7 @@ params {
             effective_genome_size = 2400000000
             chrom_sizes = "${params.index_dir}/mm10_basic/indexes/mm10.fa.sizes"
             gene_info = "${params.index_dir}/mm10_basic/geneinfo.bed"
-            chromosomes_dir = "${params.index_dir}/mm10_basic/Chromsomes/*"
-        }
-        'test' { // blank files for testing stubs on GitHub Actions
-            blacklist = 'test.blacklist'
-            blacklist_files = "${params.index_dir}/test.blacklist"
-            reference_files = "${params.index_dir}/test/*"
-            effective_genome_size = 2700000000
-            chrom_sizes = "${params.index_dir}/test.fa.sizes"
-            gene_info = "${params.index_dir}/geneinfo.bed"
-            chromosomes_dir = "${params.index_dir}/chroms/*"
+            chromosomes_dir = "${params.index_dir}/mm10_basic/Chromsomes/"
         }
     }
 }

conf/modules.config

@@ -15,10 +15,4 @@ process {
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
-    withName: FASTQ_SCREEN {
-        // use bind for singularity and volume for docker
-        containerOptions = { workflow.containerEngine == 'singularity' ?
-            "--mount type=bind,src=$params.fastq_screen.db_dir,dst=$params.fastq_screen.db_dir" :
-            "--volume $params.fastq_screen.db_dir:$params.fastq_screen.db_dir "}
-    }
 }

conf/slurmint.config

@@ -11,15 +11,18 @@ params {
     }
     index_dir = '/data/CCBR_Pipeliner/db/PipeDB/Indices'
 }
+
 process {
-    scratch = '/lscratch/$SLURM_JOBID'
+    //scratch = '/lscratch/$SLURM_JOBID'
 }
+
 singularity {
     enabled = true
     autoMounts = true
     cacheDir = "/data/CCBR_Pipeliner/SIFS"
     envWhitelist='https_proxy,http_proxy,ftp_proxy,DISPLAY,SLURM_JOBID,SINGULARITY_BINDPATH'
 }
+
 env {
     SINGULARITY_CACHEDIR = "/data/CCBR_Pipeliner/SIFS"
 }

main.nf

@@ -37,17 +37,19 @@ workflow {
     INPUT_CHECK.out.reads.set { raw_fastqs }
     raw_fastqs | TRIM_SE
     TRIM_SE.out.set{ trimmed_fastqs }
-    blacklist_files = Channel
-                        .fromPath(params.genomes[ params.genome ].blacklist_files)
-                        .collect()
+
+    Channel.fromPath(params.genomes[ params.genome ].blacklist_files, checkIfExists: true)
+        .collect()
+        .set{ blacklist_files }
     ALIGN_BLACKLIST(trimmed_fastqs, blacklist_files)
-    reference_files = Channel
-                        .fromPath(params.genomes[ params.genome ].reference_files)
-                        .collect()
+    Channel.fromPath(params.genomes[ params.genome ].reference_files, checkIfExists: true)
+        .collect()
+        .set{ reference_files }
     ALIGN_GENOME(ALIGN_BLACKLIST.out.reads, reference_files)
     ALIGN_GENOME.out.bam.set{ aligned_bam }
 
-    chrom_sizes = Channel.fromPath(params.genomes[ params.genome ].chrom_sizes)
+    Channel.fromPath(params.genomes[ params.genome ].chrom_sizes, checkIfExists: true)
+        .set{ chrom_sizes }
     aligned_bam.combine(chrom_sizes) | DEDUPLICATE
     DEDUPLICATE.out.bam.set{ deduped_bam }
     DEDUPLICATE.out.tag_align.set{ deduped_tagalign }
@@ -62,12 +64,11 @@ workflow {
            deduped_bam, DEDUPLICATE.out.flagstat,
            PHANTOM_PEAKS.out.spp, frag_lengths
            )
-    }
-
-    if (params.run.normalize_input) {
-        // Create channel: [ meta, [ ip_bam, control_bam ] [ ip_bai, control_bai ] ]
         QC.out.bigwigs.set{ ch_ip_ctrl_bigwig }
-        ch_ip_ctrl_bigwig | NORMALIZE_INPUT
+
+        if (params.run.normalize_input) {
+            ch_ip_ctrl_bigwig | NORMALIZE_INPUT
+        }
     }
 
     if (params.run.call_peaks) {

modules/local/qc.nf

@@ -35,7 +35,8 @@ process FASTQ_SCREEN {
     container = "${params.containers.fastq_screen}"
 
     input:
-        tuple val(meta), path(fastq), path(conf)
+        tuple val(meta), path(fastq), path(conf), path(db_dir)
+
     output:
         path("${meta.id}*_screen.*"), emit: screen
 

nextflow.config

@@ -37,7 +37,7 @@ params {
         normalize_using = "RPGC"
         excluded_chroms = "chrM chrX chrY"
     }
-    multiqc_config = "${baseDir}/assets/multiqc_config.yaml"
+    multiqc_config = "${baseDir}assets/multiqc_config.yaml"
     min_fragment_length = 200 // https://github.com/CCBR/Pipeliner/blob/86c6ccaa3d58381a0ffd696bbf9c047e4f991f9e/Rules/InitialChIPseqQC.snakefile#L539
     gem_read_dists = 'https://groups.csail.mit.edu/cgs/gem/download/Read_Distribution_default.txt'
 

pyproject.toml

@@ -61,7 +61,7 @@ Changelog = "https://github.com/CCBR/CHAMPAGNE/blob/main/docs/CHANGELOG.md"
 champagne = "."
 
 [tool.setuptools.package-data]
-"*" = ["CITATION.cff", "LICENSE", "VERSION", "main.nf", "nextflow.conf", "assets/", "bin/", "conf/", "modules/*/*", "subworkflows/*/*", "tests/"]
+"*" = ["CITATION.cff", "LICENSE", "VERSION", "main.nf", "nextflow.conf", "assets/", "bin/", "conf/", "modules/*/*", "subworkflows/*/*", "tests/*/*"]
 
 [tool.setuptools.dynamic]
 version = {file = "src/VERSION"}

subworkflows/local/peaks.nf

@@ -32,10 +32,11 @@ workflow CALL_PEAKS {
                 meta1, tag1, meta2, tag2 ->
                     meta1.control == meta2.id ? [ meta1, tag1, tag2 ]: null
             }
-            .combine(Channel.fromPath(params.gem_read_dists))
+            .combine(Channel.fromPath(params.gem_read_dists, checkIfExists: true))
             .combine(chrom_sizes)
             .set { ch_gem }
-            chrom_files = Channel.fromPath(params.genomes[ params.genome ].chromosomes_dir).collect()
+        Channel.fromPath("${params.genomes[ params.genome ].chromosomes_dir}", type: 'dir', checkIfExists: true)
+            .set{ chrom_files }
 
         ch_tagalign | MACS_BROAD
         ch_tagalign | MACS_NARROW

subworkflows/local/qc.nf

@@ -33,7 +33,10 @@ workflow QC {
     main:
         raw_fastqs.combine(Channel.value("raw")) | FASTQC_RAW
         trimmed_fastqs.combine(Channel.value("trimmed")) | FASTQC_TRIMMED
-        trimmed_fastqs.combine(Channel.fromPath(params.fastq_screen.conf)) | FASTQ_SCREEN
+        trimmed_fastqs
+          .combine(Channel.fromPath(params.fastq_screen.conf, checkIfExists: true))
+          .combine(Channel.fromPath(params.fastq_screen.db_dir,
+                                    type: 'dir', checkIfExists: true)) | FASTQ_SCREEN
 
         PRESEQ(aligned_bam)
         // when preseq fails, write NAs for the stats that are calculated from its log
@@ -77,8 +80,9 @@ workflow QC {
                     meta1.control == meta2.id ? [ meta1, [ bam1, bam2 ], [ bai1, bai2 ] ] : null
             }
             .set { ch_ip_ctrl_bam_bai }
-        PLOT_FINGERPRINT(ch_ip_ctrl_bam_bai)
-        BED_PROTEIN_CODING(Channel.fromPath(params.genomes[ params.genome ].gene_info))
+        ch_ip_ctrl_bam_bai | PLOT_FINGERPRINT
+        Channel.fromPath(params.genomes[ params.genome ].gene_info,
+                         checkIfExists: true) | BED_PROTEIN_CODING
         COMPUTE_MATRIX(bigwig_list,
                        BED_PROTEIN_CODING.out.bed.combine(Channel.from('metagene','TSS'))
         )
@@ -98,7 +102,7 @@ workflow QC {
             .set { ch_ip_ctrl_bigwig }
 
         MULTIQC(
-            Channel.fromPath(params.multiqc_config),
+            Channel.fromPath(params.multiqc_config, checkIfExists: true),
             FASTQC_RAW.out.zip.collect(),
             FASTQC_TRIMMED.out.zip.collect(),
             FASTQ_SCREEN.out.screen.collect(),