Sgalera

.gitignore

@@ -7,3 +7,11 @@ testing/
 testing*
 *.pyc
 .nf-test/
+null/
+smallest_run_SampleSheet.csv
+just_testing*
+NovaSeq_run_SampleSheet.csv
+dragen_demultiplex*.out
+tests/
+*.tar.gz
+info.txt

README.md

@@ -83,6 +83,24 @@ nextflow run nf-core/demultiplex \
 > Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_;
 > see [docs](https://nf-co.re/usage/configuration#custom-configuration-files).
 
+<<<<<<< HEAD
+> [!NOTE]
+> If you are about to run the pipeline inside the HPC (NASERTIC Urederra) cluster, we have created a [config file](conf/test_dragen.config) which runs the pipeline with  minimal possible requirements
+
+```bash
+nextflow run main.nf -profile test_dragen,singularity
+```
+
+## TODO
+```bash
+nextflow run main.nf \
+ --input /data/scratch/LAB/temp_demultiplex/nfcore_demultiplex/sgalera/nfcore_demultiplex/sample_sheet.csv \
+ --outdir /data/scratch/LAB/temp_demultiplex/to_delete_whatever \
+ -c hpc.conf
+```
+
+=======
+>>>>>>> c1ff1a76cd7c07f4006b89e1f4c35e7be8f3b81b
 For more details and further functionality, please refer to the [usage documentation](https://nf-co.re/demultiplex/usage) and the [parameter documentation](https://nf-co.re/demultiplex/parameters).
 
 ## Pipeline output

assets/sample_sheet.csv

@@ -0,0 +1,2 @@
+id,samplesheet,lane,flowcell
+smallest_run_iSeq,./assets/smallest_run_SampleSheet.csv,5,/data/medper/LAB/illumina_iSeq/Runs/20231109_FS10001385_17_BRL95625-0922/

assets/sample_sheet_NovaSeq.csv

@@ -0,0 +1,2 @@
+id,samplesheet,lane,flowcell
+smallest_run_NovaSeq,./assets/NovaSeq_run_SampleSheet.csv,5,/data/medper/LAB/illumina_NovaSeq/Runs/240306_A01215_0124_BHL3LFDRX3/

assets/sample_sheet_NovaSeq_split.csv

@@ -0,0 +1,3 @@
+id,samplesheet,lane,flowcell
+smallest_run_NovaSeq_1,./assets/NovaSeq_run_SampleSheet.csv,1,/data/medper/LAB/illumina_NovaSeq/Runs/240306_A01215_0124_BHL3LFDRX3/
+smallest_run_NovaSeq_2,./assets/NovaSeq_run_SampleSheet.csv,1,/data/medper/LAB/illumina_NovaSeq/Runs/240306_A01215_0124_BHL3LFDRX3/

assets/sample_sheet_split.csv

@@ -0,0 +1,3 @@
+id,samplesheet,lane,flowcell
+smallest_run_iSeq_1,./assets/smallest_run_SampleSheet.csv,1,/data/medper/LAB/illumina_iSeq/Runs/20231109_FS10001385_17_BRL95625-0922/
+smallest_run_iSeq_2,./assets/smallest_run_SampleSheet.csv,1,/data/medper/LAB/illumina_iSeq/Runs/20231109_FS10001385_17_BRL95625-0922/

conf/modules.config

@@ -2,6 +2,7 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     Config file for defining DSL2 per module options and publishing paths
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
     Available keys to override module options:
         ext.args   = Additional arguments appended to command in module.
         ext.args2  = Second set of arguments appended to command in module (multi-tool modules).
@@ -80,6 +81,43 @@ process {
         ]
     }
 
+    withName: DRAGEN_DEMULTIPLEXER {
+        ext.args = {
+            def argsList = []
+            if (meta.lane) {
+                def laneInt = meta.lane.toInteger()
+                if (laneInt >= 5) {
+                    argsList.add("--no-lane-splitting=true")
+                } else {
+                    argsList.add("--bcl-only-lane=$meta.lane")
+                }
+            }
+            return argsList.join(" ").trim()
+        }
+
+        publishDir = [
+            [
+                // Gather and write InterOp files
+                path: { meta.lane.toInteger() >= 5 ? "${params.outdir}/InterOp/" : "${params.outdir}/${meta.id}/InterOp" },
+                mode: params.publish_dir_mode,
+                pattern: "**.bin",
+                saveAs: {filename -> filename.split("/")[-1] }
+            ],
+            [
+                // Gather and write Reports and Stats
+                path: { meta.lane.toInteger() >= 5 ? "${params.outdir}" : "${params.outdir}/${meta.id}/" },
+                mode: params.publish_dir_mode,
+                pattern: "{Reports,Stats}",
+                saveAs: {filename -> filename.split("/")[-1] }
+            ],
+            [
+                path: { meta.lane.toInteger() >= 5 || meta.lane.toInteger() == 0 ? "${params.outdir}/" : "${params.outdir}/${meta.id}" },
+                mode: params.publish_dir_mode,
+                pattern: "**.fastq.gz",
+            ]
+        ]
+    }
+
     withName: BASES2FASTQ {
         ext.args = [ "--strict",
                     params.trim_fastq ? "--settings 'R1AdapterTrim,true' --settings 'R2AdapterTrim,true'" : "",
@@ -107,16 +145,16 @@ process {
     }
 
     withName: FASTP {
+
         publishDir = [
             [
-                path: { "${params.outdir}/${meta.fcid}" },
+                path: { meta.lane.toInteger() >= 5 ? "${params.outdir}/fastp" : "${params.outdir}/${meta.fcid}/fastp" },
                 mode: params.publish_dir_mode,
                 saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
                 pattern: "*.{html,json}"
             ],
             [
-                // if ! params.trim_fastq, only output reports but not trimmed fastq files
-                path: { "${params.outdir}/${meta.fcid}" },
+                path: { meta.lane.toInteger() >= 5 ? "${params.outdir}/fastp" : "${params.outdir}/${meta.fcid}/fastp" },
                 mode: params.publish_dir_mode,
                 saveAs: { filename -> params.trim_fastq ? filename : null },
                 pattern: "*.fastq.gz"
@@ -126,15 +164,31 @@ process {
 
     withName: FALCO {
         publishDir = [
-            path: { "${params.outdir}/${meta.fcid}" },
+            path: { meta.lane.toInteger() >= 5 ? "${params.outdir}/falco" : "${params.outdir}/${meta.fcid}/falco" },
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
+    withName: KRAKEN2 {
+        publishDir = [
+            path: { meta.lane.toInteger() >= 5 ? "${params.outdir}/kraken2" : "${params.outdir}/${meta.fcid}/kraken2" },
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.equals('versions.yml') ? null: filename }
+        ]
+    }
+
+    withName: FASTQ_SCREEN {
+        publishDir = [
+            path: { meta.lane.toInteger() >= 5 ? "${params.outdir}/fastq_screen" : "${params.outdir}/${meta.fcid}/fastq_screen" },
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+        ]
+    }
+
 
     withName: MD5SUM {
         publishDir = [
-            path: { "${params.outdir}/${meta.fcid}" },
+            path: { meta.lane.toInteger() >= 5 ? "${params.outdir}/md5sum" : "${params.outdir}/${meta.fcid}/md5sum" },
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]

conf/test_dragen.config

@@ -0,0 +1,48 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Nextflow config file for running minimal tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Defines input files and everything required to run a fast and simple pipeline test.
+
+    Use as follows:
+        nextflow run nf-core/demultiplex -profile test_dragen,<docker/singularity> --outdir <OUTDIR>
+
+----------------------------------------------------------------------------------------
+*/
+
+includeConfig '../hpc.conf'
+
+params {
+    config_profile_name        = 'Test dragen demultiplex profile'
+    config_profile_description = 'Minimal test dataset to check pipeline function with dragen'
+
+    // Limit resources so that this can run on GitHub Actions
+    max_cpus                    =   2
+    max_memory                  =   '6.GB'
+    max_time                    =   '6.h'
+
+    // TODO remove this flag when pipeline is correct
+    NovaSeq                     =   false
+    split                       =   false
+
+    // Common parameters
+    demultiplexer               =   'dragen'
+    sample_size                 =   1000
+    kraken_db                   =   '/data/scratch/LAB/references/dbkraken2/k2_standard_20240112'
+    fastq_screen_subset         =   1000
+
+    // Input data
+    if (params.NovaSeq && params.split){
+        input                   =   './assets/sample_sheet_NovaSeq_split.csv'
+        outdir                  =   '/data/scratch/LAB/temp_demultiplex/to_delete/to_delete_NovaSeq_split'
+    } else if (params.NovaSeq){
+        input                   =   './assets/sample_sheet_NovaSeq.csv'
+        outdir                  =   '/data/scratch/LAB/temp_demultiplex/to_delete/to_delete_NovaSeq'
+    } else if (params.split) {
+        input                   =   './assets/sample_sheet_split.csv'
+        outdir                  =   '/data/scratch/LAB/temp_demultiplex/to_delete/to_delete_split'
+    } else {
+        input                   =   './assets/sample_sheet.csv'
+        outdir                  =   '/data/scratch/LAB/temp_demultiplex/to_delete/to_delete_no_split'
+    }
+}

hpc.conf

@@ -0,0 +1,50 @@
+//Profile config names for nf-core/configs
+params {
+    config_profile_description = 'Nasertic HPC cluster profile'
+    config_profile_contact = 'NASERTIC (@sgaleraalq @mansego @Faitero @Ibai13)'
+    config_profile_url = 'https://hpc.nasertic.es/ngs/secuenciacion'
+}
+
+singularity {
+    enabled = true
+    autoMounts = true
+    cacheDir = '/home/projects/LAB/nf-core-workflows/nf-core/singularity_cache'
+    runOptions = '-B /home/projects/LAB/nf-core-workflows'
+    runOptions = '-B /data'
+}
+
+process {
+    executor = 'slurm'
+    withLabel:process_high {
+        memory = 32.GB
+        cpus = 12
+        time = 24.h
+    }
+    withLabel: process_medium {
+        memory = 10.GB
+        cpus = 2
+        time = 7.h
+    }
+    withLabel: process_low {
+        memory = 5.GB
+        cpus = 1
+        time = 5.h
+    }
+    withLabel: dragen {
+        memory = 240.GB
+        cpus = 48
+        time = 3.h
+    }
+    withLabel: kraken2 {
+        memory = 115.GB
+        cpus = 12
+        time = 3.h
+    }
+    withLabel:local { executor = "local" }
+}
+
+params {
+    max_memory = 110.GB
+    max_cpus = 20
+    max_time = 167.h
+}

modules.json

@@ -7,57 +7,93 @@
                 "nf-core": {
                     "bases2fastq": {
                         "branch": "master",
-                        "git_sha": "575e1bc54b083fb15e7dd8b5fcc40bea60e8ce83",
+                        "git_sha": "c2342b45bbfc5d6fe19cc4154b38fa57177c67dd",
                         "installed_by": ["modules"]
                     },
                     "bcl2fastq": {
                         "branch": "master",
                         "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5",
-                        "installed_by": ["bcl_demultiplex"]
+                        "installed_by": [
+                            "bcl_demultiplex"
+                        ]
                     },
                     "bclconvert": {
                         "branch": "master",
                         "git_sha": "e1e42c1312400df268a0c92e14bd33552e782448",
-                        "installed_by": ["bcl_demultiplex"]
+                        "installed_by": [
+                            "bcl_demultiplex"
+                        ]
                     },
                     "custom/dumpsoftwareversions": {
                         "branch": "master",
                         "git_sha": "de45447d060b8c8b98575bc637a4a575fd0638e1",
-                        "installed_by": ["modules"]
+                        "installed_by": [
+                            "modules"
+                        ]
                     },
                     "falco": {
                         "branch": "master",
                         "git_sha": "a236d2c2ec9e68db9b8501105d804acb359552b5",
-                        "installed_by": ["modules"]
+                        "installed_by": [
+                            "modules"
+                        ]
                     },
                     "fastp": {
                         "branch": "master",
-                        "git_sha": "003920c7f9a8ae19b69a97171922880220bedf56",
+                        "git_sha": "95cf5fe0194c7bf5cb0e3027a2eb7e7c89385080",
                         "installed_by": ["modules"]
                     },
                     "fqtk": {
                         "branch": "master",
                         "git_sha": "97a796133cd9644ba3dd83d3c0fd09a9a63b1a84",
-                        "installed_by": ["modules"]
+                        "installed_by": [
+                            "modules"
+                        ]
+                    },
+                    "kraken2/kraken2": {
+                        "branch": "master",
+                        "git_sha": "8fae4ee738f645812384d6aba9d0dc651ad79ff9",
+                        "installed_by": [
+                            "modules"
+                        ]
+                    },
+                    "kraken2/kraken2": {
+                        "branch": "master",
+                        "git_sha": "653218e79ffa76fde20319e9062f8b8da5cf7555",
+                        "installed_by": ["fastq_contam_seqtk_kraken"]
+                    },
+                    "kraken2/kraken2": {
+                        "branch": "master",
+                        "git_sha": "653218e79ffa76fde20319e9062f8b8da5cf7555",
+                        "installed_by": ["fastq_contam_seqtk_kraken"]
                     },
                     "md5sum": {
                         "branch": "master",
                         "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5",
-                        "installed_by": ["modules"]
+                        "installed_by": [
+                            "modules"
+                        ]
                     },
                     "multiqc": {
                         "branch": "master",
-                        "git_sha": "9e71d8519dfbfc328c078bba14d4bd4c99e39a94",
+                        "git_sha": "b7ebe95761cd389603f9cc0e0dc384c0f663815a",
                         "installed_by": ["modules"]
                     },
+                    "seqtk/sample": {
+                        "branch": "master",
+                        "git_sha": "7f88aae93c69586c0789322b77743ee0ef469502",
+                        "installed_by": ["fastq_contam_seqtk_kraken"]
+                    },
                     "sgdemux": {
                         "branch": "master",
                         "git_sha": "574074e6ea2497b53df1f226473a8c71524af0d0",
-                        "installed_by": ["modules"]
+                        "installed_by": [
+                            "modules"
+                        ]
                     },
                     "untar": {
                         "branch": "master",
-                        "git_sha": "e719354ba77df0a1bd310836aa2039b45c29d620",
+                        "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa",
                         "installed_by": ["modules"]
                     }
                 }
@@ -68,6 +104,11 @@
                         "branch": "master",
                         "git_sha": "cfd937a668919d948f6fcbf4218e79de50c2f36f",
                         "installed_by": ["subworkflows"]
+                    },
+                    "fastq_contam_seqtk_kraken": {
+                        "branch": "master",
+                        "git_sha": "dedc0e31087f3306101c38835d051bf49789445a",
+                        "installed_by": ["subworkflows"]
                     }
                 }
             }