diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7035fe9c..4b3c26a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,10 @@ on: env: NXF_ANSI_LOG: false - NFTEST_VER: "0.8.1" + NFT_VER: "0.9.0" + NFT_WORKDIR: "~" + NFT_DIFF: "pdiff" + NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2" concurrency: group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" @@ -42,29 +45,30 @@ jobs: - name: Check out pipeline code uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - - name: Install Nextflow - uses: nf-core/setup-nextflow@v2 + - uses: actions/setup-python@v4 with: - version: "${{ matrix.NXF_VER }}" + python-version: "3.11" + architecture: "x64" + + - name: Install pdiff to see diff between nf-test snapshots + run: | + python -m pip install --upgrade pip + pip install pdiff - - name: Cache nf-test installation - id: cache-software - uses: actions/cache@v3 + - uses: nf-core/setup-nextflow@v2 with: - path: | - /usr/local/bin/nf-test - /home/runner/.nf-test/nf-test.jar - key: ${{ runner.os }}-${{ env.NFTEST_VER }}-nftest + version: "${{ matrix.NXF_VER }}" - - name: Install nf-test - if: steps.cache-software.outputs.cache-hit != 'true' - run: | - wget -qO- https://code.askimed.com/install/nf-test | bash - sudo mv nf-test /usr/local/bin/ + - uses: nf-core/setup-nf-test@v1 + with: + version: ${{ env.NFT_VER }} - name: Run nf-test run: | - nf-test test tests/main_pipeline_${{ matrix.profile }}.test --junitxml=test.xml + nf-test test \ + --ci \ + --junitxml=test.xml \ + tests/main_pipeline_${{ matrix.profile }}.nf.test - name: Output log on failure if: failure() diff --git a/CHANGELOG.md b/CHANGELOG.md index cd2a2235..fd1b6e41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## v2.7.1 - 2024-08-13 + +- Fix that tests have not been executed with nf-test v0.9 ([#359](https://github.com/nf-core/scrnaseq/pull/359)) +- Add support for 10XV4 chemistry ([#348](https://github.com/nf-core/scrnaseq/pull/348)) +- Fix issues with predefined STAR index ([#350](https://github.com/nf-core/scrnaseq/pull/350)) +- Update modules ([#351](https://github.com/nf-core/scrnaseq/pull/351)) +- Fix resource specifications for `cellranger mkref`/`cellrangerarc mkref` ([#352](https://github.com/nf-core/scrnaseq/pull/352)) + ## v2.7.0 - 2024-06-03 - Apply `check_max` to AlevinQC time limit ([#335](https://github.com/nf-core/scrnaseq/pull/335)) diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml index 1e86ad34..d5fcc74a 100644 --- a/assets/multiqc_config.yml +++ b/assets/multiqc_config.yml @@ -1,7 +1,7 @@ report_comment: > - This report has been generated by the nf-core/scrnaseq + This report has been generated by the nf-core/scrnaseq analysis pipeline. For information about how to interpret these results, please see the - documentation. + documentation. report_section_order: "nf-core-scrnaseq-methods-description": order: -1000 diff --git a/assets/protocols.json b/assets/protocols.json index d8da2f8a..0552f8d5 100644 --- a/assets/protocols.json +++ b/assets/protocols.json @@ -12,6 +12,10 @@ "protocol": "10xv3", "whitelist": "assets/whitelist/10x_V3_barcode_whitelist.txt.gz" }, + "10XV4": { + "protocol": "10xv4", + "whitelist": "assets/whitelist/10x_V4_barcode_whitelist.txt.gz" + }, "dropseq": { "protocol": "dropseq" } @@ -28,6 +32,9 @@ }, "10XV3": { "protocol": "SC3Pv3" + }, + "10XV4": { + "protocol": "SC3Pv4" } }, "cellrangerarc": { @@ -51,6 +58,11 @@ "extra_args": "--soloUMIlen 12", "whitelist": "assets/whitelist/10x_V3_barcode_whitelist.txt.gz" }, + "10XV4": { + "protocol": "CB_UMI_Simple", + "extra_args": "--soloUMIlen 12", + "whitelist": "assets/whitelist/10x_V4_barcode_whitelist.txt.gz" + }, "dropseq": { "protocol": "CB_UMI_Simple" }, @@ -68,6 +80,9 @@ "10XV3": { "protocol": "10XV3" }, + "10XV4": { + "protocol": "10XV4" + }, "dropseq": { "protocol": "DROPSEQ" }, @@ -88,6 +103,9 @@ "10XV3": { "protocol": "10x-v3" }, + "10XV4": { + "protocol": "10x-v4" + }, "dropseq": { "protocol": "dropseq" } diff --git a/assets/whitelist/10x_V4_barcode_whitelist.txt.gz b/assets/whitelist/10x_V4_barcode_whitelist.txt.gz new file mode 100644 index 00000000..5571b835 Binary files /dev/null and b/assets/whitelist/10x_V4_barcode_whitelist.txt.gz differ diff --git a/docs/usage.md b/docs/usage.md index 36120b36..499e404d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -85,7 +85,7 @@ The single-cell protocol used in the experiment can be specified using the `--pr For cellranger, it is recommended to stick with the default value `'auto'` for automatic detection of the protocol. For all other aligner, you need to specify the protocol manually. -The three 10x Genomics protocols 3' v1 (`10XV1`), 3' v2 (`10XV2`) and 3' v3 (`10XV3`) are universally supported +The three 10x Genomics protocols 3' v1 (`10XV1`), 3' v2 (`10XV2`), 3' v3 (`10XV3`), and 3' v4 (`10XV4`) are universally supported by all aligners in the pipeline and mapped to the correct options automatically. If the protocol is unknown to the nf-core pipeline, the value specified to `--protocol` is passed to the aligner _in verbatim_ to support additional protocols. diff --git a/modules.json b/modules.json index dd4c8feb..aa186d98 100644 --- a/modules.json +++ b/modules.json @@ -7,17 +7,17 @@ "nf-core": { "cellranger/count": { "branch": "master", - "git_sha": "e66183d2ab6a5c2f3fd66b2bee942287cf65536c", + "git_sha": "90dad5491658049282ceb287a3d7732c1ce39837", "installed_by": ["modules"] }, "cellranger/mkgtf": { "branch": "master", - "git_sha": "e66183d2ab6a5c2f3fd66b2bee942287cf65536c", + "git_sha": "90dad5491658049282ceb287a3d7732c1ce39837", "installed_by": ["modules"] }, "cellranger/mkref": { "branch": "master", - "git_sha": "e66183d2ab6a5c2f3fd66b2bee942287cf65536c", + "git_sha": "3549a361ce3401b6afd238a266389d78392a53f6", "installed_by": ["modules"] }, "cellranger/mkvdjref": { @@ -27,7 +27,7 @@ }, "cellranger/multi": { "branch": "master", - "git_sha": "5f12fc2128f419a8750c5b0620e4b54d7aa33fec", + "git_sha": "90dad5491658049282ceb287a3d7732c1ce39837", "installed_by": ["modules"] }, "cellrangerarc/count": { @@ -42,12 +42,12 @@ }, "cellrangerarc/mkref": { "branch": "master", - "git_sha": "4196b1b2e7ce265892f3979eabf7a9ddc030702f", + "git_sha": "3549a361ce3401b6afd238a266389d78392a53f6", "installed_by": ["modules"] }, "fastqc": { "branch": "master", - "git_sha": "285a50500f9e02578d90b3ce6382ea3c30216acd", + "git_sha": "46eca555142d6e597729fcb682adcc791796f514", "installed_by": ["modules"] }, "gffread": { @@ -57,7 +57,7 @@ }, "gunzip": { "branch": "master", - "git_sha": "3a5fef109d113b4997c9822198664ca5f2716208", + "git_sha": "4e5f4687318f24ba944a13609d3ea6ebd890737d", "installed_by": ["modules"] }, "kallistobustools/count": { @@ -72,12 +72,12 @@ }, "multiqc": { "branch": "master", - "git_sha": "b7ebe95761cd389603f9cc0e0dc384c0f663815a", + "git_sha": "b80f5fd12ff7c43938f424dd76392a2704fa2396", "installed_by": ["modules"] }, "star/genomegenerate": { "branch": "master", - "git_sha": "a21faa6a3481af92a343a10926f59c189a2c16c9", + "git_sha": "46eca555142d6e597729fcb682adcc791796f514", "installed_by": ["modules"] }, "universc": { @@ -87,7 +87,7 @@ }, "unzip": { "branch": "master", - "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", + "git_sha": "b0c3ff2485534c09d9debbf20125d9c6b72ce118", "installed_by": ["modules"] } } diff --git a/modules/nf-core/cellranger/count/templates/cellranger_count.py b/modules/nf-core/cellranger/count/templates/cellranger_count.py index 53360f23..7cd68e27 100644 --- a/modules/nf-core/cellranger/count/templates/cellranger_count.py +++ b/modules/nf-core/cellranger/count/templates/cellranger_count.py @@ -4,6 +4,7 @@ Copyright (c) Gregor Sturm 2023 - MIT License """ + from subprocess import run from pathlib import Path from textwrap import dedent @@ -34,11 +35,11 @@ def chunk_iter(seq, size): # Match R1 in the filename, but only if it is followed by a non-digit or non-character # match "file_R1.fastq.gz", "file.R1_000.fastq.gz", etc. but # do not match "SRR12345", "file_INFIXR12", etc -filename_pattern = r'([^a-zA-Z0-9])R1([^a-zA-Z0-9])' +filename_pattern = r"([^a-zA-Z0-9])R1([^a-zA-Z0-9])" for i, (r1, r2) in enumerate(chunk_iter(fastqs, 2), start=1): # double escapes are required because nextflow processes this python 'template' - if re.sub(filename_pattern, r'\\1R2\\2', r1.name) != r2.name: + if re.sub(filename_pattern, r"\\1R2\\2", r1.name) != r2.name: raise AssertionError( dedent( f"""\ @@ -55,8 +56,8 @@ def chunk_iter(seq, size): r1.rename(fastq_all / f"{sample_id}_S1_L{i:03d}_R1_001.fastq.gz") r2.rename(fastq_all / f"{sample_id}_S1_L{i:03d}_R2_001.fastq.gz") +# fmt: off run( - # fmt: off [ "cellranger", "count", "--id", "${prefix}", @@ -64,11 +65,11 @@ def chunk_iter(seq, size): "--transcriptome", "${reference.name}", "--localcores", "${task.cpus}", "--localmem", "${task.memory.toGiga()}", - *shlex.split("""${args}""") + *shlex.split("""${args}"""), ], - # fmt: on check=True, ) +# fmt: on # Output version information version = run( diff --git a/modules/nf-core/cellranger/count/tests/main.nf.test b/modules/nf-core/cellranger/count/tests/main.nf.test index dc8a58a5..3029c0ca 100644 --- a/modules/nf-core/cellranger/count/tests/main.nf.test +++ b/modules/nf-core/cellranger/count/tests/main.nf.test @@ -19,7 +19,7 @@ nextflow_process { script "../../mkgtf/main.nf" process { """ - input[0] = file(params.test_data['homo_sapiens']['genome']['genome_gtf'], checkIfExists: true) + input[0] = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.gtf', checkIfExists: true) """ } } @@ -27,7 +27,7 @@ nextflow_process { script "../../mkref/main.nf" process { """ - input[0] = file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true) + input[0] = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) input[1] = CELLRANGER_MKGTF.out.gtf input[2] = "homo_sapiens_reference" """ @@ -43,8 +43,8 @@ nextflow_process { input[0] = [ [ id:'test_10x', single_end:false, strandedness:'auto' ], // meta map [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_gex1_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_gex1_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/gex_1/subsampled_5k_human_antiCMV_T_TBNK_connect_GEX_1_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/gex_1/subsampled_5k_human_antiCMV_T_TBNK_connect_GEX_1_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] ] input[1] = CELLRANGER_MKREF.out.reference @@ -81,8 +81,8 @@ nextflow_process { input[0] = [ [ id:'test_10x', single_end:false, strandedness:'auto' ], // meta map [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_gex1_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_gex1_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/gex_1/subsampled_5k_human_antiCMV_T_TBNK_connect_GEX_1_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/gex_1/subsampled_5k_human_antiCMV_T_TBNK_connect_GEX_1_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] ] input[1] = CELLRANGER_MKREF.out.reference diff --git a/modules/nf-core/cellranger/mkgtf/tests/main.nf.test b/modules/nf-core/cellranger/mkgtf/tests/main.nf.test index bba46b45..8ede4df1 100644 --- a/modules/nf-core/cellranger/mkgtf/tests/main.nf.test +++ b/modules/nf-core/cellranger/mkgtf/tests/main.nf.test @@ -15,7 +15,7 @@ nextflow_process { when { process { """ - input[0] = file(params.test_data['homo_sapiens']['genome']['genome_gtf'], checkIfExists: true) + input[0] = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.gtf', checkIfExists: true) """ } } @@ -36,7 +36,7 @@ nextflow_process { when { process { """ - input[0] = file(params.test_data['homo_sapiens']['genome']['genome_gtf'], checkIfExists: true) + input[0] = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.gtf', checkIfExists: true) """ } } diff --git a/modules/nf-core/cellranger/mkref/main.nf b/modules/nf-core/cellranger/mkref/main.nf index 4325c308..a719b778 100644 --- a/modules/nf-core/cellranger/mkref/main.nf +++ b/modules/nf-core/cellranger/mkref/main.nf @@ -22,6 +22,9 @@ process CELLRANGER_MKREF { error "CELLRANGER_MKREF module does not support Conda. Please use Docker / Singularity / Podman instead." } def args = task.ext.args ?: '' + // --localcores is passed to the martian runtime and specifies the number of allocated jobs + // --nthreads is passed to the STAR index generation. + // see also https://github.com/nf-core/scrnaseq/issues/329 """ cellranger \\ mkref \\ @@ -30,6 +33,7 @@ process CELLRANGER_MKREF { --genes=$gtf \\ --localcores=${task.cpus} \\ --localmem=${task.memory.toGiga()} \\ + --nthreads=${task.cpus} \\ $args cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/cellranger/mkref/tests/main.nf.test b/modules/nf-core/cellranger/mkref/tests/main.nf.test index 5282f3f6..dfbcd222 100644 --- a/modules/nf-core/cellranger/mkref/tests/main.nf.test +++ b/modules/nf-core/cellranger/mkref/tests/main.nf.test @@ -16,8 +16,8 @@ nextflow_process { when { process { """ - input[0] = file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true) - input[1] = file(params.test_data['homo_sapiens']['genome']['genome_gtf'], checkIfExists: true) + input[0] = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + input[1] = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.gtf', checkIfExists: true) input[2] = "homo_sapiens_reference" """ } @@ -36,9 +36,9 @@ nextflow_process { path( "${outputDir}/homo_sapiens_reference/star/geneInfo.tab" ), path( "${outputDir}/homo_sapiens_reference/star/exonGeTrInfo.tab" ), path( "${outputDir}/homo_sapiens_reference/star/exonInfo.tab" ), - path( "${outputDir}/homo_sapiens_reference/star/sjdbList.out.tab" ) - ).match(), - { assert snapshot(process.out.versions).match("versions") } + path( "${outputDir}/homo_sapiens_reference/star/sjdbList.out.tab" ), + process.out.versions + ).match() } ) } @@ -62,8 +62,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.reference).match() }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out).match() } ) } diff --git a/modules/nf-core/cellranger/mkref/tests/main.nf.test.snap b/modules/nf-core/cellranger/mkref/tests/main.nf.test.snap index 046fefbd..2db410f5 100644 --- a/modules/nf-core/cellranger/mkref/tests/main.nf.test.snap +++ b/modules/nf-core/cellranger/mkref/tests/main.nf.test.snap @@ -15,38 +15,42 @@ "geneInfo.tab:md5,8b608537307443ffaee4927d2b428805", "exonGeTrInfo.tab:md5,72d4dd88d25e3c5b0bb72b12b4ac99a8", "exonInfo.tab:md5,0d560290fab688b7268d88d5494bf9fe", - "sjdbList.out.tab:md5,9e4f991abbbfeb3935a2bb21b9e258f1" - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-03-25T09:41:24.407740353" - }, - "homo_sapiens - reference - stub": { - "content": [ + "sjdbList.out.tab:md5,9e4f991abbbfeb3935a2bb21b9e258f1", [ - [ - "empty_file:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + "versions.yml:md5,e9d88519c551d8ccbbdc6bd2673fc41f" ] ], "meta": { "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nextflow": "24.04.3" }, - "timestamp": "2024-03-25T09:41:46.135604587" + "timestamp": "2024-07-31T14:50:38.928938" }, - "versions": { + "homo_sapiens - reference - stub": { "content": [ - [ - "versions.yml:md5,e9d88519c551d8ccbbdc6bd2673fc41f" - ] + { + "0": [ + [ + "empty_file:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + "versions.yml:md5,e9d88519c551d8ccbbdc6bd2673fc41f" + ], + "reference": [ + [ + "empty_file:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,e9d88519c551d8ccbbdc6bd2673fc41f" + ] + } ], "meta": { "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nextflow": "24.04.3" }, - "timestamp": "2024-03-25T09:41:46.149227444" + "timestamp": "2024-07-31T14:50:44.599257" } -} \ No newline at end of file +} diff --git a/modules/nf-core/cellranger/multi/tests/main.nf.test b/modules/nf-core/cellranger/multi/tests/main.nf.test index 072e3cc4..0259e1b3 100644 --- a/modules/nf-core/cellranger/multi/tests/main.nf.test +++ b/modules/nf-core/cellranger/multi/tests/main.nf.test @@ -19,7 +19,7 @@ nextflow_process { script "modules/nf-core/cellranger/mkgtf/main.nf" process { """ - input[0] = file(params.test_data['homo_sapiens']['genome']['genome_gtf'], checkIfExists: true) + input[0] = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.gtf', checkIfExists: true) """ } } @@ -28,7 +28,7 @@ nextflow_process { script "modules/nf-core/cellranger/mkref/main.nf" process { """ - input[0] = file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true) + input[0] = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) input[1] = CELLRANGER_MKGTF.out.gtf input[2] = 'homo_sapiens_chr22_reference' """ @@ -52,27 +52,27 @@ nextflow_process { // stage B cell FASTQ test data bcell_fastqs_10k_pbmc = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_b_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_b_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/bcell/subsampled_sc5p_v2_hs_PBMC_10k_b_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/bcell/subsampled_sc5p_v2_hs_PBMC_10k_b_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] def bcell_fastq_samplename_10k_pbmc = "subsampled_sc5p_v2_hs_PBMC_10k" // stage 5' gene expression FASTQ test data fivepgex_fastqs_10k_pbmc = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_5gex_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_5gex_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5gex/subsampled_sc5p_v2_hs_PBMC_10k_5gex_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5gex/subsampled_sc5p_v2_hs_PBMC_10k_5gex_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] def fivepgex_fastq_samplename_10k_pbmc = "subsampled_sc5p_v2_hs_PBMC_10k" // stage 5' feature barcode (antibody capture) FASTQ test data ab_fastqs_10k_pbmc = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_5fb_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_5fb_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5fb/subsampled_sc5p_v2_hs_PBMC_10k_5fb_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5fb/subsampled_sc5p_v2_hs_PBMC_10k_5fb_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] def fivepab_fastq_samplename_10k_pbmc = "subsampled_sc5p_v2_hs_PBMC_10k" // stage feature barcode reference for antibody capture - fb_reference_10k_pbmc = file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_feature_ref_csv'], checkIfExists: true) + fb_reference_10k_pbmc = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/sc5p_v2_hs_PBMC_10k_multi_5gex_5fb_b_t_feature_ref.csv', checkIfExists: true) /*******************************/ /*** end stage 10k PBMC data ***/ @@ -87,20 +87,20 @@ nextflow_process { // stage 3' CMO FASTQ test data cmo_fastqs_10k_pbmc_cmo = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_cmo_cmo_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_cmo_cmo_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/cmo/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_multiplexing_capture_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/cmo/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_multiplexing_capture_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] def cmo_fastq_samplename_10k_pbmc_cmo = "subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K" // stage 3' gene expression FASTQ test data threepgex_fastqs_10k_pbmc_cmo = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_cmo_gex1_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_cmo_gex1_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/gex_1/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_gex_S2_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/gex_1/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_gex_S2_L001_R2_001.fastq.gz', checkIfExists: true) ] def threepgex_fastq_samplename_10k_pbmc_cmo = "subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K" // stage feature barcode reference for antibody capture - cmo_reference_10k_pbmc_cmo = file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_cmo_feature_ref_csv'], checkIfExists: true) + cmo_reference_10k_pbmc_cmo = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/10k_pbmc_cmo_count_feature_reference.csv', checkIfExists: true) // CMO needs a barcode file cmo_barcodes_csv = file("cmo_barcodes.csv") @@ -119,27 +119,27 @@ nextflow_process { // stage antibody capture data ab_fastqs_5k_cmvpos_tcells = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_ab_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_ab_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/ab/subsampled_5k_human_antiCMV_T_TBNK_connect_AB_S2_L004_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/ab/subsampled_5k_human_antiCMV_T_TBNK_connect_AB_S2_L004_R2_001.fastq.gz', checkIfExists: true) ] def ab_fastq_samplename_5k_cmvpos_tcells = "subsampled_5k_human_antiCMV_T_TBNK_connect" // stage GEX data gex_fastqs_5k_cmvpos_tcells = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_gex1_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_gex1_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/gex_1/subsampled_5k_human_antiCMV_T_TBNK_connect_GEX_1_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/gex_1/subsampled_5k_human_antiCMV_T_TBNK_connect_GEX_1_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] def gex_fastq_samplename_5k_cmvpos_tcells = "subsampled_5k_human_antiCMV_T_TBNK_connect" // stage VDJ data vdj_fastqs_5k_cmvpos_tcells = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_vdj_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_vdj_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/vdj/subsampled_5k_human_antiCMV_T_TBNK_connect_VDJ_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/vdj/subsampled_5k_human_antiCMV_T_TBNK_connect_VDJ_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] def vdj_fastq_samplename_5k_cmvpos_tcells = "subsampled_5k_human_antiCMV_T_TBNK_connect" // stage feature barcode reference for antibody capture - fb_reference_5k_cmvpos_tcells = file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_feature_ref_csv'], checkIfExists: true) + fb_reference_5k_cmvpos_tcells = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/5k_human_antiCMV_T_TBNK_connect_Multiplex_count_feature_reference.csv', checkIfExists: true) /*************************************/ /*** end stage 5k CMV+ T-cell data ***/ @@ -150,9 +150,9 @@ nextflow_process { /*** stage VDJ reference ***/ /***************************/ - vdj_json = file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_vdj_ref_json'] , checkIfExists: true) - vdj_fasta = file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_vdj_ref_fasta'] , checkIfExists: true) - vdj_suppfasta = file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_vdj_ref_suppfasta'], checkIfExists: true) + vdj_json = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/references/vdj/refdata-cellranger-vdj-GRCh38-alts-ensembl-5.0.0/reference.json', checkIfExists: true) + vdj_fasta = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/references/vdj/refdata-cellranger-vdj-GRCh38-alts-ensembl-5.0.0/fasta/regions.fa', checkIfExists: true) + vdj_suppfasta = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/references/vdj/refdata-cellranger-vdj-GRCh38-alts-ensembl-5.0.0/fasta/supp_regions.fa', checkIfExists: true) // awkwardly restage VDJ ref to enforce directory structure expected by cellranger vdj_reference = file( "${workDir}/vdj_reference" ) @@ -262,27 +262,27 @@ nextflow_process { // stage B cell FASTQ test data bcell_fastqs_10k_pbmc = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_b_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_b_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/bcell/subsampled_sc5p_v2_hs_PBMC_10k_b_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/bcell/subsampled_sc5p_v2_hs_PBMC_10k_b_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] def bcell_fastq_samplename_10k_pbmc = "subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K" // stage 5' gene expression FASTQ test data fivepgex_fastqs_10k_pbmc = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_5gex_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_5gex_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5gex/subsampled_sc5p_v2_hs_PBMC_10k_5gex_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5gex/subsampled_sc5p_v2_hs_PBMC_10k_5gex_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] def fivepgex_fastq_samplename_10k_pbmc = "subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K" // stage 5' feature barcode (antibody capture) FASTQ test data ab_fastqs_10k_pbmc = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_5fb_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_5fb_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5fb/subsampled_sc5p_v2_hs_PBMC_10k_5fb_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5fb/subsampled_sc5p_v2_hs_PBMC_10k_5fb_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] def fivepab_fastq_samplename_10k_pbmc = "subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K" // stage feature barcode reference for antibody capture - fb_reference_10k_pbmc = file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_feature_ref_csv'], checkIfExists: true) + fb_reference_10k_pbmc = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/sc5p_v2_hs_PBMC_10k_multi_5gex_5fb_b_t_feature_ref.csv', checkIfExists: true) /*******************************/ /*** end stage 10k PBMC data ***/ @@ -297,20 +297,20 @@ nextflow_process { // stage 3' CMO FASTQ test data cmo_fastqs_10k_pbmc_cmo = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_cmo_cmo_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_cmo_cmo_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/cmo/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_multiplexing_capture_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/cmo/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_multiplexing_capture_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] def cmo_fastq_samplename_10k_pbmc_cmo = "subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K" // stage 3' gene expression FASTQ test data threepgex_fastqs_10k_pbmc_cmo = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_cmo_gex1_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_cmo_gex1_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/gex_1/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_gex_S2_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/gex_1/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_gex_S2_L001_R2_001.fastq.gz', checkIfExists: true) ] def threepgex_fastq_samplename_10k_pbmc_cmo = "subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K" // stage feature barcode reference for antibody capture - cmo_reference_10k_pbmc_cmo = file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_cmo_feature_ref_csv'], checkIfExists: true) + cmo_reference_10k_pbmc_cmo = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/10k_pbmc_cmo_count_feature_reference.csv', checkIfExists: true) // CMO needs a barcode file cmo_barcodes_csv = file("cmo_barcodes.csv") @@ -329,27 +329,27 @@ nextflow_process { // stage antibody capture data ab_fastqs_5k_cmvpos_tcells = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_ab_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_ab_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/ab/subsampled_5k_human_antiCMV_T_TBNK_connect_AB_S2_L004_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/ab/subsampled_5k_human_antiCMV_T_TBNK_connect_AB_S2_L004_R2_001.fastq.gz', checkIfExists: true) ] def ab_fastq_samplename_5k_cmvpos_tcells = "subsampled_5k_human_antiCMV_T_TBNK_connect" // stage GEX data gex_fastqs_5k_cmvpos_tcells = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_gex1_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_gex1_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/gex_1/subsampled_5k_human_antiCMV_T_TBNK_connect_GEX_1_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/gex_1/subsampled_5k_human_antiCMV_T_TBNK_connect_GEX_1_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] def gex_fastq_samplename_5k_cmvpos_tcells = "subsampled_5k_human_antiCMV_T_TBNK_connect" // stage VDJ data vdj_fastqs_5k_cmvpos_tcells = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_vdj_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_vdj_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/vdj/subsampled_5k_human_antiCMV_T_TBNK_connect_VDJ_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/vdj/subsampled_5k_human_antiCMV_T_TBNK_connect_VDJ_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] def vdj_fastq_samplename_5k_cmvpos_tcells = "subsampled_5k_human_antiCMV_T_TBNK_connect" // stage feature barcode reference for antibody capture - fb_reference_5k_cmvpos_tcells = file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_feature_ref_csv'], checkIfExists: true) + fb_reference_5k_cmvpos_tcells = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/5k_human_antiCMV_T_TBNK_connect_Multiplex_count_feature_reference.csv', checkIfExists: true) /*************************************/ /*** end stage 5k CMV+ T-cell data ***/ @@ -362,8 +362,8 @@ nextflow_process { /***************************/ // will build this as done in cellranger count test - gex_ref_fasta = file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true) - gex_ref_gtf = file(params.test_data['homo_sapiens']['genome']['genome_gtf'] , checkIfExists: true) + gex_ref_fasta = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + gex_ref_gtf = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.gtf', checkIfExists: true) def gex_ref_name = "homo_sapiens_chr22_reference" /*******************************/ @@ -375,9 +375,9 @@ nextflow_process { /*** stage VDJ reference ***/ /***************************/ - vdj_json = file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_vdj_ref_json'] , checkIfExists: true) - vdj_fasta = file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_vdj_ref_fasta'] , checkIfExists: true) - vdj_suppfasta = file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_vdj_ref_suppfasta'], checkIfExists: true) + vdj_json = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/references/vdj/refdata-cellranger-vdj-GRCh38-alts-ensembl-5.0.0/reference.json', checkIfExists: true) + vdj_fasta = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/references/vdj/refdata-cellranger-vdj-GRCh38-alts-ensembl-5.0.0/fasta/regions.fa', checkIfExists: true) + vdj_suppfasta = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/references/vdj/refdata-cellranger-vdj-GRCh38-alts-ensembl-5.0.0/fasta/supp_regions.fa', checkIfExists: true) // awkwardly restage VDJ ref to enforce directory structure expected by cellranger vdj_reference = file( "${workDir}/vdj_reference" ) @@ -502,27 +502,27 @@ nextflow_process { // stage B cell FASTQ test data bcell_fastqs_10k_pbmc = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_b_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_b_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/bcell/subsampled_sc5p_v2_hs_PBMC_10k_b_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/bcell/subsampled_sc5p_v2_hs_PBMC_10k_b_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] def bcell_fastq_samplename_10k_pbmc = "subsampled_5k_human_antiCMV_T_TBNK_connect" // stage 5' gene expression FASTQ test data fivepgex_fastqs_10k_pbmc = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_5gex_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_5gex_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5gex/subsampled_sc5p_v2_hs_PBMC_10k_5gex_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5gex/subsampled_sc5p_v2_hs_PBMC_10k_5gex_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] def fivepgex_fastq_samplename_10k_pbmc = "subsampled_5k_human_antiCMV_T_TBNK_connect" // stage 5' feature barcode (antibody capture) FASTQ test data ab_fastqs_10k_pbmc = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_5fb_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_5fb_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5fb/subsampled_sc5p_v2_hs_PBMC_10k_5fb_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5fb/subsampled_sc5p_v2_hs_PBMC_10k_5fb_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] def fivepab_fastq_samplename_10k_pbmc = "subsampled_5k_human_antiCMV_T_TBNK_connect" // stage feature barcode reference for antibody capture - fb_reference_10k_pbmc = file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_feature_ref_csv'], checkIfExists: true) + fb_reference_10k_pbmc = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/sc5p_v2_hs_PBMC_10k_multi_5gex_5fb_b_t_feature_ref.csv', checkIfExists: true) /*******************************/ /*** end stage 10k PBMC data ***/ @@ -537,20 +537,20 @@ nextflow_process { // stage 3' CMO FASTQ test data cmo_fastqs_10k_pbmc_cmo = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_cmo_cmo_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_cmo_cmo_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/cmo/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_multiplexing_capture_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/cmo/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_multiplexing_capture_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] def cmo_fastq_samplename_10k_pbmc_cmo = "subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K" // stage 3' gene expression FASTQ test data threepgex_fastqs_10k_pbmc_cmo = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_cmo_gex1_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_cmo_gex1_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/gex_1/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_gex_S2_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/gex_1/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_gex_S2_L001_R2_001.fastq.gz', checkIfExists: true) ] def threepgex_fastq_samplename_10k_pbmc_cmo = "subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K" // stage feature barcode reference for antibody capture - cmo_reference_10k_pbmc_cmo = file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_10k_pbmc_cmo_feature_ref_csv'], checkIfExists: true) + cmo_reference_10k_pbmc_cmo = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/10k_pbmc_cmo_count_feature_reference.csv', checkIfExists: true) // CMO needs a barcode file cmo_barcodes_csv = file("cmo_barcodes.csv") @@ -569,27 +569,27 @@ nextflow_process { // stage antibody capture data ab_fastqs_5k_cmvpos_tcells = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_ab_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_ab_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/ab/subsampled_5k_human_antiCMV_T_TBNK_connect_AB_S2_L004_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/ab/subsampled_5k_human_antiCMV_T_TBNK_connect_AB_S2_L004_R2_001.fastq.gz', checkIfExists: true) ] def ab_fastq_samplename_5k_cmvpos_tcells = "subsampled_5k_human_antiCMV_T_TBNK_connect" // stage GEX data gex_fastqs_5k_cmvpos_tcells = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_gex1_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_gex1_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/gex_1/subsampled_5k_human_antiCMV_T_TBNK_connect_GEX_1_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/gex_1/subsampled_5k_human_antiCMV_T_TBNK_connect_GEX_1_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] def gex_fastq_samplename_5k_cmvpos_tcells = "subsampled_5k_human_antiCMV_T_TBNK_connect" // stage VDJ data vdj_fastqs_5k_cmvpos_tcells = [ - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_vdj_fastq_1_gz'], checkIfExists: true), - file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_vdj_fastq_2_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/vdj/subsampled_5k_human_antiCMV_T_TBNK_connect_VDJ_S1_L001_R1_001.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/vdj/subsampled_5k_human_antiCMV_T_TBNK_connect_VDJ_S1_L001_R2_001.fastq.gz', checkIfExists: true) ] def vdj_fastq_samplename_5k_cmvpos_tcells = "subsampled_5k_human_antiCMV_T_TBNK_connect" // stage feature barcode reference for antibody capture - fb_reference_5k_cmvpos_tcells = file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_5k_cmvpos_tcells_feature_ref_csv'], checkIfExists: true) + fb_reference_5k_cmvpos_tcells = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/5k_human_antiCMV_T_TBNK_connect_Multiplex_count_feature_reference.csv', checkIfExists: true) /*************************************/ /*** end stage 5k CMV+ T-cell data ***/ @@ -602,8 +602,8 @@ nextflow_process { /***************************/ // will build this as done in cellranger count test - gex_ref_fasta = file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true) - gex_ref_gtf = file(params.test_data['homo_sapiens']['genome']['genome_gtf'] , checkIfExists: true) + gex_ref_fasta = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + gex_ref_gtf = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.gtf', checkIfExists: true) def gex_ref_name = "homo_sapiens_chr22_reference" /*******************************/ @@ -615,9 +615,9 @@ nextflow_process { /*** stage VDJ reference ***/ /***************************/ - vdj_json = file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_vdj_ref_json'] , checkIfExists: true) - vdj_fasta = file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_vdj_ref_fasta'] , checkIfExists: true) - vdj_suppfasta = file(params.test_data['homo_sapiens']['10xgenomics']['cellranger']['test_10x_vdj_ref_suppfasta'], checkIfExists: true) + vdj_json = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/references/vdj/refdata-cellranger-vdj-GRCh38-alts-ensembl-5.0.0/reference.json', checkIfExists: true) + vdj_fasta = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/references/vdj/refdata-cellranger-vdj-GRCh38-alts-ensembl-5.0.0/fasta/regions.fa', checkIfExists: true) + vdj_suppfasta = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/10xgenomics/cellranger/references/vdj/refdata-cellranger-vdj-GRCh38-alts-ensembl-5.0.0/fasta/supp_regions.fa', checkIfExists: true) // awkwardly restage VDJ ref to enforce directory structure expected by cellranger vdj_reference = file( "${workDir}/vdj_reference" ) diff --git a/modules/nf-core/cellrangerarc/mkref/main.nf b/modules/nf-core/cellrangerarc/mkref/main.nf index 079776ba..b030ae92 100644 --- a/modules/nf-core/cellrangerarc/mkref/main.nf +++ b/modules/nf-core/cellrangerarc/mkref/main.nf @@ -35,8 +35,9 @@ process CELLRANGERARC_MKREF { reference_name = "cellrangerarc_reference" } + // unlike cellranger mkref and spaceranger mkref, cellranger-arc mkref is not *yet* implemented in the + // 10x martian runtime. It is therefore not necessary to specify --localcores and --localmem """ - python3 < versions.yml diff --git a/modules/nf-core/fastqc/tests/main.nf.test b/modules/nf-core/fastqc/tests/main.nf.test index 70edae4d..e9d79a07 100644 --- a/modules/nf-core/fastqc/tests/main.nf.test +++ b/modules/nf-core/fastqc/tests/main.nf.test @@ -23,17 +23,14 @@ nextflow_process { then { assertAll ( - { assert process.success }, - - // NOTE The report contains the date inside it, which means that the md5sum is stable per day, but not longer than that. So you can't md5sum it. - // looks like this:
Mon 2 Oct 2023
test.gz
- // https://github.com/nf-core/modules/pull/3903#issuecomment-1743620039 - - { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, - { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, - { assert path(process.out.html[0][1]).text.contains("File typeConventional base calls") }, - - { assert snapshot(process.out.versions).match("fastqc_versions_single") } + { assert process.success }, + // NOTE The report contains the date inside it, which means that the md5sum is stable per day, but not longer than that. So you can't md5sum it. + // looks like this:
Mon 2 Oct 2023
test.gz
+ // https://github.com/nf-core/modules/pull/3903#issuecomment-1743620039 + { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("File typeConventional base calls") }, + { assert snapshot(process.out.versions).match() } ) } } @@ -54,16 +51,14 @@ nextflow_process { then { assertAll ( - { assert process.success }, - - { assert process.out.html[0][1][0] ==~ ".*/test_1_fastqc.html" }, - { assert process.out.html[0][1][1] ==~ ".*/test_2_fastqc.html" }, - { assert process.out.zip[0][1][0] ==~ ".*/test_1_fastqc.zip" }, - { assert process.out.zip[0][1][1] ==~ ".*/test_2_fastqc.zip" }, - { assert path(process.out.html[0][1][0]).text.contains("File typeConventional base calls") }, - { assert path(process.out.html[0][1][1]).text.contains("File typeConventional base calls") }, - - { assert snapshot(process.out.versions).match("fastqc_versions_paired") } + { assert process.success }, + { assert process.out.html[0][1][0] ==~ ".*/test_1_fastqc.html" }, + { assert process.out.html[0][1][1] ==~ ".*/test_2_fastqc.html" }, + { assert process.out.zip[0][1][0] ==~ ".*/test_1_fastqc.zip" }, + { assert process.out.zip[0][1][1] ==~ ".*/test_2_fastqc.zip" }, + { assert path(process.out.html[0][1][0]).text.contains("File typeConventional base calls") }, + { assert path(process.out.html[0][1][1]).text.contains("File typeConventional base calls") }, + { assert snapshot(process.out.versions).match() } ) } } @@ -83,13 +78,11 @@ nextflow_process { then { assertAll ( - { assert process.success }, - - { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, - { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, - { assert path(process.out.html[0][1]).text.contains("File typeConventional base calls") }, - - { assert snapshot(process.out.versions).match("fastqc_versions_interleaved") } + { assert process.success }, + { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("File typeConventional base calls") }, + { assert snapshot(process.out.versions).match() } ) } } @@ -109,13 +102,11 @@ nextflow_process { then { assertAll ( - { assert process.success }, - - { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, - { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, - { assert path(process.out.html[0][1]).text.contains("File typeConventional base calls") }, - - { assert snapshot(process.out.versions).match("fastqc_versions_bam") } + { assert process.success }, + { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("File typeConventional base calls") }, + { assert snapshot(process.out.versions).match() } ) } } @@ -138,22 +129,20 @@ nextflow_process { then { assertAll ( - { assert process.success }, - - { assert process.out.html[0][1][0] ==~ ".*/test_1_fastqc.html" }, - { assert process.out.html[0][1][1] ==~ ".*/test_2_fastqc.html" }, - { assert process.out.html[0][1][2] ==~ ".*/test_3_fastqc.html" }, - { assert process.out.html[0][1][3] ==~ ".*/test_4_fastqc.html" }, - { assert process.out.zip[0][1][0] ==~ ".*/test_1_fastqc.zip" }, - { assert process.out.zip[0][1][1] ==~ ".*/test_2_fastqc.zip" }, - { assert process.out.zip[0][1][2] ==~ ".*/test_3_fastqc.zip" }, - { assert process.out.zip[0][1][3] ==~ ".*/test_4_fastqc.zip" }, - { assert path(process.out.html[0][1][0]).text.contains("File typeConventional base calls") }, - { assert path(process.out.html[0][1][1]).text.contains("File typeConventional base calls") }, - { assert path(process.out.html[0][1][2]).text.contains("File typeConventional base calls") }, - { assert path(process.out.html[0][1][3]).text.contains("File typeConventional base calls") }, - - { assert snapshot(process.out.versions).match("fastqc_versions_multiple") } + { assert process.success }, + { assert process.out.html[0][1][0] ==~ ".*/test_1_fastqc.html" }, + { assert process.out.html[0][1][1] ==~ ".*/test_2_fastqc.html" }, + { assert process.out.html[0][1][2] ==~ ".*/test_3_fastqc.html" }, + { assert process.out.html[0][1][3] ==~ ".*/test_4_fastqc.html" }, + { assert process.out.zip[0][1][0] ==~ ".*/test_1_fastqc.zip" }, + { assert process.out.zip[0][1][1] ==~ ".*/test_2_fastqc.zip" }, + { assert process.out.zip[0][1][2] ==~ ".*/test_3_fastqc.zip" }, + { assert process.out.zip[0][1][3] ==~ ".*/test_4_fastqc.zip" }, + { assert path(process.out.html[0][1][0]).text.contains("File typeConventional base calls") }, + { assert path(process.out.html[0][1][1]).text.contains("File typeConventional base calls") }, + { assert path(process.out.html[0][1][2]).text.contains("File typeConventional base calls") }, + { assert path(process.out.html[0][1][3]).text.contains("File typeConventional base calls") }, + { assert snapshot(process.out.versions).match() } ) } } @@ -173,21 +162,18 @@ nextflow_process { then { assertAll ( - { assert process.success }, - - { assert process.out.html[0][1] ==~ ".*/mysample_fastqc.html" }, - { assert process.out.zip[0][1] ==~ ".*/mysample_fastqc.zip" }, - { assert path(process.out.html[0][1]).text.contains("File typeConventional base calls") }, - - { assert snapshot(process.out.versions).match("fastqc_versions_custom_prefix") } + { assert process.success }, + { assert process.out.html[0][1] ==~ ".*/mysample_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/mysample_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("File typeConventional base calls") }, + { assert snapshot(process.out.versions).match() } ) } } test("sarscov2 single-end [fastq] - stub") { - options "-stub" - + options "-stub" when { process { """ @@ -201,12 +187,123 @@ nextflow_process { then { assertAll ( - { assert process.success }, - { assert snapshot(process.out.html.collect { file(it[1]).getName() } + - process.out.zip.collect { file(it[1]).getName() } + - process.out.versions ).match("fastqc_stub") } + { assert process.success }, + { assert snapshot(process.out).match() } ) } } + test("sarscov2 paired-end [fastq] - stub") { + + options "-stub" + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) ] + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } + + test("sarscov2 interleaved [fastq] - stub") { + + options "-stub" + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_interleaved.fastq.gz', checkIfExists: true) + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } + + test("sarscov2 paired-end [bam] - stub") { + + options "-stub" + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } + + test("sarscov2 multiple [fastq] - stub") { + + options "-stub" + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test2_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test2_2.fastq.gz', checkIfExists: true) ] + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } + + test("sarscov2 custom_prefix - stub") { + + options "-stub" + when { + process { + """ + input[0] = Channel.of([ + [ id:'mysample', single_end:true ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } } diff --git a/modules/nf-core/fastqc/tests/main.nf.test.snap b/modules/nf-core/fastqc/tests/main.nf.test.snap index 86f7c311..d5db3092 100644 --- a/modules/nf-core/fastqc/tests/main.nf.test.snap +++ b/modules/nf-core/fastqc/tests/main.nf.test.snap @@ -1,88 +1,392 @@ { - "fastqc_versions_interleaved": { + "sarscov2 custom_prefix": { "content": [ [ "versions.yml:md5,e1cc25ca8af856014824abd842e93978" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.3" }, - "timestamp": "2024-01-31T17:40:07.293713" + "timestamp": "2024-07-22T11:02:16.374038" }, - "fastqc_stub": { + "sarscov2 single-end [fastq] - stub": { "content": [ - [ - "test.html", - "test.zip", - "versions.yml:md5,e1cc25ca8af856014824abd842e93978" - ] + { + "0": [ + [ + { + "id": "test", + "single_end": true + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": true + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "html": [ + [ + { + "id": "test", + "single_end": true + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "zip": [ + [ + { + "id": "test", + "single_end": true + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.3" + }, + "timestamp": "2024-07-22T11:02:24.993809" + }, + "sarscov2 custom_prefix - stub": { + "content": [ + { + "0": [ + [ + { + "id": "mysample", + "single_end": true + }, + "mysample.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "mysample", + "single_end": true + }, + "mysample.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "html": [ + [ + { + "id": "mysample", + "single_end": true + }, + "mysample.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "zip": [ + [ + { + "id": "mysample", + "single_end": true + }, + "mysample.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.3" }, - "timestamp": "2024-01-31T17:31:01.425198" + "timestamp": "2024-07-22T11:03:10.93942" }, - "fastqc_versions_multiple": { + "sarscov2 interleaved [fastq]": { "content": [ [ "versions.yml:md5,e1cc25ca8af856014824abd842e93978" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.3" }, - "timestamp": "2024-01-31T17:40:55.797907" + "timestamp": "2024-07-22T11:01:42.355718" }, - "fastqc_versions_bam": { + "sarscov2 paired-end [bam]": { "content": [ [ "versions.yml:md5,e1cc25ca8af856014824abd842e93978" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.3" }, - "timestamp": "2024-01-31T17:40:26.795862" + "timestamp": "2024-07-22T11:01:53.276274" }, - "fastqc_versions_single": { + "sarscov2 multiple [fastq]": { "content": [ [ "versions.yml:md5,e1cc25ca8af856014824abd842e93978" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.3" }, - "timestamp": "2024-01-31T17:39:27.043675" + "timestamp": "2024-07-22T11:02:05.527626" }, - "fastqc_versions_paired": { + "sarscov2 paired-end [fastq]": { "content": [ [ "versions.yml:md5,e1cc25ca8af856014824abd842e93978" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.3" + }, + "timestamp": "2024-07-22T11:01:31.188871" + }, + "sarscov2 paired-end [fastq] - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": false + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "html": [ + [ + { + "id": "test", + "single_end": false + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "zip": [ + [ + { + "id": "test", + "single_end": false + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.3" + }, + "timestamp": "2024-07-22T11:02:34.273566" + }, + "sarscov2 multiple [fastq] - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": false + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "html": [ + [ + { + "id": "test", + "single_end": false + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "zip": [ + [ + { + "id": "test", + "single_end": false + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.3" }, - "timestamp": "2024-01-31T17:39:47.584191" + "timestamp": "2024-07-22T11:03:02.304411" }, - "fastqc_versions_custom_prefix": { + "sarscov2 single-end [fastq]": { "content": [ [ "versions.yml:md5,e1cc25ca8af856014824abd842e93978" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.3" + }, + "timestamp": "2024-07-22T11:01:19.095607" + }, + "sarscov2 interleaved [fastq] - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": false + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "html": [ + [ + { + "id": "test", + "single_end": false + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "zip": [ + [ + { + "id": "test", + "single_end": false + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.3" + }, + "timestamp": "2024-07-22T11:02:44.640184" + }, + "sarscov2 paired-end [bam] - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": false + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "html": [ + [ + { + "id": "test", + "single_end": false + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "zip": [ + [ + { + "id": "test", + "single_end": false + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.3" }, - "timestamp": "2024-01-31T17:41:14.576531" + "timestamp": "2024-07-22T11:02:53.550742" } } \ No newline at end of file diff --git a/modules/nf-core/gunzip/environment.yml b/modules/nf-core/gunzip/environment.yml index 25910b34..dfc02a7b 100644 --- a/modules/nf-core/gunzip/environment.yml +++ b/modules/nf-core/gunzip/environment.yml @@ -4,4 +4,6 @@ channels: - bioconda - defaults dependencies: - - conda-forge::sed=4.7 + - conda-forge::grep=3.11 + - conda-forge::sed=4.8 + - conda-forge::tar=1.34 diff --git a/modules/nf-core/gunzip/main.nf b/modules/nf-core/gunzip/main.nf index 468a6f28..5e67e3b9 100644 --- a/modules/nf-core/gunzip/main.nf +++ b/modules/nf-core/gunzip/main.nf @@ -4,8 +4,8 @@ process GUNZIP { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : - 'nf-core/ubuntu:20.04' }" + 'https://depot.galaxyproject.org/singularity/ubuntu:22.04' : + 'nf-core/ubuntu:22.04' }" input: tuple val(meta), path(archive) @@ -18,8 +18,11 @@ process GUNZIP { task.ext.when == null || task.ext.when script: - def args = task.ext.args ?: '' - gunzip = archive.toString() - '.gz' + def args = task.ext.args ?: '' + def extension = ( archive.toString() - '.gz' ).tokenize('.')[-1] + def name = archive.toString() - '.gz' - ".$extension" + def prefix = task.ext.prefix ?: name + gunzip = prefix + ".$extension" """ # Not calling gunzip itself because it creates files # with the original group ownership rather than the @@ -37,7 +40,11 @@ process GUNZIP { """ stub: - gunzip = archive.toString() - '.gz' + def args = task.ext.args ?: '' + def extension = ( archive.toString() - '.gz' ).tokenize('.')[-1] + def name = archive.toString() - '.gz' - ".$extension" + def prefix = task.ext.prefix ?: name + gunzip = prefix + ".$extension" """ touch $gunzip cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/gunzip/meta.yml b/modules/nf-core/gunzip/meta.yml index 231034f2..f32973a0 100644 --- a/modules/nf-core/gunzip/meta.yml +++ b/modules/nf-core/gunzip/meta.yml @@ -37,3 +37,4 @@ maintainers: - "@joseespinosa" - "@drpatelh" - "@jfy133" + - "@gallvp" diff --git a/modules/nf-core/gunzip/tests/main.nf.test b/modules/nf-core/gunzip/tests/main.nf.test index 6406008e..776211ad 100644 --- a/modules/nf-core/gunzip/tests/main.nf.test +++ b/modules/nf-core/gunzip/tests/main.nf.test @@ -33,4 +33,89 @@ nextflow_process { } + test("Should run without failures - prefix") { + + config './nextflow.config' + + when { + params { + outdir = "$outputDir" + } + process { + """ + input[0] = Channel.of([ + [ id: 'test' ], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) + ] + ) + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("Should run without failures - stub") { + + options '-stub' + + when { + params { + outdir = "$outputDir" + } + process { + """ + input[0] = Channel.of([ + [], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) + ] + ) + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("Should run without failures - prefix - stub") { + + options '-stub' + config './nextflow.config' + + when { + params { + outdir = "$outputDir" + } + process { + """ + input[0] = Channel.of([ + [ id: 'test' ], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) + ] + ) + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + } diff --git a/modules/nf-core/gunzip/tests/main.nf.test.snap b/modules/nf-core/gunzip/tests/main.nf.test.snap index 720fd9ff..069967e7 100644 --- a/modules/nf-core/gunzip/tests/main.nf.test.snap +++ b/modules/nf-core/gunzip/tests/main.nf.test.snap @@ -1,4 +1,70 @@ { + "Should run without failures - prefix - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.xyz.fastq:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + "versions.yml:md5,54376d32aca20e937a4ec26dac228e84" + ], + "gunzip": [ + [ + { + "id": "test" + }, + "test.xyz.fastq:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,54376d32aca20e937a4ec26dac228e84" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.2" + }, + "timestamp": "2024-06-25T11:35:10.861293" + }, + "Should run without failures - stub": { + "content": [ + { + "0": [ + [ + [ + + ], + "test_1.fastq:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + "versions.yml:md5,54376d32aca20e937a4ec26dac228e84" + ], + "gunzip": [ + [ + [ + + ], + "test_1.fastq:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,54376d32aca20e937a4ec26dac228e84" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.2" + }, + "timestamp": "2024-06-25T11:35:05.857145" + }, "Should run without failures": { "content": [ { @@ -26,6 +92,43 @@ ] } ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.2" + }, "timestamp": "2023-10-17T15:35:37.690477896" + }, + "Should run without failures - prefix": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.xyz.fastq:md5,4161df271f9bfcd25d5845a1e220dbec" + ] + ], + "1": [ + "versions.yml:md5,54376d32aca20e937a4ec26dac228e84" + ], + "gunzip": [ + [ + { + "id": "test" + }, + "test.xyz.fastq:md5,4161df271f9bfcd25d5845a1e220dbec" + ] + ], + "versions": [ + "versions.yml:md5,54376d32aca20e937a4ec26dac228e84" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.2" + }, + "timestamp": "2024-06-25T11:33:32.921739" } } \ No newline at end of file diff --git a/modules/nf-core/gunzip/tests/nextflow.config b/modules/nf-core/gunzip/tests/nextflow.config new file mode 100644 index 00000000..dec77642 --- /dev/null +++ b/modules/nf-core/gunzip/tests/nextflow.config @@ -0,0 +1,5 @@ +process { + withName: GUNZIP { + ext.prefix = { "${meta.id}.xyz" } + } +} diff --git a/modules/nf-core/multiqc/environment.yml b/modules/nf-core/multiqc/environment.yml index ca39fb67..2121492d 100644 --- a/modules/nf-core/multiqc/environment.yml +++ b/modules/nf-core/multiqc/environment.yml @@ -4,4 +4,4 @@ channels: - bioconda - defaults dependencies: - - bioconda::multiqc=1.21 + - bioconda::multiqc=1.23 diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index 47ac352f..459dfea5 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -3,14 +3,16 @@ process MULTIQC { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.21--pyhdfd78af_0' : - 'biocontainers/multiqc:1.21--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.23--pyhdfd78af_0' : + 'biocontainers/multiqc:1.23--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" path(multiqc_config) path(extra_multiqc_config) path(multiqc_logo) + path(replace_names) + path(sample_names) output: path "*multiqc_report.html", emit: report @@ -26,6 +28,8 @@ process MULTIQC { def config = multiqc_config ? "--config $multiqc_config" : '' def extra_config = extra_multiqc_config ? "--config $extra_multiqc_config" : '' def logo = multiqc_logo ? /--cl-config 'custom_logo: "${multiqc_logo}"'/ : '' + def replace = replace_names ? "--replace-names ${replace_names}" : '' + def samples = sample_names ? "--sample-names ${sample_names}" : '' """ multiqc \\ --force \\ @@ -33,6 +37,8 @@ process MULTIQC { $config \\ $extra_config \\ $logo \\ + $replace \\ + $samples \\ . cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/multiqc/meta.yml b/modules/nf-core/multiqc/meta.yml index 45a9bc35..382c08cb 100644 --- a/modules/nf-core/multiqc/meta.yml +++ b/modules/nf-core/multiqc/meta.yml @@ -29,6 +29,19 @@ input: type: file description: Optional logo file for MultiQC pattern: "*.{png}" + - replace_names: + type: file + description: | + Optional two-column sample renaming file. First column a set of + patterns, second column a set of corresponding replacements. Passed via + MultiQC's `--replace-names` option. + pattern: "*.{tsv}" + - sample_names: + type: file + description: | + Optional TSV file with headers, passed to the MultiQC --sample_names + argument. + pattern: "*.{tsv}" output: - report: type: file diff --git a/modules/nf-core/multiqc/tests/main.nf.test b/modules/nf-core/multiqc/tests/main.nf.test index f1c4242e..6aa27f4c 100644 --- a/modules/nf-core/multiqc/tests/main.nf.test +++ b/modules/nf-core/multiqc/tests/main.nf.test @@ -17,6 +17,8 @@ nextflow_process { input[1] = [] input[2] = [] input[3] = [] + input[4] = [] + input[5] = [] """ } } @@ -41,6 +43,8 @@ nextflow_process { input[1] = Channel.of(file("https://github.com/nf-core/tools/raw/dev/nf_core/pipeline-template/assets/multiqc_config.yml", checkIfExists: true)) input[2] = [] input[3] = [] + input[4] = [] + input[5] = [] """ } } @@ -66,6 +70,8 @@ nextflow_process { input[1] = [] input[2] = [] input[3] = [] + input[4] = [] + input[5] = [] """ } } diff --git a/modules/nf-core/multiqc/tests/main.nf.test.snap b/modules/nf-core/multiqc/tests/main.nf.test.snap index bfebd802..45e95e5d 100644 --- a/modules/nf-core/multiqc/tests/main.nf.test.snap +++ b/modules/nf-core/multiqc/tests/main.nf.test.snap @@ -2,14 +2,14 @@ "multiqc_versions_single": { "content": [ [ - "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" + "versions.yml:md5,87904cd321df21fac35d18f0fc01bb19" ] ], "meta": { "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nextflow": "24.04.2" }, - "timestamp": "2024-02-29T08:48:55.657331" + "timestamp": "2024-07-10T12:41:34.562023" }, "multiqc_stub": { "content": [ @@ -17,25 +17,25 @@ "multiqc_report.html", "multiqc_data", "multiqc_plots", - "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" + "versions.yml:md5,87904cd321df21fac35d18f0fc01bb19" ] ], "meta": { "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nextflow": "24.04.2" }, - "timestamp": "2024-02-29T08:49:49.071937" + "timestamp": "2024-07-10T11:27:11.933869532" }, "multiqc_versions_config": { "content": [ [ - "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" + "versions.yml:md5,87904cd321df21fac35d18f0fc01bb19" ] ], "meta": { "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nextflow": "24.04.2" }, - "timestamp": "2024-02-29T08:49:25.457567" + "timestamp": "2024-07-10T11:26:56.709849369" } } \ No newline at end of file diff --git a/modules/nf-core/star/genomegenerate/tests/main.nf.test b/modules/nf-core/star/genomegenerate/tests/main.nf.test index c17c8ba4..4d619c47 100644 --- a/modules/nf-core/star/genomegenerate/tests/main.nf.test +++ b/modules/nf-core/star/genomegenerate/tests/main.nf.test @@ -28,15 +28,15 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(file(process.out.index[0][1]).listFiles().collect { it.getName() }.sort().toString()).match("fasta_gtf_index") }, - { assert snapshot(process.out.versions).match("fasta_gtf_versions") } + { assert snapshot( + file(process.out.index[0][1]).listFiles().collect { it.getName() }.sort().toString(), + process.out.versions) + .match() } ) } } - test("fasta_gtf_stub") { - - options '-stub' + test("fasta") { when { process { @@ -45,10 +45,7 @@ nextflow_process { [ id:'test_fasta' ], [ file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) ] ]) - input[1] = Channel.of([ - [ id:'test_gtf' ], - [ file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.gtf', checkIfExists: true) ] - ]) + input[1] = Channel.of([ [], [] ]) """ } } @@ -56,13 +53,17 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(file(process.out.index[0][1]).listFiles().collect { it.getName() }.sort().toString()).match("fasta_gtf_stub_index") }, - { assert snapshot(process.out.versions).match("fasta_gtf_stub_versions") } + { assert snapshot( + file(process.out.index[0][1]).listFiles().collect { it.getName() }.sort().toString(), + process.out.versions + ).match() } ) } } - test("fasta") { + test("fasta_gtf_stub") { + + options '-stub' when { process { @@ -71,7 +72,10 @@ nextflow_process { [ id:'test_fasta' ], [ file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) ] ]) - input[1] = Channel.of([ [], [] ]) + input[1] = Channel.of([ + [ id:'test_gtf' ], + [ file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.gtf', checkIfExists: true) ] + ]) """ } } @@ -79,11 +83,9 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(file(process.out.index[0][1]).listFiles().collect { it.getName() }.sort().toString()).match("fasta_index") }, - { assert snapshot(process.out.versions).match("fasta_versions") } + { assert snapshot(process.out).match() } ) } - } test("fasta_stub") { @@ -105,11 +107,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(file(process.out.index[0][1]).listFiles().collect { it.getName() }.sort().toString()).match("fasta_stub_index") }, - { assert snapshot(process.out.versions).match("fasta_stub_versions") } + { assert snapshot(process.out).match() } ) } - } - } diff --git a/modules/nf-core/star/genomegenerate/tests/main.nf.test.snap b/modules/nf-core/star/genomegenerate/tests/main.nf.test.snap index 5653d6e6..207f4b4f 100644 --- a/modules/nf-core/star/genomegenerate/tests/main.nf.test.snap +++ b/modules/nf-core/star/genomegenerate/tests/main.nf.test.snap @@ -1,90 +1,148 @@ { - "fasta_gtf_versions": { + "fasta_gtf": { "content": [ + "[Genome, Log.out, SA, SAindex, chrLength.txt, chrName.txt, chrNameLength.txt, chrStart.txt, exonGeTrInfo.tab, exonInfo.tab, geneInfo.tab, genomeParameters.txt, sjdbInfo.txt, sjdbList.fromGTF.out.tab, sjdbList.out.tab, transcriptInfo.tab]", [ "versions.yml:md5,46b8f1f34bb7f23892cd1eb249ed4d7f" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.04.3" + "nf-test": "0.9.0", + "nextflow": "24.04.3" }, - "timestamp": "2024-02-01T15:54:31.798555" + "timestamp": "2024-07-22T14:55:35.478401" }, - "fasta_stub_versions": { + "fasta_gtf_stub": { "content": [ - [ - "versions.yml:md5,46b8f1f34bb7f23892cd1eb249ed4d7f" - ] - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.04.3" - }, - "timestamp": "2024-02-01T15:55:07.521209" - }, - "fasta_gtf_stub_index": { - "content": [ - "[Genome, Log.out, SA, SAindex, chrLength.txt, chrName.txt, chrNameLength.txt, chrStart.txt, exonGeTrInfo.tab, exonInfo.tab, geneInfo.tab, genomeParameters.txt, sjdbInfo.txt, sjdbList.fromGTF.out.tab, sjdbList.out.tab, transcriptInfo.tab]" - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.04.3" - }, - "timestamp": "2024-02-01T15:54:46.478098" - }, - "fasta_gtf_stub_versions": { - "content": [ - [ - "versions.yml:md5,46b8f1f34bb7f23892cd1eb249ed4d7f" - ] + { + "0": [ + [ + { + "id": "test_fasta" + }, + [ + "Genome:md5,d41d8cd98f00b204e9800998ecf8427e", + "Log.out:md5,d41d8cd98f00b204e9800998ecf8427e", + "SA:md5,d41d8cd98f00b204e9800998ecf8427e", + "SAindex:md5,d41d8cd98f00b204e9800998ecf8427e", + "chrLength.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "chrName.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "chrNameLength.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "chrStart.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "exonGeTrInfo.tab:md5,d41d8cd98f00b204e9800998ecf8427e", + "exonInfo.tab:md5,d41d8cd98f00b204e9800998ecf8427e", + "geneInfo.tab:md5,d41d8cd98f00b204e9800998ecf8427e", + "genomeParameters.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sjdbInfo.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sjdbList.fromGTF.out.tab:md5,d41d8cd98f00b204e9800998ecf8427e", + "sjdbList.out.tab:md5,d41d8cd98f00b204e9800998ecf8427e", + "transcriptInfo.tab:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "1": [ + "versions.yml:md5,46b8f1f34bb7f23892cd1eb249ed4d7f" + ], + "index": [ + [ + { + "id": "test_fasta" + }, + [ + "Genome:md5,d41d8cd98f00b204e9800998ecf8427e", + "Log.out:md5,d41d8cd98f00b204e9800998ecf8427e", + "SA:md5,d41d8cd98f00b204e9800998ecf8427e", + "SAindex:md5,d41d8cd98f00b204e9800998ecf8427e", + "chrLength.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "chrName.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "chrNameLength.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "chrStart.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "exonGeTrInfo.tab:md5,d41d8cd98f00b204e9800998ecf8427e", + "exonInfo.tab:md5,d41d8cd98f00b204e9800998ecf8427e", + "geneInfo.tab:md5,d41d8cd98f00b204e9800998ecf8427e", + "genomeParameters.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sjdbInfo.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sjdbList.fromGTF.out.tab:md5,d41d8cd98f00b204e9800998ecf8427e", + "sjdbList.out.tab:md5,d41d8cd98f00b204e9800998ecf8427e", + "transcriptInfo.tab:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "versions": [ + "versions.yml:md5,46b8f1f34bb7f23892cd1eb249ed4d7f" + ] + } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.04.3" + "nf-test": "0.9.0", + "nextflow": "24.04.3" }, - "timestamp": "2024-02-01T15:54:46.491657" + "timestamp": "2024-07-22T14:55:57.247585" }, - "fasta_index": { + "fasta_stub": { "content": [ - "[Genome, Log.out, SA, SAindex, chrLength.txt, chrName.txt, chrNameLength.txt, chrStart.txt, genomeParameters.txt]" + { + "0": [ + [ + { + "id": "test_fasta" + }, + [ + "Genome:md5,d41d8cd98f00b204e9800998ecf8427e", + "Log.out:md5,d41d8cd98f00b204e9800998ecf8427e", + "SA:md5,d41d8cd98f00b204e9800998ecf8427e", + "SAindex:md5,d41d8cd98f00b204e9800998ecf8427e", + "chrLength.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "chrName.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "chrNameLength.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "chrStart.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "genomeParameters.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "1": [ + "versions.yml:md5,46b8f1f34bb7f23892cd1eb249ed4d7f" + ], + "index": [ + [ + { + "id": "test_fasta" + }, + [ + "Genome:md5,d41d8cd98f00b204e9800998ecf8427e", + "Log.out:md5,d41d8cd98f00b204e9800998ecf8427e", + "SA:md5,d41d8cd98f00b204e9800998ecf8427e", + "SAindex:md5,d41d8cd98f00b204e9800998ecf8427e", + "chrLength.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "chrName.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "chrNameLength.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "chrStart.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "genomeParameters.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "versions": [ + "versions.yml:md5,46b8f1f34bb7f23892cd1eb249ed4d7f" + ] + } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.04.3" + "nf-test": "0.9.0", + "nextflow": "24.04.3" }, - "timestamp": "2024-02-01T15:54:57.552329" + "timestamp": "2024-07-22T14:56:07.01742" }, - "fasta_versions": { + "fasta": { "content": [ + "[Genome, Log.out, SA, SAindex, chrLength.txt, chrName.txt, chrNameLength.txt, chrStart.txt, genomeParameters.txt]", [ "versions.yml:md5,46b8f1f34bb7f23892cd1eb249ed4d7f" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.04.3" - }, - "timestamp": "2024-02-01T15:54:57.560541" - }, - "fasta_gtf_index": { - "content": [ - "[Genome, Log.out, SA, SAindex, chrLength.txt, chrName.txt, chrNameLength.txt, chrStart.txt, exonGeTrInfo.tab, exonInfo.tab, geneInfo.tab, genomeParameters.txt, sjdbInfo.txt, sjdbList.fromGTF.out.tab, sjdbList.out.tab, transcriptInfo.tab]" - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.04.3" - }, - "timestamp": "2024-02-01T15:54:31.786814" - }, - "fasta_stub_index": { - "content": [ - "[Genome, Log.out, SA, SAindex, chrLength.txt, chrName.txt, chrNameLength.txt, chrStart.txt, genomeParameters.txt]" - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.04.3" + "nf-test": "0.9.0", + "nextflow": "24.04.3" }, - "timestamp": "2024-02-01T15:55:07.517472" + "timestamp": "2024-07-22T14:55:45.48784" } } \ No newline at end of file diff --git a/modules/nf-core/unzip/main.nf b/modules/nf-core/unzip/main.nf index 08cfc3c4..a0c02109 100644 --- a/modules/nf-core/unzip/main.nf +++ b/modules/nf-core/unzip/main.nf @@ -20,7 +20,6 @@ process UNZIP { script: def args = task.ext.args ?: '' if ( archive instanceof List && archive.name.size > 1 ) { error "[UNZIP] error: 7za only accepts a single archive as input. Please check module input." } - prefix = task.ext.prefix ?: ( meta.id ? "${meta.id}" : archive.baseName) """ 7za \\ @@ -34,4 +33,17 @@ process UNZIP { 7za: \$(echo \$(7za --help) | sed 's/.*p7zip Version //; s/(.*//') END_VERSIONS """ + + stub: + def args = task.ext.args ?: '' + if ( archive instanceof List && archive.name.size > 1 ) { error "[UNZIP] error: 7za only accepts a single archive as input. Please check module input." } + prefix = task.ext.prefix ?: ( meta.id ? "${meta.id}" : archive.baseName) + """ + mkdir "${prefix}" + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + 7za: \$(echo \$(7za --help) | sed 's/.*p7zip Version //; s/(.*//') + END_VERSIONS + """ } diff --git a/modules/nf-core/unzip/tests/main.nf.test b/modules/nf-core/unzip/tests/main.nf.test new file mode 100644 index 00000000..238b68d8 --- /dev/null +++ b/modules/nf-core/unzip/tests/main.nf.test @@ -0,0 +1,54 @@ +nextflow_process { + + name "Test Process UNZIP" + script "../main.nf" + process "UNZIP" + + tag "modules" + tag "modules_nfcore" + tag "unzip" + + test("generic [tar] [tar_gz]") { + + when { + process { + """ + input[0] = [ + [ id: 'hello' ], + file(params.modules_testdata_base_path + 'generic/tar/hello.tar.gz', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } + + test("generic [tar] [tar_gz] stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id: 'hello' ], + file(params.modules_testdata_base_path + 'generic/tar/hello.tar.gz', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } +} diff --git a/modules/nf-core/unzip/tests/main.nf.test.snap b/modules/nf-core/unzip/tests/main.nf.test.snap new file mode 100644 index 00000000..cdd2ab16 --- /dev/null +++ b/modules/nf-core/unzip/tests/main.nf.test.snap @@ -0,0 +1,76 @@ +{ + "generic [tar] [tar_gz] stub": { + "content": [ + { + "0": [ + [ + { + "id": "hello" + }, + [ + + ] + ] + ], + "1": [ + "versions.yml:md5,52c55ce814e8bc9edc5a6c625ed794b8" + ], + "unzipped_archive": [ + [ + { + "id": "hello" + }, + [ + + ] + ] + ], + "versions": [ + "versions.yml:md5,52c55ce814e8bc9edc5a6c625ed794b8" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.2" + }, + "timestamp": "2024-06-30T19:16:37.11550986" + }, + "generic [tar] [tar_gz]": { + "content": [ + { + "0": [ + [ + { + "id": "hello" + }, + [ + "hello.tar:md5,80c66db79a773bc87b3346035ff9593e" + ] + ] + ], + "1": [ + "versions.yml:md5,52c55ce814e8bc9edc5a6c625ed794b8" + ], + "unzipped_archive": [ + [ + { + "id": "hello" + }, + [ + "hello.tar:md5,80c66db79a773bc87b3346035ff9593e" + ] + ] + ], + "versions": [ + "versions.yml:md5,52c55ce814e8bc9edc5a6c625ed794b8" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.2" + }, + "timestamp": "2024-06-30T19:16:25.120242571" + } +} \ No newline at end of file diff --git a/modules/nf-core/unzip/tests/tags.yml b/modules/nf-core/unzip/tests/tags.yml new file mode 100644 index 00000000..7f5647e1 --- /dev/null +++ b/modules/nf-core/unzip/tests/tags.yml @@ -0,0 +1,2 @@ +unzip: + - "modules/nf-core/unzip/**" diff --git a/nextflow.config b/nextflow.config index 665fb381..e1b608d2 100644 --- a/nextflow.config +++ b/nextflow.config @@ -19,6 +19,8 @@ params { genome = null transcript_fasta = null txp2gene = null + fasta = null + gtf = null // salmon alevin parameters (simpleaf) simpleaf_rlen = 91 @@ -252,7 +254,7 @@ singularity.registry = 'quay.io' // Nextflow plugins plugins { - id 'nf-validation@1.1.3' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-validation@1.1.4' // Validation of pipeline parameters and creation of an input channel from a sample sheet } // Load igenomes.config if required @@ -303,7 +305,7 @@ manifest { description = """Pipeline for processing 10x Genomics single cell rnaseq data""" mainScript = 'main.nf' nextflowVersion = '!>=23.04.0' - version = '2.7.0' + version = '2.7.1' doi = '10.5281/zenodo.3568187' } diff --git a/nextflow_schema.json b/nextflow_schema.json index 386f57a4..e5fb71b5 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -65,8 +65,8 @@ }, "protocol": { "type": "string", - "description": "The protocol that was used to generate the single cell data, e.g. 10x Genomics v2 Chemistry.\n\n Can be 'auto' (cellranger only), '10XV1', '10XV2', '10XV3', or any other protocol string that will get directly passed the respective aligner.", - "help_text": "The default is to auto-detect the protocol when running cellranger. For all other aligners the protocol MUST be manually specified. \n\n The following protocols are recognized by the pipeline and mapped to the corresponding protocol name of the respective aligner: '10XV1', '10XV2', '10XV3'. \n\nAny other protocol value is passed to the aligner in verbatim to support other sequencing platforms. See the [kallisto](https://pachterlab.github.io/kallisto/manual#bus), [simpleaf](https://simpleaf.readthedocs.io/en/latest/quant-command.html#a-note-on-the-chemistry-flag), [starsolo](https://gensoft.pasteur.fr/docs/STAR/2.7.9a/STARsolo.html), and [universc](https://github.com/minoda-lab/universc#pre-set-configurations) documentations for more details.", + "description": "The protocol that was used to generate the single cell data, e.g. 10x Genomics v2 Chemistry.\n\n Can be 'auto' (cellranger only), '10XV1', '10XV2', '10XV3', '10XV4', or any other protocol string that will get directly passed the respective aligner.", + "help_text": "The default is to auto-detect the protocol when running cellranger. For all other aligners the protocol MUST be manually specified. \n\n The following protocols are recognized by the pipeline and mapped to the corresponding protocol name of the respective aligner: '10XV1', '10XV2', '10XV3', '10XV4'. \n\nAny other protocol value is passed to the aligner in verbatim to support other sequencing platforms. See the [kallisto](https://pachterlab.github.io/kallisto/manual#bus), [simpleaf](https://simpleaf.readthedocs.io/en/latest/quant-command.html#a-note-on-the-chemistry-flag), [starsolo](https://gensoft.pasteur.fr/docs/STAR/2.7.9a/STARsolo.html), and [universc](https://github.com/minoda-lab/universc#pre-set-configurations) documentations for more details.", "default": "auto", "fa_icon": "fas fa-cogs" } diff --git a/subworkflows/local/align_cellrangermulti.nf b/subworkflows/local/align_cellrangermulti.nf index f853c001..977bf478 100644 --- a/subworkflows/local/align_cellrangermulti.nf +++ b/subworkflows/local/align_cellrangermulti.nf @@ -43,7 +43,11 @@ workflow CELLRANGER_MULTI_ALIGN { vdj: meta.feature_type == "vdj" return [ meta, fastq ] ab: meta.feature_type == "ab" - return [ meta, fastq ] + if (params.fb_reference){ + return [ meta, fastq ] + } else { + error ("Antibody reference was not specified. Please provide a reference file for feature barcoding (e.g. antibody measurements).\nPlease refer to https://www.10xgenomics.com/support/software/cell-ranger/latest/analysis/inputs/cr-feature-ref-csv for more details.") + } beam: meta.feature_type == "beam" return [ meta, fastq ] crispr: meta.feature_type == "crispr" diff --git a/tests/main_pipeline_alevin.test b/tests/main_pipeline_alevin.nf.test similarity index 100% rename from tests/main_pipeline_alevin.test rename to tests/main_pipeline_alevin.nf.test diff --git a/tests/main_pipeline_alevin.test.snap b/tests/main_pipeline_alevin.nf.test.snap similarity index 100% rename from tests/main_pipeline_alevin.test.snap rename to tests/main_pipeline_alevin.nf.test.snap diff --git a/tests/main_pipeline_cellranger.test b/tests/main_pipeline_cellranger.nf.test similarity index 100% rename from tests/main_pipeline_cellranger.test rename to tests/main_pipeline_cellranger.nf.test diff --git a/tests/main_pipeline_cellranger.test.snap b/tests/main_pipeline_cellranger.nf.test.snap similarity index 100% rename from tests/main_pipeline_cellranger.test.snap rename to tests/main_pipeline_cellranger.nf.test.snap diff --git a/tests/main_pipeline_cellrangermulti.test b/tests/main_pipeline_cellrangermulti.nf.test similarity index 100% rename from tests/main_pipeline_cellrangermulti.test rename to tests/main_pipeline_cellrangermulti.nf.test diff --git a/tests/main_pipeline_cellrangermulti.test.snap b/tests/main_pipeline_cellrangermulti.nf.test.snap similarity index 100% rename from tests/main_pipeline_cellrangermulti.test.snap rename to tests/main_pipeline_cellrangermulti.nf.test.snap diff --git a/tests/main_pipeline_kallisto.test b/tests/main_pipeline_kallisto.nf.test similarity index 100% rename from tests/main_pipeline_kallisto.test rename to tests/main_pipeline_kallisto.nf.test diff --git a/tests/main_pipeline_kallisto.test.snap b/tests/main_pipeline_kallisto.nf.test.snap similarity index 100% rename from tests/main_pipeline_kallisto.test.snap rename to tests/main_pipeline_kallisto.nf.test.snap diff --git a/tests/main_pipeline_star.test b/tests/main_pipeline_star.nf.test similarity index 100% rename from tests/main_pipeline_star.test rename to tests/main_pipeline_star.nf.test diff --git a/tests/main_pipeline_star.test.snap b/tests/main_pipeline_star.nf.test.snap similarity index 100% rename from tests/main_pipeline_star.test.snap rename to tests/main_pipeline_star.nf.test.snap diff --git a/workflows/scrnaseq.nf b/workflows/scrnaseq.nf index 5c789c57..10ced221 100644 --- a/workflows/scrnaseq.nf +++ b/workflows/scrnaseq.nf @@ -32,8 +32,12 @@ workflow SCRNASEQ { error "Only cellranger supports `protocol = 'auto'`. Please specify the protocol manually!" } - ch_genome_fasta = params.fasta ? file(params.fasta, checkIfExists: true) : ( params.genome ? file( getGenomeAttribute('fasta'), checkIfExists: true ) : [] ) - ch_gtf = params.gtf ? file(params.gtf , checkIfExists: true) : ( params.genome ? file( getGenomeAttribute('gtf') , checkIfExists: true ) : [] ) + params.fasta = getGenomeAttribute('fasta') + params.gtf = getGenomeAttribute('gtf') + params.star_index = getGenomeAttribute('star') + + ch_genome_fasta = params.fasta ? file(params.fasta, checkIfExists: true) : [] + ch_gtf = params.gtf ? file(params.gtf, checkIfExists: true) : [] // general input and params ch_transcript_fasta = params.transcript_fasta ? file(params.transcript_fasta): [] @@ -67,7 +71,8 @@ workflow SCRNASEQ { ch_salmon_index = params.salmon_index ? file(params.salmon_index) : [] //star params - ch_star_index = params.star_index ? file(params.star_index) : [] + star_index = params.star_index ? file(params.star_index, checkIfExists: true) : null + ch_star_index = star_index ? [[id: star_index.baseName], star_index] : [] star_feature = params.star_feature //cellranger params @@ -346,7 +351,9 @@ workflow SCRNASEQ { ch_multiqc_files.collect(), ch_multiqc_config.toList(), ch_multiqc_custom_config.toList(), - ch_multiqc_logo.toList() + ch_multiqc_logo.toList(), + [], + [] ) emit: