diff --git a/CHANGELOG.md b/CHANGELOG.md index 18705ca6..0613fa25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # biobox x.x.x +## NEW FUNCTIONALITY + +* `agat`: + - `agat/agat_convert_genscan2gff`: convert a genscan file into a GFF file (PR #100). + +* `bd_rhapsody/bd_rhapsody_sequence_analysis`: BD Rhapsody Sequence Analysis CWL pipeline (PR #96). + +* `rsem/rsem_calculate_expression`: Calculate expression levels (PR #93). + +## MINOR CHANGES + +* Upgrade to Viash 0.9.0. + +# biobox 0.2.0 + ## BREAKING CHANGES * `star/star_align_reads`: Change all arguments from `--camelCase` to `--snake_case` (PR #62). @@ -50,6 +65,16 @@ * `fastqc`: High throughput sequence quality control analysis tool (PR #92). +* `sortmerna`: Local sequence alignment tool for mapping, clustering, and filtering rRNA from + metatranscriptomic data (PR #146). + +* `fq_subsample`: Sample a subset of records from single or paired FASTQ files (PR #147). + +* `kallisto`: + - `kallisto_index`: Create a kallisto index (PR #149). + - `kallisto_quant`: Quantifying abundances of transcripts from RNA-Seq data, or more generally of target sequences using high-throughput sequencing reads (PR #152). + + * `trimgalore`: Quality and adapter trimming for fastq files (PR #117). ## MINOR CHANGES @@ -139,7 +164,8 @@ - `samtools/samtools_fastq`: Converts a SAM/BAM/CRAM file to FASTA (PR #53). * `umi_tools`: - -`umi_tools/umi_tools_extract`: Flexible removal of UMI sequences from fastq reads (PR #71). + - `umi_tools/umi_tools_extract`: Flexible removal of UMI sequences from fastq reads (PR #71). + - `umi_tools/umi_tools_prepareforrsem`: Fix paired-end reads in name sorted BAM file to prepare for RSEM (PR #148). * `falco`: A C++ drop-in replacement of FastQC to assess the quality of sequence read data (PR #43). @@ -147,10 +173,8 @@ - `bedtools_getfasta`: extract sequences from a FASTA file for each of the intervals defined in a BED/GFF/VCF file (PR #59). -* `sortmerna`: Local sequence alignment tool for mapping, clustering, and filtering rRNA from metatranscriptomic - data. (PR #146) -* `fq_subsample`: Sample a subset of records from single or paired FASTQ files (PR #147). + ## MINOR CHANGES diff --git a/_viash.yaml b/_viash.yaml index ab4f3828..d08f2fb2 100644 --- a/_viash.yaml +++ b/_viash.yaml @@ -7,7 +7,7 @@ links: issue_tracker: https://github.com/viash-hub/biobox/issues repository: https://github.com/viash-hub/biobox -viash_version: 0.9.0-RC7 +viash_version: 0.9.0 config_mods: | .requirements.commands := ['ps'] diff --git a/src/agat/agat_convert_genscan2gff/config.vsh.yaml b/src/agat/agat_convert_genscan2gff/config.vsh.yaml new file mode 100644 index 00000000..2adce1da --- /dev/null +++ b/src/agat/agat_convert_genscan2gff/config.vsh.yaml @@ -0,0 +1,95 @@ +name: agat_convert_genscan2gff +namespace: agat +description: | + The script takes a GENSCAN file as input, and will translate it in gff + format. The GENSCAN format is described [here](http://genome.crg.es/courses/Bioinformatics2003_genefinding/results/genscan.html). + + **Known problem** + + You must have submited only DNA sequence, without any header!! Indeed the tool expects only DNA + sequences and does not crash/warn if an header is submited along the + sequence. e.g If you have an header ">seq" s-e-q are seen as the 3 first + nucleotides of the sequence. Then all prediction location are shifted + accordingly. (checked only on the [online version](http://argonaute.mit.edu/GENSCAN.html). + I don't know if there is the same problem elsewhere.) +keywords: [gene annotations, GFF conversion, GENSCAN] +links: + homepage: https://github.com/NBISweden/AGAT + documentation: https://agat.readthedocs.io/en/latest/tools/agat_convert_genscan2gff.html + issue_tracker: https://github.com/NBISweden/AGAT/issues + repository: https://github.com/NBISweden/AGAT +references: + doi: 10.5281/zenodo.3552717 +license: GPL-3.0 +requirements: + - commands: [agat] +authors: + - __merge__: /src/_authors/leila_paquay.yaml + roles: [ author, maintainer ] + +argument_groups: + - name: Inputs + arguments: + - name: --genscan + alternatives: [-g] + description: Input genscan bed file that will be converted. + type: file + required: true + direction: input + - name: Outputs + arguments: + - name: --output + alternatives: [-o, --out, --outfile, --gff] + description: Output GFF file. If no output file is specified, the output will be written to STDOUT. + type: file + direction: output + required: true + example: output.gff + - name: Arguments + arguments: + - name: --source + description: | + The source informs about the tool used to produce the data and is stored in 2nd field of a gff file. Example: Stringtie, Maker, Augustus, etc. [default: data] + type: string + required: false + example: Stringtie + - name: --primary_tag + description: | + The primary_tag corresponds to the data type and is stored in 3rd field of a gff file. Example: gene, mRNA, CDS, etc. [default: gene] + type: string + required: false + example: gene + - name: --inflate_type + description: | + Feature type (3rd column in gff) created when inflate parameter activated [default: exon]. + type: string + required: false + example: exon + - name: --verbose + description: add verbosity + type: boolean_true + - name: --config + alternatives: [-c] + description: | + AGAT config file. By default AGAT takes the original agat_config.yaml shipped with AGAT. The `--config` option gives you the possibility to use your own AGAT config file (located elsewhere or named differently). + type: file + required: false + example: custom_agat_config.yaml +resources: + - type: bash_script + path: script.sh +test_resources: + - type: bash_script + path: test.sh + - type: file + path: test_data +engines: + - type: docker + image: quay.io/biocontainers/agat:1.4.0--pl5321hdfd78af_0 + setup: + - type: docker + run: | + agat --version | sed 's/AGAT\s\(.*\)/agat: "\1"/' > /var/software_versions.txt +runners: + - type: executable + - type: nextflow \ No newline at end of file diff --git a/src/agat/agat_convert_genscan2gff/help.txt b/src/agat/agat_convert_genscan2gff/help.txt new file mode 100644 index 00000000..8a9e9f52 --- /dev/null +++ b/src/agat/agat_convert_genscan2gff/help.txt @@ -0,0 +1,94 @@ +```sh +agat_convert_genscan2gff.pl --help +``` + ------------------------------------------------------------------------------ +| Another GFF Analysis Toolkit (AGAT) - Version: v1.4.0 | +| https://github.com/NBISweden/AGAT | +| National Bioinformatics Infrastructure Sweden (NBIS) - www.nbis.se | + ------------------------------------------------------------------------------ + +Name: + agat_convert_genscan2gff.pl + +Description: + The script takes a genscan file as input, and will translate it in gff + format. The genscan format is described here: + http://genome.crg.es/courses/Bioinformatics2003_genefinding/results/gens + can.html /!\ vvv Known problem vvv /!\ You must have submited only DNA + sequence, wihtout any header!! Indeed the tool expects only DNA + sequences and does not crash/warn if an header is submited along the + sequence. e.g If you have an header ">seq" s-e-q are seen as the 3 first + nucleotides of the sequence. Then all prediction location are shifted + accordingly. (checked only on the online version + http://argonaute.mit.edu/GENSCAN.html. I don't know if there is the same + pronlem elsewhere.) /!\ ^^^ Known problem ^^^^ /!\ + +Usage: + agat_convert_genscan2gff.pl --genscan infile.bed [ -o outfile ] + agat_convert_genscan2gff.pl -h + +Options: + --genscan or -g + Input genscan bed file that will be convert. + + --source + The source informs about the tool used to produce the data and + is stored in 2nd field of a gff file. Example: + Stringtie,Maker,Augustus,etc. [default: data] + + --primary_tag + The primary_tag corresponf to the data type and is stored in 3rd + field of a gff file. Example: gene,mRNA,CDS,etc. [default: gene] + + --inflate_off + By default we inflate the block fields (blockCount, blockSizes, + blockStarts) to create subfeatures of the main feature + (primary_tag). Type of subfeature created based on the + inflate_type parameter. If you don't want this inflating + behaviour you can deactivate it by using the option + --inflate_off. + + --inflate_type + Feature type (3rd column in gff) created when inflate parameter + activated [default: exon]. + + --verbose + add verbosity + + -o , --output , --out , --outfile or --gff + Output GFF file. If no output file is specified, the output will + be written to STDOUT. + + -c or --config + String - Input agat config file. By default AGAT takes as input + agat_config.yaml file from the working directory if any, + otherwise it takes the orignal agat_config.yaml shipped with + AGAT. To get the agat_config.yaml locally type: "agat config + --expose". The --config option gives you the possibility to use + your own AGAT config file (located elsewhere or named + differently). + + -h or --help + Display this helpful text. + +Feedback: + Did you find a bug?: + Do not hesitate to report bugs to help us keep track of the bugs and + their resolution. Please use the GitHub issue tracking system available + at this address: + + https://github.com/NBISweden/AGAT/issues + + Ensure that the bug was not already reported by searching under Issues. + If you're unable to find an (open) issue addressing the problem, open a new one. + Try as much as possible to include in the issue when relevant: + - a clear description, + - as much relevant information as possible, + - the command used, + - a data sample, + - an explanation of the expected behaviour that is not occurring. + + Do you want to contribute?: + You are very welcome, visit this address for the Contributing + guidelines: + https://github.com/NBISweden/AGAT/blob/master/CONTRIBUTING.md diff --git a/src/agat/agat_convert_genscan2gff/script.sh b/src/agat/agat_convert_genscan2gff/script.sh new file mode 100644 index 00000000..38afb084 --- /dev/null +++ b/src/agat/agat_convert_genscan2gff/script.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -eo pipefail + +## VIASH START +## VIASH END + +# unset flags +[[ "$par_inflate_off" == "true" ]] && unset par_inflate_off +[[ "$par_verbose" == "false" ]] && unset par_verbose + +# run agat_convert_genscan2gff +agat_convert_genscan2gff.pl \ + --genscan "$par_genscan" \ + --output "$par_output" \ + ${par_source:+--source "${par_source}"} \ + ${par_primary_tag:+--primary_tag "${par_primary_tag}"} \ + ${par_inflate_off:+--inflate_off} \ + ${par_inflate_type:+--inflate_type "${par_inflate_type}"} \ + ${par_verbose:+--verbose} \ + ${par_config:+--config "${par_config}"} \ No newline at end of file diff --git a/src/agat/agat_convert_genscan2gff/test.sh b/src/agat/agat_convert_genscan2gff/test.sh new file mode 100644 index 00000000..b666dacf --- /dev/null +++ b/src/agat/agat_convert_genscan2gff/test.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -eo pipefail + +## VIASH START +## VIASH END + +test_dir="${meta_resources_dir}/test_data" + +# create temporary directory and clean up on exit +TMPDIR=$(mktemp -d "$meta_temp_dir/$meta_name-XXXXXX") +function clean_up { + [[ -d "$TMPDIR" ]] && rm -rf "$TMPDIR" +} +trap clean_up EXIT + +echo "> Run $meta_name with test data" +"$meta_executable" \ + --genscan "$test_dir/test.genscan" \ + --output "$TMPDIR/output.gff" + +echo ">> Checking output" +[ ! -f "$TMPDIR/output.gff" ] && echo "Output file output.gff does not exist" && exit 1 + +echo ">> Check if output is empty" +[ ! -s "$TMPDIR/output.gff" ] && echo "Output file output.gff is empty" && exit 1 + +echo ">> Check if output matches expected output" +diff "$TMPDIR/output.gff" "$test_dir/agat_convert_genscan2gff_1.gff" +if [ $? -ne 0 ]; then + echo "Output file output.gff does not match expected output" + exit 1 +fi + +echo "> Test successful" diff --git a/src/agat/agat_convert_genscan2gff/test_data/agat_convert_genscan2gff_1.gff b/src/agat/agat_convert_genscan2gff/test_data/agat_convert_genscan2gff_1.gff new file mode 100644 index 00000000..695fb46c --- /dev/null +++ b/src/agat/agat_convert_genscan2gff/test_data/agat_convert_genscan2gff_1.gff @@ -0,0 +1,25 @@ +##gff-version 3 +unknown genscan gene 2223 4605 75.25 + . ID=gene_1 +unknown genscan mRNA 2223 4605 75.25 + . ID=mrna_1;Parent=gene_1 +unknown genscan exon 2223 3020 75.25 + . ID=exon_1;Parent=mrna_1 +unknown genscan exon 4249 4605 13.03 + . ID=exon_2;Parent=mrna_1 +unknown genscan CDS 2223 3020 75.25 + 0 ID=cds_1;Parent=mrna_1 +unknown genscan CDS 4249 4605 13.03 + 0 ID=cds_2;Parent=mrna_1 +unknown genscan gene 6829 8789 20.06 - . ID=gene_2 +unknown genscan mRNA 6829 8789 20.06 - . ID=mrna_2;Parent=gene_2 +unknown genscan exon 6829 7297 20.06 - . ID=exon_3;Parent=mrna_2 +unknown genscan exon 7730 7888 12.78 - . ID=exon_4;Parent=mrna_2 +unknown genscan exon 8029 8185 7.45 - . ID=exon_5;Parent=mrna_2 +unknown genscan exon 8278 8546 17.45 - . ID=exon_6;Parent=mrna_2 +unknown genscan exon 8647 8789 18.65 - . ID=exon_7;Parent=mrna_2 +unknown genscan CDS 6829 7297 20.06 - 1 ID=cds_3;Parent=mrna_2 +unknown genscan CDS 7730 7888 12.78 - 1 ID=cds_4;Parent=mrna_2 +unknown genscan CDS 8029 8185 7.45 - 2 ID=cds_5;Parent=mrna_2 +unknown genscan CDS 8278 8546 17.45 - 1 ID=cds_6;Parent=mrna_2 +unknown genscan CDS 8647 8789 18.65 - 0 ID=cds_7;Parent=mrna_2 +unknown genscan gene 10209 11924 16.18 + . ID=gene_3 +unknown genscan mRNA 10209 11924 16.18 + . ID=mrna_3;Parent=gene_3 +unknown genscan exon 10209 11313 16.18 + . ID=exon_8;Parent=mrna_3 +unknown genscan exon 11850 11924 3.27 + . ID=exon_9;Parent=mrna_3 +unknown genscan CDS 10209 11313 16.18 + 0 ID=cds_8;Parent=mrna_3 +unknown genscan CDS 11850 11924 3.27 + 2 ID=cds_9;Parent=mrna_3 diff --git a/src/agat/agat_convert_genscan2gff/test_data/script.sh b/src/agat/agat_convert_genscan2gff/test_data/script.sh new file mode 100755 index 00000000..c1693653 --- /dev/null +++ b/src/agat/agat_convert_genscan2gff/test_data/script.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# clone repo +if [ ! -d /tmp/agat_source ]; then + git clone --depth 1 --single-branch --branch master https://github.com/NBISweden/AGAT /tmp/agat_source +fi + +# copy test data +cp -r /tmp/agat_source/t/scripts_output/in/test.genscan src/agat/agat_convert_genscan2gff/test_data/test.genscan +cp -r /tmp/agat_source/t/scripts_output/out/agat_convert_genscan2gff_1.gff src/agat/agat_convert_genscan2gff/test_data/agat_convert_genscan2gff_1.gff + diff --git a/src/agat/agat_convert_genscan2gff/test_data/test.genscan b/src/agat/agat_convert_genscan2gff/test_data/test.genscan new file mode 100644 index 00000000..a88037db --- /dev/null +++ b/src/agat/agat_convert_genscan2gff/test_data/test.genscan @@ -0,0 +1,127 @@ +GENSCAN 1.0 Date run: 7-Mar-120 Time: 14:46:49 + + + +Sequence /tmp/03_07_20-14:46:49.fasta : 12217 bp : 42.83% C+G : Isochore 1 ( 0 - 43 C+G%) + + + +Parameter matrix: HumanIso.smat + + + +Predicted genes/exons: + + + +Gn.Ex Type S .Begin ...End .Len Fr Ph I/Ac Do/T CodRg P.... Tscr.. + +----- ---- - ------ ------ ---- -- -- ---- ---- ----- ----- ------ + + + + 1.01 Init + 2223 3020 798 2 0 55 2 924 0.940 75.25 + + 1.02 Term + 4249 4605 357 0 0 26 38 307 0.976 13.03 + + 1.03 PlyA + 4711 4716 6 -0.45 + + + + 2.06 PlyA - 4852 4847 6 -0.45 + + 2.05 Term - 7297 6829 469 0 1 13 42 387 0.281 20.06 + + 2.04 Intr - 7888 7730 159 0 0 85 93 144 0.998 12.78 + + 2.03 Intr - 8185 8029 157 2 1 65 60 144 0.787 7.45 + + 2.02 Intr - 8546 8278 269 1 2 36 65 287 0.946 17.45 + + 2.01 Init - 8789 8647 143 2 2 94 96 176 0.550 18.65 + + 2.00 Prom - 9720 9681 40 -6.55 + + + + 3.00 Prom + 10160 10199 40 -11.84 + + 3.01 Init + 10209 11313 1105 2 1 66 57 269 0.512 16.18 + + 3.02 Intr + 11850 11924 75 1 0 80 86 57 0.507 3.27 + + + +Suboptimal exons with probability > 1.000 + + + +Exnum Type S .Begin ...End .Len Fr Ph B/Ac Do/T CodRg P.... Tscr.. + +----- ---- - ------ ------ ---- -- -- ---- ---- ----- ----- ------ + + + +NO EXONS FOUND AT GIVEN PROBABILITY CUTOFF + + + + + +Predicted peptide sequence(s): + + + + + +>/tmp/03_03_20-07:33:11.fasta|GENSCAN_predicted_peptide_1|384_aa + +MSSKNKVSKQDIDSIVESLMKKQKSYFEPRLAQIQQVGMENVQKLSAIHAELALLTASIS + +TVKSDVDKLKCKVENNFSAIDGHDQAFGELELKMADMEDRSRRCNIRVIGLKERLEGFNA + +IQYLTHSLPKWFPALADVPVEVMSAHRIYSDAKRGDNRTLIFNVLRYTTRQAILRAAKKD + +PLSVDDRKVRFSPDYSNFTVKRCQAFHQAKDAARNKCLDFFLLYPATLKIKEGAQYRSFT + +SPKEAEDYVNSAASNHAATPASPRQHGTILTIYRRIHSLYDGERARKIQLLEQAASVALT + +GDNWTSVRNDNYLGVTAHFIDNVWKLRCFALEVKKKKKHSRHTAEDCAEEFIDVSNRWEI + +NGKLTTLGTDSALIMLAAARLLPF + + + +>/tmp/03_03_20-07:33:11.fasta|GENSCAN_predicted_peptide_2|398_aa + +MASTMPSSSSTEDEENTPECLNKDHYHFHHYTMEYIQDKPTNVARVGGFTDKKSIAKVER + +CLARERQEATEDHEAIPSTSGATSLTKKLRSRSGLPIAGSGLVLPALCIICQKKEKFINR + +AGKRQRDPLSKAETLTVGQLQKAAELKDDQSILLHIKDKDCVALEVQYHKGCYNQYTRFM + +TRPEKPEKEQNEPTFDVGYKILCERIIRQRLLVNQEVLRMGQLRMAFIELVKANEGLDAS + +NYSIKNLERSRRADAGSQRIQIFDPDQRTPTQWKKFLSEGTKKEALAEFLYVAWKNADLT + +IVGKNLCLYIAHTNQCHCVTVKEGVQSVRVVEDLLLFLHAQHAAREHKAVIIKSSDTDVA + +VIAVSVQTDLPCSLYVFTGTGNRTRIIDITKVSSANKI + + + +>/tmp/03_03_20-07:33:11.fasta|GENSCAN_predicted_peptide_3|394_aa + +MQRGRAAGINGIPPEFYVAFWEQLSPFFLHMINFSIEKGGFLRDVNTALISLLMKKDKNP + +TDCSSYRPLSLLNSDVKIFAKLLPLRLEPHMPELVSSDQTGFIKSRTAADNIRRLLHIIA + +AAPGCETPMSVLSLDAMKAFDRLEWSFLWSVLEAMGFISTFIGMVKVLYSNPSARVLTGQ + +TFSSLFPVSRSSRQGCPLSPALFVLSLEPLAQAVRLSNLVLPICICDTQHKLSLFADDVI + +VFLEHPTQSLPHFLSICEEFRKLSGFKMNWSKSALMHLNDNARKSVTPVNIPLVGQLKYL + +GIEVFPSLNQIVKHNYSLAFTNVLKDMDRWISLPMSIQARISIIKMNGLPRIHFVSSMVP + +LPPPSDYWIKISAQGVRCPLAKPFTHSPYSKTKX diff --git a/src/bd_rhapsody/bd_rhapsody_make_reference/config.vsh.yaml b/src/bd_rhapsody/bd_rhapsody_make_reference/config.vsh.yaml index e596bf06..dc71262b 100644 --- a/src/bd_rhapsody/bd_rhapsody_make_reference/config.vsh.yaml +++ b/src/bd_rhapsody/bd_rhapsody_make_reference/config.vsh.yaml @@ -116,12 +116,11 @@ argument_groups: resources: - type: python_script path: script.py - - path: make_rhap_reference_2.2.1_nodocker.cwl test_resources: - type: bash_script path: test.sh - - path: test_data + - path: ../test_data requirements: commands: [ "cwl-runner" ] @@ -131,12 +130,19 @@ engines: image: bdgenomics/rhapsody:2.2.1 setup: - type: apt - packages: [procps] + packages: [procps, git] - type: python packages: [cwlref-runner, cwl-runner] - type: docker run: | - echo "bdgenomics/rhapsody: 2.2.1" > /var/software_versions.txt + mkdir /var/bd_rhapsody_cwl && \ + cd /var/bd_rhapsody_cwl && \ + git clone https://bitbucket.org/CRSwDev/cwl.git . && \ + git checkout 8feeace1141b24749ea6003f8e6ad6d3ad5232de + - type: docker + run: + - VERSION=$(ls -v /var/bd_rhapsody_cwl | grep '^v' | sed 's#v##' | tail -1) + - 'echo "bdgenomics/rhapsody: \"$VERSION\"" > /var/software_versions.txt' runners: - type: executable diff --git a/src/bd_rhapsody/bd_rhapsody_make_reference/make_rhap_reference_2.2.1_nodocker.cwl b/src/bd_rhapsody/bd_rhapsody_make_reference/make_rhap_reference_2.2.1_nodocker.cwl deleted file mode 100644 index fead2c02..00000000 --- a/src/bd_rhapsody/bd_rhapsody_make_reference/make_rhap_reference_2.2.1_nodocker.cwl +++ /dev/null @@ -1,115 +0,0 @@ -requirements: - InlineJavascriptRequirement: {} -class: CommandLineTool -label: Reference Files Generator for BD Rhapsodyâ„¢ Sequencing Analysis Pipeline -cwlVersion: v1.2 -doc: >- - The Reference Files Generator creates an archive containing Genome Index and Transcriptome annotation files needed for the BD Rhapsodyâ„¢ Sequencing Analysis Pipeline. The app takes as input one or more FASTA and GTF files and produces a compressed archive in the form of a tar.gz file. The archive contains:\n - STAR index\n - Filtered GTF file - - -baseCommand: run_reference_generator.sh -inputs: - Genome_fasta: - type: File[] - label: Reference Genome - doc: |- - Reference genome file in FASTA format. The BD Rhapsodyâ„¢ Sequencing Analysis Pipeline uses GRCh38 for Human and GRCm39 for Mouse. - inputBinding: - prefix: --reference-genome - shellQuote: false - Gtf: - type: File[] - label: Transcript Annotations - doc: |- - Transcript annotation files in GTF format. The BD Rhapsodyâ„¢ Sequencing Analysis Pipeline uses Gencode v42 for Human and M31 for Mouse. - inputBinding: - prefix: --gtf - shellQuote: false - Extra_sequences: - type: File[]? - label: Extra Sequences - doc: |- - Additional sequences in FASTA format to use when building the STAR index. (E.g. phiX genome) - inputBinding: - prefix: --extra-sequences - shellQuote: false - Mitochondrial_Contigs: - type: string[]? - default: ["chrM", "chrMT", "M", "MT"] - label: Mitochondrial Contig Names - doc: |- - Names of the Mitochondrial contigs in the provided Reference Genome. Fragments originating from contigs other than these are identified as 'nuclear fragments' in the ATACseq analysis pipeline. - inputBinding: - prefix: --mitochondrial-contigs - shellQuote: false - Filtering_off: - type: boolean? - label: Turn off filtering - doc: |- - By default the input Transcript Annotation files are filtered based on the gene_type/gene_biotype attribute. Only features having the following attribute values are are kept: - - protein_coding - - lncRNA (lincRNA and antisense for Gencode < v31/M22/Ensembl97) - - IG_LV_gene - - IG_V_gene - - IG_V_pseudogene - - IG_D_gene - - IG_J_gene - - IG_J_pseudogene - - IG_C_gene - - IG_C_pseudogene - - TR_V_gene - - TR_V_pseudogene - - TR_D_gene - - TR_J_gene - - TR_J_pseudogene - - TR_C_gene - If you have already pre-filtered the input Annotation files and/or wish to turn-off the filtering, please set this option to True. - inputBinding: - prefix: --filtering-off - shellQuote: false - WTA_Only: - type: boolean? - label: WTA only index - doc: Build a WTA only index, otherwise builds a WTA + ATAC index. - inputBinding: - prefix: --wta-only-index - shellQuote: false - Archive_prefix: - type: string? - label: Archive Prefix - doc: |- - A prefix for naming the compressed archive file containing the Reference genome index and annotation files. The default value is constructed based on the input Reference files. - inputBinding: - prefix: --archive-prefix - shellQuote: false - Extra_STAR_params: - type: string? - label: Extra STAR Params - doc: |- - Additional parameters to pass to STAR when building the genome index. Specify exactly like how you would on the command line. - Example: - --limitGenomeGenerateRAM 48000 --genomeSAindexNbases 11 - inputBinding: - prefix: --extra-star-params - shellQuote: true - - Maximum_threads: - type: int? - label: Maximum Number of Threads - doc: |- - The maximum number of threads to use in the pipeline. By default, all available cores are used. - inputBinding: - prefix: --maximum-threads - shellQuote: false - -outputs: - - Archive: - type: File - doc: |- - A Compressed archive containing the Reference Genome Index and annotation GTF files. This archive is meant to be used as an input in the BD Rhapsodyâ„¢ Sequencing Analysis Pipeline. - id: Reference_Archive - label: Reference Files Archive - outputBinding: - glob: '*.tar.gz' - diff --git a/src/bd_rhapsody/bd_rhapsody_make_reference/script.py b/src/bd_rhapsody/bd_rhapsody_make_reference/script.py index ca635508..dcbfe933 100644 --- a/src/bd_rhapsody/bd_rhapsody_make_reference/script.py +++ b/src/bd_rhapsody/bd_rhapsody_make_reference/script.py @@ -83,21 +83,21 @@ def generate_config(par: dict[str, Any], meta, config) -> str: for config_key, arg_type, par_value in config_key_value_pairs: if arg_type == "file": - str = strip_margin(f"""\ + content = strip_margin(f"""\ |{config_key}: |""") if isinstance(par_value, list): for file in par_value: - str += strip_margin(f"""\ + content += strip_margin(f"""\ | - class: File | location: "{file}" |""") else: - str += strip_margin(f"""\ + content += strip_margin(f"""\ | class: File | location: "{par_value}" |""") - content_list.append(str) + content_list.append(content) else: content_list.append(strip_margin(f"""\ |{config_key}: {par_value} @@ -108,9 +108,9 @@ def generate_config(par: dict[str, Any], meta, config) -> str: def get_cwl_file(meta: dict[str, Any]) -> str: # create cwl file (if need be) - cwl_file=os.path.join(meta["resources_dir"], "make_rhap_reference_2.2.1_nodocker.cwl") + cwl_file="/var/bd_rhapsody_cwl/v2.2.1/Extra_Utilities/make_rhap_reference_2.2.1.cwl" - return cwl_file + return os.path.abspath(cwl_file) def main(par: dict[str, Any], meta: dict[str, Any]): config = read_config(meta["config"]) diff --git a/src/bd_rhapsody/bd_rhapsody_make_reference/test_data/script.sh b/src/bd_rhapsody/bd_rhapsody_make_reference/test_data/script.sh deleted file mode 100644 index 8d468064..00000000 --- a/src/bd_rhapsody/bd_rhapsody_make_reference/test_data/script.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -TMP_DIR=/tmp/bd_rhapsody_make_reference -OUT_DIR=src/bd_rhapsody/bd_rhapsody_make_reference/test_data - -# check if seqkit is installed -if ! command -v seqkit &> /dev/null; then - echo "seqkit could not be found" - exit 1 -fi - -# create temporary directory and clean up on exit -mkdir -p $TMP_DIR -function clean_up { - rm -rf "$TMP_DIR" -} -trap clean_up EXIT - -# fetch reference -ORIG_FA=$TMP_DIR/reference.fa.gz -if [ ! -f $ORIG_FA ]; then - wget https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_41/GRCh38.primary_assembly.genome.fa.gz \ - -O $ORIG_FA -fi - -ORIG_GTF=$TMP_DIR/reference.gtf.gz -if [ ! -f $ORIG_GTF ]; then - wget https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_41/gencode.v41.annotation.gtf.gz \ - -O $ORIG_GTF -fi - -# create small reference -START=30000 -END=31500 -CHR=chr1 - -# subset to small region -seqkit grep -r -p "^$CHR\$" "$ORIG_FA" | \ - seqkit subseq -r "$START:$END" > $OUT_DIR/reference_small.fa - -zcat "$ORIG_GTF" | \ - awk -v FS='\t' -v OFS='\t' " - \$1 == \"$CHR\" && \$4 >= $START && \$5 <= $END { - \$4 = \$4 - $START + 1; - \$5 = \$5 - $START + 1; - print; - }" > $OUT_DIR/reference_small.gtf diff --git a/src/bd_rhapsody/bd_rhapsody_sequence_analysis/_process_cwl.R b/src/bd_rhapsody/bd_rhapsody_sequence_analysis/_process_cwl.R new file mode 100644 index 00000000..e33b8ea7 --- /dev/null +++ b/src/bd_rhapsody/bd_rhapsody_sequence_analysis/_process_cwl.R @@ -0,0 +1,116 @@ +# Extract arguments from CWL file and write them to arguments.yaml +# +# This script: +# - reads the CWL file +# - extracts the main workflow arguments +# - compares cwl arguments to viash config arguments +# - writes the arguments to arguments.yaml +# +# It can be used to update the arguments in the viash config after an +# update to the CWL file has been made. +# +# Dependencies: tidyverse, jsonlite, yaml, dynutils +# +# Install dependencies: +# ```R +# install.packages(c("tidyverse", "jsonlite", "yaml", "dynutils")) +# ``` +# +# Usage: +# ```bash +# Rscript src/bd_rhapsody/bd_rhapsody_sequence_analysis/_process_cwl.R +# ``` + +library(tidyverse) + +# fetch and read cwl file +lines <- read_lines("https://bitbucket.org/CRSwDev/cwl/raw/8feeace1141b24749ea6003f8e6ad6d3ad5232de/v2.2.1/rhapsody_pipeline_2.2.1.cwl") +cwl_header <- lines[[1]] +cwl_obj <- jsonlite::fromJSON(lines[-1], simplifyVector = FALSE) + +# detect main workflow arguments +gr <- dynutils::list_as_tibble(cwl_obj$`$graph`) + +gr %>% print(n = 100) + +main <- gr %>% filter(gr$id == "#main") + +main_inputs <- main$inputs[[1]] + +input_ids <- main_inputs %>% map_chr("id") %>% gsub("^#main/", "", .) + +# check whether in config +config <- yaml::read_yaml("src/bd_rhapsody/bd_rhapsody_sequence_analysis/config.vsh.yaml") +config$all_arguments <- config$argument_groups %>% map("arguments") %>% list_flatten() +arg_names <- config$all_arguments %>% map_chr("name") %>% gsub("^--", "", .) + +# arguments in cwl but not in config +setdiff(tolower(input_ids), arg_names) + +# arguments in config but not in cwl +setdiff(arg_names, tolower(input_ids)) + +# create arguments from main_inputs +arguments <- map(main_inputs, function(main_input) { + input_id <- main_input$id %>% gsub("^#main/", "", .) + input_type <- main_input$type[[2]] + + if (is.list(input_type) && input_type$type == "array") { + multiple <- TRUE + input_type <- input_type$items + } else { + multiple <- FALSE + } + + if (is.list(input_type) && input_type$type == "enum") { + choices <- input_type$symbols %>% + gsub(paste0(input_type$name, "/"), "", .) + input_type <- "enum" + } else { + choices <- NULL + } + + description <- + if (is.null(main_input$label)) { + main_input$doc + } else if (is.null(main_input$doc)) { + main_input$label + } else { + paste0(main_input$label, ". ", main_input$doc) + } + + type_map <- c( + "float" = "double", + "int" = "integer", + "string" = "string", + "boolean" = "boolean", + "File" = "file", + "enum" = "string" + ) + + out <- list( + name = paste0("--", tolower(input_id)), + type = type_map[input_type], + # TODO: use summary when viash 0.9 is released + # summary = main_input$doc, + # description = main_input$doc, + description = description, + multiple = multiple, + choices = choices, + info = list( + config_key = input_id + ) + ) + + out[!sapply(out, is.null)] +}) + + + +yaml::write_yaml( + arguments, + "src/bd_rhapsody/bd_rhapsody_sequence_analysis/arguments.yaml", + handlers = list( + logical = yaml::verbatim_logical + ) +) diff --git a/src/bd_rhapsody/bd_rhapsody_sequence_analysis/config.vsh.yaml b/src/bd_rhapsody/bd_rhapsody_sequence_analysis/config.vsh.yaml new file mode 100644 index 00000000..eb3eaf38 --- /dev/null +++ b/src/bd_rhapsody/bd_rhapsody_sequence_analysis/config.vsh.yaml @@ -0,0 +1,661 @@ +name: bd_rhapsody_sequence_analysis +namespace: bd_rhapsody +description: | + BD Rhapsody Sequence Analysis CWL pipeline v2.2. + + This pipeline performs analysis of single-cell multiomic sequence read (FASTQ) data. The supported + sequencing libraries are those generated by the BD Rhapsody™ assay kits, including: Whole Transcriptome + mRNA (WTA), Targeted mRNA, AbSeq Antibody-Oligonucleotides (ABC), Single-Cell Multiplexing (SMK), + TCR/BCR (VDJ), and ATAC-Seq. +keywords: [rna-seq, single-cell, multiomic, atac-seq, targeted, abseq, tcr, bcr] +links: + repository: https://bitbucket.org/CRSwDev/cwl/src/master/v2.2.1 + documentation: https://bd-rhapsody-bioinfo-docs.genomics.bd.com +license: Unknown +authors: + - __merge__: /src/_authors/robrecht_cannoodt.yaml + roles: [ author, maintainer ] + - __merge__: /src/_authors/weiwei_schultz.yaml + roles: [ contributor ] + +argument_groups: + - name: Inputs + arguments: + - name: "--reads" + type: file + description: | + Reads (optional) - Path to your FASTQ.GZ formatted read files from libraries that may include: + + - WTA mRNA + - Targeted mRNA + - AbSeq + - Sample Multiplexing + - VDJ + + You may specify as many R1/R2 read pairs as you want. + required: false + multiple: true + example: + - WTALibrary_S1_L001_R1_001.fastq.gz + - WTALibrary_S1_L001_R2_001.fastq.gz + info: + config_key: Reads + - name: "--reads_atac" + type: file + description: | + Path to your FASTQ.GZ formatted read files from ATAC-Seq libraries. + You may specify as many R1/R2/I2 files as you want. + required: false + multiple: true + example: + - ATACLibrary_S2_L001_R1_001.fastq.gz + - ATACLibrary_S2_L001_R2_001.fastq.gz + - ATACLibrary_S2_L001_I2_001.fastq.gz + info: + config_key: Reads_ATAC + - name: References + description: | + Assay type will be inferred from the provided reference(s). + Do not provide both reference_archive and targeted_reference at the same time. + + Valid reference input combinations: + - reference_archive: WTA only + - reference_archive & abseq_reference: WTA + AbSeq + - reference_archive & supplemental_reference: WTA + extra transgenes + - reference_archive & abseq_reference & supplemental_reference: WTA + AbSeq + extra transgenes + - reference_archive: WTA + ATAC or ATAC only + - reference_archive & supplemental_reference: WTA + ATAC + extra transgenes + - targeted_reference: Targeted only + - targeted_reference & abseq_reference: Targeted + AbSeq + - abseq_reference: AbSeq only + + The reference_archive can be generated with the bd_rhapsody_make_reference component. + Alternatively, BD also provides standard references which can be downloaded from these locations: + + - Human: https://bd-rhapsody-public.s3.amazonaws.com/Rhapsody-WTA/Pipeline-version2.x_WTA_references/RhapRef_Human_WTA_2023-02.tar.gz + - Mouse: https://bd-rhapsody-public.s3.amazonaws.com/Rhapsody-WTA/Pipeline-version2.x_WTA_references/RhapRef_Mouse_WTA_2023-02.tar.gz + arguments: + - name: "--reference_archive" + type: file + description: | + Path to Rhapsody WTA Reference in the tar.gz format. + + Structure of the reference archive: + + - `BD_Rhapsody_Reference_Files/`: top level folder + - `star_index/`: sub-folder containing STAR index, that is files created with `STAR --runMode genomeGenerate` + - GTF for gene-transcript-annotation e.g. "gencode.v43.primary_assembly.annotation.gtf" + example: "RhapRef_Human_WTA_2023-02.tar.gz" + required: false + info: + config_key: Reference_Archive + - name: "--targeted_reference" + type: file + description: | + Path to the targeted reference file in FASTA format. + example: "BD_Rhapsody_Immune_Response_Panel_Hs.fasta" + multiple: true + info: + config_key: Targeted_Reference + - name: "--abseq_reference" + type: file + description: Path to the AbSeq reference file in FASTA format. Only needed if BD AbSeq Ab-Oligos are used. + example: "AbSeq_reference.fasta" + multiple: true + info: + config_key: AbSeq_Reference + - name: "--supplemental_reference" + type: file + alternatives: [-s] + description: Path to the supplemental reference file in FASTA format. Only needed if there are additional transgene sequences to be aligned against in a WTA assay experiment. + example: "supplemental_reference.fasta" + multiple: true + info: + config_key: Supplemental_Reference + - name: Outputs + description: Outputs for all pipeline runs + # based on https://bd-rhapsody-bioinfo-docs.genomics.bd.com/outputs/top_outputs.html + arguments: + - name: "--output_dir" + type: file + direction: output + alternatives: [-o] + description: "The unprocessed output directory containing all the outputs from the pipeline." + required: true + example: output_dir/ + - name: "--output_seurat" + type: file + direction: output + description: "Single-cell analysis tool inputs. Seurat (.rds) input file containing RSEC molecules data table and all cell annotation metadata." + example: output_seurat.rds + required: false + info: + template: "[sample_name]_Seurat.rds" + - name: "--output_mudata" + type: file + direction: output + description: "Single-cell analysis tool inputs. Scanpy / Muon input file containing RSEC molecules data table and all cell annotation metadata." + example: output_mudata.h5mu + required: false + info: + template: "[sample_name].h5mu" + - name: "--metrics_summary" + type: file + direction: output + description: "Metrics Summary. Report containing sequencing, molecules, and cell metrics." + example: metrics_summary.csv + required: false + info: + template: "[sample_name]_Metrics_Summary.csv" + - name: "--pipeline_report" + type: file + direction: output + description: "Pipeline Report. Summary report containing the results from the sequencing analysis pipeline run." + example: pipeline_report.html + required: false + info: + template: "[sample_name]_Pipeline_Report.html" + - name: "--rsec_mols_per_cell" + type: file + direction: output + description: "Molecules per bioproduct per cell bassed on RSEC" + example: RSEC_MolsPerCell_MEX.zip + required: false + info: + template: "[sample_name]_RSEC_MolsPerCell_MEX.zip" + - name: "--dbec_mols_per_cell" + type: file + direction: output + description: "Molecules per bioproduct per cell bassed on DBEC. DBEC data table is only output if the experiment includes targeted mRNA or AbSeq bioproducts." + example: DBEC_MolsPerCell_MEX.zip + required: false + info: + template: "[sample_name]_DBEC_MolsPerCell_MEX.zip" + - name: "--rsec_mols_per_cell_unfiltered" + type: file + direction: output + description: "Unfiltered tables containing all cell labels with ≥10 reads." + example: RSEC_MolsPerCell_Unfiltered_MEX.zip + required: false + info: + template: "[sample_name]_RSEC_MolsPerCell_Unfiltered_MEX.zip" + - name: "--bam" + type: file + direction: output + description: "Alignment file of R2 with associated R1 annotations for Bioproduct." + example: BioProduct.bam + required: false + info: + template: "[sample_name]_Bioproduct.bam" + - name: "--bam_index" + type: file + direction: output + description: "Index file for the alignment file." + example: BioProduct.bam.bai + required: false + info: + template: "[sample_name]_Bioproduct.bam.bai" + - name: "--bioproduct_stats" + type: file + direction: output + description: "Bioproduct Stats. Metrics from RSEC and DBEC Unique Molecular Identifier adjustment algorithms on a per-bioproduct basis." + example: Bioproduct_Stats.csv + required: false + info: + template: "[sample_name]_Bioproduct_Stats.csv" + - name: "--dimred_tsne" + type: file + direction: output + description: "t-SNE dimensionality reduction coordinates per cell index" + example: tSNE_coordinates.csv + required: false + info: + template: "[sample_name]_(assay)_tSNE_coordinates.csv" + - name: "--dimred_umap" + type: file + direction: output + description: "UMAP dimensionality reduction coordinates per cell index" + example: UMAP_coordinates.csv + required: false + info: + template: "[sample_name]_(assay)_UMAP_coordinates.csv" + - name: "--immune_cell_classification" + type: file + direction: output + description: "Immune Cell Classification. Cell type classification based on the expression of immune cell markers." + example: Immune_Cell_Classification.csv + required: false + info: + template: "[sample_name]_(assay)_cell_type_experimental.csv" + - name: Multiplex outputs + description: Outputs when multiplex option is selected + arguments: + - name: "--sample_tag_metrics" + type: file + direction: output + description: "Sample Tag Metrics. Metrics from the sample determination algorithm." + example: Sample_Tag_Metrics.csv + required: false + info: + template: "[sample_name]_Sample_Tag_Metrics.csv" + - name: "--sample_tag_calls" + type: file + direction: output + description: "Sample Tag Calls. Assigned Sample Tag for each putative cell" + example: Sample_Tag_Calls.csv + required: false + info: + template: "[sample_name]_Sample_Tag_Calls.csv" + - name: "--sample_tag_counts" + type: file + direction: output + description: "Sample Tag Counts. Separate data tables and metric summary for cells assigned to each sample tag. Note: For putative cells that could not be assigned a specific Sample Tag, a Multiplet_and_Undetermined.zip file is also output." + example: Sample_Tag1.zip + required: false + multiple: true + info: + template: "[sample_name]_Sample_Tag[number].zip" + - name: "--sample_tag_counts_unassigned" + type: file + direction: output + description: "Sample Tag Counts Unassigned. Data table and metric summary for cells that could not be assigned a specific Sample Tag." + example: Multiplet_and_Undetermined.zip + required: false + info: + template: "[sample_name]_Multiplet_and_Undetermined.zip" + - name: VDJ Outputs + description: Outputs when VDJ option selected + arguments: + - name: "--vdj_metrics" + type: file + direction: output + description: "VDJ Metrics. Overall metrics from the VDJ analysis." + example: VDJ_Metrics.csv + required: false + info: + template: "[sample_name]_VDJ_Metrics.csv" + - name: "--vdj_per_cell" + type: file + direction: output + description: "VDJ Per Cell. Cell specific read and molecule counts, VDJ gene segments, CDR3 sequences, paired chains, and cell type." + example: VDJ_perCell.csv + required: false + info: + template: "[sample_name]_VDJ_perCell.csv" + - name: "--vdj_per_cell_uncorrected" + type: file + direction: output + description: "VDJ Per Cell Uncorrected. Cell specific read and molecule counts, VDJ gene segments, CDR3 sequences, paired chains, and cell type." + example: VDJ_perCell_uncorrected.csv + required: false + info: + template: "[sample_name]_VDJ_perCell_uncorrected.csv" + - name: "--vdj_dominant_contigs" + type: file + direction: output + description: "VDJ Dominant Contigs. Dominant contig for each cell label chain type combination (putative cells only)." + example: VDJ_Dominant_Contigs_AIRR.csv + required: false + info: + template: "[sample_name]_VDJ_Dominant_Contigs_AIRR.csv" + - name: "--vdj_unfiltered_contigs" + type: file + direction: output + description: "VDJ Unfiltered Contigs. All contigs that were assembled and annotated successfully (all cells)." + example: VDJ_Unfiltered_Contigs_AIRR.csv + required: false + info: + template: "[sample_name]_VDJ_Unfiltered_Contigs_AIRR.csv" + - name: "ATAC-Seq outputs" + description: Outputs when ATAC-Seq option selected + arguments: + - name: "--atac_metrics" + type: file + direction: output + description: "ATAC Metrics. Overall metrics from the ATAC-Seq analysis." + example: ATAC_Metrics.csv + required: false + info: + template: "[sample_name]_ATAC_Metrics.csv" + - name: "--atac_metrics_json" + type: file + direction: output + description: "ATAC Metrics JSON. Overall metrics from the ATAC-Seq analysis in JSON format." + example: ATAC_Metrics.json + required: false + info: + template: "[sample_name]_ATAC_Metrics.json" + - name: "--atac_fragments" + type: file + direction: output + description: "ATAC Fragments. Chromosomal location, cell index, and read support for each fragment detected" + example: ATAC_Fragments.bed.gz + required: false + info: + template: "[sample_name]_ATAC_Fragments.bed.gz" + - name: "--atac_fragments_index" + type: file + direction: output + description: "Index of ATAC Fragments." + example: ATAC_Fragments.bed.gz.tbi + required: false + info: + template: "[sample_name]_ATAC_Fragments.bed.gz.tbi" + - name: "--atac_transposase_sites" + type: file + direction: output + description: "ATAC Transposase Sites. Chromosomal location, cell index, and read support for each transposase site detected" + example: ATAC_Transposase_Sites.bed.gz + required: false + info: + template: "[sample_name]_ATAC_Transposase_Sites.bed.gz" + - name: "--atac_transposase_sites_index" + type: file + direction: output + description: "Index of ATAC Transposase Sites." + example: ATAC_Transposase_Sites.bed.gz.tbi + required: false + info: + template: "[sample_name]_ATAC_Transposase_Sites.bed.gz.tbi" + - name: "--atac_peaks" + type: file + direction: output + description: "ATAC Peaks. Peak regions of transposase activity" + example: ATAC_Peaks.bed.gz + required: false + info: + template: "[sample_name]_ATAC_Peaks.bed.gz" + - name: "--atac_peaks_index" + type: file + direction: output + description: "Index of ATAC Peaks." + example: ATAC_Peaks.bed.gz.tbi + required: false + info: + template: "[sample_name]_ATAC_Peaks.bed.gz.tbi" + - name: "--atac_peak_annotation" + type: file + direction: output + description: "ATAC Peak Annotation. Estimated annotation of peak-to-gene connections" + example: peak_annotation.tsv.gz + required: false + info: + template: "[sample_name]_peak_annotation.tsv.gz" + - name: "--atac_cell_by_peak" + type: file + direction: output + description: "ATAC Cell by Peak. Peak regions of transposase activity per cell" + example: ATAC_Cell_by_Peak_MEX.zip + required: false + info: + template: "[sample_name]_ATAC_Cell_by_Peak_MEX.zip" + - name: "--atac_cell_by_peak_unfiltered" + type: file + direction: output + description: "ATAC Cell by Peak Unfiltered. Unfiltered file containing all cell labels with >=1 transposase sites in peaks." + example: ATAC_Cell_by_Peak_Unfiltered_MEX.zip + required: false + info: + template: "[sample_name]_ATAC_Cell_by_Peak_Unfiltered_MEX.zip" + - name: "--atac_bam" + type: file + direction: output + description: "ATAC BAM. Alignment file for R1 and R2 with associated I2 annotations for ATAC-Seq. Only output if the BAM generation flag is set to true." + example: ATAC.bam + required: false + info: + template: "[sample_name]_ATAC.bam" + - name: "--atac_bam_index" + type: file + direction: output + description: "Index of ATAC BAM." + example: ATAC.bam.bai + required: false + info: + template: "[sample_name]_ATAC.bam.bai" + - name: AbSeq Cell Calling outputs + description: Outputs when Cell Calling Abseq is selected + arguments: + - name: "--protein_aggregates_experimental" + type: file + direction: output + description: "Protein Aggregates Experimental" + example: Protein_Aggregates_Experimental.csv + required: false + info: + template: "[sample_name]_Protein_Aggregates_Experimental.csv" + - name: Putative Cell Calling Settings + arguments: + - name: "--cell_calling_data" + type: string + description: | + Specify the dataset to be used for putative cell calling: mRNA, AbSeq, ATAC, mRNA_and_ATAC + + For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above. + + For putative cell calling using an ATAC dataset, please provide a WTA+ATAC-Seq Reference_Archive file above. + + The default data for putative cell calling, will be determined the following way: + + - If mRNA Reads and ATAC Reads exist: mRNA_and_ATAC + - If only ATAC Reads exist: ATAC + - Otherwise: mRNA + choices: [mRNA, AbSeq, ATAC, mRNA_and_ATAC] + example: mRNA + info: + config_key: Cell_Calling_Data + - name: "--cell_calling_bioproduct_algorithm" + type: string + description: | + Specify the bioproduct algorithm to be used for putative cell calling: Basic or Refined + + By default, the Basic algorithm will be used for putative cell calling. + choices: [Basic, Refined] + example: Basic + info: + config_key: Cell_Calling_Bioproduct_Algorithm + - name: "--cell_calling_atac_algorithm" + type: string + description: | + Specify the ATAC-seq algorithm to be used for putative cell calling: Basic or Refined + + By default, the Basic algorithm will be used for putative cell calling. + choices: [Basic, Refined] + example: Basic + info: + config_key: Cell_Calling_ATAC_Algorithm + - name: "--exact_cell_count" + type: integer + description: | + Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count + example: 10000 + min: 1 + info: + config_key: Exact_Cell_Count + - name: "--expected_cell_count" + type: integer + description: | + Guide the basic putative cell calling algorithm by providing an estimate of the number of cells expected. Usually this can be the number of cells loaded into the Rhapsody cartridge. If there are multiple inflection points on the second derivative cumulative curve, this will ensure the one selected is near the expected. + example: 20000 + min: 1 + info: + config_key: Expected_Cell_Count + - name: Intronic Reads Settings + arguments: + - name: --exclude_intronic_reads + type: boolean + description: | + By default, the flag is false, and reads aligned to exons and introns are considered and represented in molecule counts. When the flag is set to true, intronic reads will be excluded. + The value can be true or false. + example: false + info: + config_key: Exclude_Intronic_Reads + - name: Multiplex Settings + arguments: + - name: "--sample_tags_version" + type: string + description: | + Specify the version of the Sample Tags used in the run: + + * If Sample Tag Multiplexing was done, specify the appropriate version: human, mouse, flex, nuclei_includes_mrna, nuclei_atac_only + * If this is an SMK + Nuclei mRNA run or an SMK + Multiomic ATAC-Seq (WTA+ATAC-Seq) run (and not an SMK + ATAC-Seq only run), choose the "nuclei_includes_mrna" option. + * If this is an SMK + ATAC-Seq only run (and not SMK + Multiomic ATAC-Seq (WTA+ATAC-Seq)), choose the "nuclei_atac_only" option. + choices: [human, mouse, flex, nuclei_includes_mrna, nuclei_atac_only] + example: human + info: + config_key: Sample_Tags_Version + - name: "--tag_names" + type: string + description: | + Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv + Do not use the special characters: &, (), [], {}, <>, ?, | + multiple: true + example: [4-mySample, 9-myOtherSample, 6-alsoThisSample] + info: + config_key: Tag_Names + - name: VDJ arguments + arguments: + - name: "--vdj_version" + type: string + description: | + If VDJ was done, specify the appropriate option: human, mouse, humanBCR, humanTCR, mouseBCR, mouseTCR + choices: [human, mouse, humanBCR, humanTCR, mouseBCR, mouseTCR] + example: human + info: + config_key: VDJ_Version + - name: ATAC options + arguments: + - name: "--predefined_atac_peaks" + type: file + description: An optional BED file containing pre-established chromatin accessibility peak regions for generating the ATAC cell-by-peak matrix. + example: predefined_peaks.bed + info: + config_key: Predefined_ATAC_Peaks + - name: Additional options + arguments: + - name: "--run_name" + type: string + description: | + Specify a run name to use as the output file base name. Use only letters, numbers, or hyphens. Do not use special characters or spaces. + default: sample + info: + config_key: Run_Name + - name: "--generate_bam" + type: boolean + description: | + Specify whether to create the BAM file output + default: false + info: + config_key: Generate_Bam + - name: "--long_reads" + type: boolean + description: | + Use STARlong (default: undefined - i.e. autodetects based on read lengths) - Specify if the STARlong aligner should be used instead of STAR. Set to true if the reads are longer than 650bp. + info: + config_key: Long_Reads + - name: Advanced options + description: | + NOTE: Only change these if you are really sure about what you are doing + arguments: + - name: "--custom_star_params" + type: string + description: | + Modify STAR alignment parameters - Set this parameter to fully override default STAR mapping parameters used in the pipeline. + For reference this is the default that is used: + + Short Reads: `--outFilterScoreMinOverLread 0 --outFilterMatchNminOverLread 0 --outFilterMultimapScoreRange 0 --clip3pAdapterSeq AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA --seedSearchStartLmax 50 --outFilterMatchNmin 25 --limitOutSJcollapsed 2000000` + Long Reads: Same as Short Reads + `--seedPerReadNmax 10000` + + This applies to fastqs provided in the Reads user input + Do NOT set any non-mapping related params like `--genomeDir`, `--outSAMtype`, `--outSAMunmapped`, `--readFilesIn`, `--runThreadN`, etc. + We use STAR version 2.7.10b + example: "--alignIntronMax 6000 --outFilterScoreMinOverLread 0.1 --limitOutSJcollapsed 2000000" + info: + config_key: Custom_STAR_Params + - name: "--custom_bwa_mem2_params" + type: string + description: | + Modify bwa-mem2 alignment parameters - Set this parameter to fully override bwa-mem2 mapping parameters used in the pipeline + The pipeline does not specify any custom mapping params to bwa-mem2 so program default values are used + This applies to fastqs provided in the Reads_ATAC user input + Do NOT set any non-mapping related params like `-C`, `-t`, etc. + We use bwa-mem2 version 2.2.1 + example: "-k 16 -w 200 -r" + info: + config_key: Custom_bwa_mem2_Params + - name: CWL-runner arguments + arguments: + - name: "--parallel" + type: boolean + description: "Run jobs in parallel." + default: true + - name: "--timestamps" + type: boolean_true + description: "Add timestamps to the errors, warnings, and notifications." + - name: Undocumented arguments + arguments: + - name: --abseq_umi + type: integer + multiple: false + info: + config_key: AbSeq_UMI + - name: --target_analysis + type: boolean + multiple: false + info: + config_key: Target_analysis + - name: --vdj_jgene_evalue + type: double + description: | + e-value threshold for J gene. The e-value threshold for J gene call by IgBlast/PyIR, default is set as 0.001 + multiple: false + info: + config_key: VDJ_JGene_Evalue + - name: --vdj_vgene_evalue + type: double + description: | + e-value threshold for V gene. The e-value threshold for V gene call by IgBlast/PyIR, default is set as 0.001 + multiple: false + info: + config_key: VDJ_VGene_Evalue + - name: --write_filtered_reads + type: boolean + multiple: false + info: + config_key: Write_Filtered_Reads +resources: + - type: python_script + path: script.py +test_resources: + - type: python_script + path: test.py + - path: ../test_data + - path: ../helpers + +requirements: + commands: [ "cwl-runner" ] + +engines: + - type: docker + image: bdgenomics/rhapsody:2.2.1 + setup: + - type: apt + packages: [procps, git] + - type: python + packages: [cwlref-runner, cwl-runner] + - type: docker + run: | + mkdir /var/bd_rhapsody_cwl && \ + cd /var/bd_rhapsody_cwl && \ + git clone https://bitbucket.org/CRSwDev/cwl.git . && \ + git checkout 8feeace1141b24749ea6003f8e6ad6d3ad5232de + - type: docker + run: + - VERSION=$(ls -v /var/bd_rhapsody_cwl | grep '^v' | sed 's#v##' | tail -1) + - 'echo "bdgenomics/rhapsody: \"$VERSION\"" > /var/software_versions.txt' + test_setup: + - type: python + packages: [biopython, gffutils] +runners: + - type: executable + - type: nextflow diff --git a/src/bd_rhapsody/bd_rhapsody_sequence_analysis/help.txt b/src/bd_rhapsody/bd_rhapsody_sequence_analysis/help.txt new file mode 100644 index 00000000..618faa3e --- /dev/null +++ b/src/bd_rhapsody/bd_rhapsody_sequence_analysis/help.txt @@ -0,0 +1,167 @@ +```bash +cwl-runner src/bd_rhapsody/bd_rhapsody_sequence_analysis/rhapsody_pipeline_2.2.1_nodocker.cwl --help +``` + +usage: src/bd_rhapsody/bd_rhapsody_sequence_analysis/rhapsody_pipeline_2.2.1_nodocker.cwl + [-h] [--AbSeq_Reference ABSEQ_REFERENCE] [--AbSeq_UMI ABSEQ_UMI] + [--Cell_Calling_ATAC_Algorithm CELL_CALLING_ATAC_ALGORITHM] + [--Cell_Calling_Bioproduct_Algorithm CELL_CALLING_BIOPRODUCT_ALGORITHM] + [--Cell_Calling_Data CELL_CALLING_DATA] + [--Custom_STAR_Params CUSTOM_STAR_PARAMS] + [--Custom_bwa_mem2_Params CUSTOM_BWA_MEM2_PARAMS] + [--Exact_Cell_Count EXACT_CELL_COUNT] [--Exclude_Intronic_Reads] + [--Expected_Cell_Count EXPECTED_CELL_COUNT] [--Generate_Bam] + [--Long_Reads] [--Maximum_Threads MAXIMUM_THREADS] + [--Predefined_ATAC_Peaks PREDEFINED_ATAC_PEAKS] [--Reads READS] + [--Reads_ATAC READS_ATAC] [--Reference_Archive REFERENCE_ARCHIVE] + [--Run_Name RUN_NAME] [--Sample_Tags_Version SAMPLE_TAGS_VERSION] + [--Supplemental_Reference SUPPLEMENTAL_REFERENCE] + [--Tag_Names TAG_NAMES] [--Target_analysis] + [--Targeted_Reference TARGETED_REFERENCE] + [--VDJ_JGene_Evalue VDJ_JGENE_EVALUE] + [--VDJ_VGene_Evalue VDJ_VGENE_EVALUE] [--VDJ_Version VDJ_VERSION] + [--Write_Filtered_Reads] + [job_order] + +The BD Rhapsody™ assays are used to create sequencing libraries from single +cell transcriptomes. After sequencing, the analysis pipeline takes the FASTQ +files and a reference file for gene alignment. The pipeline generates +molecular counts per cell, read counts per cell, metrics, and an alignment +file. + +positional arguments: + job_order Job input json file + +options: + -h, --help show this help message and exit + --AbSeq_Reference ABSEQ_REFERENCE + AbSeq Reference + --AbSeq_UMI ABSEQ_UMI + --Cell_Calling_ATAC_Algorithm CELL_CALLING_ATAC_ALGORITHM + Specify the ATAC algorithm to be used for ATAC + putative cell calling. The Basic algorithm is the + default. + --Cell_Calling_Bioproduct_Algorithm CELL_CALLING_BIOPRODUCT_ALGORITHM + Specify the bioproduct algorithm to be used for + mRNA/AbSeq putative cell calling. The Basic algorithm + is the default. + --Cell_Calling_Data CELL_CALLING_DATA + Specify the data to be used for putative cell calling. + The default data for putative cell calling will be + determined the following way: - If mRNA and ATAC Reads + exist, mRNA_and_ATAC is the default. - If only ATAC + Reads exist, ATAC is the default. - Otherwise, mRNA is + the default. + --Custom_STAR_Params CUSTOM_STAR_PARAMS + Allows you to specify custom STAR aligner mapping + parameters. Only the mapping parameters you provide + here will be used with STAR, meaning that you must + provide the complete list of parameters that you want + to take effect. For reference, the parameters used by + default in the pipeline are: 1. Short Reads: + --outFilterScoreMinOverLread 0 + --outFilterMatchNminOverLread 0 + --outFilterMultimapScoreRange 0 --clip3pAdapterSeq + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + --seedSearchStartLmax 50 --outFilterMatchNmin 25 + --limitOutSJcollapsed 2000000 2. Long Reads: Same + options as short reads + --seedPerReadNmax 10000 + Example input: --alignIntronMax 500000 + --outFilterScoreMinOverLread 0 --limitOutSJcollapsed + 2000000 Important: 1. This applies to fastqs provided + in the Reads user input 2. Please do not specify any + non-mapping related params like: --runThreadN, + --genomeDir --outSAMtype, etc. 3. Please only use + params supported by STAR version 2.7.10b + --Custom_bwa_mem2_Params CUSTOM_BWA_MEM2_PARAMS + Allows you to specify custom bwa-mem2 mapping + parameters. Only the mapping parameters you provide + here will be used with bwa-mem2, meaning that you must + provide the complete list of parameters that you want + to take effect. The pipeline uses program default + mapping parameters. Example input: -k 15 -w 200 -r 2 + Important: 1. This applies to fastqs provided in the + Reads_ATAC user input 2. Please do not specify any + non-mapping related params like: -C, -t, etc. 3. + Please only use params supported by bwa-mem2 version + 2.2.1 + --Exact_Cell_Count EXACT_CELL_COUNT + Set a specific number (>=1) of cells as putative, + based on those with the highest error-corrected read + count + --Exclude_Intronic_Reads + By default, reads aligned to exons and introns are + considered and represented in molecule counts. + Including intronic reads may increase sensitivity, + resulting in an increase in molecule counts and the + number of genes per cell for both cellular and nuclei + samples. Intronic reads may indicate unspliced mRNAs + and are also useful, for example, in the study of + nuclei and RNA velocity. When set to true, intronic + reads will be excluded. + --Expected_Cell_Count EXPECTED_CELL_COUNT + Optional. Guide the basic putative cell calling + algorithm by providing an estimate of the number of + cells expected. Usually this can be the number of + cells loaded into the Rhapsody cartridge. If there are + multiple inflection points on the second derivative + cumulative curve, this will ensure the one selected is + near the expected. + --Generate_Bam Default: false. A Bam read alignment file contains + reads from all the input libraries, but creating it + can consume a lot of compute and disk resources. By + setting this field to true, the Bam file will be + created. This option is shared for both Bioproduct and + ATAC libraries. + --Long_Reads By default, we detect if there are any reads longer + than 650bp and then flag QualCLAlign to use STARlong + instead of STAR. This flag can be explicitly set if it + is known in advance that there are reads longer than + 650bp. + --Maximum_Threads MAXIMUM_THREADS + The maximum number of threads to use in the pipeline. + By default, all available cores are used. + --Predefined_ATAC_Peaks PREDEFINED_ATAC_PEAKS + An optional BED file containing pre-established + chromatin accessibility peak regions for generating + the ATAC cell-by-peak matrix. Only applies to ATAC + assays. + --Reads READS FASTQ files from libraries that may include WTA mRNA, + Targeted mRNA, AbSeq, Sample Multiplexing, and related + technologies + --Reads_ATAC READS_ATAC + FASTQ files from libraries generated using the ATAC + assay protocol. Each lane of a library is expected to + have 3 FASTQs - R1, R2 and I1/I2, where the index read + contains the Cell Barcode and UMI sequence. Only + applies to ATAC assays. + --Reference_Archive REFERENCE_ARCHIVE + Reference Files Archive + --Run_Name RUN_NAME This is a name for output files, for example + Experiment1_Metrics_Summary.csv. Default if left empty + is to name run based on a library. Any non-alpha + numeric characters will be changed to a hyphen. + --Sample_Tags_Version SAMPLE_TAGS_VERSION + The sample multiplexing kit version. This option + should only be set for a multiplexed experiment. + --Supplemental_Reference SUPPLEMENTAL_REFERENCE + Supplemental Reference + --Tag_Names TAG_NAMES + Specify the Sample Tag number followed by - (hyphen) + and a sample name to appear in the output files. For + example: 4-Ramos. Should be alpha numeric, with + - + and _ allowed. Any special characters: &, (), [], {}, + <>, ?, | will be corrected to underscores. + --Target_analysis + --Targeted_Reference TARGETED_REFERENCE + Targeted Reference + --VDJ_JGene_Evalue VDJ_JGENE_EVALUE + The e-value threshold for J gene call by IgBlast/PyIR, + default is set as 0.001 + --VDJ_VGene_Evalue VDJ_VGENE_EVALUE + The e-value threshold for V gene call by IgBlast/PyIR, + default is set as 0.001 + --VDJ_Version VDJ_VERSION + The VDJ species and chain types. This option should + only be set for VDJ experiment. + --Write_Filtered_Reads diff --git a/src/bd_rhapsody/bd_rhapsody_sequence_analysis/pipeline_inputs_template_2.2.1.yaml b/src/bd_rhapsody/bd_rhapsody_sequence_analysis/pipeline_inputs_template_2.2.1.yaml new file mode 100644 index 00000000..19728a57 --- /dev/null +++ b/src/bd_rhapsody/bd_rhapsody_sequence_analysis/pipeline_inputs_template_2.2.1.yaml @@ -0,0 +1,203 @@ +#!/usr/bin/env cwl-runner + +cwl:tool: rhapsody + +# This is a template YML file used to specify the inputs for a BD Rhapsody Sequence Analysis pipeline run. +# See the BD Rhapsody Sequence Analysis Pipeline User Guide for more details. Enter the following information: + + +## Reads (optional) - Path to your FASTQ.GZ formatted read files from libraries that may include: +# - WTA mRNA +# - Targeted mRNA +# - AbSeq +# - Sample Multiplexing +# - VDJ +# You may specify as many R1/R2 read pairs as you want. +Reads: + + - class: File + location: "test/WTALibrary_S1_L001_R1_001.fastq.gz" + + - class: File + location: "test/WTALibrary_S1_L001_R2_001.fastq.gz" + +## Reads_ATAC (optional) - Path to your FASTQ.GZ formatted read files from ATAC-Seq libraries. +## You may specify as many R1/R2/I2 files as you want. +Reads_ATAC: + + - class: File + location: "test/ATACLibrary_S2_L001_R1_001.fastq.gz" + + - class: File + location: "test/ATACLibrary_S2_L001_R2_001.fastq.gz" + + - class: File + location: "test/ATACLibrary_S2_L001_I2_001.fastq.gz" + + +## Assay type will be inferred from the provided reference(s) +## Do not provide both Reference_Archive and Targeted_Reference at the same time +## +## Valid reference input combinations: +## WTA Reference_Archive (WTA only) +## WTA Reference_Archive + AbSeq_Reference (WTA + AbSeq) +## WTA Reference_Archive + Supplemental_Reference (WTA + extra transgenes) +## WTA Reference_Archive + AbSeq_Reference + Supplemental_Reference (WTA + AbSeq + extra transgenes) +## WTA+ATAC-Seq Reference_Archive (WTA + ATAC, ATAC only) +## WTA+ATAC-Seq Reference_Archive + Supplemental_Reference (WTA + ATAC + extra transgenes) +## Targeted_Reference (Targeted only) +## Targeted_Reference + AbSeq_Reference (Targeted + AbSeq) +## AbSeq_Reference (AbSeq only) + +## See the BD Rhapsody Sequence Analysis Pipeline User Guide for instructions on how to: +## - Obtain a pre-built Rhapsody Reference file +## - Create a custom Rhapsody Reference file + +## WTA Reference_Archive (required for WTA mRNA assay) - Path to Rhapsody WTA Reference in the tar.gz format. +## +## --Structure of reference archive-- +## BD_Rhapsody_Reference_Files/ # top level folder +## star_index/ # sub-folder containing STAR index +## [files created with STAR --runMode genomeGenerate] +## [GTF for gene-transcript-annotation e.g. "gencode.v43.primary_assembly.annotation.gtf"] +## +## WTA+ATAC-Seq Reference_Archive (required for ATAC-Seq or Multiomic ATAC-Seq (WTA+ATAC-Seq) assays) - Path to Rhapsody WTA+ATAC-Seq Reference in the tar.gz format. +## +## --Structure of reference archive-- +## BD_Rhapsody_Reference_Files/ # top level folder +## star_index/ # sub-folder containing STAR index +## [files created with STAR --runMode genomeGenerate] +## [GTF for gene-transcript-annotation e.g. "gencode.v43.primary_assembly.annotation.gtf"] +## +## mitochondrial_contigs.txt # mitochondrial contigs in the reference genome - one contig name per line. e.g. chrMT or chrM, etc. +## +## bwa-mem2_index/ # sub-folder containing bwa-mem2 index +## [files created with bwa-mem2 index] +## +Reference_Archive: + class: File + location: "test/RhapRef_Human_WTA_2023-02.tar.gz" +# location: "test/RhapRef_Human_WTA-ATAC_2023-08.tar.gz" + +## Targeted_Reference (required for Targeted mRNA assay) - Path to the targeted reference file in FASTA format. +#Targeted_Reference: +# - class: File +# location: "test/BD_Rhapsody_Immune_Response_Panel_Hs.fasta" + +## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format. Only needed if BD AbSeq Ab-Oligos are used. +## For putative cell calling using an AbSeq dataset, please provide an AbSeq reference fasta file as the AbSeq_Reference. +#AbSeq_Reference: +# - class: File +# location: "test/AbSeq_reference.fasta" + +## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format. Only needed if there are additional transgene sequences to be aligned against in a WTA assay experiment +#Supplemental_Reference: +# - class: File +# location: "test/supplemental_reference.fasta" + +#################################### +## Putative Cell Calling Settings ## +#################################### + +## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA, AbSeq, ATAC, mRNA_and_ATAC +## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above. +## For putative cell calling using an ATAC dataset, please provide a WTA+ATAC-Seq Reference_Archive file above. +## The default data for putative cell calling, will be determined the following way: +## If mRNA Reads and ATAC Reads exist: +## Cell_Calling_Data: mRNA_and_ATAC +## If only ATAC Reads exist: +## Cell_Calling_Data: ATAC +## Otherwise: +## Cell_Calling_Data: mRNA +#Cell_Calling_Data: mRNA + +## Putative cell calling bioproduct algorithm (optional) - Specify the bioproduct algorithm to be used for putative cell calling: Basic or Refined +## By default, the Basic algorithm will be used for putative cell calling. +#Cell_Calling_Bioproduct_Algorithm: Basic + +## Putative cell calling ATAC algorithm (optional) - Specify the ATAC-seq algorithm to be used for putative cell calling: Basic or Refined +## By default, the Basic algorithm will be used for putative cell calling. +#Cell_Calling_ATAC_Algorithm: Basic + +## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count +#Exact_Cell_Count: 10000 + +## Expected Cell Count (optional) - Guide the basic putative cell calling algorithm by providing an estimate of the number of cells expected. Usually this can be the number of cells loaded into the Rhapsody cartridge. If there are multiple inflection points on the second derivative cumulative curve, this will ensure the one selected is near the expected. +#Expected_Cell_Count: 20000 + + +#################################### +## Intronic Reads Settings ## +#################################### + +## Exclude_Intronic_Reads (optional) +## By default, the flag is false, and reads aligned to exons and introns are considered and represented in molecule counts. When the flag is set to true, intronic reads will be excluded. +## The value can be true or false. +#Exclude_Intronic_Reads: true + +####################### +## Multiplex options ## +####################### + +## Sample Tags Version (optional) - If Sample Tag Multiplexing was done, specify the appropriate version: human, mouse, flex, nuclei_includes_mrna, nuclei_atac_only +## If this is an SMK + Nuclei mRNA run or an SMK + Multiomic ATAC-Seq (WTA+ATAC-Seq) run (and not an SMK + ATAC-Seq only run), choose the "nuclei_includes_mrna" option. +## If this is an SMK + ATAC-Seq only run (and not SMK + Multiomic ATAC-Seq (WTA+ATAC-Seq)), choose the "nuclei_atac_only" option. +#Sample_Tags_Version: human + +## Tag_Names (optional) - Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv +# Do not use the special characters: &, (), [], {}, <>, ?, | +#Tag_Names: [4-mySample, 9-myOtherSample, 6-alsoThisSample] + +################ +## VDJ option ## +################ + +## VDJ Version (optional) - If VDJ was done, specify the appropriate option: human, mouse, humanBCR, humanTCR, mouseBCR, mouseTCR +#VDJ_Version: human + +################## +## ATAC options ## +################## + +## Predefined ATAC Peaks - An optional BED file containing pre-established chromatin accessibility peak regions for generating the ATAC cell-by-peak matrix. +#Predefined_ATAC_Peaks: +# class: File +# location: "path/predefined_peaks.bed" + +######################## +## Additional Options ## +######################## + +## Run Name (optional)- Specify a run name to use as the output file base name. Use only letters, numbers, or hyphens. Do not use special characters or spaces. +#Run_Name: my-experiment + +## Generate Bam (optional, default: false) - Specify whether to create the BAM file output +#Generate_Bam: true + +## Maximum_Threads (integer, optional, default: [use all cores of CPU]) - Set the maximum number of threads to use in the read processing steps of the pipeline: QualCLAlign, AlignmentAnalysis, VDJ assembly +#Maximum_Threads: 16 + +## Use STARlong (optional, default: "auto" - i.e. autodetects based on read lengths) - Specify if the STARlong aligner should be used instead of STAR. Set to true if the reads are longer than 650bp. +## The value can be true or false. +#Long_Reads: true + +######################## +## Advanced Options ## +######################## +## NOTE: Only change these if you are really sure about what you are doing + +## Modify STAR alignment parameters - Set this parameter to fully override default STAR mapping parameters used in the pipeline. +## For reference this is the default that is used: +## Short Reads: --outFilterScoreMinOverLread 0 --outFilterMatchNminOverLread 0 --outFilterMultimapScoreRange 0 --clip3pAdapterSeq AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA --seedSearchStartLmax 50 --outFilterMatchNmin 25 --limitOutSJcollapsed 2000000 +## Long Reads: Same as Short Reads + --seedPerReadNmax 10000 +## This applies to fastqs provided in the Reads user input +## Do NOT set any non-mapping related params like --genomeDir, --outSAMtype, --outSAMunmapped, --readFilesIn, --runThreadN, etc. +## We use STAR version 2.7.10b +#Custom_STAR_Params: --alignIntronMax 6000 --outFilterScoreMinOverLread 0.1 --limitOutSJcollapsed 2000000 + +## Modify bwa-mem2 alignment parameters - Set this parameter to fully override bwa-mem2 mapping parameters used in the pipeline +## The pipeline does not specify any custom mapping params to bwa-mem2 so program default values are used +## This applies to fastqs provided in the Reads_ATAC user input +## Do NOT set any non-mapping related params like -C, -t, etc. +## We use bwa-mem2 version 2.2.1 +#Custom_bwa_mem2_Params: -k 16 -w 200 -r diff --git a/src/bd_rhapsody/bd_rhapsody_sequence_analysis/script.py b/src/bd_rhapsody/bd_rhapsody_sequence_analysis/script.py new file mode 100644 index 00000000..cbddf6bf --- /dev/null +++ b/src/bd_rhapsody/bd_rhapsody_sequence_analysis/script.py @@ -0,0 +1,243 @@ +import os +import re +import subprocess +import tempfile +from typing import Any +import yaml +import shutil +import glob + +## VIASH START +par = { + 'reads': [ + 'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz', + 'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz' + ], + 'reads_atac': None, + 'reference_archive': "resources_test/reference_gencodev41_chr1/reference_bd_rhapsody.tar.gz", + 'targeted_reference': [], + 'abseq_reference': [], + 'supplemental_reference': [], + 'output': 'output_dir', + 'cell_calling_data': None, + 'cell_calling_bioproduct_algorithm': None, + 'cell_calling_atac_algorithm': None, + 'exact_cell_count': None, + 'expected_cell_count': None, + 'exclude_intronic_reads': None, + 'sample_tags_version': None, + 'tag_names': [], + 'vdj_version': None, + 'predefined_atac_peaks': None, + 'run_name': "sample", + 'generate_bam': None, + 'alignment_star_params': None, + 'alignment_bwa_mem2_params': None, + 'parallel': True, + 'timestamps': False, + 'dryrun': False +} +meta = { + 'config': "target/nextflow/bd_rhaspody/bd_rhaspody_sequence_analysis/.config.vsh.yaml", + 'resources_dir': os.path.abspath('src/bd_rhaspody/bd_rhaspody_sequence_analysis'), + 'temp_dir': os.getenv("VIASH_TEMP"), + 'memory_mb': None, + 'cpus': None +} +## VIASH END + +def clean_arg(argument): + argument["clean_name"] = argument["name"].lstrip("-") + return argument + +def read_config(path: str) -> dict[str, Any]: + with open(path, 'r') as f: + config = yaml.safe_load(f) + + config["arguments"] = [ + clean_arg(arg) + for grp in config["argument_groups"] + for arg in grp["arguments"] + ] + + return config + +def strip_margin(text: str) -> str: + return re.sub('(\n?)[ \t]*\|', '\\1', text) + +def process_params(par: dict[str, Any], config, temp_dir: str) -> str: + # check input parameters + assert par["reads"] or par["reads_atac"], "Pass at least one set of inputs to --reads or --reads_atac." + + # output to temp dir if output_dir was not passed + if not par["output_dir"]: + par["output_dir"] = os.path.join(temp_dir, "output") + + # checking sample prefix + if par["run_name"] and re.match("[^A-Za-z0-9]", par["run_name"]): + print("--run_name should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.", flush=True) + par["run_name"] = re.sub("[^A-Za-z0-9\\-]", "-", par["run_name"]) + + # make paths absolute + for argument in config["arguments"]: + arg_clean_name = argument["clean_name"] + if not par[arg_clean_name] or not argument["type"] == "file": + continue + par_value = par[arg_clean_name] + if isinstance(par_value, list): + par_value_absolute = list(map(os.path.abspath, par_value)) + else: + par_value_absolute = os.path.abspath(par_value) + par[arg_clean_name] = par_value_absolute + + return par + +def generate_config(par: dict[str, Any], config) -> str: + content_list = [strip_margin(f"""\ + |#!/usr/bin/env cwl-runner + | + |cwl:tool: rhapsody + |""")] + + for argument in config["arguments"]: + arg_clean_name = argument["clean_name"] + arg_par_value = par[arg_clean_name] + arg_info = argument.get("info") or {} # Note: .info might be None + config_key = arg_info.get("config_key") + if arg_par_value and config_key: + + if argument["type"] == "file": + content = strip_margin(f"""\ + |{config_key}: + |""") + if isinstance(arg_par_value, list): + for file in arg_par_value: + content += strip_margin(f"""\ + | - class: File + | location: "{file}" + |""") + else: + content += strip_margin(f"""\ + | class: File + | location: "{arg_par_value}" + |""") + content_list.append(content) + else: + content_list.append(strip_margin(f"""\ + |{config_key}: {arg_par_value} + |""")) + + ## Write config to file + return ''.join(content_list) + +def generate_config_file(par: dict[str, Any], config: dict[str, Any], temp_dir: str) -> str: + config_file = os.path.join(temp_dir, "config.yml") + config_content = generate_config(par, config) + with open(config_file, "w") as f: + f.write(config_content) + return config_file + +def generate_cwl_file(meta: dict[str, Any], dir: str) -> str: + # create cwl file (if need be) + # orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_pipeline_2.2.1_nodocker.cwl") + orig_cwl_file="/var/bd_rhapsody_cwl/v2.2.1/rhapsody_pipeline_2.2.1.cwl" + + if not meta["memory_mb"] and not meta["cpus"]: + return os.path.abspath(orig_cwl_file) + + # Inject computational requirements into pipeline + cwl_file = os.path.join(dir, "pipeline.cwl") + + # Read in the file + with open(orig_cwl_file, 'r') as file : + cwl_data = file.read() + + # Inject computational requirements into pipeline + if meta["memory_mb"]: + memory = int(meta["memory_mb"]) - 2000 # keep 2gb for OS + cwl_data = re.sub('"ramMin": [^\n]*[^,](,?)\n', f'"ramMin": {memory}\\1\n', cwl_data) + if meta["cpus"]: + cwl_data = re.sub('"coresMin": [^\n]*[^,](,?)\n', f'"coresMin": {meta["cpus"]}\\1\n', cwl_data) + + # Write the file out again + with open(cwl_file, 'w') as file: + file.write(cwl_data) + + return os.path.abspath(cwl_file) + +def copy_outputs(par: dict[str, Any], config: dict[str, Any]): + for arg in config["arguments"]: + par_value = par[arg["clean_name"]] + if par_value and arg["type"] == "file" and arg["direction"] == "output": + # example template: '[sample_name]_(assay)_cell_type_experimental.csv' + template = (arg.get("info") or {}).get("template") # Note: .info might be None + if template: + template_glob = template\ + .replace("[sample_name]", par["run_name"])\ + .replace("(assay)", "*")\ + .replace("[number]", "*") + files = glob.glob(os.path.join(par["output_dir"], template_glob)) + if not files and arg["required"]: + raise ValueError(f"Expected output file '{template_glob}' not found.") + elif len(files) > 1 and not arg["multiple"]: + raise ValueError(f"Expected single output file '{template_glob}', but found multiple.") + + if not arg["multiple"]: + shutil.copy(files[0], par_value) + else: + # replace '*' in par_value with index + for i, file in enumerate(files): + shutil.copy(file, par_value.replace("*", str(i))) + + +def main(par: dict[str, Any], meta: dict[str, Any], temp_dir: str): + config = read_config(meta["config"]) + + # Preprocess params + par = process_params(par, config, temp_dir) + + ## Process parameters + cmd = [ + "cwl-runner", + "--no-container", + "--preserve-entire-environment", + "--outdir", par["output_dir"], + ] + + if par["parallel"]: + cmd.append("--parallel") + + if par["timestamps"]: + cmd.append("--timestamps") + + # Create cwl file (if need be) + cwl_file = generate_cwl_file(meta, temp_dir) + cmd.append(cwl_file) + + # Create params file + config_file = generate_config_file(par, config, temp_dir) + cmd.append(config_file) + + # keep environment variables but set TMPDIR to temp_dir + env = dict(os.environ) + env["TMPDIR"] = temp_dir + + # Create output dir if not exists + if not os.path.exists(par["output_dir"]): + os.makedirs(par["output_dir"]) + + # Run command + print("> " + ' '.join(cmd), flush=True) + _ = subprocess.run( + cmd, + cwd=os.path.dirname(config_file), + env=env, + check=True + ) + + # Copy outputs + copy_outputs(par, config) + +if __name__ == "__main__": + with tempfile.TemporaryDirectory(prefix="cwl-bd_rhapsody-", dir=meta["temp_dir"]) as temp_dir: + main(par, meta, temp_dir) diff --git a/src/bd_rhapsody/bd_rhapsody_sequence_analysis/test.py b/src/bd_rhapsody/bd_rhapsody_sequence_analysis/test.py new file mode 100644 index 00000000..aed8e80b --- /dev/null +++ b/src/bd_rhapsody/bd_rhapsody_sequence_analysis/test.py @@ -0,0 +1,494 @@ +import subprocess +import gzip +from pathlib import Path +from typing import Tuple +import numpy as np +import random +import mudata as md + +## VIASH START +meta = { + "name": "bd_rhapsody_sequence_analysis", + "executable": "target/docker/bd_rhapsody/bd_rhapsody_sequence_analysis/bd_rhapsody_sequence_analysis", + "resources_dir": "src/bd_rhapsody", + "cpus": 8, + "memory_mb": 4096, +} +## VIASH END + +import sys +sys.path.append(meta["resources_dir"]) + +from helpers.rhapsody_cell_label import index_to_sequence + +meta["executable"] = Path(meta["executable"]) +meta["resources_dir"] = Path(meta["resources_dir"]) + +######################################################################################### + +# Generate index +print("> Generate index", flush=True) +# cwl_file = meta["resources_dir"] / "bd_rhapsody_make_reference.cwl" +cwl_file = "/var/bd_rhapsody_cwl/v2.2.1/Extra_Utilities/make_rhap_reference_2.2.1.cwl" +reference_small_gtf = meta["resources_dir"] / "test_data" / "reference_small.gtf" +reference_small_fa = meta["resources_dir"] / "test_data" / "reference_small.fa" +bdabseq_panel_fa = meta["resources_dir"] / "test_data" / "BDAbSeq_ImmuneDiscoveryPanel.fasta" +sampletagsequences_fa = meta["resources_dir"] / "test_data" / "SampleTagSequences_HomoSapiens_ver1.fasta" + +config_file = Path("reference_config.yml") +reference_file = Path("Rhap_reference.tar.gz") + +subprocess.run([ + "cwl-runner", + "--no-container", + "--preserve-entire-environment", + "--outdir", + ".", + str(cwl_file), + "--Genome_fasta", + str(reference_small_fa), + "--Gtf", + str(reference_small_gtf), + "--Extra_STAR_params", + "--genomeSAindexNbases 4" +]) + +######################################################################################### +# Load reference in memory + +from Bio import SeqIO +import gffutils + +# Load FASTA sequence +with open(str(reference_small_fa), "r") as handle: + reference_fasta_dict = SeqIO.to_dict(SeqIO.parse(handle, "fasta")) +with open(str(bdabseq_panel_fa), "r") as handle: + bdabseq_panel_fasta_dict = SeqIO.to_dict(SeqIO.parse(handle, "fasta")) +with open(str(sampletagsequences_fa), "r") as handle: + sampletagsequences_fasta_dict = SeqIO.to_dict(SeqIO.parse(handle, "fasta")) + +# create in memory db +reference_gtf_db = gffutils.create_db( + str(reference_small_gtf), + dbfn=":memory:", + force=True, + keep_order=True, + merge_strategy="merge", + sort_attribute_values=True, + disable_infer_transcripts=True, + disable_infer_genes=True +) + +############################################# +# TODO: move helper functions to separate helper file + + +def generate_bd_read_metadata( + instrument_id: str = "A00226", + run_id: str = "970", + flowcell_id: str = "H5FGVMXY", + lane: int = 1, + tile: int = 1101, + x: int = 1000, + y: int = 1000, + illumina_flag: str = "1:N:0", + sample_id: str = "CAGAGAGG", +) -> str: + """ + Generate a FASTQ metadata line for a BD Rhapsody FASTQ file. + + Args: + instrument_id: The instrument ID. + run_id: The run ID. + flowcell_id: The flowcell ID. + lane: The lane number. + tile: The tile number. Between 1101 and 1112 in the used example data. + x: The x-coordinate. Between 1000 and 32967 in the used example data. + y: The y-coordinate. Between 1000 and 37059 in the used example data. + illumina_flag: The Illumina flag. Either 1:N:0 or 2:N:0 in the used example data. + sample_id: The sample ID. + """ + # format: @A00226:970:H5FGVDMXY:1:1101:2645:1000 2:N:0:CAGAGAGG + return f"@{instrument_id}:{run_id}:{flowcell_id}:{lane}:{tile}:{x}:{y} {illumina_flag}:{sample_id}" + + +def generate_bd_wta_transcript( + transcript_length: int = 42, +) -> str: + """ + Generate a WTA transcript from a given GTF and FASTA file. + """ + + # Randomly select a gene + gene = random.choice(list(reference_gtf_db.features_of_type("gene"))) + + # Find all exons within the gene + exons = list(reference_gtf_db.children(gene, featuretype="exon", order_by="start")) + + # Calculate total exon length + total_exon_length = sum(exon.end - exon.start + 1 for exon in exons) + + # If total exon length is less than desired transcript length, use it as is + max_transcript_length = min(total_exon_length, transcript_length) + + # Build the WTA transcript sequence + sequence = "" + for exon in exons: + exon_seq = str(reference_fasta_dict[exon.seqid].seq[exon.start - 1 : exon.end]) + sequence += exon_seq + + # Break if desired length is reached + if len(sequence) >= max_transcript_length: + sequence = sequence[:max_transcript_length] + break + + # add padding if need be + if len(sequence) < max_transcript_length: + sequence += "N" * (max_transcript_length - len(sequence)) + + return sequence + + +def generate_bd_wta_read( + cell_index: int = 0, + bead_version: str = "EnhV2", + umi_length: int = 14, + transcript_length: int = 42, +) -> Tuple[str, str]: + """ + Generate a BD Rhapsody WTA read pair for a given cell index. + + Args: + cell_index: The cell index to generate reads for. + bead_version: The bead version to use for generating the cell label. + umi_length: The length of the UMI to generate. + transcript_length: The length of the transcript to generate + + Returns: + A tuple of two strings, the first string being the R1 read and the second string being the R2 read. + + More info: + + See structure of reads: + - https://bd-rhapsody-bioinfo-docs.genomics.bd.com/steps/top_steps.html + - https://bd-rhapsody-bioinfo-docs.genomics.bd.com/steps/steps_cell_label.html + - https://scomix.bd.com/hc/en-us/articles/360057714812-All-FAQ + R1 is Cell Label + UMI + PolyT -> 60 bp + actually, CLS1 + "GTGA" + CLS2 + "GACA" + CLS3 + UMI + R2 is the actual read -> 42 bp + + Example R1 + CLS1 Link CLS2 Link CLS3 UMI + AAAATCCTGT GTGA AACCAAAGT GACA GATAGAGGAG CGCATGTTTATAAC + """ + + # generate metadata + per_row = np.floor((32967 - 1000) / 9) + per_col = np.floor((37059 - 1000) / 9) + + assert cell_index >= 0 and cell_index < per_row * per_col, f"cell_index must be between 0 and {per_row} * {per_col}" + x = 1000 + (cell_index % per_row) * 9 + y = 1000 + (cell_index // per_row) * 9 + instrument_id = "A00226" + run_id = "970" + flowcell_id = "H5FGVMXY" + meta_r1 = generate_bd_read_metadata(instrument_id=instrument_id, run_id=run_id, flowcell_id=flowcell_id, x=x, y=y, illumina_flag="1:N:0") + meta_r2 = generate_bd_read_metadata(instrument_id=instrument_id, run_id=run_id, flowcell_id=flowcell_id, x=x, y=y, illumina_flag="2:N:0") + + # generate r1 (cls1 + link + cls2 + link + cls3 + umi) + assert cell_index >= 0 and cell_index < 384 * 384 * 384 + cell_label = index_to_sequence(cell_index + 1, bead_version=bead_version) + # sample random umi + umi = "".join(random.choices("ACGT", k=umi_length)) + quality_r1 = "I" * (len(cell_label) + len(umi)) + r1 = f"{meta_r1}\n{cell_label}{umi}\n+\n{quality_r1}\n" + + # generate r2 by extracting sequence from fasta and gtf + wta_transcript = generate_bd_wta_transcript(transcript_length=transcript_length) + quality_r2 = "I" * transcript_length + r2 = f"{meta_r2}\n{wta_transcript}\n+\n{quality_r2}\n" + + return r1, r2 + +def generate_bd_wta_fastq_files( + num_cells: int = 100, + num_reads_per_cell: int = 1000, +) -> Tuple[str, str]: + """ + Generate BD Rhapsody WTA FASTQ files for a given number of cells and transcripts per cell. + + Args: + num_cells: The number of cells to generate + num_reads_per_cell: The number of reads to generate per cell + + Returns: + A tuple of two strings, the first string being the R1 reads and the second string being the R2 reads. + """ + r1_reads = "" + r2_reads = "" + for cell_index in range(num_cells): + for _ in range(num_reads_per_cell): + r1, r2 = generate_bd_wta_read(cell_index) + r1_reads += r1 + r2_reads += r2 + + return r1_reads, r2_reads + +def generate_bd_abc_read( + cell_index: int = 0, + bead_version: str = "EnhV2", + umi_length: int = 14, + transcript_length: int = 72, +) -> Tuple[str, str]: + """ + Generate a BD Rhapsody ABC read pair for a given cell index. + + Args: + cell_index: The cell index to generate reads for. + bead_version: The bead version to use for generating the cell label. + umi_length: The length of the UMI to generate. + transcript_length: The length of the transcript to generate + + Returns: + A tuple of two strings, the first string being the R1 read and the second string being the R2 read. + """ + # generate metadata + per_row = np.floor((32967 - 1000) / 9) + per_col = np.floor((37059 - 1000) / 9) + + assert cell_index >= 0 and cell_index < per_row * per_col, f"cell_index must be between 0 and {per_row} * {per_col}" + x = 1000 + (cell_index % per_row) * 9 + y = 1000 + (cell_index // per_row) * 9 + instrument_id = "A01604" + run_id = "19" + flowcell_id = "HMKLYDRXY" + meta_r1 = generate_bd_read_metadata(instrument_id=instrument_id, run_id=run_id, flowcell_id=flowcell_id, x=x, y=y, illumina_flag="1:N:0") + meta_r2 = generate_bd_read_metadata(instrument_id=instrument_id, run_id=run_id, flowcell_id=flowcell_id, x=x, y=y, illumina_flag="2:N:0") + + # generate r1 (cls1 + link + cls2 + link + cls3 + umi) + assert cell_index >= 0 and cell_index < 384 * 384 * 384 + cell_label = index_to_sequence(cell_index + 1, bead_version=bead_version) + # sample random umi + umi = "".join(random.choices("ACGT", k=umi_length)) + quality_r1 = "I" * (len(cell_label) + len(umi)) + r1 = f"{meta_r1}\n{cell_label}{umi}\n+\n{quality_r1}\n" + + # generate r2 by sampling sequence from bdabseq_panel_fa + abseq_seq = str(random.choice(list(bdabseq_panel_fasta_dict.values())).seq) + abc_suffix = "AAAAAAAAAAAAAAAAAAAAAAA" + abc_data = abseq_seq[:transcript_length - len(abc_suffix) - 1] + abc_prefix = "N" + "".join(random.choices("ACGT", k=transcript_length - len(abc_data) - len(abc_suffix) - 1)) + + abc_transcript = f"{abc_prefix}{abc_data}{abc_suffix}" + + quality_r2 = "#" + "I" * (len(abc_transcript) - 1) + r2 = f"{meta_r2}\n{abc_transcript}\n+\n{quality_r2}\n" + + return r1, r2 + +def generate_bd_abc_fastq_files( + num_cells: int = 100, + num_reads_per_cell: int = 1000, +) -> Tuple[str, str]: + """ + Generate BD Rhapsody ABC FASTQ files for a given number of cells and transcripts per cell. + + Args: + num_cells: The number of cells to generate + num_reads_per_cell: The number of reads to generate per cell + + Returns: + A tuple of two strings, the first string being the R1 reads and the second string being the R2 reads. + """ + r1_reads = "" + r2_reads = "" + for cell_index in range(num_cells): + for _ in range(num_reads_per_cell): + r1, r2 = generate_bd_abc_read(cell_index) + r1_reads += r1 + r2_reads += r2 + + return r1_reads, r2_reads + +def generate_bd_smk_read( + cell_index: int = 0, + bead_version: str = "EnhV2", + umi_length: int = 14, + transcript_length: int = 72, + num_sample_tags: int = 3, +): + """ + Generate a BD Rhapsody SMK read pair for a given cell index. + + Args: + cell_index: The cell index to generate reads for. + bead_version: The bead version to use for generating the cell label. + umi_length: The length of the UMI to generate. + transcript_length: The length of the transcript to generate + num_sample_tags: The number of sample tags to use + + Returns: + A tuple of two strings, the first string being the R1 read and the second string being the R2 read. + """ + # generate metadata + per_row = np.floor((32967 - 1000) / 9) + per_col = np.floor((37059 - 1000) / 9) + + assert cell_index >= 0 and cell_index < per_row * per_col, f"cell_index must be between 0 and {per_row} * {per_col}" + x = 1000 + (cell_index % per_row) * 9 + y = 1000 + (cell_index // per_row) * 9 + instrument_id = "A00226" + run_id = "970" + flowcell_id = "H5FGVDMXY" + + meta_r1 = generate_bd_read_metadata(instrument_id=instrument_id, run_id=run_id, flowcell_id=flowcell_id, x=x, y=y, illumina_flag="1:N:0") + meta_r2 = generate_bd_read_metadata(instrument_id=instrument_id, run_id=run_id, flowcell_id=flowcell_id, x=x, y=y, illumina_flag="2:N:0") + + # generate r1 (cls1 + link + cls2 + link + cls3 + umi) + assert cell_index >= 0 and cell_index < 384 * 384 * 384 + cell_label = index_to_sequence(cell_index + 1, bead_version=bead_version) + # sample random umi + umi = "".join(random.choices("ACGT", k=umi_length)) + quality_r1 = "I" * (len(cell_label) + len(umi)) + r1 = f"{meta_r1}\n{cell_label}{umi}\n+\n{quality_r1}\n" + + # generate r2 by selecting the cell_index %% num_sample_tags sample tags + sampletag_index = cell_index % num_sample_tags + sampletag_seq = str(list(sampletagsequences_fasta_dict.values())[sampletag_index].seq) + smk_data = sampletag_seq[:transcript_length] + smk_suffix = "A" * (transcript_length - len(smk_data)) + quality_r2 = "I" * len(smk_data) + "#" * len(smk_suffix) + r2 = f"{meta_r2}\n{smk_data}{smk_suffix}\n+\n{quality_r2}\n" + + return r1, r2 + +def generate_bd_smk_fastq_files( + num_cells: int = 100, + num_reads_per_cell: int = 1000, + num_sample_tags: int = 3, +) -> Tuple[str, str]: + """ + Generate BD Rhapsody SMK FASTQ files for a given number of cells and transcripts per cell. + + Args: + num_cells: The number of cells to generate + num_reads_per_cell: The number of reads to generate per cell + num_sample_tags: The number of sample tags to use + + Returns: + A tuple of two strings, the first string being the R1 reads and the second string being the R2 reads. + """ + r1_reads = "" + r2_reads = "" + for cell_index in range(num_cells): + for _ in range(num_reads_per_cell): + r1, r2 = generate_bd_smk_read(cell_index, num_sample_tags=num_sample_tags) + r1_reads += r1 + r2_reads += r2 + + return r1_reads, r2_reads + +######################################################################################### + +# Prepare WTA, ABC, and SMK test data +print("> Prepare WTA test data", flush=True) +wta_reads_r1_str, wta_reads_r2_str = generate_bd_wta_fastq_files(num_cells=100, num_reads_per_cell=1000) +with gzip.open("WTAreads_R1.fq.gz", "wt") as f: + f.write(wta_reads_r1_str) +with gzip.open("WTAreads_R2.fq.gz", "wt") as f: + f.write(wta_reads_r2_str) + +print("> Prepare ABC test data", flush=True) +abc_reads_r1_str, abc_reads_r2_str = generate_bd_abc_fastq_files(num_cells=100, num_reads_per_cell=1000) +with gzip.open("ABCreads_R1.fq.gz", "wt") as f: + f.write(abc_reads_r1_str) +with gzip.open("ABCreads_R2.fq.gz", "wt") as f: + f.write(abc_reads_r2_str) + +print("> Prepare SMK test data", flush=True) +smk_reads_r1_str, smk_reads_r2_str = generate_bd_smk_fastq_files(num_cells=100, num_reads_per_cell=1000, num_sample_tags=3) +with gzip.open("SMKreads_R1.fq.gz", "wt") as f: + f.write(smk_reads_r1_str) +with gzip.open("SMKreads_R2.fq.gz", "wt") as f: + f.write(smk_reads_r2_str) + +######################################################################################### + +# Run executable +print(f">> Run {meta['name']}", flush=True) +output_dir = Path("output") +subprocess.run([ + meta['executable'], + "--reads=WTAreads_R1.fq.gz;WTAreads_R2.fq.gz", + f"--reference_archive={reference_file}", + "--reads=ABCreads_R1.fq.gz;ABCreads_R2.fq.gz", + f"--abseq_reference={bdabseq_panel_fa}", + "--reads=SMKreads_R1.fq.gz;SMKreads_R2.fq.gz", + "--tag_names=1-Sample1;2-Sample2;3-Sample3", + "--sample_tags_version=human", + "--output_dir=output", + "--exact_cell_count=100", + f"---cpus={meta['cpus'] or 1}", + f"---memory={meta['memory_mb'] or 2048}mb", + # "--output_seurat=seurat.rds", + "--output_mudata=mudata.h5mu", + "--metrics_summary=metrics_summary.csv", + "--pipeline_report=pipeline_report.html", +]) + + +# Check if output exists +print(">> Check if output exists", flush=True) +assert (output_dir / "sample_Bioproduct_Stats.csv").exists() +assert (output_dir / "sample_Metrics_Summary.csv").exists() +assert (output_dir / "sample_Pipeline_Report.html").exists() +assert (output_dir / "sample_RSEC_MolsPerCell_MEX.zip").exists() +assert (output_dir / "sample_RSEC_MolsPerCell_Unfiltered_MEX.zip").exists() +# seurat object is not generated when abc data is added +# assert (output_dir / "sample_Seurat.rds").exists() +assert (output_dir / "sample.h5mu").exists() + +# check individual outputs +# assert Path("seurat.rds").exists() +assert Path("mudata.h5mu").exists() +assert Path("metrics_summary.csv").exists() +assert Path("pipeline_report.html").exists() + +print(">> Check contents of output", flush=True) +data = md.read_h5mu("mudata.h5mu") + +assert data.n_obs == 100, "Number of cells is incorrect" +assert "rna" in data.mod, "RNA data is missing" +assert "prot" in data.mod, "Protein data is missing" + +# check rna data +data_rna = data.mod["rna"] +assert data_rna.n_vars == 1, "Number of genes is incorrect" +assert data_rna.X.sum(axis=1).min() > 950, "Number of reads per cell is incorrect" +# assert data_rna.var.Raw_Reads.sum() == 100000, "Number of reads is incorrect" +assert data_rna.var.Raw_Reads.sum() >= 99990 and data_rna.var.Raw_Reads.sum() <= 100010, \ + f"Expected 100000 RNA reads, got {data_rna.var.Raw_Reads.sum()}" + +# check prot data +data_prot = data.mod["prot"] +assert data_prot.n_vars == len(bdabseq_panel_fasta_dict), "Number of proteins is incorrect" +assert data_prot.X.sum(axis=1).min() > 950, "Number of reads per cell is incorrect" +assert data_prot.var.Raw_Reads.sum() >= 99990 and data_prot.var.Raw_Reads.sum() <= 100010, \ + f"Expected 100000 Prot reads, got {data_prot.var.Raw_Reads.sum()}" + + +# check smk data +expected_sample_tags = (["SampleTag01_hs", "SampleTag02_hs", "SampleTag03_hs"] * 34)[:100] +expected_sample_names = (["Sample1", "Sample2", "Sample3"] * 34)[:100] +sample_tags = data_rna.obs["Sample_Tag"] +assert sample_tags.nunique() == 3, "Number of sample tags is incorrect" +assert sample_tags.tolist() == expected_sample_tags, "Sample tags are incorrect" +sample_names = data_rna.obs["Sample_Name"] +assert sample_names.nunique() == 3, "Number of sample names is incorrect" +assert sample_names.tolist() == expected_sample_names, "Sample names are incorrect" + +# TODO: add VDJ, ATAC, and targeted RNA to test + +######################################################################################### + +print("> Test successful", flush=True) diff --git a/src/bd_rhapsody/helpers/rhapsody_cell_label.py b/src/bd_rhapsody/helpers/rhapsody_cell_label.py new file mode 100644 index 00000000..601ce7be --- /dev/null +++ b/src/bd_rhapsody/helpers/rhapsody_cell_label.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python + +# copied from https://bd-rhapsody-public.s3.amazonaws.com/CellLabel/rhapsody_cell_label.py.txt +# documented at https://bd-rhapsody-bioinfo-docs.genomics.bd.com/steps/steps_cell_label.html + +""" +Rhapsody cell label structure +Information on the cell label is captured by the combination of bases in three cell label sections (CLS1, CLS2, CLS3). +Two common linker sequences (L1, L2) separate the three CLS. + +--CLS1---|-L1-|--CLS2---|-L2-|--CL3---|--UMI---|-CaptureSequence- + + +Each cell label section has a whitelist of 96 or 384 possible 9 base sequences. +All the capture oligos from a single bead will have the same cell label. + +---------------- + +V1 beads: + +[A96_cell_key1] + [v1_linker1] + [A96_cell_key2] + [v1_linker2] + [A96_cell_key3] + [8 random base UMI] + [18 base polyT capture] + + +---------------- + +Enhanced beads: +Enhanced beads contain two different capture oligo types, polyT and 5prime. On any one bead, the two different capture oligo types have the same cell label sequences. +Compared to the V1 bead, enhanced beads have shorter linker sequences, longer polyT, and 0-3 diversity insert bases at the beginning of the sequence. +The cell label sections use the same 3 sequence whitelists as V1 beads. + +polyT capture oligo: +[Enh_insert 0-3 bases] + [A96_cell_key1] + [Enh_linker1] + [A96_cell_key2] + [Enh_linker2] + [A96_cell_key3] + [8 random base UMI] + [25 base polyT capture] + +5prime capture oligo: +[Enh_5p_primer] + [A96_cell_key1] + [Enh_5p_linker1] + [A96_cell_key2] + [Enh_5p_linker2] + [A96_cell_key3] + [8 random base UMI] + [Tso_capture_seq] + + +---------------- + +Enhanced V2/V3 beads: +Enhanced V2/V3 beads have the same structure as Enhanced beads, but the cell label sections have been updated with increased diversity + + +polyT capture oligo: +[Enh_insert 0-3 bases] + [B384_cell_key1] + [Enh_linker1] + [B384_cell_key2] + [Enh_linker2] + [B384_cell_key3] + [8 random base UMI] + [25 base polyT capture] + +5prime capture oligo: +[Enh_5p_primer] + [B384_cell_key1] + [Enh_5p_linker1] + [B384_cell_key2] + [Enh_5p_linker2] + [B384_cell_key3] + [8 random base UMI] + [Tso_capture_seq] + + +The only difference between Enh V2 and Enh V3 beads is a different Tso_capture_seq. + +---------------- + +The Rhapsody Sequence Analysis Pipeline will convert each cell label into a single integer representing a unique cell label sequence - which is used in the output files as the 'Cell_index'. +This cell index integer is deterministic and derived from the 3 part cell label as follows: + +- Get the 1-based index for each cell label section from the python sets of sequences below +- Apply this equation: + (CLS1index - 1) * 384 * 384 + (CLS2index - 1) * 384 + CLS3index + +(See label_sections_to_index() function below) + + +Example: Enhanced bead sequence: +ACACATTGCAGTGAAGATAGTTCGACACTCAAGACA + +Each part identified: +A CACATTGCA GTGA AGATAGTTC GACA CTCAAGACA +DiversityInsert A96_cell_key1-33 Linker1 A96_cell_key2-78 Linker2 A96_cell_key3-21 + +33-78-21 +(33 - 1) * 384 * 384 + (78 - 1) * 384 + 21 +=4748181 + + +The original sequences of cell label can be determined from the cell index integer by reversing this conversion. +See index_to_label_sections() and index_to_sequence() functions below. + +""" + +v1_linker1 = 'ACTGGCCTGCGA' +v1_linker2 = 'GGTAGCGGTGACA' + +Enh_linker1 = 'GTGA' +Enh_linker2 = 'GACA' + +Enh_5p_primer = "ACAGGAAACTCATGGTGCGT" + +Enh_5p_linker1 = "AATG" +Enh_5p_linker2 = "CCAC" + +Enh_inserts = ["", "A", "GT", "TCA"] + +Tso_capture_seq_Enh_EnhV2 = "TATGCGTAGTAGGTATG" +Tso_capture_seq_EnhV3 = "GTGGAGTCGTGATTATA" + +A96_cell_key1 = ("GTCGCTATA","CTTGTACTA","CTTCACATA","ACACGCCGG","CGGTCCAGG","AATCGAATG","CCTAGTATA","ATTGGCTAA","AAGACATGC","AAGGCGATC", + "GTGTCCTTA","GGATTAGGA","ATGGATCCA","ACATAAGCG","AACTGTATT","ACCTTGCGG","CAGGTGTAG","AGGAGATTA","GCGATTACA","ACCGGATAG", + "CCACTTGGA","AGAGAAGTT","TAAGTTCGA","ACGGATATT","TGGCTCAGA","GAATCTGTA","ACCAAGGAC","AGTATCTGT","CACACACTA","ATTAAGTGC", + "AAGTAACCC","AAATCCTGT","CACATTGCA","GCACTGTCA","ATACTTAGG","GCAATCCGA","ACGCAATCA","GAGTATTAG","GACGGATTA","CAGCTGACA", + "CAACATATT","AACTTCTCC","CTATGAAAT","ATTATTACC","TACCGAGCA","TCTCTTCAA","TAAGCGTTA","GCCTTACAA","AGCACACAG","ACAGTTCCG", + "AGTAAAGCC","CAGTTTCAC","CGTTACTAA","TTGTTCCAA","AGAAGCACT","CAGCAAGAT","CAAACCGCC","CTAACTCGC","AATATTGGG","AGAACTTCC", + "CAAAGGCAC","AAGCTCAAC","TCCAGTCGA","AGCCATCAC","AACGAGAAG","CTACAGAAC","AGAGCTATG","GAGGATGGA","TGTACCTTA","ACACACAAA", + "TCAGGAGGA","GAGGTGCTA","ACCCTGACC","ACAAGGATC","ATCCCGGAG","TATGTGGCA","GCTGCCAAT","ATCAGAGCT","TCGAAGTGA","ATAGACGAG", + "AGCCCAATC","CAGAATCGT","ATCTCCACA","ACGAAAGGT","TAGCTTGTA","ACACGAGAT","AACCGCCTC","ATTTAGATG","CAAGCAAGC","CAAAGTGTG", + "GGCAAGCAA","GAGCCAATA","ATGTAATGG","CCTGAGCAA","GAGTACATT","TGCGATCTA" + ) + +A96_cell_key2 = ("TACAGGATA","CACCAGGTA","TGTGAAGAA","GATTCATCA","CACCCAAAG","CACAAAGGC","GTGTGTCGA","CTAGGTCCT","ACAGTGGTA","TCGTTAGCA", + "AGCGACACC","AAGCTACTT","TGTTCTCCA","ACGCGAAGC","CAGAAATCG","ACCAAAATG","AGTGTTGTC","TAGGGATAC","AGGGCTGGT","TCATCCTAA", + "AATCCTGAA","ATCCTAGGA","ACGACCACC","TTCCATTGA","TAGTCTTGA","ACTGTTAGA","ATTCATCGT","ACTTCGAGC","TTGCGTACA","CAGTGCCCG", + "GACACTTAA","AGGAGGCGC","GCCTGTTCA","GTACATCTA","AATCAGTTT","ACGATGAAT","TGACAGACA","ATTAGGCAT","GGAGTCTAA","TAGAACACA", + "AAATAAATA","CCGACAAGA","CACCTACCC","AAGAGTAGA","TCATTGAGA","GACCTTAGA","CAAGACCTA","GGAATGATA","AAACGTACC","ACTATCCTC", + "CCGTATCTA","ACACATGTC","TTGGTATGA","GTGCAGTAA","AGGATTCAA","AGAATGGAG","CTCTCTCAA","GCTAACTCA","ATCAACCGA","ATGAGTTAC", + "ACTTGATGA","ACTTTAACT","TTGGAGGTA","GCCAATGTA","ATCCAACCG","GATGAACTG","CCATGCACA","TAGTGACTA","AAACTGCGC","ATTACCAAG", + "CACTCGAGA","AACTCATTG","CTTGCTTCA","ACCTGAGTC","AGGTTCGCT","AAGGACTAT","CGTTCGGTA","AGATAGTTC","CAATTGATC","GCATGGCTA", + "ACCAGGTGT","AGCTGCCGT","TATAGCCCT","AGAGGACCA","ACAATATGG","CAGCACTTC","CACTTATGT","AGTGAAAGG","AACCCTCGG","AGGCAGCTA", + "AACCAAAGT","GAGTGCGAA","CGCTAAGCA","AATTATAAC","TACTAGTCA","CAACAACGG" + ) + +A96_cell_key3 = ("AAGCCTTCT","ATCATTCTG","CACAAGTAT","ACACCTTAG","GAACGACAA","AGTCTGTAC","AAATTACAG","GGCTACAGA","AATGTATCG","CAAGTAGAA", + "GATCTCTTA","AACAACGCG","GGTGAGTTA","CAGGGAGGG","TCCGTCTTA","TGCATAGTA","ACTTACGAT","TGTATGCGA","GCTCCTTGA","GGCACAACA", + "CTCAAGACA","ACGCTGTTG","ATATTGTAA","AAGTTTACG","CAGCCTGGC","CTATTAGCC","CAAACGTGG","AAAGTCATT","GTCTTGGCA","GATCAGCGA", + "ACATTCGGC","AGTAATTAG","TGAAGCCAA","TCTACGACA","CATAACGTT","ATGGGACTC","GATAGAGGA","CTACATGCG","CAACGATCT","GTTAGCCTA", + "AGTTGCATC","AAGGGAACT","ACTACATAT","CTAAGCTTC","ACGAACCAG","TACTTCGGA","AACATCCAT","AGCCTGGTT","CAAGTTTCC","CAGGCATTT", + "ACGTGGGAG","TCTCACGGA","GCAACATTA","ATGGTCCGT","CTATCATGA","CAATACAAG","AAAGAGGCC","GTAGAAGCA","GCTATGGAA","ACTCCAGGG", + "ACAAGTGCA","GATGGTCCA","TCCTCAATA","AATAAACAA","CTGTACGGA","CTAGATAGA","AGCTATGTG","AAATGGAGG","AGCCGCAAG","ACAGTAAAC", + "AACGTGTGA","ACTGAATTC","AAGGGTCAG","TGTCTATCA","TCAGATTCA","CACGATCCG","AACAGAAAC","CATGAATGA","CGTACTACG","TTCAGCTCA", + "AAGGCCGCA","GGTTGGACA","CGTCTAGGT","AATTCGGCG","CAACCTCCA","CAATAGGGT","ACAGGCTCC","ACAACTAGT","AGTTGTTCT","AATTACCGG", + "ACAAACTTT","TCTCGGTTA","ACTAGACCG","ACTCATACG","ATCGAGTCT","CATAGGTCA" + ) + +B384_cell_key1 = ("TGTGTTCGC","TGTGGCGCC","TGTCTAGCG","TGGTTGTCC","TGGTTCCTC","TGGTGTGCT","TGGCGACCG","TGCTGTGGC","TGCTGGCAC","TGCTCTTCC", + "TGCCTCACC","TGCCATTAT","TGATGTCTC","TGATGGCCT","TGATGCTTG","TGAAGGACC","TCTGTCTCC","TCTGATTAT","TCTGAGGTT","TCTCGTTCT", + "TCTCATCCG","TCCTGGATT","TCAGCATTC","TCACGCCTT","TATGTGCAC","TATGCGGCC","TATGACGAG","TATCTCGTG","TATATGACC","TAGGCTGTG", + "TACTGCGTT","TACGTGTCC","TAATCACAT","GTTGTGTTG","GTTGTGGCT","GTTGTCTGT","GTTGTCGAG","GTTGTCCTC","GTTGTATCC","GTTGGTTCT", + "GTTGGCGTT","GTTGGAGCG","GTTGCTGCC","GTTGCGCAT","GTTGCAGGT","GTTGCACTG","GTTGATGAT","GTTGATACG","GTTGAAGTC","GTTCTGTGC", + "GTTCTCTCG","GTTCTATAT","GTTCGTATG","GTTCGGCCT","GTTCGCGGC","GTTCGATTC","GTTCCGGTT","GTTCCGACG","GTTCACGCT","GTTATCACC", + "GTTAGTCCG","GTTAGGTGT","GTTAGAGAC","GTTAGACTT","GTTACCTCT","GTTAATTCC","GTTAAGCGC","GTGTTGCTT","GTGTTCGGT","GTGTTCCAG", + "GTGTTCATC","GTGTCACAC","GTGTCAAGT","GTGTACTGC","GTGGTTAGT","GTGGTACCG","GTGGCGATC","GTGCTTCTG","GTGCGTTCC","GTGCGGTAT", + "GTGCGCCTT","GTGCGAACT","GTGCAGCCG","GTGCAATTG","GTGCAAGGC","GTCTTGCGC","GTCTGGCCG","GTCTGAGGC","GTCTCAGAT","GTCTCAACC", + "GTCTATCGT","GTCGGTGTG","GTCGGAATC","GTCGCTCCG","GTCCTCGCC","GTCCTACCT","GTCCGCTTG","GTCCATTCT","GTCCAATAC","GTCATGTAT", + + "GTCAGTGGT","GTCAGATAG","GTATTAACT","GTATCAGTC","GTATAGCCT","GTATACTTG","GTATAAGGT","GTAGCATCG","GTACCGTCC","GTACACCTC", + "GTAAGTGCC","GTAACAGAG","GGTTGTGTC","GGTTGGCTG","GGTTGACGC","GGTTCGTCG","GGTTCAGTT","GGTTATATT","GGTTAATAC","GGTGTACGT", + "GGTGCCGCT","GGTGCATGC","GGTCGTTGC","GGTCGAGGT","GGTAGGCAC","GGTAGCTTG","GGTACATAG","GGTAATCTG","GGCTTGGCC","GGCTTCACG", + "GGCTTATGT","GGCTTACTC","GGCTGTCTT","GGCTCTGTG","GGCTCCGGT","GGCTCACCT","GGCGTTGAG","GGCGTGTAC","GGCGTGCTG","GGCGTATCG", + "GGCGCTCGT","GGCGCTACC","GGCGAGCCT","GGCGAGATC","GGCGACTTG","GGCCTCTTC","GGCCTACAG","GGCCAGCGC","GGCCAACTT","GGCATTCCT", + "GGCATCCGC","GGCATAACC","GGCAACGAT","GGATGTCCG","GGATGAGAG","GGATCTGGC","GGATCCATG","GGATAGGTT","GGAGTCGTG","GGAGAAGGC", + "GGACTCCTT","GGACTAGTC","GGACCGTTG","GGAATTAGT","GGAATCTCT","GGAATCGAC","GGAAGCCTC","GCTTGTAGC","GCTTGACCG","GCTTCGGAC", + "GCTTCACAT","GCTTAGTCT","GCTGGATAT","GCTGGAACC","GCTGCGATG","GCTGATCAG","GCTGAGCGT","GCTCTTGTC","GCTCTCCTG","GCTCGGTCC", + "GCTCCAATT","GCTATTCGC","GCTATGAGT","GCTAGTGTT","GCTAGGATC","GCTAGCACT","GCTACGTAT","GCTAACCTT","GCGTTCCGC","GCGTGTGCC", + "GCGTGCATT","GCGTCGGTT","GCGTATGTG","GCGTATACT","GCGGTTCAC","GCGGTCTTG","GCGGCGTCG","GCGGCACCT","GCGCTGGAC","GCGCTCTCC", + + "GCGCGGCAG","GCGCGATAC","GCGCCGACC","GCGAGCGAG","GCGAGAGGT","GCGAATTAC","GCCTTGCAT","GCCTGCGCT","GCCTAACTG","GCCGTCCGT", + "GCCGCTGTC","GCCATGCCG","GCCAGCTAT","GCCAACCAG","GCATGGTTG","GCATCGACG","GCAGGCTAG","GCAGGACGC","GCAGCCATC","GCAGATACC", + "GCAGACGTT","GCACTATGT","GCACACGAG","GATTGTCAT","GATTGGTAG","GATTGCACC","GATTCTACT","GATTCGCTT","GATTAGGCC","GATTACGGT", + "GATGTTGGC","GATGTTATG","GATGGCCAG","GATCGTTCG","GATCGGAGC","GATCGCCTC","GATCCTCTG","GATCCAGCG","GATACACGC","GAGTTACCT", + "GAGTCGTAT","GAGTCGCCG","GAGGTGTAG","GAGGCATTG","GAGCGGACG","GAGCCTGAG","GAGATCTGT","GAGATAATT","GAGACGGCT","GACTTCGTG", + "GACTGTTCT","GACTCTTAG","GACCGCATT","GAATTGAGC","GAATATTGC","GAAGGCTCT","GAAGAGACT","GAACTGCCG","GAACGCGTG","CTTGTGTAT", + "CTTGTGCGC","CTTGTCATG","CTTGGTCTT","CTTGGTACC","CTTGGATGT","CTTGCTCAC","CTTGCAATC","CTTGAGGCC","CTTGACGGT","CTTCTGATC", + "CTTCTCGTT","CTTCTAGGC","CTTCGTTAG","CTTATGTCC","CTTATGCTT","CTTATATAG","CTTAGGTTG","CTTAGGAGC","CTTACTTAT","CTGTTCTCG", + "CTGTGCCTC","CTGTCGCAT","CTGTCGAGC","CTGTAGCTG","CTGTACGTT","CTGCTTGCC","CTGCGTAGT","CTGCACACC","CTGATGGAT","CTGAGTCAT", + "CTGACGCCG","CTGAACGAG","CTCTTGTAG","CTCTTAGTT","CTCTTACCG","CTCTGCACC","CTCTCGTCC","CTCGTATTG","CTCGACTAT","CTCCTGACG", + + "CTCACTAGC","CTATACGGC","CGTTCGCTC","CGTTCACCG","CGTATAGTT","CGGTGTTCC","CGGTGTCAG","CGGTCCTGC","CGGCGACTC","CGGCACGGT", + "CGGATAGCC","CGGAGAGAT","CGCTAATAG","CGCGTTGGC","CGCGCAGAG","CGCACTGCC","CCTTGTCTC","CCTTGGCGT","CCTTCTGAG","CCTTCTCCT", + "CCTTCGACC","CCTTACTTG","CCTGTTCGT","CCTGTATGC","CCTCGGCCG","CCGTTAATT","CCATGTGCG","CCAGTGGTT","CCAGGCATT","CCAGGATCC", + "CCAGCGTTG","CATTCCGAT","CATTATACC","CATGTTGAG","ATTGCGTGT","ATTGCGGAC","ATTGCGCCG","ATTGACTTG","ATTCGGCTG","ATTCGCGAG", + "ATTCCAAGT","ATTATCTTC","ATTACTGTT","ATTACACTC","ATGTTCTAT","ATGTTACGC","ATGTGTATC","ATGTGGCAG","ATGTCTGTG","ATGGTGCAT", + "ATGCTTACT","ATGCTGTCC","ATGCTCGGC","ATGAGGTTC","ATGAGAGTG","ATCTTGGCT","ATCTGTGCG","ATCGGTTCC","ATCATGCTC","ATCATCACT", + "ATATCTTAT","ATAGGCGCC","AGTTGGTAT","AGTTGAGCC","AGTGCGACC","AGGTGCTAC","AGGCTTGCG","AGGCCTTCC","AGGCACCTT","AGGAATATG", + "AGCGGCCAG","AGCCTGGTC","AGCCTGACT","AGCAATCCG","AGAGATGTT","AGAGAATTC","ACTCGCTTG","ACTCGACCT","ACGTACACC","ACGGATGGT", + "ACCAGTCTG","ACATTCGGC","ACATGAGGT","ACACTAATT" + ) + +B384_cell_key2 = ("TTGTGTTGT","TTGTGGTAG","TTGTGCGGA","TTGTCTGTT","TTGTCTAAG","TTGTCATAT","TTGTCACGA","TTGTATGAA","TTGTACAGT","TTGGTTAAT", + "TTGGTGCAA","TTGGTCGAG","TTGGTATTA","TTGGCACAG","TTGGATACA","TTGGAAGTG","TTGCGGTTA","TTGCCATTG","TTGCACGCG","TTGCAAGGT", + "TTGATGTAT","TTGATAATT","TTGAGACGT","TTGACTACT","TTGACCGAA","TTCTGGTCT","TTCTGCACA","TTCTCCTTA","TTCTCCGCT","TTCTAGGTA", + "TTCTAATCG","TTCGTCGTA","TTCGTAGAT","TTCGGCTTG","TTCGGAATA","TTCGCCAGA","TTCGATTGT","TTCGATCAG","TTCCTCGGT","TTCCGGCAG", + "TTCCGCATT","TTCCAATTA","TTCATTGAA","TTCATGCTG","TTCAGGAGT","TTCACTATA","TTCAACTCT","TTCAACGTG","TTATGCGTT","TTATGATTG", + "TTATCCTGT","TTATCCGAG","TTATATTAT","TTAGGCGCG","TTACTGGAA","TTACTAGTT","TTACGTGGT","TTACGATAT","TTACCTAGA","TTACATGAG", + "TTACAGCGT","TTACACGGA","TTACACACT","TTAATCAGT","TTAATAGGA","TTAAGTGTG","TTAACCTTG","TTAACACAA","TGTTCACTT","TGTTCAAGA", + "TGTTAAGTG","TGTGTTATG","TGTGTCCAA","TGTGGAGCG","TGTCAGTTA","TGTCAGAAG","TGGTTAGTT","TGGTTACAA","TGGCGTTAT","TGGCGCCAA", + "TGGAGTCTT","TGCGTATTG","TGATAGAGA","TGAGGTATT","TGAGAATCT","TCTTGGTAA","TCTTCATAG","TCTGTCCTT","TCTGGAATT","TCTACCGCG", + "TCGTTCGAA","TCGTCAGTG","TCGACGAGA","TCATGGCTT","TCACACTTA","TATTCCGAA","TATTATGGT","TATGCTATT","TATCAAGGA","TAGTTCAAT", + + "TAGCTGCTT","TAGAGGAAG","TACCTGTTA","TACACCTGT","GTTGTGCGT","GTTGGCTAT","GTTGCCAAG","GTTGACCTT","GTTCTGCTA","GTTCTGAAT", + "GTTCTATCA","GTTCGCGTG","GTTCCTTAT","GTTAGCAGT","GTTACTGTG","GTTACTCAA","GTTAAGAGA","GTTAACTTA","GTGTCGGCA","GTGTCCATT", + "GTGCTTGAG","GTGCTCGTT","GTGCTCACA","GTGCCTGGA","GTCTTGTCG","GTCTTGATT","GTCTTCCGT","GTCTTAAGA","GTCTCATCT","GTCTACGAG", + "GTCGTTGCT","GTCGTGTTA","GTCGGTAAT","GTCGGATGT","GTCGAGCTG","GTCCGGACT","GTCCAACAT","GTCAGACGA","GTCAGAATT","GTCACTCTT", + "GTCAAGGAA","GTATGTCTT","GTATGTACA","GTATCGGTT","GTATATGTA","GTATACAAT","GTAGTTAAG","GTAGTCGAT","GTAGCCTTA","GTAGATACT", + "GTACGATTA","GTACAGTCT","GTAATTCGT","GCTTGGCAG","GCTTGCTTG","GCTTGAGGA","GCTTCATTA","GCTTATGCG","GCTGTGTAG","GCTGTCATG", + "GCTGGTTGT","GCTGGACTG","GCTGCCTAA","GCTGATATT","GCTCTTAGT","GCTCTATTG","GCTCGCCGT","GCTCCGCTG","GCTATTCTG","GCTATACGA", + "GCTACTAAG","GCTACATGT","GCTAACTCT","GCGTTGTAA","GCGTTCTCT","GCGTGCGTA","GCGTCTTGA","GCGTCCGAT","GCGTAAGAG","GCGCTTACG", + "GCGCGGATT","GCGCCATAT","GCGCATGAA","GCGATCAAT","GCGAGCCTT","GCGAGATTG","GCGAGAACA","GCCTTGGTA","GCCTTCTAG","GCCTTCACA", + "GCCTGAGTG","GCCTCACGT","GCCGGCGAA","GCCGCACAA","GCCATGCTT","GCCATATAT","GCCAATTCG","GCATTCGTT","GCATGATGT","GCAGTTGGA", + + "GCAGTGTCT","GCACTTGTG","GCAATCTGT","GCAACACTT","GATTGTATT","GATTGCGAG","GATTCCAGT","GATTCATAT","GATTATCAG","GATTAGGTT", + "GATGTTGCG","GATGGATCT","GATGCTGAT","GATGCCTTG","GATCTCCTT","GATCGCTTA","GATATTGAA","GATATTACT","GAGTGTTAT","GAGCTCAGT", + "GAGCGTGCT","GAGCGTCGA","GAGCGGTTG","GAGCGACTT","GAGCCGAAT","GAGATAGAT","GAGACCTAT","GACGGTCGT","GACGCAGGT","GACGATATG", + "GACCTATCT","GAATTAGGA","GAATCAGCT","GAAGTTCAT","GAAGTGGTT","GAAGTATTG","GAAGGCATT","GAACGCTGT","CTTGTCCAG","CTTGGATTG", + "CTTGCTGAA","CTTGCCGTG","CTTGATTCT","CTTCTGTCG","CTTCGGCGT","CTTATGAGT","CTTACCGAT","CTGTTAGGT","CTGTCGTCT","CTGTATAAT", + "CTGGCTCAT","CTGGATGCG","CTGCGTGTG","CTGCGCGGT","CTGCCGATT","CTGCATTGT","CTGATTAAG","CTGAGATAT","CTGACCTGT","CTCGTATCT", + "CTCGGCAAG","CTCGCAATT","CTCCTGCTT","CTCCTAAGT","CTCCGGATG","CTCCGAGCG","CTCACAGGT","CTATTCTAT","CTATTAGTG","CTATGAATT", + "CTACATATT","CGTGGCATT","CGTCTTAAT","CGTCTGGTT","CGTCACTGT","CGTAGGTCT","CGGTTCGAG","CGGTTCATT","CGGTGCTCT","CGGTAATTG", + "CGGCCTGAT","CGGATATAG","CGGAATATT","CGCTCCAAT","CGCGTTCGT","CGCAGGTTG","CGAGGATGT","CGAGCTGTT","CGACGGCTT","CCTTGTGTG", + "CCTGTCTCA","CCTGACTAT","CCTACCTTG","CCGTAGATT","CCGGCTGGT","CATCGGACG","CATCGATAA","CATCCTTCT","CAGTTCTGT","CAGTGCCAG", + + "CAGGCACTG","CAGCCTCTT","CACTTATAT","CACTGGTCG","CACTGCATG","CACGCGTTG","CACGATGTT","CACCATCTG","CACAGGCGT","ATTGTACAA", + "ATTGGTATG","ATTGCTAAT","ATTGCATAG","ATTGCAGTT","ATTCTGCAG","ATTCTACGT","ATTCGGATT","ATTCCGTTG","ATTCATCAA","ATTCAAGAG", + "ATTAGCCTT","ATTAATATT","ATGTTAGAG","ATGTTAACT","ATGTAGTCG","ATGGTGTAG","ATGGATTAT","ATCTTGAAG","ATCTGATAT","ATCTCAGAA", + "ATCGCTCAA","ATCGCGTCG","ATCCATGGT","ATCATGAGA","ATCATAGTT","ATCAGCGAG","ATCACCATT","ATAGTAATT","ATAGCTGTG","ATACTCTCG", + "ATACCTCAT","AGTTGCGCG","AGTTGAATT","AGTTATGAT","AGTGTCCGT","AGTGGCTTG","AGTGCTTCT","AGTATCATT","AGTACACAA","AGGTATGCG", + "AGGTATAGT","AGGCTACTT","AGGCCAGGT","AGGAGCGAT","AGCTTATAG","AGCTCTAGA","AGCGTGTAT","AGCGTCACA","AGCCTTCAT","AGCCTGTCG", + "AGCCTCGAG","AGCACTGAA","AGATGTACG","AGAGTTAAT","AGACCTCTG","ACTTCTATA","ACTGTCGAG","ACTGTATGT","ACTCTGTAA","ACTCGCGAA", + "ACTAGATCT","ACTAACGTT","ACGTTACTG","ACGTGGAAT","ACGGACTCT","ACGCCTAAT","ACGCCGTTA","ACGACGTGT","ACCTCGCAT","ACCATCATA", + "ACATATATT","ACAGGCACA","ACACCTGAG","ACACATTCT" + ) + +B384_cell_key3 = ("TTGTGGCTG","TTGTGGAGT","TTGTGCGAC","TTGTCTTCA","TTGTAAGAT","TTGGTTCTG","TTGGTGCGT","TTGGTCTAC","TTGGTAACT","TTGGCGTGC", + "TTGGATTAG","TTGGAGACG","TTGGAATCA","TTGCGGCGA","TTGCGCTCG","TTGCCTTAC","TTGCCGGAT","TTGCATGCT","TTGCACGTC","TTGCACCAT", + "TTGAACCTG","TTCTCGCGT","TTCTCAACT","TTCTACTCA","TTCGTCCAT","TTCGGATAC","TTCGGACGT","TTCGCAATC","TTCCGGTGC","TTCCGACTG", + "TTCATTATG","TTCATGGAT","TTCAGCGCA","TTCACCTCG","TTCAAGCAG","TTCAACTAC","TTATGCCAG","TTATGCATC","TTATCGTAC","TTATACCTA", + "TTATAATAG","TTATAAGTC","TTAGTTAGC","TTAGCTCAT","TTAGCACTA","TTAGATATG","TTACTACGA","TTACCGTCA","TTACAGAGC","TTAATTGCA", + "TTAACAGAT","TGTTGGCTA","TGTTGATGA","TGTTAAGCT","TGTGGCCGA","TGTGCTAGC","TGTGCGTCA","TGTCGCAGT","TGTCGAGCA","TGTACAACG", + "TGGTTCCGA","TGGTTCACT","TGGTCAAGT","TGGCTTGTA","TGGCTGTCG","TGGCGTATG","TGGCGCGCT","TGGATGTAC","TGGACTTGC","TGGAATACT", + "TGCTAGCGA","TGCGTTGCT","TGCGGTCTG","TGCGCTTAG","TGCGCGACG","TGCCTGCAT","TGCCTAGAC","TGCACGAGT","TGAGTGTGC","TGAGGCTCG", + "TCTTCCGTC","TCTTATAGT","TCTTACCAT","TCTGTTGTC","TCTGTTACT","TCTGGCTAG","TCTCAGATC","TCTAGTTGA","TCTAGTACG","TCGTACTAC", + "TCGGTGTAG","TCGGCTGCT","TCGCTACTG","TCGATCACG","TCGAGGCAT","TCCGGCGTC","TCCGGAGCT","TCCGCTCGT","TCCGAGTAC","TCCATTCAT", + + "TCCATGGTC","TCCAAGTCG","TCATTACGT","TCATGCACT","TCAGGTTGC","TCAGACCGT","TCACTCAGT","TCAAGCTCA","TATTGCGCA","TATTCGGCT", + "TATTCCAGC","TATTCATCA","TATGTTCAG","TATGGTATG","TATGCAAGT","TATCTGGTC","TATCTGACT","TATCCAGAT","TATCAGTCG","TATCACGCT", + "TAGGCGCGA","TAGGCACAT","TAGGATCGT","TAGCATTGC","TAGAGTTAC","TAGACTGAT","TACTTGTCG","TACGTCCGA","TACCGTACT","TACCGCGAT", + "TACCAGGAC","TACAGAAGT","TAAGTGCAT","TAAGCTACT","GTTGACCGA","GTTCTCGAC","GTTCCTGCT","GTTATGATG","GTGCTTGCA","GTGCCGCGT", + "GTATTGCTG","GTATTCCGA","GTATTAAGC","GTATGACGT","GTAGTTGTC","GTAGTACAT","GTAGCTCGA","GGTTGCTCA","GGTTGAGTA","GGTTAACGT", + "GGTGTGGCA","GGTCTTCAG","GGTCGTCTA","GGTCGGCGT","GGTCCGACT","GGTCATGTC","GGTCACATG","GGTAGTGCT","GGTAGCGTC","GGTACCAGT", + "GGTAAGGAT","GGCTTGTGC","GGCTTGACT","GGCTTACGA","GGCTGTAGT","GGCTGGCAG","GGCTCCATC","GGCGTGGAT","GGCGTAATC","GGCGCAAGT", + "GGCGAGTAG","GGCGACCGT","GGCCTGTCA","GGCCATTGC","GGCACTCTG","GGATGTCAT","GGAGTAACT","GGAGAACGA","GGACTGGCT","GGACGTTCA", + "GGAACGTGC","GCTGTCCAT","GCTGGTTCA","GCTGCAACT","GCTCGTTAC","GCTATAGAT","GCTAGTCGT","GCTACCATG","GCGTTCTGA","GCGTGTTAG", + "GCGGTATCG","GCGGAGCAT","GCGCGGTGC","GCGCCTAGT","GCGCCGGCT","GCCTTCATG","GCCATACTG","GCATGTTGA","GCATGCTAC","GCAGTATAC", + + "GCAGGTACT","GCAGCGCGT","GCACCTCAT","GCAATTCGA","GATTGCCGT","GATGAACAT","GATCTTCGA","GATCTGCAT","GAGTGGCAT","GAGTCGGAC", + "GAGTATGAT","GAGGCGAGT","GAGGCAACG","GAGCGCACT","GAATAGGCT","ATTGTCACT","ATTGTATCA","ATTGGTCAG","ATTGGCGAT","ATTGATCGT", + "ATTCGTAGT","ATTCATACG","ATTCAGGAC","ATTACTTCA","ATTAATTAG","ATTAAGCAT","ATGTCTCTA","ATGTAGCGT","ATGGCATAC","ATGGAGATC", + "ATGGACTCG","ATGGAACGA","ATGCTTCAT","ATGCTCGCT","ATGCGACGT","ATGCCGTAG","ATGAGTTCG","ATGACTATC","ATGACCGAC","ATCTTATGC", + "ATCTTACTA","ATCTATCAG","ATCGTGTAC","ATCGTCTGA","ATCGGCATG","ATCGCGAGC","ATCGCAACG","ATCGATGCT","ATCGAATAG","ATCCTTCTG", + "ATCCTGCGT","ATCCGCACT","ATCCATTAC","ATCCAAGCA","ATCAGATCA","ATCACACAT","ATCAACGTC","ATCAACCGA","ATATTGAGT","ATATTCGTC", + "ATATTACAG","ATATCTTGA","ATATCGCAT","ATATCAATC","ATAGTCCTG","ATAGGTCTA","ATAGCTGAC","ATAGCGGTA","AGTTCGCTG","AGTTACAGC", + "AGTTAACTA","AGTGCAATC","AGTCTGGTA","AGTCTGAGC","AGTCTACAT","AGTCGAACT","AGTCCATCG","AGTCATTCA","AGTATCCAG","AGTAGACTG", + "AGTAATCGA","AGTAAGTGC","AGGTTGGCT","AGGTTCTAG","AGGTGTTCA","AGGTGCCAT","AGGTCTGAT","AGGTCGTAC","AGGTCAGCA","AGGCTTATC", + "AGGCTATGA","AGGCCGACG","AGGCCAAGC","AGGCAGGTC","AGGCAAGAT","AGGAGCAGT","AGGACCGCT","AGGAATTAC","AGCTTGGAC","AGCTTAAGT", + + "AGCTACACG","AGCGTTACG","AGCGGTGCA","AGCGGAGTC","AGCGGACGA","AGCGCGCTA","AGCGATAGC","AGCGACTCA","AGCCTCTAC","AGCCGTCGT", + "AGCATGATC","AGCACTTCG","AGCACGGCA","AGATTCTGA","AGATTAGAT","AGATGATAG","AGATATGTA","AGATACCGT","AGAGTGCGT","AGAGCCGAT", + "AGACTCACT","ACTTGCCTA","ACTTGAGCA","ACTTCTAGC","ACTTCGACT","ACTTAGTAC","ACTGTTGAT","ACTGTAACG","ACTGGTATC","ACTGACGTC", + "ACTGAAGCT","ACTCTGATG","ACTCCTGAC","ACTCCGCTA","ACTCAACTG","ACTATTGCA","ACTAGGCAG","ACTACGCGT","ACTAATACT","ACGTTCGTA", + "ACGTGTGCT","ACGTGTATG","ACGTGGAGC","ACGTCTTCG","ACGTCAGTC","ACGGTCTCA","ACGGTCCGT","ACGGTACAG","ACGGCGCTG","ACGCTGCGA", + "ACGCGTGTA","ACGCGCCAG","ACGATGTCG","ACGATGGAT","ACGATCTAC","ACGAGCTGA","ACGAGCATC","ACGAATCGT","ACGAACGCA","ACCTTGTAG", + "ACCTGTTGC","ACCTGTCAT","ACCTCGATC","ACCTAGGTA","ACCTACTGA","ACCTAATCG","ACCGTAGCA","ACCGGTAGT","ACCGGCTAC","ACCGCTTCA", + "ACATTGTGC","ACATTCTCG","ACATGGCTG","ACATGACGA","ACATATGAT","ACATATACG","ACAGCGTAC","ACACTTGCT","ACACTATCA","ACACGCATG", + "ACACCAGTA","ACACCAACT","ACACATAGT","ACACACCTA" + ) + + +def label_sections_to_index(label): + """ + Return the cell_index integer based on input 3 part cell label string + + """ + + cl1, cl2, cl3 = [int(n) for n in label.split('-')] + return (cl1 - 1) * 384 * 384 + (cl2 - 1) * 384 + (cl3 - 1) + 1 + + +# print(label_sections_to_index('1-1-1')) +# print(label_sections_to_index('33-78-21')) +# print(label_sections_to_index('43-12-77')) +# print(label_sections_to_index('96-96-96')) +# print(label_sections_to_index('135-43-344')) +# print(label_sections_to_index('384-384-384')) +# print('-') + +#---------------------------------- + + +def index_to_label_sections(index): + + zerobased = int(index) - 1 + + cl1 = (int((zerobased) / 384 / 384) % 384) + 1 + cl2 = (int((zerobased) / 384) % 384) + 1 + cl3 = (zerobased % 384) + 1 + + return f'{cl1}-{cl2}-{cl3}' + + +# print(index_to_label_sections(1)) +# print(index_to_label_sections(4748181)) +# print(index_to_label_sections(6197453)) +# print(index_to_label_sections(14044896)) +# print(index_to_label_sections(19775576)) +# print(index_to_label_sections(56623104)) +# print('-') +#---------------------------------- + + +def index_to_sequence(index, bead_version): + + zerobased = int(index) - 1 + + cl1 = (int((zerobased) / 384 / 384) % 384) + 1 + cl2 = (int((zerobased) / 384) % 384) + 1 + cl3 = (zerobased % 384) + 1 + + if bead_version == 'v1': + cls1_sequence = A96_cell_key1[cl1-1] + cls2_sequence = A96_cell_key2[cl2-1] + cls3_sequence = A96_cell_key3[cl3-1] + + return f'{cls1_sequence}{v1_linker1}{cls2_sequence}{v1_linker2}{cls3_sequence}' + + elif bead_version == 'Enh': + + diversityInsert = '' + + if 1 <= cl1 <= 24: + diversityInsert = '' + elif 25 <= cl1 <= 48: + diversityInsert = 'A' + elif 49 <= cl1 <= 72: + diversityInsert = 'GT' + else: # 73 <= cl1 <= 96: + diversityInsert = 'TCA' + + cls1_sequence = A96_cell_key1[cl1-1] + cls2_sequence = A96_cell_key2[cl2-1] + cls3_sequence = A96_cell_key3[cl3-1] + + return f'{diversityInsert}{cls1_sequence}{Enh_linker1}{cls2_sequence}{Enh_linker2}{cls3_sequence}' + + elif bead_version == 'EnhV2': + + diversityInsert = '' + subIndex = ((cl1-1) % 96) + 1 + + if 1 <= subIndex <= 24: + diversityInsert = '' + elif 25 <= subIndex <= 48: + diversityInsert = 'A' + elif 49 <= subIndex <= 72: + diversityInsert = 'GT' + else: # 73 <= subIndex <= 96: + diversityInsert = 'TCA' + + cls1_sequence = B384_cell_key1[cl1-1] + cls2_sequence = B384_cell_key2[cl2-1] + cls3_sequence = B384_cell_key3[cl3-1] + + return f'{diversityInsert}{cls1_sequence}{Enh_linker1}{cls2_sequence}{Enh_linker2}{cls3_sequence}' + + +# print(index_to_sequence(4748181, 'Enh')) +# print(index_to_sequence(52923177, 'EnhV2')) + +#---------------------------------- + + +def create_cell_index_fasta_V1(): + with open('Rhapsody_cellBarcodeV1_IndexToSequence.fasta', 'w') as f: + for cl1 in range(1, 96+1): + for cl2 in range(1, 96+1): + for cl3 in range(1, 96+1): + index = label_sections_to_index(f'{cl1}-{cl2}-{cl3}') + sequence = index_to_sequence(index, 'v1') + f.write(f'>{index}\n') + f.write(f'{sequence}\n') + +#create_cell_index_fasta_V1() + + +def create_cell_index_fasta_Enh(): + with open('Rhapsody_cellBarcodeEnh_IndexToSequence.fasta', 'w') as f: + for cl1 in range(1, 96+1): + for cl2 in range(1, 96+1): + for cl3 in range(1, 96+1): + index = label_sections_to_index(f'{cl1}-{cl2}-{cl3}') + sequence = index_to_sequence(index, 'Enh') + f.write(f'>{index}\n') + f.write(f'{sequence}\n') + +#create_cell_index_fasta_Enh() + +def create_cell_index_fasta_EnhV2(): + with open('Rhapsody_cellBarcodeEnhV2_IndexToSequence.fasta', 'w') as f: + for cl1 in range(1, 384+1): + for cl2 in range(1, 384+1): + for cl3 in range(1, 384+1): + index = label_sections_to_index(f'{cl1}-{cl2}-{cl3}') + sequence = index_to_sequence(index, 'EnhV2') + f.write(f'>{index}\n') + f.write(f'{sequence}\n') + +#create_cell_index_fasta_EnhV2() diff --git a/src/bd_rhapsody/test_data/BDAbSeq_ImmuneDiscoveryPanel.fasta b/src/bd_rhapsody/test_data/BDAbSeq_ImmuneDiscoveryPanel.fasta new file mode 100644 index 00000000..930add4a --- /dev/null +++ b/src/bd_rhapsody/test_data/BDAbSeq_ImmuneDiscoveryPanel.fasta @@ -0,0 +1,60 @@ +>CD11c:B-LY6|ITGAX|AHS0056|pAbO Catalog_940024 +ATGCGTTGCGAGAGATATGCGTAGGTTGCTGATTGG +>CD14:MPHIP9|CD14|AHS0037|pAbO Catalog_940005 +TGGCCCGTGGTAGCGCAATGTGAGATCGTAATAAGT +>CXCR5|CXCR5|AHS0039|pAbO Catalog_940042 +AGGAAGGTCGATTGTATAACGCGGCATTGTAACGGC +>CD19:SJ25C1|CD19|AHS0030|pAbO Catalog_940004 +TAGTAATGTGTTCGTAGCCGGTAATAATCTTCGTGG +>CD25:2A3|IL2RA|AHS0026|pAbO Catalog_940009 +AGTTGTATGGGTTAGCCGAGAGTAGTGCGTATGATT +>CD27:M-T271|CD27|AHS0025|pAbO Catalog_940018 +TGTCCGGTTTAGCGAATTGGGTTGAGTCACGTAGGT +>CD278|ICOS|AHS0012|pAbO Catalog_940043 +ATAGTCCGCCGTAATCGTTGTGTCGCTGAAAGGGTT +>CD279:EH12-1|PDCD1|AHS0014|pAbO Catalog_940015 +ATGGTAGTATCACGACGTAGTAGGGTAATTGGCAGT +>CD3:UCHT1|CD3E|AHS0231|pAbO Catalog_940307 +AGCTAGGTGTTATCGGCAAGTTGTACGGTGAAGTCG +>GITR|TNFRSF18|AHS0104|pAbO Catalog_940096 +TCTGTGTGTCGGGTTGAATCGTAGTGAGTTAGCGTG +>Tim3|HAVCR2|AHS0016|pAbO Catalog_940066 +TAGGTAGTAGTCCCGTATATCCGATCCGTGTTGTTT +>CD4:SK3|CD4|AHS0032|pAbO Catalog_940001 +TCGGTGTTATGAGTAGGTCGTCGTGCGGTTTGATGT +>CD45RA:HI100|PTPRC|AHS0009|pAbO Catalog_940011 +AAGCGATTGCGAAGGGTTAGTCAGTACGTTATGTTG +>CD56:NCAM16.2|NCAM1|AHS0019|pAbO Catalog_940007 +AGAGGTTGAGTCGTAATAATAATCGGAAGGCGTTGG +>CD62L:DREG-56|SELL|AHS0049|pAbO Catalog_940041 +ATGGTAAATATGGGCGAATGCGGGTTGTGCTAAAGT +>CCR7|CCR7|AHS0273|pAbO Catalog_940394 +AATGTGTGATCGGCAAAGGGTTCTCGGGTTAATATG +>CXCR6|CXCR6|AHS0148|pAbO Catalog_940234 +GTGGTTGGTTATTCGGACGGTTCTATTGTGAGCGCT +>CD127|IL7R|AHS0028|pAbO Catalog_940012 +AGTTATTAGGCTCGTAGGTATGTTTAGGTTATCGCG +>CD134:ACT35|TNFRSF4|AHS0013|pAbO Catalog_940060 +GGTGTTGGTAAGACGGACGGAGTAGATATTCGAGGT +>CD28:L293|CD28|AHS0138|pAbO Catalog_940226 +TTGTTGAGGATACGATGAAGCGGTTTAAGGGTGTGG +>CD272|BTLA|AHS0052|pAbO Catalog_940105 +GTAGGTTGATAGTCGGCGATAGTGCGGTTGAAAGCT +>CD8:SK1|CD8A|AHS0228|pAbO Catalog_940305 +AGGACATAGAGTAGGACGAGGTAGGCTTAAATTGCT +>HLA-DR|CD74|AHS0035|pAbO Catalog_940010 +TGTTGGTTATTCGTTAGTGCATCCGTTTGGGCGTGG +>CD16:3G8|FCGR3A|AHS0053|pAbO Catalog_940006 +TAAATCTAATCGCGGTAACATAACGGTGGGTAAGGT +>CD183|CXCR3|AHS0031|pAbO Catalog_940030 +AAAGTGTTGGCGTTATGTGTTCGTTAGCGGTGTGGG +>CD196|CCR6|AHS0034|pAbO Catalog_940033 +ACGTGTTATGGTGTTGTTCGAATTGTGGTAGTCAGT +>CD137|TNFRSF9|AHS0003|pAbO Catalog_940055 +TGACAAGCAACGAGCGATACGAAAGGCGAAATTAGT +>CD161:HP-3G10|KLRB1|AHS0205|pAbO Catalog_940283 +TTTAGGACGATTAGTTGTGCGGCATAGGAGGTGTTC +>IgM|IGHM|AHS0198|pAbO Catalog_940276 +TTTGGAGGGTAGCTAGTTGCAGTTCGTGGTCGTTTC +>IgD|IGHD|AHS0058|pAbO Catalog_940026 +TGAGGGATGTATAGCGAGAATTGCGACCGTAGACTT diff --git a/src/bd_rhapsody/test_data/SampleTagSequences_HomoSapiens_ver1.fasta b/src/bd_rhapsody/test_data/SampleTagSequences_HomoSapiens_ver1.fasta new file mode 100644 index 00000000..3d5a42fa --- /dev/null +++ b/src/bd_rhapsody/test_data/SampleTagSequences_HomoSapiens_ver1.fasta @@ -0,0 +1,24 @@ +>SampleTag01_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGATTCAAGGGCAGCCGCGTCACGATTGGATACGACTGTTGGACCGG +>SampleTag02_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGTGGATGGGATAAGTGCGTGATGGACCGAAGGGACCTCGTGGCCGG +>SampleTag03_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGCGGCTCGTGCTGCGTCGTCTCAAGTCCAGAAACTCCGTGTATCCT +>SampleTag04_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGATTGGGAGGCTTTCGTACCGCTGCCGCCACCAGGTGATACCCGCT +>SampleTag05_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGCTCCCTGGTGTTCAATACCCGATGTGGTGGGCAGAATGTGGCTGG +>SampleTag06_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGTTACCCGCAGGAAGACGTATACCCCTCGTGCCAGGCGACCAATGC +>SampleTag07_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGTGTCTACGTCGGACCGCAAGAAGTGAGTCAGAGGCTGCACGCTGT +>SampleTag08_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGCCCCACCAGGTTGCTTTGTCGGACGAGCCCGCACAGCGCTAGGAT +>SampleTag09_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGGTGATCCGCGCAGGCACACATACCGACTCAGATGGGTTGTCCAGG +>SampleTag10_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGGCAGCCGGCGTCGTACGAGGCACAGCGGAGACTAGATGAGGCCCC +>SampleTag11_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGCGCGTCCAATTTCCGAAGCCCCGCCCTAGGAGTTCCCCTGCGTGC +>SampleTag12_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGGCCCATTCATTGCACCCGCCAGTGATCGACCCTAGTGGAGCTAAG diff --git a/src/bd_rhapsody/bd_rhapsody_make_reference/test_data/reference_small.fa b/src/bd_rhapsody/test_data/reference_small.fa similarity index 100% rename from src/bd_rhapsody/bd_rhapsody_make_reference/test_data/reference_small.fa rename to src/bd_rhapsody/test_data/reference_small.fa diff --git a/src/bd_rhapsody/bd_rhapsody_make_reference/test_data/reference_small.gtf b/src/bd_rhapsody/test_data/reference_small.gtf similarity index 100% rename from src/bd_rhapsody/bd_rhapsody_make_reference/test_data/reference_small.gtf rename to src/bd_rhapsody/test_data/reference_small.gtf diff --git a/src/bd_rhapsody/test_data/script.sh b/src/bd_rhapsody/test_data/script.sh new file mode 100644 index 00000000..f8db0313 --- /dev/null +++ b/src/bd_rhapsody/test_data/script.sh @@ -0,0 +1,141 @@ +#!/bin/bash + +TMP_DIR=/tmp/bd_rhapsody_make_reference +OUT_DIR=src/bd_rhapsody/test_data + +# check if seqkit is installed +if ! command -v seqkit &> /dev/null; then + echo "seqkit could not be found" + exit 1 +fi + +# create temporary directory and clean up on exit +mkdir -p $TMP_DIR +function clean_up { + rm -rf "$TMP_DIR" +} +trap clean_up EXIT + +# fetch reference +ORIG_FA=$TMP_DIR/reference.fa.gz +if [ ! -f $ORIG_FA ]; then + wget https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_41/GRCh38.primary_assembly.genome.fa.gz \ + -O $ORIG_FA +fi + +ORIG_GTF=$TMP_DIR/reference.gtf.gz +if [ ! -f $ORIG_GTF ]; then + wget https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_41/gencode.v41.annotation.gtf.gz \ + -O $ORIG_GTF +fi + +# create small reference +START=30000 +END=31500 +CHR=chr1 + +# subset to small region +seqkit grep -r -p "^$CHR\$" "$ORIG_FA" | \ + seqkit subseq -r "$START:$END" > $OUT_DIR/reference_small.fa + +zcat "$ORIG_GTF" | \ + awk -v FS='\t' -v OFS='\t' " + \$1 == \"$CHR\" && \$4 >= $START && \$5 <= $END { + \$4 = \$4 - $START + 1; + \$5 = \$5 - $START + 1; + print; + }" > $OUT_DIR/reference_small.gtf + +# download bdabseq immunediscoverypanel fasta +# note: was contained in http://bd-rhapsody-public.s3.amazonaws.com/Rhapsody-Demo-Data-Inputs/12WTA-ABC-SMK-EB-5kJRT.tar +cat > $OUT_DIR/BDAbSeq_ImmuneDiscoveryPanel.fasta <CD11c:B-LY6|ITGAX|AHS0056|pAbO Catalog_940024 +ATGCGTTGCGAGAGATATGCGTAGGTTGCTGATTGG +>CD14:MPHIP9|CD14|AHS0037|pAbO Catalog_940005 +TGGCCCGTGGTAGCGCAATGTGAGATCGTAATAAGT +>CXCR5|CXCR5|AHS0039|pAbO Catalog_940042 +AGGAAGGTCGATTGTATAACGCGGCATTGTAACGGC +>CD19:SJ25C1|CD19|AHS0030|pAbO Catalog_940004 +TAGTAATGTGTTCGTAGCCGGTAATAATCTTCGTGG +>CD25:2A3|IL2RA|AHS0026|pAbO Catalog_940009 +AGTTGTATGGGTTAGCCGAGAGTAGTGCGTATGATT +>CD27:M-T271|CD27|AHS0025|pAbO Catalog_940018 +TGTCCGGTTTAGCGAATTGGGTTGAGTCACGTAGGT +>CD278|ICOS|AHS0012|pAbO Catalog_940043 +ATAGTCCGCCGTAATCGTTGTGTCGCTGAAAGGGTT +>CD279:EH12-1|PDCD1|AHS0014|pAbO Catalog_940015 +ATGGTAGTATCACGACGTAGTAGGGTAATTGGCAGT +>CD3:UCHT1|CD3E|AHS0231|pAbO Catalog_940307 +AGCTAGGTGTTATCGGCAAGTTGTACGGTGAAGTCG +>GITR|TNFRSF18|AHS0104|pAbO Catalog_940096 +TCTGTGTGTCGGGTTGAATCGTAGTGAGTTAGCGTG +>Tim3|HAVCR2|AHS0016|pAbO Catalog_940066 +TAGGTAGTAGTCCCGTATATCCGATCCGTGTTGTTT +>CD4:SK3|CD4|AHS0032|pAbO Catalog_940001 +TCGGTGTTATGAGTAGGTCGTCGTGCGGTTTGATGT +>CD45RA:HI100|PTPRC|AHS0009|pAbO Catalog_940011 +AAGCGATTGCGAAGGGTTAGTCAGTACGTTATGTTG +>CD56:NCAM16.2|NCAM1|AHS0019|pAbO Catalog_940007 +AGAGGTTGAGTCGTAATAATAATCGGAAGGCGTTGG +>CD62L:DREG-56|SELL|AHS0049|pAbO Catalog_940041 +ATGGTAAATATGGGCGAATGCGGGTTGTGCTAAAGT +>CCR7|CCR7|AHS0273|pAbO Catalog_940394 +AATGTGTGATCGGCAAAGGGTTCTCGGGTTAATATG +>CXCR6|CXCR6|AHS0148|pAbO Catalog_940234 +GTGGTTGGTTATTCGGACGGTTCTATTGTGAGCGCT +>CD127|IL7R|AHS0028|pAbO Catalog_940012 +AGTTATTAGGCTCGTAGGTATGTTTAGGTTATCGCG +>CD134:ACT35|TNFRSF4|AHS0013|pAbO Catalog_940060 +GGTGTTGGTAAGACGGACGGAGTAGATATTCGAGGT +>CD28:L293|CD28|AHS0138|pAbO Catalog_940226 +TTGTTGAGGATACGATGAAGCGGTTTAAGGGTGTGG +>CD272|BTLA|AHS0052|pAbO Catalog_940105 +GTAGGTTGATAGTCGGCGATAGTGCGGTTGAAAGCT +>CD8:SK1|CD8A|AHS0228|pAbO Catalog_940305 +AGGACATAGAGTAGGACGAGGTAGGCTTAAATTGCT +>HLA-DR|CD74|AHS0035|pAbO Catalog_940010 +TGTTGGTTATTCGTTAGTGCATCCGTTTGGGCGTGG +>CD16:3G8|FCGR3A|AHS0053|pAbO Catalog_940006 +TAAATCTAATCGCGGTAACATAACGGTGGGTAAGGT +>CD183|CXCR3|AHS0031|pAbO Catalog_940030 +AAAGTGTTGGCGTTATGTGTTCGTTAGCGGTGTGGG +>CD196|CCR6|AHS0034|pAbO Catalog_940033 +ACGTGTTATGGTGTTGTTCGAATTGTGGTAGTCAGT +>CD137|TNFRSF9|AHS0003|pAbO Catalog_940055 +TGACAAGCAACGAGCGATACGAAAGGCGAAATTAGT +>CD161:HP-3G10|KLRB1|AHS0205|pAbO Catalog_940283 +TTTAGGACGATTAGTTGTGCGGCATAGGAGGTGTTC +>IgM|IGHM|AHS0198|pAbO Catalog_940276 +TTTGGAGGGTAGCTAGTTGCAGTTCGTGGTCGTTTC +>IgD|IGHD|AHS0058|pAbO Catalog_940026 +TGAGGGATGTATAGCGAGAATTGCGACCGTAGACTT +EOF + +# this was obtained by running the command: +# docker run bdgenomics/rhapsody:2.2.1 cat /rhapsody/control_files/SampleTagSequences_HomoSapiens_ver1.fasta +cat > $OUT_DIR/SampleTagSequences_HomoSapiens_ver1.fasta <SampleTag01_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGATTCAAGGGCAGCCGCGTCACGATTGGATACGACTGTTGGACCGG +>SampleTag02_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGTGGATGGGATAAGTGCGTGATGGACCGAAGGGACCTCGTGGCCGG +>SampleTag03_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGCGGCTCGTGCTGCGTCGTCTCAAGTCCAGAAACTCCGTGTATCCT +>SampleTag04_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGATTGGGAGGCTTTCGTACCGCTGCCGCCACCAGGTGATACCCGCT +>SampleTag05_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGCTCCCTGGTGTTCAATACCCGATGTGGTGGGCAGAATGTGGCTGG +>SampleTag06_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGTTACCCGCAGGAAGACGTATACCCCTCGTGCCAGGCGACCAATGC +>SampleTag07_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGTGTCTACGTCGGACCGCAAGAAGTGAGTCAGAGGCTGCACGCTGT +>SampleTag08_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGCCCCACCAGGTTGCTTTGTCGGACGAGCCCGCACAGCGCTAGGAT +>SampleTag09_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGGTGATCCGCGCAGGCACACATACCGACTCAGATGGGTTGTCCAGG +>SampleTag10_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGGCAGCCGGCGTCGTACGAGGCACAGCGGAGACTAGATGAGGCCCC +>SampleTag11_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGCGCGTCCAATTTCCGAAGCCCCGCCCTAGGAGTTCCCCTGCGTGC +>SampleTag12_hs|stAbO +GTTGTCAAGATGCTACCGTTCAGAGGCCCATTCATTGCACCCGCCAGTGATCGACCCTAGTGGAGCTAAG +EOF diff --git a/src/kallisto/kallisto_index/Kallisto b/src/kallisto/kallisto_index/Kallisto new file mode 100644 index 00000000..3c7b5b2b Binary files /dev/null and b/src/kallisto/kallisto_index/Kallisto differ diff --git a/src/kallisto/kallisto_index/config.vsh.yaml b/src/kallisto/kallisto_index/config.vsh.yaml new file mode 100644 index 00000000..2c4f65c7 --- /dev/null +++ b/src/kallisto/kallisto_index/config.vsh.yaml @@ -0,0 +1,94 @@ +name: kallisto_index +namespace: kallisto +description: | + Build a Kallisto index for the transcriptome to use Kallisto in the mapping-based mode. +keywords: [kallisto, index] +links: + homepage: https://pachterlab.github.io/kallisto/about + documentation: https://pachterlab.github.io/kallisto/manual + repository: https://github.com/pachterlab/kallisto + issue_tracker: https://github.com/pachterlab/kallisto/issues +references: + doi: https://doi.org/10.1038/nbt.3519 +license: BSD 2-Clause License + +argument_groups: +- name: "Input" + arguments: + - name: "--input" + type: file + description: | + Path to a FASTA-file containing the transcriptome sequences, either in plain text or + compressed (.gz) format. + required: true + - name: "--d_list" + type: file + description: | + Path to a FASTA-file containing sequences to mask from quantification. + +- name: "Output" + arguments: + - name: "--index" + type: file + direction: output + example: Kallisto_index + +- name: "Options" + arguments: + - name: "--kmer_size" + type: integer + description: | + Kmer length passed to indexing step of pseudoaligners (default: '31'). + example: 31 + - name: "--make_unique" + type: boolean_true + description: | + Replace repeated target names with unique names. + - name: "--aa" + type: boolean_true + description: | + Generate index from a FASTA-file containing amino acid sequences. + - name: "--distiguish" + type: boolean_true + description: | + Generate index where sequences are distinguished by the sequence names. + - name: "--min_size" + alternatives: ["-m"] + type: integer + description: | + Length of minimizers (default: automatically chosen). + - name: "--ec_max_size" + alternatives: ["-e"] + type: integer + description: | + Maximum number of targets in an equivalence class (default: no maximum). + - name: "--tmp" + alternatives: ["-T"] + type: string + description: | + Path to a directory for temporary files. + example: "tmp" + +resources: + - type: bash_script + path: script.sh + +test_resources: + - type: bash_script + path: test.sh + - path: test_data + +engines: + - type: docker + image: ubuntu:22.04 + setup: + - type: docker + run: | + apt-get update && \ + apt-get install -y --no-install-recommends wget && \ + wget --no-check-certificate https://github.com/pachterlab/kallisto/releases/download/v0.50.1/kallisto_linux-v0.50.1.tar.gz && \ + tar -xzf kallisto_linux-v0.50.1.tar.gz && \ + mv kallisto/kallisto /usr/local/bin/ +runners: + - type: executable + - type: nextflow diff --git a/src/kallisto/kallisto_index/help.txt b/src/kallisto/kallisto_index/help.txt new file mode 100644 index 00000000..28778ac0 --- /dev/null +++ b/src/kallisto/kallisto_index/help.txt @@ -0,0 +1,21 @@ +``` +kallisto index +``` +kallisto 0.50.1 +Builds a kallisto index + +Usage: kallisto index [arguments] FASTA-files + +Required argument: +-i, --index=STRING Filename for the kallisto index to be constructed + +Optional argument: +-k, --kmer-size=INT k-mer (odd) length (default: 31, max value: 31) +-t, --threads=INT Number of threads to use (default: 1) +-d, --d-list=STRING Path to a FASTA-file containing sequences to mask from quantification + --make-unique Replace repeated target names with unique names + --aa Generate index from a FASTA-file containing amino acid sequences + --distinguish Generate index where sequences are distinguished by the sequence name +-T, --tmp=STRING Temporary directory (default: tmp) +-m, --min-size=INT Length of minimizers (default: automatically chosen) +-e, --ec-max-size=INT Maximum number of targets in an equivalence class (default: no maximum) diff --git a/src/kallisto/kallisto_index/script.sh b/src/kallisto/kallisto_index/script.sh new file mode 100644 index 00000000..59a5d3de --- /dev/null +++ b/src/kallisto/kallisto_index/script.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +## VIASH START +## VIASH END + +set -eo pipefail + +unset_if_false=( par_make_unique par_aa par_distinguish ) + +for var in "${unset_if_false[@]}"; do + temp_var="${!var}" + [[ "$temp_var" == "false" ]] && unset $var +done + +if [ -n "$par_kmer_size" ]; then + if [[ "$par_kmer_size" -lt 1 || "$par_kmer_size" -gt 31 || $(( par_kmer_size % 2 )) -eq 0 ]]; then + echo "Error: Kmer size must be an odd number between 1 and 31." + exit 1 + fi +fi + +kallisto index \ + -i "${par_index}" \ + ${par_kmer_size:+--kmer-size "${par_kmer_size}"} \ + ${par_make_unique:+--make-unique} \ + ${par_aa:+--aa} \ + ${par_distinguish:+--distinguish} \ + ${par_min_size:+--min-size "${par_min_size}"} \ + ${par_ec_max_size:+--ec-max-size "${par_ec_max_size}"} \ + ${par_d_list:+--d-list "${par_d_list}"} \ + ${meta_cpus:+--cpu "${meta_cpus}"} \ + ${par_tmp:+--tmp "${par_tmp}"} \ + "${par_input}" + diff --git a/src/kallisto/kallisto_index/test.sh b/src/kallisto/kallisto_index/test.sh new file mode 100644 index 00000000..2646dcd8 --- /dev/null +++ b/src/kallisto/kallisto_index/test.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +echo ">>>Test 1: Testing $meta_functionality_name with non-default k-mer size" + +"$meta_executable" \ + --input "$meta_resources_dir/test_data/transcriptome.fasta" \ + --index Kallisto \ + --kmer_size 21 + + +echo ">>> Checking whether output exists and is correct" +[ ! -f "Kallisto" ] && echo "Kallisto index does not exist!" && exit 1 +[ ! -s "Kallisto" ] && echo "Kallisto index is empty!" && exit 1 + +kallisto inspect Kallisto 2> test.txt +grep "number of k-mers: 989" test.txt || { echo "The content of the index seems to be incorrect." && exit 1; } + +################################################################################ + +echo ">>>Test 2: Testing $meta_functionality_name with d_list argument" + +"$meta_executable" \ + --input "$meta_resources_dir/test_data/transcriptome.fasta" \ + --index Kallisto \ + --d_list "$meta_resources_dir/test_data/d_list.fasta" + +echo ">>> Checking whether output exists and is correct" +[ ! -f "Kallisto" ] && echo "Kallisto index does not exist!" && exit 1 +[ ! -s "Kallisto" ] && echo "Kallisto index is empty!" && exit 1 + +kallisto inspect Kallisto 2> test.txt +grep "number of k-mers: 959" test.txt || { echo "The content of the index seems to be incorrect." && exit 1; } + +echo "All tests succeeded!" +exit 0 diff --git a/src/kallisto/kallisto_index/test_data/d_list.fasta b/src/kallisto/kallisto_index/test_data/d_list.fasta new file mode 100644 index 00000000..ad5e05bf --- /dev/null +++ b/src/kallisto/kallisto_index/test_data/d_list.fasta @@ -0,0 +1,5 @@ +>YAL067W-A CDS=1-228 +ATGCCAATTATAGGGGTGCCGAGGTGCCTTATAAAACCCTTTTCTGTGCCTGTGACATTTCCTTTTTCGG +TCAAAAAGAATATCCGAATTTTAGATTTGGACCCTCGTACAGAAGCTTATTGTCTAAGCCTGAATTCAGT +CTGCTTTAAACGGCTTCCGCGGAGGAAATATTTCCATCTCTTGAATTCGTACAACATTAAACGTGTGTTG +GGAGTCGTATACTGTTAG diff --git a/src/kallisto/kallisto_index/test_data/transcriptome.fasta b/src/kallisto/kallisto_index/test_data/transcriptome.fasta new file mode 100644 index 00000000..94c06163 --- /dev/null +++ b/src/kallisto/kallisto_index/test_data/transcriptome.fasta @@ -0,0 +1,23 @@ +>YAL069W CDS=1-315 +ATGATCGTAAATAACACACACGTGCTTACCCTACCACTTTATACCACCACCACATGCCATACTCACCCTC +ACTTGTATACTGATTTTACGTACGCACACGGATGCTACAGTATATACCATCTCAAACTTACCCTACTCTC +AGATTCCACTTCACTCCATGGCCCATCTCTCACTGAATCAGTACCAAATGCACTCACATCATTATGCACG +GCACTTGCCTCAGCGGTCTATACCCTGTGCCATTTACCCATAACGCCCATCATTATCCACATTTTGATAT +CTATATCTCATTCGGCGGTCCCAAATATTGTATAA +>YAL068W-A CDS=1-255 +ATGCACGGCACTTGCCTCAGCGGTCTATACCCTGTGCCATTTACCCATAACGCCCATCATTATCCACATT +TTGATATCTATATCTCATTCGGCGGTCCCAAATATTGTATAACTGCCCTTAATACATACGTTATACCACT +TTTGCACCATATACTTACCACTCCATTTATATACACTTATGTCAATATTACAGAAAAATCCCCACAAAAA +TCACCTAAACATAAAAATATTCTACTTTTCAACAATAATACATAA +>YAL068C CDS=1-363 +ATGGTCAAATTAACTTCAATCGCCGCTGGTGTCGCTGCCATCGCTGCTACTGCTTCTGCAACCACCACTC +TAGCTCAATCTGACGAAAGAGTCAACTTGGTGGAATTGGGTGTCTACGTCTCTGATATCAGAGCTCACTT +AGCCCAATACTACATGTTCCAAGCCGCCCACCCAACTGAAACCTACCCAGTCGAAGTTGCTGAAGCCGTT +TTCAACTACGGTGACTTCACCACCATGTTGACCGGTATTGCTCCAGACCAAGTGACCAGAATGATCACCG +GTGTTCCATGGTACTCCAGCAGATTAAAGCCAGCCATCTCCAGTGCTCTATCCAAGGACGGTATCTACAC +TATCGCAAACTAG +>YAL067W-A CDS=1-228 +ATGCCAATTATAGGGGTGCCGAGGTGCCTTATAAAACCCTTTTCTGTGCCTGTGACATTTCCTTTTTCGG +TCAAAAAGAATATCCGAATTTTAGATTTGGACCCTCGTACAGAAGCTTATTGTCTAAGCCTGAATTCAGT +CTGCTTTAAACGGCTTCCGCGGAGGAAATATTTCCATCTCTTGAATTCGTACAACATTAAACGTGTGTTG +GGAGTCGTATACTGTTAG \ No newline at end of file diff --git a/src/kallisto/kallisto_quant/config.vsh.yaml b/src/kallisto/kallisto_quant/config.vsh.yaml new file mode 100644 index 00000000..e92ac6b3 --- /dev/null +++ b/src/kallisto/kallisto_quant/config.vsh.yaml @@ -0,0 +1,105 @@ +name: kallisto_quant +namespace: kallisto +description: | + Quantifying abundances of transcripts from RNA-Seq data, or more generally of target sequences using high-throughput sequencing reads. +keywords: [kallisto, quant, pseudoalignment] +links: + homepage: https://pachterlab.github.io/kallisto/about + documentation: https://pachterlab.github.io/kallisto/manual + repository: https://github.com/pachterlab/kallisto + issue_tracker: https://github.com/pachterlab/kallisto/issues +references: + doi: 10.1038/nbt.3519 +license: BSD 2-Clause License + +argument_groups: +- name: "Input" + arguments: + - name: "--input" + type: file + description: List of input FastQ files of size 1 and 2 for single-end and paired-end data, respectively. + direction: "input" + multiple: true + required: true + - name: "--index" + alternatives: ["-i"] + type: file + description: Kallisto genome index. + must_exist: true + required: true + +- name: "Output" + arguments: + - name: "--output_dir" + alternatives: ["-o"] + type: string + description: Directory to write output to. + required: true + +- name: "Options" + arguments: + - name: "--single" + type: boolean_true + description: Single end mode. + - name: "--single_overhang" + type: boolean_true + description: Include reads where unobserved rest of fragment is predicted to lie outside a transcript. + - name: "--fr_stranded" + type: boolean_true + description: Strand specific reads, first read forward. + - name: "--rf_stranded" + type: boolean_true + description: Strand specific reads, first read reverse. + - name: "--fragment_length" + alternatives: ["-l"] + type: double + description: The estimated average fragment length. + - name: "--sd" + alternatives: ["-s"] + type: double + description: | + The estimated standard deviation of the fragment length (default: -l, -s values are estimated + from paired end data, but are required when using --single). + - name: "--plaintext" + type: boolean_true + description: Output plaintext instead of HDF5. + - name: "--bootstrap_samples" + alternatives: ["-b"] + type: integer + description: | + Number of bootstrap samples to draw. Default: '0' + example: 0 + - name: "--seed" + type: integer + description: | + Random seed for bootstrap. Default: '42' + example: 42 + + +resources: +- type: bash_script + path: script.sh + +test_resources: +- type: bash_script + path: test.sh +- type: file + path: test_data + +engines: + - type: docker + image: ubuntu:22.04 + setup: + - type: docker + run: | + apt-get update && \ + apt-get install -y --no-install-recommends wget && \ + wget --no-check-certificate https://github.com/pachterlab/kallisto/releases/download/v0.50.1/kallisto_linux-v0.50.1.tar.gz && \ + tar -xzf kallisto_linux-v0.50.1.tar.gz && \ + mv kallisto/kallisto /usr/local/bin/ + - type: docker + run: | + echo "kallisto: $(kallisto version | sed 's/kallisto, version //')" > /var/software_versions.txt +runners: + - type: executable + - type: nextflow diff --git a/src/kallisto/kallisto_quant/help.txt b/src/kallisto/kallisto_quant/help.txt new file mode 100644 index 00000000..7022571b --- /dev/null +++ b/src/kallisto/kallisto_quant/help.txt @@ -0,0 +1,33 @@ +``` +kallisto quant +``` + +kallisto 0.50.1 +Computes equivalence classes for reads and quantifies abundances + +Usage: kallisto quant [arguments] FASTQ-files + +Required arguments: +-i, --index=STRING Filename for the kallisto index to be used for + quantification +-o, --output-dir=STRING Directory to write output to + +Optional arguments: +-b, --bootstrap-samples=INT Number of bootstrap samples (default: 0) + --seed=INT Seed for the bootstrap sampling (default: 42) + --plaintext Output plaintext instead of HDF5 + --single Quantify single-end reads + --single-overhang Include reads where unobserved rest of fragment is + predicted to lie outside a transcript + --fr-stranded Strand specific reads, first read forward + --rf-stranded Strand specific reads, first read reverse +-l, --fragment-length=DOUBLE Estimated average fragment length +-s, --sd=DOUBLE Estimated standard deviation of fragment length + (default: -l, -s values are estimated from paired + end data, but are required when using --single) +-p, --priors Priors for the EM algorithm, either as raw counts or as + probabilities. Pseudocounts are added to raw reads to + prevent zero valued priors. Supplied in the same order + as the transcripts in the transcriptome +-t, --threads=INT Number of threads to use (default: 1) + --verbose Print out progress information every 1M proccessed reads \ No newline at end of file diff --git a/src/kallisto/kallisto_quant/script.sh b/src/kallisto/kallisto_quant/script.sh new file mode 100644 index 00000000..a7105cd1 --- /dev/null +++ b/src/kallisto/kallisto_quant/script.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +## VIASH START +## VIASH END + +set -eo pipefail + +unset_if_false=( par_single par_single_overhang par_rf_stranded par_fr_stranded par_plaintext ) + +for var in "${unset_if_false[@]}"; do + temp_var="${!var}" + [[ "$temp_var" == "false" ]] && unset $var +done + +IFS=";" read -ra input <<< $par_input + +# Check if par_single is not set and ensure even number of input files +if [ -z "$par_single" ]; then + if [ $((${#input[@]} % 2)) -ne 0 ]; then + echo "Error: When running in paired-end mode, the number of input files must be even." + echo "Number of input files provided: ${#input[@]}" + exit 1 + fi +fi + + +mkdir -p $par_output_dir + + +kallisto quant \ + ${meta_cpus:+--threads $meta_cpus} \ + -i $par_index \ + ${par_gtf:+--gtf "${par_gtf}"} \ + ${par_single:+--single} \ + ${par_single_overhang:+--single-overhang} \ + ${par_fr_stranded:+--fr-stranded} \ + ${par_rf_stranded:+--rf-stranded} \ + ${par_plaintext:+--plaintext} \ + ${par_bootstrap_samples:+--bootstrap-samples "${par_bootstrap_samples}"} \ + ${par_fragment_length:+--fragment-length "${par_fragment_length}"} \ + ${par_sd:+--sd "${par_sd}"} \ + ${par_seed:+--seed "${par_seed}"} \ + -o $par_output_dir \ + ${input[*]} + + diff --git a/src/kallisto/kallisto_quant/test.sh b/src/kallisto/kallisto_quant/test.sh new file mode 100644 index 00000000..28e2e3ad --- /dev/null +++ b/src/kallisto/kallisto_quant/test.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +echo ">>> Testing $meta_functionality_name" + +echo ">>> Test 1: Testing for paired-end reads" +"$meta_executable" \ + --index "$meta_resources_dir/test_data/index/transcriptome.idx" \ + --rf_stranded \ + --output_dir . \ + --input "$meta_resources_dir/test_data/reads/A_R1.fastq;$meta_resources_dir/test_data/reads/A_R2.fastq" + +echo ">>> Checking whether output exists" +[ ! -f "run_info.json" ] && echo "run_info.json does not exist!" && exit 1 +[ ! -s "run_info.json" ] && echo "run_info.json is empty!" && exit 1 +[ ! -f "abundance.tsv" ] && echo "abundance.tsv does not exist!" && exit 1 +[ ! -s "abundance.tsv" ] && echo "abundance.tsv is empty!" && exit 1 +[ ! -f "abundance.h5" ] && echo "abundance.h5 does not exist!" && exit 1 +[ ! -s "abundance.h5" ] && echo "abundance.h5 is empty!" && exit 1 + +echo ">>> Checking if output is correct" +diff "abundance.tsv" "$meta_resources_dir/test_data/abundance_1.tsv" || { echo "abundance.tsv is not correct"; exit 1; } + +rm -rf abundance.tsv abundance.h5 run_info.json + +################################################################################ + +echo ">>> Test 2: Testing for single-end reads" +"$meta_executable" \ + --index "$meta_resources_dir/test_data/index/transcriptome.idx" \ + --rf_stranded \ + --output_dir . \ + --single \ + --input "$meta_resources_dir/test_data/reads/A_R1.fastq" \ + --fragment_length 101 \ + --sd 50 + +echo ">>> Checking whether output exists" +[ ! -f "run_info.json" ] && echo "run_info.json does not exist!" && exit 1 +[ ! -s "run_info.json" ] && echo "run_info.json is empty!" && exit 1 +[ ! -f "abundance.tsv" ] && echo "abundance.tsv does not exist!" && exit 1 +[ ! -s "abundance.tsv" ] && echo "abundance.tsv is empty!" && exit 1 +[ ! -f "abundance.h5" ] && echo "abundance.h5 does not exist!" && exit 1 +[ ! -s "abundance.h5" ] && echo "abundance.h5 is empty!" && exit 1 + +echo ">>> Checking if output is correct" +diff "abundance.tsv" "$meta_resources_dir/test_data/abundance_2.tsv" || { echo "abundance.tsv is not correct"; exit 1; } + +rm -rf abundance.tsv abundance.h5 run_info.json + +################################################################################ + +echo "All tests succeeded!" +exit 0 diff --git a/src/kallisto/kallisto_quant/test_data/abundance_1.tsv b/src/kallisto/kallisto_quant/test_data/abundance_1.tsv new file mode 100644 index 00000000..1de99e54 --- /dev/null +++ b/src/kallisto/kallisto_quant/test_data/abundance_1.tsv @@ -0,0 +1,2 @@ +target_id length eff_length est_counts tpm +Sheila 35 36 0 -nan diff --git a/src/kallisto/kallisto_quant/test_data/abundance_2.tsv b/src/kallisto/kallisto_quant/test_data/abundance_2.tsv new file mode 100644 index 00000000..6b3e9055 --- /dev/null +++ b/src/kallisto/kallisto_quant/test_data/abundance_2.tsv @@ -0,0 +1,2 @@ +target_id length eff_length est_counts tpm +Sheila 35 15.0373 0 -nan diff --git a/src/kallisto/kallisto_quant/test_data/index/transcriptome.idx b/src/kallisto/kallisto_quant/test_data/index/transcriptome.idx new file mode 100644 index 00000000..194fec14 Binary files /dev/null and b/src/kallisto/kallisto_quant/test_data/index/transcriptome.idx differ diff --git a/src/kallisto/kallisto_quant/test_data/reads/A_R1.fastq b/src/kallisto/kallisto_quant/test_data/reads/A_R1.fastq new file mode 100644 index 00000000..999ed649 --- /dev/null +++ b/src/kallisto/kallisto_quant/test_data/reads/A_R1.fastq @@ -0,0 +1,4 @@ +@1 +GCTAGCTCAGAAAAAAAAAATCGTCGCGTGCGCGT ++ +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! diff --git a/src/kallisto/kallisto_quant/test_data/reads/A_R2.fastq b/src/kallisto/kallisto_quant/test_data/reads/A_R2.fastq new file mode 100644 index 00000000..999ed649 --- /dev/null +++ b/src/kallisto/kallisto_quant/test_data/reads/A_R2.fastq @@ -0,0 +1,4 @@ +@1 +GCTAGCTCAGAAAAAAAAAATCGTCGCGTGCGCGT ++ +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! diff --git a/src/kallisto/kallisto_quant/test_data/script.sh b/src/kallisto/kallisto_quant/test_data/script.sh new file mode 100755 index 00000000..6d684b29 --- /dev/null +++ b/src/kallisto/kallisto_quant/test_data/script.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# clone repo +if [ ! -d /tmp/snakemake-wrappers ]; then + git clone --depth 1 --single-branch --branch master https://github.com/snakemake/snakemake-wrappers /tmp/snakemake-wrappers +fi + +# copy test data +cp -r /tmp/snakemake-wrappers/bio/kallisto/quant/test/* src/kallisto/kallisto_quant/test_data + +rm src/kallisto/kallisto_quant/test_data/Snakefile \ No newline at end of file diff --git a/src/rsem/rsem_calculate_expression/config.vsh.yaml b/src/rsem/rsem_calculate_expression/config.vsh.yaml new file mode 100644 index 00000000..2cd950cb --- /dev/null +++ b/src/rsem/rsem_calculate_expression/config.vsh.yaml @@ -0,0 +1,479 @@ +name: "rsem_calculate_expression" +namespace: "rsem" +description: | + Calculate expression with RSEM. +keywords: [Transcriptome, Index, Alignment, RSEM] +links: + homepage: https://deweylab.github.io/RSEM/ + documentation: https://deweylab.github.io/RSEM/rsem-calculate-expression.html + repository: https://github.com/deweylab/RSEM +references: + doi: https://doi.org/10.1186/1471-2105-12-323 +license: GPL-3.0 + + +argument_groups: +- name: "Input" + arguments: + - name: "--id" + type: string + description: Sample ID. + - name: "--strandedness" + type: string + description: Sample strand-specificity. Must be one of unstranded, forward, reverse + choices: [forward, reverse, unstranded] + - name: "--paired" + type: boolean_true + description: Paired-end reads or not? + - name: "--input" + type: file + description: Input reads for quantification. + multiple: true + - name: "--index" + type: file + must_exist: false + description: RSEM index. + - name: "--extra_args" + type: string + description: Extra rsem-calculate-expression arguments in addition to the examples. + +- name: "Output" + arguments: + - name: "--counts_gene" + type: file + description: Expression counts on gene level + example: $id.genes.results + direction: output + - name: "--counts_transcripts" + type: file + description: Expression counts on transcript level + example: $id.isoforms.results + direction: output + - name: "--stat" + type: file + description: RSEM statistics + example: $id.stat + direction: output + - name: "--logs" + type: file + description: RSEM logs + example: $id.log + direction: output + - name: "--bam_star" + type: file + description: BAM file generated by STAR (optional) + example: $id.STAR.genome.bam + direction: output + - name: "--bam_genome" + type: file + description: Genome BAM file (optional) + example: $id.genome.bam + direction: output + - name: "--bam_transcript" + type: file + description: Transcript BAM file (optional) + example: $id.transcript.bam + direction: output + - name: "--sort_bam_by_read_name" + type: boolean_true + description: | + Sort BAM file aligned under transcript coordidate by read name. Setting this option on will produce + deterministic maximum likelihood estimations from independent runs. Note that sorting will take long + time and lots of memory. + - name: "--no_bam_output" + type: boolean_true + description: Do not output any BAM file. + - name: "--sampling_for_bam" + type: boolean_true + description: | + When RSEM generates a BAM file, instead of outputting all alignments a read has with their posterior + probabilities, one alignment is sampled according to the posterior probabilities. The sampling procedure + includes the alignment to the "noise" transcript, which does not appear in the BAM file. Only the + sampled alignment has a weight of 1. All other alignments have weight 0. If the "noise" transcript is + sampled, all alignments appeared in the BAM file should have weight 0. + - name: "--output_genome_bam" + type: boolean_true + description: | + Generate a BAM file, 'sample_name.genome.bam', with alignments mapped to genomic coordinates and + annotated with their posterior probabilities. In addition, RSEM will call samtools (included in RSEM + package) to sort and index the bam file. 'sample_name.genome.sorted.bam' and 'sample_name.genome.sorted.bam.bai' + will be generated. + - name: "--sort_bam_by_coordinate" + type: boolean_true + description: | + Sort RSEM generated transcript and genome BAM files by coordinates and build associated indices. + +- name: "Basic Options" + arguments: + - name: "--no_qualities" + type: boolean_true + description: Input reads do not contain quality scores. + - name: "--alignments" + type: boolean_true + description: | + Input file contains alignments in SAM/BAM/CRAM format. The exact file format will be determined + automatically. + - name: "--fai" + type: file + description: | + If the header section of input alignment file does not contain reference sequence information, + this option should be turned on. is a FAI format file containing each reference sequence's + name and length. Please refer to the SAM official website for the details of FAI format. + - name: "--bowtie2" + type: boolean_true + description: | + Use Bowtie 2 instead of Bowtie to align reads. Since currently RSEM does not handle indel, local + and discordant alignments, the Bowtie2 parameters are set in a way to avoid those alignments. In + particular, we use options '--sensitive --dpad 0 --gbar 99999999 --mp 1,1 --np 1 --score_min L,0,-0.1' + by default. The last parameter of '--score_min', '-0.1', is the negative of maximum mismatch rate. + This rate can be set by option '--bowtie2_mismatch_rate'. If reads are paired-end, we additionally + use options '--no_mixed' and '--no_discordant'. + - name: "--star" + type: boolean_true + description: | + Use STAR to align reads. Alignment parameters are from ENCODE3's STAR-RSEM pipeline. To save + computational time and memory resources, STAR's Output BAM file is unsorted. It is stored in RSEM's + temporary directory with name as 'sample_name.bam'. Each STAR job will have its own private copy of + the genome in memory. + - name: "--hisat2_hca" + type: boolean_true + description: | + Use HISAT2 to align reads to the transcriptome according to Human Cell Atlast. + - name: "--append_names" + type: boolean_true + description: | + If gene_name/transcript_name is available, append it to the end of gene_id/transcript_id (separated + by '_') in files 'sample_name.isoforms.results' and 'sample_name.genes.results'. + - name: "--seed" + type: integer + description: | + Set the seed for the random number generators used in calculating posterior mean estimates and + credibility intervals. The seed must be a non-negative 32 bit integer. + - name: "--single_cell_prior" + type: boolean_true + description: | + By default, RSEM uses Dirichlet(1) as the prior to calculate posterior mean estimates and credibility + intervals. However, much less genes are expressed in single cell RNA-Seq data. Thus, if you want to + compute posterior mean estimates and/or credibility intervals and you have single-cell RNA-Seq data, + you are recommended to turn on this option. Then RSEM will use Dirichlet(0.1) as the prior which + encourage the sparsity of the expression levels. + - name: "--calc_pme" + type: boolean_true + description: Run RSEM's collapsed Gibbs sampler to calculate posterior mean estimates. + - name: "--calc_ci" + type: boolean_true + description: | + Calculate 95% credibility intervals and posterior mean estimates. The credibility level can be + changed by setting '--ci_credibility_level'. + - name: "--quiet" + alternatives: "-q" + type: boolean_true + description: Suppress the output of logging information. + +- name: "Aligner Options" + arguments: + - name: "--seed_length" + type: integer + description: | + Seed length used by the read aligner. Providing the correct value is important for RSEM. If RSEM + runs Bowtie, it uses this value for Bowtie's seed length parameter. Any read with its or at least + one of its mates' (for paired-end reads) length less than this value will be ignored. If the + references are not added poly(A) tails, the minimum allowed value is 5, otherwise, the minimum + allowed value is 25. Note that this script will only check if the value >= 5 and give a warning + message if the value < 25 but >= 5. (Default: 25) + example: 25 + - name: "--phred64_quals" + type: boolean_true + description: | + Input quality scores are encoded as Phred+64 (default for GA Pipeline ver. >= 1.3). This option is + used by Bowtie, Bowtie 2 and HISAT2. Otherwise, quality score will be encoded as Phred+33. (Default: false) + - name: "--solexa_quals" + type: boolean_true + description: | + Input quality scores are solexa encoded (from GA Pipeline ver. < 1.3). This option is used by + Bowtie, Bowtie 2 and HISAT2. Otherwise, quality score will be encoded as Phred+33. (Default: false) + - name: "--bowtie_n" + type: integer + description: | + (Bowtie parameter) max # of mismatches in the seed. (Range: 0-3, Default: 2) + choices: [0, 1, 2, 3] + example: 2 + - name: "--bowtie_e" + type: integer + description: | + (Bowtie parameter) max sum of mismatch quality scores across the alignment. (Default: 99999999) + example: 99999999 + - name: "--bowtie_m" + type: integer + description: | + (Bowtie parameter) suppress all alignments for a read if > valid alignments exist. (Default: 200) + example: 200 + - name: "--bowtie_chunkmbs" + type: integer + description: | + (Bowtie parameter) memory allocated for best first alignment calculation (Default: 0 - use Bowtie's default) + example: 0 + - name: "--bowtie2_mismatch_rate" + type: double + description: | + (Bowtie 2 parameter) The maximum mismatch rate allowed. (Default: 0.1) + example: 0.1 + - name: "--bowtie2_k" + type: integer + description: | + (Bowtie 2 parameter) Find up to alignments per read. (Default: 200) + example: 200 + - name: "--bowtie2_sensitivity_level" + type: string + description: | + (Bowtie 2 parameter) Set Bowtie 2's preset options in --end-to-end mode. This option controls how + hard Bowtie 2 tries to find alignments. must be one of "very_fast", "fast", "sensitive" + and "very_sensitive". The four candidates correspond to Bowtie 2's "--very-fast", "--fast", + "--sensitive" and "--very-sensitive" options. (Default: "sensitive" - use Bowtie 2's default) + choices: ["very_fast", "fast", "sensitive", "very_sensitive"] + example: sensitive + - name: "--star_gzipped_read_file" + type: boolean_true + description: | + Input read file(s) is compressed by gzip. (Default: false) + - name: "--star_bzipped_read_file" + type: boolean_true + description: | + Input read file(s) is compressed by bzip2. (Default: false) + - name: "--star_output_genome_bam" + type: boolean_true + description: | + Save the BAM file from STAR alignment under genomic coordinate to 'sample_name.STAR.genome.bam'. + This file is NOT sorted by genomic coordinate. In this file, according to STAR's manual, 'paired + ends of an alignment are always adjacent, and multiple alignments of a read are adjacent as well'. + (Default: false) + +- name: "Advanced Options" + arguments: + - name: "--tag" + type: string + description: | + The name of the optional field used in the SAM input for identifying a read with too many valid + alignments. The field should have the format :i:, where a bigger than 0 + indicates a read with too many alignments. (Default: "") + example: "" + - name: "--fragment_length_min" + type: integer + description: | + Minimum read/insert length allowed. This is also the value for the Bowtie/Bowtie2 -I option. + (Default: 1) + example: 1 + - name: "--fragment_length_max" + type: integer + description: | + Maximum read/insert length allowed. This is also the value for the Bowtie/Bowtie 2 -X option. + (Default: 1000) + example: 1000 + - name: "--fragment_length_mean" + type: integer + description: | + (single-end data only) The mean of the fragment length distribution, which is assumed to be a + Gaussian. (Default: -1, which disables use of the fragment length distribution) + example: -1 + - name: "--gragment_length_sd" + type: double + description: | + (single-end data only) The standard deviation of the fragment length distribution, which is + assumed to be a Gaussian. (Default: 0, which assumes that all fragments are of the same length, + given by the rounded value of --fragment_length_mean). + example: 0 + - name: "--estimate_rspd" + type: boolean_true + description: | + Set this option if you want to estimate the read start position distribution (RSPD) from data. + Otherwise, RSEM will use a uniform RSPD. + - name: "--num_rspd_bins" + type: integer + description: | + Number of bins in the RSPD. Only relevant when '--estimate_rspd' is specified. Use of the default + setting is recommended. (Default: 20) + example: 20 + - name: "--gibbs_burnin" + type: integer + description: | + The number of burn-in rounds for RSEM's Gibbs sampler. Each round passes over the entire data set + once. If RSEM can use multiple threads, multiple Gibbs samplers will start at the same time and all + samplers share the same burn-in number. (Default: 200) + example: 200 + - name: "--gibbs_number_of_samples" + type: integer + description: | + The total number of count vectors RSEM will collect from its Gibbs samplers. (Default: 1000) + example: 1000 + - name: "--gibbs_sampling_gap" + type: integer + description: | + The number of rounds between two succinct count vectors RSEM collects. If the count vector after + round N is collected, the count vector after round N + will also be collected. (Default: 1) + example: 1 + - name: "--ci_credibility_level" + type: double + description: | + The credibility level for credibility intervals. (Default: 0.95) + example: 0.95 + - name: "--ci_number_of_samples_per_count_vector" + type: integer + description: | + The number of read generating probability vectors sampled per sampled count vector. The crebility + intervals are calculated by first sampling P(C | D) and then sampling P(Theta | C) for each sampled + count vector. This option controls how many Theta vectors are sampled per sampled count vector. + (Default: 50) + example: 50 + - name: "--keep_intermediate_files" + type: boolean_true + description: | + Keep temporary files generated by RSEM. RSEM creates a temporary directory, 'sample_name.temp', + into which it puts all intermediate output files. If this directory already exists, RSEM overwrites + all files generated by previous RSEM runs inside of it. By default, after RSEM finishes, the + temporary directory is deleted. Set this option to prevent the deletion of this directory and the + intermediate files inside of it. + - name: "--temporary_folder" + type: string + description: | + Set where to put the temporary files generated by RSEM. If the folder specified does not exist, + RSEM will try to create it. (Default: sample_name.temp) + example: sample_name.temp + - name: "--time" + type: boolean_true + description: | + Output time consumed by each step of RSEM to 'sample_name.time'. + +- name: "Prior-Enhanced RSEM Options" + arguments: + - name: "--run_pRSEM" + type: boolean_true + description: | + Running prior-enhanced RSEM (pRSEM). Prior parameters, i.e. isoform's initial pseudo-count for + RSEM's Gibbs sampling, will be learned from input RNA-seq data and an external data set. When pRSEM + needs and only needs ChIP-seq peak information to partition isoforms (e.g. in pRSEM's default + partition model), either ChIP-seq peak file (with the '--chipseq_peak_file' option) or ChIP-seq + FASTQ files for target and input and the path for Bowtie executables are required (with the + '--chipseq_target_read_files ', '--chipseq_control_read_files ', and '--bowtie_path + options), otherwise, ChIP-seq FASTQ files for target and control and the path to Bowtie + executables are required. + - name: "--chipseq_peak_file" + type: file + must_exist: true + description: | + Full path to a ChIP-seq peak file in ENCODE's narrowPeak, i.e. BED6+4, format. This file is used + when running prior-enhanced RSEM in the default two-partition model. It partitions isoforms by + whether they have ChIP-seq overlapping with their transcription start site region or not. Each + partition will have its own prior parameter learned from a training set. This file can be either + gzipped or ungzipped. + - name: "--chipseq_target_read_files" + type: file + must_exist: true + description: | + Comma-separated full path of FASTQ read file(s) for ChIP-seq target. This option is used when running + prior-enhanced RSEM. It provides information to calculate ChIP-seq peaks and signals. The file(s) + can be either ungzipped or gzipped with a suffix '.gz' or '.gzip'. The options '--bowtie_path ' + and '--chipseq_control_read_files ' must be defined when this option is specified. + - name: "--chipseq_control_read_files" + type: file + must_exist: true + description: | + Comma-separated full path of FASTQ read file(s) for ChIP-seq conrol. This option is used when running + prior-enhanced RSEM. It provides information to call ChIP-seq peaks. The file(s) can be either + ungzipped or gzipped with a suffix '.gz' or '.gzip'. The options '--bowtie_path ' and + '--chipseq_target_read_files ' must be defined when this option is specified. + - name: "--chipseq_read_files_multi_targets" + type: file + must_exist: true + description: | + Comma-separated full path of FASTQ read files for multiple ChIP-seq targets. This option is used when + running prior-enhanced RSEM, where prior is learned from multiple complementary data sets. It provides + information to calculate ChIP-seq signals. All files can be either ungzipped or gzipped with a suffix + '.gz' or '.gzip'. When this option is specified, the option '--bowtie_path ' must be defined and + the option '--partition_model ' will be set to 'cmb_lgt' automatically. + - name: "--chipseq_bed_files_multi_targets" + type: file + must_exist: true + description: | + Comma-separated full path of BED files for multiple ChIP-seq targets. This option is used when running + prior-enhanced RSEM, where prior is learned from multiple complementary data sets. It provides information + of ChIP-seq signals and must have at least the first six BED columns. All files can be either ungzipped + or gzipped with a suffix '.gz' or '.gzip'. When this option is specified, the option '--partition_model + ' will be set to 'cmb_lgt' automatically. + - name: "--cap_stacked_chipseq_reads" + type: boolean_true + description: | + Keep a maximum number of ChIP-seq reads that aligned to the same genomic interval. This option is used + when running prior-enhanced RSEM, where prior is learned from multiple complementary data sets. This + option is only in use when either '--chipseq_read_files_multi_targets ' or + '--chipseq_bed_files_multi_targets ' is specified. + - name: "--n_max_stacked_chipseq_reads" + type: integer + description: | + The maximum number of stacked ChIP-seq reads to keep. This option is used when running prior-enhanced + RSEM, where prior is learned from multiple complementary data sets. This option is only in use when the + option '--cap_stacked_chipseq_reads' is set. + - name: "--partition_model" + type: string + description: | + A keyword to specify the partition model used by prior-enhanced RSEM. It must be one of the following + keywords: + * pk + * pk_lgtnopk + * lm3, lm4, lm5, or lm6 + * nopk_lm2pk, nopk_lm3pk, nopk_lm4pk, or nopk_lm5pk + * pk_lm2nopk, pk_lm3nopk, pk_lm4nopk, or pk_lm5nopk + * cmb_lgt + Parameters for all the above models are learned from a training set. For detailed explanations, please + see prior-enhanced RSEM's paper. (Default: 'pk') + example: "pk" + + +resources: + - type: bash_script + path: script.sh + +test_resources: + - type: bash_script + path: test.sh + +engines: + - type: docker + image: ubuntu:22.04 + setup: + - type: apt + packages: + - build-essential + - gcc + - g++ + - make + - wget + - zlib1g-dev + - unzip + - type: docker + env: + - STAR_VERSION=2.7.11b + - RSEM_VERSION=1.3.3 + run: | + apt-get update && \ + apt-get clean && \ + wget --no-check-certificate https://github.com/alexdobin/STAR/archive/refs/tags/2.7.11a.zip && \ + unzip 2.7.11a.zip && \ + cp STAR-2.7.11a/bin/Linux_x86_64_static/STAR /usr/local/bin && \ + cd && \ + wget --no-check-certificate https://github.com/deweylab/RSEM/archive/refs/tags/v1.3.3.zip && \ + unzip v1.3.3.zip && \ + cd RSEM-1.3.3 && \ + make && \ + make install + - type: docker + run: | + echo "RSEM: `rsem-calculate-expression --version | sed -e 's/Current version: RSEM v//g'`" > /var/software_versions.txt && \ + echo "STAR: `STAR --version`" >> /var/software_versions.txt && \ + echo "bowtie2: `bowtie2 --version | grep -oP '\d+\.\d+\.\d+'`" >> /var/software_versions.txt && \ + echo "bowtie: `bowtie --version | grep -oP 'bowtie-align-s version \K\d+\.\d+\.\d+'`" >> /var/software_versions.txt && \ + echo "HISAT2: `hisat2 --version | grep -oP 'hisat2-align-s version \K\d+\.\d+\.\d+'`" >> /var/software_versions.txt +runners: + - type: executable + - type: nextflow + + diff --git a/src/rsem/rsem_calculate_expression/help.txt b/src/rsem/rsem_calculate_expression/help.txt new file mode 100644 index 00000000..edfa3333 --- /dev/null +++ b/src/rsem/rsem_calculate_expression/help.txt @@ -0,0 +1,1002 @@ +NAME + rsem-calculate-expression - Estimate gene and isoform expression from + RNA-Seq data. + +SYNOPSIS + rsem-calculate-expression [options] upstream_read_file(s) reference_name sample_name + rsem-calculate-expression [options] --paired-end upstream_read_file(s) downstream_read_file(s) reference_name sample_name + rsem-calculate-expression [options] --alignments [--paired-end] input reference_name sample_name + +ARGUMENTS + upstream_read_files(s) + Comma-separated list of files containing single-end reads or + upstream reads for paired-end data. By default, these files are + assumed to be in FASTQ format. If the --no-qualities option is + specified, then FASTA format is expected. + + downstream_read_file(s) + Comma-separated list of files containing downstream reads which are + paired with the upstream reads. By default, these files are assumed + to be in FASTQ format. If the --no-qualities option is specified, + then FASTA format is expected. + + input + SAM/BAM/CRAM formatted input file. If "-" is specified for the + filename, the input is instead assumed to come from standard input. + RSEM requires all alignments of the same read group together. For + paired-end reads, RSEM also requires the two mates of any alignment + be adjacent. In addition, RSEM does not allow the SEQ and QUAL + fields to be empty. See Description section for how to make input + file obey RSEM's requirements. + + reference_name + The name of the reference used. The user must have run + 'rsem-prepare-reference' with this reference_name before running + this program. + + sample_name + The name of the sample analyzed. All output files are prefixed by + this name (e.g., sample_name.genes.results) + +BASIC OPTIONS + --paired-end + Input reads are paired-end reads. (Default: off) + + --no-qualities + Input reads do not contain quality scores. (Default: off) + + --strandedness + This option defines the strandedness of the RNA-Seq reads. It + recognizes three values: 'none', 'forward', and 'reverse'. 'none' + refers to non-strand-specific protocols. 'forward' means all + (upstream) reads are derived from the forward strand. 'reverse' + means all (upstream) reads are derived from the reverse strand. If + 'forward'/'reverse' is set, the '--norc'/'--nofw' Bowtie/Bowtie 2 + option will also be enabled to avoid aligning reads to the opposite + strand. For Illumina TruSeq Stranded protocols, please use + 'reverse'. (Default: 'none') + + -p/--num-threads + Number of threads to use. Both Bowtie/Bowtie2, expression estimation + and 'samtools sort' will use this many threads. (Default: 1) + + --alignments + Input file contains alignments in SAM/BAM/CRAM format. The exact + file format will be determined automatically. (Default: off) + + --fai + If the header section of input alignment file does not contain + reference sequence information, this option should be turned on. + is a FAI format file containing each reference sequence's + name and length. Please refer to the SAM official website for the + details of FAI format. (Default: off) + + --bowtie2 + Use Bowtie 2 instead of Bowtie to align reads. Since currently RSEM + does not handle indel, local and discordant alignments, the Bowtie2 + parameters are set in a way to avoid those alignments. In + particular, we use options '--sensitive --dpad 0 --gbar 99999999 + --mp 1,1 --np 1 --score-min L,0,-0.1' by default. The last parameter + of '--score-min', '-0.1', is the negative of maximum mismatch rate. + This rate can be set by option '--bowtie2-mismatch-rate'. If reads + are paired-end, we additionally use options '--no-mixed' and + '--no-discordant'. (Default: off) + + --star + Use STAR to align reads. Alignment parameters are from ENCODE3's + STAR-RSEM pipeline. To save computational time and memory resources, + STAR's Output BAM file is unsorted. It is stored in RSEM's temporary + directory with name as 'sample_name.bam'. Each STAR job will have + its own private copy of the genome in memory. (Default: off) + + --hisat2-hca + Use HISAT2 to align reads to the transcriptome according to Human + Cell Atlast SMART-Seq2 pipeline. In particular, we use HISAT + parameters "-k 10 --secondary --rg-id=$sampleToken --rg + SM:$sampleToken --rg LB:$sampleToken --rg PL:ILLUMINA --rg + PU:$sampleToken --new-summary --summary-file $sampleName.log + --met-file $sampleName.hisat2.met.txt --met 5 --mp 1,1 --np 1 + --score-min L,0,-0.1 --rdg 99999999,99999999 --rfg 99999999,99999999 + --no-spliced-alignment --no-softclip --seed 12345". If inputs are + paired-end reads, we additionally use parameters "--no-mixed + --no-discordant". (Default: off) + + --append-names + If gene_name/transcript_name is available, append it to the end of + gene_id/transcript_id (separated by '_') in files + 'sample_name.isoforms.results' and 'sample_name.genes.results'. + (Default: off) + + --seed + Set the seed for the random number generators used in calculating + posterior mean estimates and credibility intervals. The seed must be + a non-negative 32 bit integer. (Default: off) + + --single-cell-prior + By default, RSEM uses Dirichlet(1) as the prior to calculate + posterior mean estimates and credibility intervals. However, much + less genes are expressed in single cell RNA-Seq data. Thus, if you + want to compute posterior mean estimates and/or credibility + intervals and you have single-cell RNA-Seq data, you are recommended + to turn on this option. Then RSEM will use Dirichlet(0.1) as the + prior which encourage the sparsity of the expression levels. + (Default: off) + + --calc-pme + Run RSEM's collapsed Gibbs sampler to calculate posterior mean + estimates. (Default: off) + + --calc-ci + Calculate 95% credibility intervals and posterior mean estimates. + The credibility level can be changed by setting + '--ci-credibility-level'. (Default: off) + + -q/--quiet + Suppress the output of logging information. (Default: off) + + -h/--help + Show help information. + + --version + Show version information. + +OUTPUT OPTIONS + --sort-bam-by-read-name + Sort BAM file aligned under transcript coordidate by read name. + Setting this option on will produce deterministic maximum likelihood + estimations from independent runs. Note that sorting will take long + time and lots of memory. (Default: off) + + --no-bam-output + Do not output any BAM file. (Default: off) + + --sampling-for-bam + When RSEM generates a BAM file, instead of outputting all alignments + a read has with their posterior probabilities, one alignment is + sampled according to the posterior probabilities. The sampling + procedure includes the alignment to the "noise" transcript, which + does not appear in the BAM file. Only the sampled alignment has a + weight of 1. All other alignments have weight 0. If the "noise" + transcript is sampled, all alignments appeared in the BAM file + should have weight 0. (Default: off) + + --output-genome-bam + Generate a BAM file, 'sample_name.genome.bam', with alignments + mapped to genomic coordinates and annotated with their posterior + probabilities. In addition, RSEM will call samtools (included in + RSEM package) to sort and index the bam file. + 'sample_name.genome.sorted.bam' and + 'sample_name.genome.sorted.bam.bai' will be generated. (Default: + off) + + --sort-bam-by-coordinate + Sort RSEM generated transcript and genome BAM files by coordinates + and build associated indices. (Default: off) + + --sort-bam-memory-per-thread + Set the maximum memory per thread that can be used by 'samtools + sort'. represents the memory and accepts suffices 'K/M/G'. + RSEM will pass to the '-m' option of 'samtools sort'. Note + that the default used here is different from the default used by + samtools. (Default: 1G) + +ALIGNER OPTIONS + --seed-length + Seed length used by the read aligner. Providing the correct value is + important for RSEM. If RSEM runs Bowtie, it uses this value for + Bowtie's seed length parameter. Any read with its or at least one of + its mates' (for paired-end reads) length less than this value will + be ignored. If the references are not added poly(A) tails, the + minimum allowed value is 5, otherwise, the minimum allowed value is + 25. Note that this script will only check if the value >= 5 and give + a warning message if the value < 25 but >= 5. (Default: 25) + + --phred33-quals + Input quality scores are encoded as Phred+33. This option is used by + Bowtie, Bowtie 2 and HISAT2. (Default: on) + + --phred64-quals + Input quality scores are encoded as Phred+64 (default for GA + Pipeline ver. >= 1.3). This option is used by Bowtie, Bowtie 2 and + HISAT2. (Default: off) + + --solexa-quals + Input quality scores are solexa encoded (from GA Pipeline ver. < + 1.3). This option is used by Bowtie, Bowtie 2 and HISAT2. (Default: + off) + + --bowtie-path + The path to the Bowtie executables. (Default: the path to the Bowtie + executables is assumed to be in the user's PATH environment + variable) + + --bowtie-n + (Bowtie parameter) max # of mismatches in the seed. (Range: 0-3, + Default: 2) + + --bowtie-e + (Bowtie parameter) max sum of mismatch quality scores across the + alignment. (Default: 99999999) + + --bowtie-m + (Bowtie parameter) suppress all alignments for a read if > + valid alignments exist. (Default: 200) + + --bowtie-chunkmbs + (Bowtie parameter) memory allocated for best first alignment + calculation (Default: 0 - use Bowtie's default) + + --bowtie2-path + (Bowtie 2 parameter) The path to the Bowtie 2 executables. (Default: + the path to the Bowtie 2 executables is assumed to be in the user's + PATH environment variable) + + --bowtie2-mismatch-rate + (Bowtie 2 parameter) The maximum mismatch rate allowed. (Default: + 0.1) + + --bowtie2-k + (Bowtie 2 parameter) Find up to alignments per read. (Default: + 200) + + --bowtie2-sensitivity-level + (Bowtie 2 parameter) Set Bowtie 2's preset options in --end-to-end + mode. This option controls how hard Bowtie 2 tries to find + alignments. must be one of "very_fast", "fast", "sensitive" + and "very_sensitive". The four candidates correspond to Bowtie 2's + "--very-fast", "--fast", "--sensitive" and "--very-sensitive" + options. (Default: "sensitive" - use Bowtie 2's default) + + --star-path + The path to STAR's executable. (Default: the path to STAR executable + is assumed to be in user's PATH environment variable) + + --star-gzipped-read-file + (STAR parameter) Input read file(s) is compressed by gzip. (Default: + off) + + --star-bzipped-read-file + (STAR parameter) Input read file(s) is compressed by bzip2. + (Default: off) + + --star-output-genome-bam + (STAR parameter) Save the BAM file from STAR alignment under genomic + coordinate to 'sample_name.STAR.genome.bam'. This file is NOT sorted + by genomic coordinate. In this file, according to STAR's manual, + 'paired ends of an alignment are always adjacent, and multiple + alignments of a read are adjacent as well'. (Default: off) + + --hisat2-path + The path to HISAT2's executable. (Default: the path to HISAT2 + executable is assumed to be in user's PATH environment variable) + +ADVANCED OPTIONS + --tag + The name of the optional field used in the SAM input for identifying + a read with too many valid alignments. The field should have the + format :i:, where a bigger than 0 indicates + a read with too many alignments. (Default: "") + + --fragment-length-min + Minimum read/insert length allowed. This is also the value for the + Bowtie/Bowtie2 -I option. (Default: 1) + + --fragment-length-max + Maximum read/insert length allowed. This is also the value for the + Bowtie/Bowtie 2 -X option. (Default: 1000) + + --fragment-length-mean + (single-end data only) The mean of the fragment length distribution, + which is assumed to be a Gaussian. (Default: -1, which disables use + of the fragment length distribution) + + --fragment-length-sd + (single-end data only) The standard deviation of the fragment length + distribution, which is assumed to be a Gaussian. (Default: 0, which + assumes that all fragments are of the same length, given by the + rounded value of --fragment-length-mean) + + --estimate-rspd + Set this option if you want to estimate the read start position + distribution (RSPD) from data. Otherwise, RSEM will use a uniform + RSPD. (Default: off) + + --num-rspd-bins + Number of bins in the RSPD. Only relevant when '--estimate-rspd' is + specified. Use of the default setting is recommended. (Default: 20) + + --gibbs-burnin + The number of burn-in rounds for RSEM's Gibbs sampler. Each round + passes over the entire data set once. If RSEM can use multiple + threads, multiple Gibbs samplers will start at the same time and all + samplers share the same burn-in number. (Default: 200) + + --gibbs-number-of-samples + The total number of count vectors RSEM will collect from its Gibbs + samplers. (Default: 1000) + + --gibbs-sampling-gap + The number of rounds between two succinct count vectors RSEM + collects. If the count vector after round N is collected, the count + vector after round N + will also be collected. (Default: 1) + + --ci-credibility-level + The credibility level for credibility intervals. (Default: 0.95) + + --ci-memory + Maximum size (in memory, MB) of the auxiliary buffer used for + computing credibility intervals (CI). (Default: 1024) + + --ci-number-of-samples-per-count-vector + The number of read generating probability vectors sampled per + sampled count vector. The crebility intervals are calculated by + first sampling P(C | D) and then sampling P(Theta | C) for each + sampled count vector. This option controls how many Theta vectors + are sampled per sampled count vector. (Default: 50) + + --keep-intermediate-files + Keep temporary files generated by RSEM. RSEM creates a temporary + directory, 'sample_name.temp', into which it puts all intermediate + output files. If this directory already exists, RSEM overwrites all + files generated by previous RSEM runs inside of it. By default, + after RSEM finishes, the temporary directory is deleted. Set this + option to prevent the deletion of this directory and the + intermediate files inside of it. (Default: off) + + --temporary-folder + Set where to put the temporary files generated by RSEM. If the + folder specified does not exist, RSEM will try to create it. + (Default: sample_name.temp) + + --time + Output time consumed by each step of RSEM to 'sample_name.time'. + (Default: off) + +PRIOR-ENHANCED RSEM OPTIONS + --run-pRSEM + Running prior-enhanced RSEM (pRSEM). Prior parameters, i.e. + isoform's initial pseudo-count for RSEM's Gibbs sampling, will be + learned from input RNA-seq data and an external data set. When pRSEM + needs and only needs ChIP-seq peak information to partition isoforms + (e.g. in pRSEM's default partition model), either ChIP-seq peak file + (with the '--chipseq-peak-file' option) or ChIP-seq FASTQ files for + target and input and the path for Bowtie executables are required + (with the '--chipseq-target-read-files ', + '--chipseq-control-read-files ', and '--bowtie-path + options), otherwise, ChIP-seq FASTQ files for target and control and + the path to Bowtie executables are required. (Default: off) + + --chipseq-peak-file + Full path to a ChIP-seq peak file in ENCODE's narrowPeak, i.e. + BED6+4, format. This file is used when running prior-enhanced RSEM + in the default two-partition model. It partitions isoforms by + whether they have ChIP-seq overlapping with their transcription + start site region or not. Each partition will have its own prior + parameter learned from a training set. This file can be either + gzipped or ungzipped. (Default: "") + + --chipseq-target-read-files + Comma-separated full path of FASTQ read file(s) for ChIP-seq target. + This option is used when running prior-enhanced RSEM. It provides + information to calculate ChIP-seq peaks and signals. The file(s) can + be either ungzipped or gzipped with a suffix '.gz' or '.gzip'. The + options '--bowtie-path ' and '--chipseq-control-read-files + ' must be defined when this option is specified. (Default: + "") + + --chipseq-control-read-files + Comma-separated full path of FASTQ read file(s) for ChIP-seq conrol. + This option is used when running prior-enhanced RSEM. It provides + information to call ChIP-seq peaks. The file(s) can be either + ungzipped or gzipped with a suffix '.gz' or '.gzip'. The options + '--bowtie-path ' and '--chipseq-target-read-files ' + must be defined when this option is specified. (Default: "") + + --chipseq-read-files-multi-targets + Comma-separated full path of FASTQ read files for multiple ChIP-seq + targets. This option is used when running prior-enhanced RSEM, where + prior is learned from multiple complementary data sets. It provides + information to calculate ChIP-seq signals. All files can be either + ungzipped or gzipped with a suffix '.gz' or '.gzip'. When this + option is specified, the option '--bowtie-path ' must be + defined and the option '--partition-model ' will be set to + 'cmb_lgt' automatically. (Default: "") + + --chipseq-bed-files-multi-targets + Comma-separated full path of BED files for multiple ChIP-seq + targets. This option is used when running prior-enhanced RSEM, where + prior is learned from multiple complementary data sets. It provides + information of ChIP-seq signals and must have at least the first six + BED columns. All files can be either ungzipped or gzipped with a + suffix '.gz' or '.gzip'. When this option is specified, the option + '--partition-model ' will be set to 'cmb_lgt' automatically. + (Default: "") + + --cap-stacked-chipseq-reads + Keep a maximum number of ChIP-seq reads that aligned to the same + genomic interval. This option is used when running prior-enhanced + RSEM, where prior is learned from multiple complementary data sets. + This option is only in use when either + '--chipseq-read-files-multi-targets ' or + '--chipseq-bed-files-multi-targets ' is specified. (Default: + off) + + --n-max-stacked-chipseq-reads + The maximum number of stacked ChIP-seq reads to keep. This option is + used when running prior-enhanced RSEM, where prior is learned from + multiple complementary data sets. This option is only in use when + the option '--cap-stacked-chipseq-reads' is set. (Default: 5) + + --partition-model + A keyword to specify the partition model used by prior-enhanced + RSEM. It must be one of the following keywords: + + - pk + Partitioned by whether an isoform has a ChIP-seq peak overlapping + with its transcription start site (TSS) region. The TSS region is + defined as [TSS-500bp, TSS+500bp]. For simplicity, we refer this + type of peak as 'TSS peak' when explaining other keywords. + + - pk_lgtnopk + First partitioned by TSS peak. Then, for isoforms in the 'no TSS + peak' set, a logistic model is employed to further classify them + into two partitions. + + - lm3, lm4, lm5, or lm6 + Based on their ChIP-seq signals, isoforms are classified into 3, + 4, 5, or 6 partitions by a linear regression model. + + - nopk_lm2pk, nopk_lm3pk, nopk_lm4pk, or nopk_lm5pk + First partitioned by TSS peak. Then, for isoforms in the 'with TSS + peak' set, a linear regression model is employed to further + classify them into 2, 3, 4, or 5 partitions. + + - pk_lm2nopk, pk_lm3nopk, pk_lm4nopk, or pk_lm5nopk + First partitioned by TSS peak. Then, for isoforms in the 'no TSS + peak' set, a linear regression model is employed to further + classify them into 2, 3, 4, or 5 partitions. + + - cmb_lgt + Using a logistic regression to combine TSS signals from multiple + complementary data sets and partition training set isoform into + 'expressed' and 'not expressed'. This partition model is only in + use when either '--chipseq-read-files-multi-targets ' or + '--chipseq-bed-files-multi-targets is specified. + + Parameters for all the above models are learned from a training set. + For detailed explanations, please see prior-enhanced RSEM's paper. + (Default: 'pk') + +DEPRECATED OPTIONS + The options in this section are deprecated. They are here only for + compatibility reasons and may be removed in future releases. + + --sam + Inputs are alignments in SAM format. (Default: off) + + --bam + Inputs are alignments in BAM format. (Default: off) + + --strand-specific + Equivalent to '--strandedness forward'. (Default: off) + + --forward-prob + Probability of generating a read from the forward strand of a + transcript. Set to 1 for a strand-specific protocol where all + (upstream) reads are derived from the forward strand, 0 for a + strand-specific protocol where all (upstream) read are derived from + the reverse strand, or 0.5 for a non-strand-specific protocol. + (Default: off) + +DESCRIPTION + In its default mode, this program aligns input reads against a reference + transcriptome with Bowtie and calculates expression values using the + alignments. RSEM assumes the data are single-end reads with quality + scores, unless the '--paired-end' or '--no-qualities' options are + specified. Alternatively, users can use STAR to align reads using the + '--star' option. RSEM has provided options in 'rsem-prepare-reference' + to prepare STAR's genome indices. Users may use an alternative aligner + by specifying '--alignments', and providing an alignment file in + SAM/BAM/CRAM format. However, users should make sure that they align + against the indices generated by 'rsem-prepare-reference' and the + alignment file satisfies the requirements mentioned in ARGUMENTS + section. + + One simple way to make the alignment file satisfying RSEM's requirements + is to use the 'convert-sam-for-rsem' script. This script accepts + SAM/BAM/CRAM files as input and outputs a BAM file. For example, type + the following command to convert a SAM file, 'input.sam', to a + ready-for-use BAM file, 'input_for_rsem.bam': + + convert-sam-for-rsem input.sam input_for_rsem + + For details, please refer to 'convert-sam-for-rsem's documentation page. + +NOTES + 1. Users must run 'rsem-prepare-reference' with the appropriate + reference before using this program. + + 2. For single-end data, it is strongly recommended that the user provide + the fragment length distribution parameters (--fragment-length-mean and + --fragment-length-sd). For paired-end data, RSEM will automatically + learn a fragment length distribution from the data. + + 3. Some aligner parameters have default values different from their + original settings. + + 4. With the '--calc-pme' option, posterior mean estimates will be + calculated in addition to maximum likelihood estimates. + + 5. With the '--calc-ci' option, 95% credibility intervals and posterior + mean estimates will be calculated in addition to maximum likelihood + estimates. + + 6. The temporary directory and all intermediate files will be removed + when RSEM finishes unless '--keep-intermediate-files' is specified. + + With the '--run-pRSEM' option and associated options (see section + 'PRIOR-ENHANCED RSEM OPTIONS' above for details), prior-enhanced RSEM + will be running. Prior parameters will be learned from supplied external + data set(s) and assigned as initial pseudo-counts for isoforms in the + corresponding partition for Gibbs sampling. + +OUTPUT + sample_name.isoforms.results + File containing isoform level expression estimates. The first line + contains column names separated by the tab character. The format of + each line in the rest of this file is: + + transcript_id gene_id length effective_length expected_count TPM + FPKM IsoPct [posterior_mean_count + posterior_standard_deviation_of_count pme_TPM pme_FPKM + IsoPct_from_pme_TPM TPM_ci_lower_bound TPM_ci_upper_bound + TPM_coefficient_of_quartile_variation FPKM_ci_lower_bound + FPKM_ci_upper_bound FPKM_coefficient_of_quartile_variation] + + Fields are separated by the tab character. Fields within "[]" are + optional. They will not be presented if neither '--calc-pme' nor + '--calc-ci' is set. + + 'transcript_id' is the transcript name of this transcript. 'gene_id' + is the gene name of the gene which this transcript belongs to + (denote this gene as its parent gene). If no gene information is + provided, 'gene_id' and 'transcript_id' are the same. + + 'length' is this transcript's sequence length (poly(A) tail is not + counted). 'effective_length' counts only the positions that can + generate a valid fragment. If no poly(A) tail is added, + 'effective_length' is equal to transcript length - mean fragment + length + 1. If one transcript's effective length is less than 1, + this transcript's both effective length and abundance estimates are + set to 0. + + 'expected_count' is the sum of the posterior probability of each + read comes from this transcript over all reads. Because 1) each read + aligning to this transcript has a probability of being generated + from background noise; 2) RSEM may filter some alignable low quality + reads, the sum of expected counts for all transcript are generally + less than the total number of reads aligned. + + 'TPM' stands for Transcripts Per Million. It is a relative measure + of transcript abundance. The sum of all transcripts' TPM is 1 + million. 'FPKM' stands for Fragments Per Kilobase of transcript per + Million mapped reads. It is another relative measure of transcript + abundance. If we define l_bar be the mean transcript length in a + sample, which can be calculated as + + l_bar = \sum_i TPM_i / 10^6 * effective_length_i (i goes through + every transcript), + + the following equation is hold: + + FPKM_i = 10^3 / l_bar * TPM_i. + + We can see that the sum of FPKM is not a constant across samples. + + 'IsoPct' stands for isoform percentage. It is the percentage of this + transcript's abandunce over its parent gene's abandunce. If its + parent gene has only one isoform or the gene information is not + provided, this field will be set to 100. + + 'posterior_mean_count', 'pme_TPM', 'pme_FPKM' are posterior mean + estimates calculated by RSEM's Gibbs sampler. + 'posterior_standard_deviation_of_count' is the posterior standard + deviation of counts. 'IsoPct_from_pme_TPM' is the isoform percentage + calculated from 'pme_TPM' values. + + 'TPM_ci_lower_bound', 'TPM_ci_upper_bound', 'FPKM_ci_lower_bound' + and 'FPKM_ci_upper_bound' are lower(l) and upper(u) bounds of 95% + credibility intervals for TPM and FPKM values. The bounds are + inclusive (i.e. [l, u]). + + 'TPM_coefficient_of_quartile_variation' and + 'FPKM_coefficient_of_quartile_variation' are coefficients of + quartile variation (CQV) for TPM and FPKM values. CQV is a robust + way of measuring the ratio between the standard deviation and the + mean. It is defined as + + CQV := (Q3 - Q1) / (Q3 + Q1), + + where Q1 and Q3 are the first and third quartiles. + + sample_name.genes.results + File containing gene level expression estimates. The first line + contains column names separated by the tab character. The format of + each line in the rest of this file is: + + gene_id transcript_id(s) length effective_length expected_count TPM + FPKM [posterior_mean_count posterior_standard_deviation_of_count + pme_TPM pme_FPKM TPM_ci_lower_bound TPM_ci_upper_bound + TPM_coefficient_of_quartile_variation FPKM_ci_lower_bound + FPKM_ci_upper_bound FPKM_coefficient_of_quartile_variation] + + Fields are separated by the tab character. Fields within "[]" are + optional. They will not be presented if neither '--calc-pme' nor + '--calc-ci' is set. + + 'transcript_id(s)' is a comma-separated list of transcript_ids + belonging to this gene. If no gene information is provided, + 'gene_id' and 'transcript_id(s)' are identical (the + 'transcript_id'). + + A gene's 'length' and 'effective_length' are defined as the weighted + average of its transcripts' lengths and effective lengths (weighted + by 'IsoPct'). A gene's abundance estimates are just the sum of its + transcripts' abundance estimates. + + sample_name.alleles.results + Only generated when the RSEM references are built with + allele-specific transcripts. + + This file contains allele level expression estimates for + allele-specific expression calculation. The first line contains + column names separated by the tab character. The format of each line + in the rest of this file is: + + allele_id transcript_id gene_id length effective_length + expected_count TPM FPKM AlleleIsoPct AlleleGenePct + [posterior_mean_count posterior_standard_deviation_of_count pme_TPM + pme_FPKM AlleleIsoPct_from_pme_TPM AlleleGenePct_from_pme_TPM + TPM_ci_lower_bound TPM_ci_upper_bound + TPM_coefficient_of_quartile_variation FPKM_ci_lower_bound + FPKM_ci_upper_bound FPKM_coefficient_of_quartile_variation] + + Fields are separated by the tab character. Fields within "[]" are + optional. They will not be presented if neither '--calc-pme' nor + '--calc-ci' is set. + + 'allele_id' is the allele-specific name of this allele-specific + transcript. + + 'AlleleIsoPct' stands for allele-specific percentage on isoform + level. It is the percentage of this allele-specific transcript's + abundance over its parent transcript's abundance. If its parent + transcript has only one allele variant form, this field will be set + to 100. + + 'AlleleGenePct' stands for allele-specific percentage on gene level. + It is the percentage of this allele-specific transcript's abundance + over its parent gene's abundance. + + 'AlleleIsoPct_from_pme_TPM' and 'AlleleGenePct_from_pme_TPM' have + similar meanings. They are calculated based on posterior mean + estimates. + + Please note that if this file is present, the fields 'length' and + 'effective_length' in 'sample_name.isoforms.results' should be + interpreted similarly as the corresponding definitions in + 'sample_name.genes.results'. + + sample_name.transcript.bam + Only generated when --no-bam-output is not specified. + + 'sample_name.transcript.bam' is a BAM-formatted file of read + alignments in transcript coordinates. The MAPQ field of each + alignment is set to min(100, floor(-10 * log10(1.0 - w) + 0.5)), + where w is the posterior probability of that alignment being the + true mapping of a read. In addition, RSEM pads a new tag ZW:f:value, + where value is a single precision floating number representing the + posterior probability. Because this file contains all alignment + lines produced by bowtie or user-specified aligners, it can also be + used as a replacement of the aligner generated BAM/SAM file. + + sample_name.transcript.sorted.bam and + sample_name.transcript.sorted.bam.bai + Only generated when --no-bam-output is not specified and + --sort-bam-by-coordinate is specified. + + 'sample_name.transcript.sorted.bam' and + 'sample_name.transcript.sorted.bam.bai' are the sorted BAM file and + indices generated by samtools (included in RSEM package). + + sample_name.genome.bam + Only generated when --no-bam-output is not specified and + --output-genome-bam is specified. + + 'sample_name.genome.bam' is a BAM-formatted file of read alignments + in genomic coordinates. Alignments of reads that have identical + genomic coordinates (i.e., alignments to different isoforms that + share the same genomic region) are collapsed into one alignment. The + MAPQ field of each alignment is set to min(100, floor(-10 * + log10(1.0 - w) + 0.5)), where w is the posterior probability of that + alignment being the true mapping of a read. In addition, RSEM pads a + new tag ZW:f:value, where value is a single precision floating + number representing the posterior probability. If an alignment is + spliced, a XS:A:value tag is also added, where value is either '+' + or '-' indicating the strand of the transcript it aligns to. + + sample_name.genome.sorted.bam and sample_name.genome.sorted.bam.bai + Only generated when --no-bam-output is not specified, and + --sort-bam-by-coordinate and --output-genome-bam are specified. + + 'sample_name.genome.sorted.bam' and + 'sample_name.genome.sorted.bam.bai' are the sorted BAM file and + indices generated by samtools (included in RSEM package). + + sample_name.time + Only generated when --time is specified. + + It contains time (in seconds) consumed by aligning reads, estimating + expression levels and calculating credibility intervals. + + sample_name.log + Only generated when --alignments is not specified. + + It captures alignment statistics outputted from the user-specified + aligner. + + sample_name.stat + This is a folder instead of a file. All model related statistics are + stored in this folder. Use 'rsem-plot-model' can generate plots + using this folder. + + 'sample_name.stat/sample_name.cnt' contains alignment statistics. + The format and meanings of each field are described in + 'cnt_file_description.txt' under RSEM directory. + + 'sample_name.stat/sample_name.model' stores RNA-Seq model parameters + learned from the data. The format and meanings of each filed of this + file are described in 'model_file_description.txt' under RSEM + directory. + + The following four output files will be generated only by + prior-enhanced RSEM + + - 'sample_name.stat/sample_name_prsem.all_tr_features' + It stores isofrom features for deriving and assigning pRSEM prior. + The first line is a header and the rest is one isoform per line. + The description for each column is: + + * trid: transcript ID from input annotation + + * geneid: gene ID from input anntation + + * chrom: isoform's chromosome name + + * strand: isoform's strand name + + * start: isoform's end with the lowest genomic loci + + * end: isoform's end with the highest genomic loci + + * tss_mpp: average mappability of [TSS-500bp, TSS+500bp], where + TSS is isoform's transcription start site, i.e. 5'-end + + * body_mpp: average mappability of (TSS+500bp, TES-500bp), where + TES is isoform's transcription end site, i.e. 3'-end + + * tes_mpp: average mappability of [TES-500bp, TES+500bp] + + * pme_count: isoform's fragment or read count from RSEM's + posterior mean estimates + + * tss: isoform's TSS loci + + * tss_pk: equal to 1 if isoform's [TSS-500bp, TSS+500bp] region + overlaps with a RNA Pol II peak; 0 otherwise + + * is_training: equal to 1 if isoform is in the training set where + Pol II prior is learned; 0 otherwise + + - 'sample_name.stat/sample_name_prsem.all_tr_prior' + It stores prior parameters for every isoform. This file does not + have a header. Each line contains a prior parameter and an + isoform's transcript ID delimited by ` # `. + + - 'sample_name.stat/sample_name_uniform_prior_1.isoforms.results' + RSEM's posterior mean estimates on the isoform level with an + initial pseudo-count of one for every isoform. It is in the same + format as the 'sample_name.isoforms.results'. + + - 'sample_name.stat/sample_name_uniform_prior_1.genes.results' + RSEM's posterior mean estimates on the gene level with an initial + pseudo-count of one for every isoform. It is in the same format as + the 'sample_name.genes.results'. + + When learning prior from multiple external data sets in + prior-enhanced RSEM, two additional output files will be generated. + + - 'sample_name.stat/sample_name.pval_LL' + It stores a p-value and a log-likelihood. The p-value indicates + whether the combination of multiple complementary data sets is + informative for RNA-seq quantification. The log-likelihood shows + how well pRSEM's Dirichlet-multinomial model fits the read counts + of partitioned training set isoforms. + + - 'sample_name.stat/sample_name.lgt_mdl.RData' + It stores an R object named 'glmmdl', which is a logistic + regression model on the training set isoforms and multiple + external data sets. + + In addition, extra columns will be added to + 'sample_name.stat/all_tr_features' + + * is_expr: equal to 1 if isoform has an abundance >= 1 TPM and a + non-zero read count from RSEM's posterior mean estimates; 0 + otherwise + + * "$external_data_set_basename": log10 of external data's signal at + [TSS-500, TSS+500]. Signal is the number of reads aligned within + that interval and normalized to RPKM by read depth and interval + length. It will be set to -4 if no read aligned to that interval. + + There are multiple columns like this one, where each represents an + external data set. + + * prd_expr_prob: predicted probability from logistic regression + model on whether this isoform is expressed or not. A probability + higher than 0.5 is considered as expressed + + * partition: group index, to which this isoforms is partitioned + + * prior: prior parameter for this isoform + +EXAMPLES + Assume the path to the bowtie executables is in the user's PATH + environment variable. Reference files are under '/ref' with name + 'mouse_125'. + + 1) '/data/mmliver.fq', single-end reads with quality scores. Quality + scores are encoded as for 'GA pipeline version >= 1.3'. We want to use 8 + threads and generate a genome BAM file. In addition, we want to append + gene/transcript names to the result files: + + rsem-calculate-expression --phred64-quals \ + -p 8 \ + --append-names \ + --output-genome-bam \ + /data/mmliver.fq \ + /ref/mouse_125 \ + mmliver_single_quals + + 2) '/data/mmliver_1.fq' and '/data/mmliver_2.fq', stranded paired-end + reads with quality scores. Suppose the library is prepared using TruSeq + Stranded Kit, which means the first mate should map to the reverse + strand. Quality scores are in SANGER format. We want to use 8 threads + and do not generate a genome BAM file: + + rsem-calculate-expression -p 8 \ + --paired-end \ + --strandedness reverse \ + /data/mmliver_1.fq \ + /data/mmliver_2.fq \ + /ref/mouse_125 \ + mmliver_paired_end_quals + + 3) '/data/mmliver.fa', single-end reads without quality scores. We want + to use 8 threads: + + rsem-calculate-expression -p 8 \ + --no-qualities \ + /data/mmliver.fa \ + /ref/mouse_125 \ + mmliver_single_without_quals + + 4) Data are the same as 1). This time we assume the bowtie executables + are under '/sw/bowtie'. We want to take a fragment length distribution + into consideration. We set the fragment length mean to 150 and the + standard deviation to 35. In addition to a BAM file, we also want to + generate credibility intervals. We allow RSEM to use 1GB of memory for + CI calculation: + + rsem-calculate-expression --bowtie-path /sw/bowtie \ + --phred64-quals \ + --fragment-length-mean 150.0 \ + --fragment-length-sd 35.0 \ + -p 8 \ + --output-genome-bam \ + --calc-ci \ + --ci-memory 1024 \ + /data/mmliver.fq \ + /ref/mouse_125 \ + mmliver_single_quals + + 5) '/data/mmliver_paired_end_quals.bam', BAM-formatted alignments for + paired-end reads with quality scores. We want to use 8 threads: + + rsem-calculate-expression --paired-end \ + --alignments \ + -p 8 \ + /data/mmliver_paired_end_quals.bam \ + /ref/mouse_125 \ + mmliver_paired_end_quals + + 6) '/data/mmliver_1.fq.gz' and '/data/mmliver_2.fq.gz', paired-end reads + with quality scores and read files are compressed by gzip. We want to + use STAR to aligned reads and assume STAR executable is '/sw/STAR'. + Suppose we want to use 8 threads and do not generate a genome BAM file: + + rsem-calculate-expression --paired-end \ + --star \ + --star-path /sw/STAR \ + --gzipped-read-file \ + --paired-end \ + -p 8 \ + /data/mmliver_1.fq.gz \ + /data/mmliver_2.fq.gz \ + /ref/mouse_125 \ + mmliver_paired_end_quals + + 7) In the above example, suppose we want to run prior-enhanced RSEM + instead. Assuming we want to learn priors from a ChIP-seq peak file + '/data/mmlive.narrowPeak.gz': + + rsem-calculate-expression --star \ + --star-path /sw/STAR \ + --gzipped-read-file \ + --paired-end \ + --calc-pme \ + --run-pRSEM \ + --chipseq-peak-file /data/mmliver.narrowPeak.gz \ + -p 8 \ + /data/mmliver_1.fq.gz \ + /data/mmliver_2.fq.gz \ + /ref/mouse_125 \ + mmliver_paired_end_quals + + 8) Similar to the example in 7), suppose we want to use the partition + model 'pk_lm2nopk' (partitioning isoforms by Pol II TSS peak first and + then partitioning 'no TSS peak' isoforms into two bins by a linear + regression model), and we want to partition isoforms by RNA Pol II's + ChIP-seq read files '/data/mmliver_PolIIRep1.fq.gz' and + '/data/mmliver_PolIIRep2.fq.gz', and the control ChIP-seq read files + '/data/mmliver_ChIPseqCtrl.fq.gz'. Also, assuming Bowtie's executables + are under '/sw/bowtie/': + + rsem-calculate-expression --star \ + --star-path /sw/STAR \ + --gzipped-read-file \ + --paired-end \ + --calc-pme \ + --run-pRSEM \ + --chipseq-target-read-files /data/mmliver_PolIIRep1.fq.gz,/data/mmliver_PolIIRep2.fq.gz \ + --chipseq-control-read-files /data/mmliver_ChIPseqCtrl.fq.gz \ + --partition-model pk_lm2nopk \ + --bowtie-path /sw/bowtie \ + -p 8 \ + /data/mmliver_1.fq.gz \ + /data/mmliver_2.fq.gz \ + /ref/mouse_125 \ + mmliver_paired_end_quals + + 9) Similar to the example in 8), suppose we want to derive prior from + four histone modification ChIP-seq read data sets: + '/data/H3K27Ac.fastq.gz', '/data/H3K4me1.fastq.gz', + '/data/H3K4me2.fastq.gz', and '/data/H3K4me3.fastq.gz'. Also, assuming + Bowtie's executables are under '/sw/bowtie/': + + rsem-calculate-expression --star \ + --star-path /sw/STAR \ + --gzipped-read-file \ + --paired-end \ + --calc-pme \ + --run-pRSEM \ + --partition-model cmb_lgt \ + --chipseq-read-files-multi-targets /data/H3K27Ac.fastq.gz,/data/H3K4me1.fastq.gz,/data/H3K4me2.fastq.gz,/data/H3K4me3.fastq.gz \ + --bowtie-path /sw/bowtie \ + -p 8 \ + /data/mmliver_1.fq.gz \ + /data/mmliver_2.fq.gz \ + /ref/mouse_125 \ + mmliver_paired_end_quals + diff --git a/src/rsem/rsem_calculate_expression/script.sh b/src/rsem/rsem_calculate_expression/script.sh new file mode 100644 index 00000000..e8c6ce5d --- /dev/null +++ b/src/rsem/rsem_calculate_expression/script.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +## VIASH START +## VIASH END + +set -eo pipefail + +function clean_up { + rm -rf "$tmpdir" +} +trap clean_up EXIT + +tmpdir=$(mktemp -d "$meta_temp_dir/$meta_functionality_name-XXXXXXXX") + +if [ "$par_strandedness" == 'forward' ]; then + strandedness='--strandedness forward' +elif [ "$par_strandedness" == 'reverse' ]; then + strandedness="--strandedness reverse" +else + strandedness='' +fi + +IFS=";" read -ra input <<< $par_input + +INDEX=$(find -L $meta_resources_dir/$par_index -name "*.grp" | sed 's/\.grp$//') + +unset_if_false=( par_paired par_quiet par_no_bam_output par_sampling_for_bam par_no_qualities + par_alignments par_bowtie2 par_star par_hisat2_hca par_append_names + par_single_cell_prior par_calc_pme par_calc_ci par_phred64_quals + par_solexa_quals par_star_gzipped_read_file par_star_bzipped_read_file + par_star_output_genome_bam par_estimate_rspd par_keep_intermediate_files + par_time par_run_pRSEM par_cap_stacked_chipseq_reads par_sort_bam_by_read_name ) + +for par in ${unset_if_false[@]}; do + test_val="${!par}" + [[ "$test_val" == "false" ]] && unset $par +done + +rsem-calculate-expression \ + ${par_quiet:+-q} \ + ${par_no_bam_output:+--no-bam-output} \ + ${par_sampling_for_bam:+--sampling-for-bam} \ + ${par_no_qualities:+--no-qualities} \ + ${par_alignments:+--alignments} \ + ${par_bowtie2:+--bowtie2} \ + ${par_star:+--star} \ + ${par_hisat2_hca:+--hisat2-hca} \ + ${par_append_names:+--append-names} \ + ${par_single_cell_prior:+--single-cell-prior} \ + ${par_calc_pme:+--calc-pme} \ + ${par_calc_ci:+--calc-ci} \ + ${par_phred64_quals:+--phred64-quals} \ + ${par_solexa_quals:+--solexa-quals} \ + ${par_star_gzipped_read_file:+--star-gzipped-read-file} \ + ${par_star_bzipped_read_file:+--star-bzipped-read-file} \ + ${par_star_output_genome_bam:+--star-output-genome-bam} \ + ${par_estimate_rspd:+--estimate-rspd} \ + ${par_keep_intermediate_files:+--keep-intermediate-files} \ + ${par_time:+--time} \ + ${par_run_pRSEM:+--run-pRSEM} \ + ${par_cap_stacked_chipseq_reads:+--cap-stacked-chipseq-reads} \ + ${par_sort_bam_by_read_name:+--sort-bam-by-read-name} \ + ${par_counts_gene:+--counts-gene "$par_counts_gene"} \ + ${par_counts_transcripts:+--counts-transcripts "$par_counts_transcripts"} \ + ${par_stat:+--stat "$par_stat"} \ + ${par_bam_star:+--bam-star "$par_bam_star"} \ + ${par_bam_genome:+--bam-genome "$par_bam_genome"} \ + ${par_bam_transcript:+--bam-transcript "$par_bam_transcript"} \ + ${par_fai:+--fai "$par_fai"} \ + ${par_seed:+--seed "$par_seed"} \ + ${par_seed_length:+--seed-length "$par_seed_length"} \ + ${par_bowtie_n:+--bowtie-n "$par_bowtie_n"} \ + ${par_bowtie_e:+--bowtie-e "$par_bowtie_e"} \ + ${par_bowtie_m:+--bowtie-m "$par_bowtie_m"} \ + ${par_bowtie_chunkmbs:+--bowtie-chunkmbs "$par_bowtie_chunkmbs"} \ + ${par_bowtie2_mismatch_rate:+--bowtie2-mismatch-rate "$par_bowtie2_mismatch_rate"} \ + ${par_bowtie2_k:+--bowtie2-k "$par_bowtie2_k"} \ + ${par_bowtie2_sensitivity_level:+--bowtie2-sensitivity-level "$par_bowtie2_sensitivity_level"} \ + ${par_tag:+--tag "$par_tag"} \ + ${par_fragment_length_min:+--fragment-length-min "$par_fragment_length_min"} \ + ${par_fragment_length_max:+--fragment-length-max "$par_fragment_length_max"} \ + ${par_fragment_length_mean:+--fragment-length-mean "$par_fragment_length_mean"} \ + ${par_fragment_length_sd:+--fragment-length-sd "$par_fragment_length_sd"} \ + ${par_num_rspd_bins:+--num-rspd-bins "$par_num_rspd_bins"} \ + ${par_gibbs_burnin:+--gibbs-burnin "$par_gibbs_burnin"} \ + ${par_gibbs_number_of_samples:+--gibbs-number-of-samples "$par_gibbs_number_of_samples"} \ + ${par_gibbs_sampling_gap:+--gibbs-sampling-gap "$par_gibbs_sampling_gap"} \ + ${par_ci_credibility_level:+--ci-credibility-level "$par_ci_credibility_level"} \ + ${par_ci_number_of_samples_per_count_vector:+--ci-number-of-samples-per-count-vector "$par_ci_number_of_samples_per_count_vector"} \ + ${par_temporary_folder:+--temporary-folder "$par_temporary_folder"} \ + ${par_chipseq_peak_file:+--chipseq-peak-file "$par_chipseq_peak_file"} \ + ${par_chipseq_target_read_files:+--chipseq-target-read-files "$par_chipseq_target_read_files"} \ + ${par_chipseq_control_read_files:+--chipseq-control-read-files "$par_chipseq_control_read_files"} \ + ${par_chipseq_read_files_multi_targets:+--chipseq-read-files-multi-targets "$par_chipseq_read_files_multi_targets"} \ + ${par_chipseq_bed_files_multi_targets:+--chipseq-bed-files-multi-targets "$par_chipseq_bed_files_multi_targets"} \ + ${par_n_max_stacked_chipseq_reads:+--n-max-stacked-chipseq-reads "$par_n_max_stacked_chipseq_reads"} \ + ${par_partition_model:+--partition-model "$par_partition_model"} \ + $strandedness \ + ${par_paired:+--paired-end} \ + ${input[*]} \ + $INDEX \ + $par_id + diff --git a/src/rsem/rsem_calculate_expression/test.sh b/src/rsem/rsem_calculate_expression/test.sh new file mode 100644 index 00000000..c9ede884 --- /dev/null +++ b/src/rsem/rsem_calculate_expression/test.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +echo ">>> Testing $meta_executable" + +test_dir="${meta_resources_dir}/test_data" + +# wget https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq3/reference/rsem.tar.gz +# gunzip -k rsem.tar.gz +# tar -xf rsem.tar +# mv $test_dir/rsem $meta_resources_dir + +echo "> Prepare test data" + +cat > reads_R1.fastq <<'EOF' +@SEQ_ID1 +ACGCTGCCTCATAAGCCTCACACAT ++ +IIIIIIIIIIIIIIIIIIIIIIIII +@SEQ_ID2 +ACCCGCAAGATTAGGCTCCGTACAC ++ +!!!!!!!!!!!!!!!!!!!!!!!!! +EOF + +cat > reads_R2.fastq <<'EOF' +@SEQ_ID1 +ATGTGTGAGGCTTATGAGGCAGCGT ++ +IIIIIIIIIIIIIIIIIIIIIIIII +@SEQ_ID2 +GTGTACGGAGCCTAATCTTGCAGGG ++ +!!!!!!!!!!!!!!!!!!!!!!!!! +EOF + +cat > genome.fasta <<'EOF' +>chr1 +TGGCATGAGCCAACGAACGCTGCCTCATAAGCCTCACACATCCGCGCCTATGTTGTGACTCTCTGTGAGCGTTCGTGGG +GCTCGTCACCACTATGGTTGGCCGGTTAGTAGTGTGACTCCTGGTTTTCTGGAGCTTCTTTAAACCGTAGTCCAGTCAA +TGCGAATGGCACTTCACGACGGACTGTCCTTAGGTGTGAGGCTTATGAGGCACTCAGGGGA +EOF + +cat > genes.gtf <<'EOF' +chr1 example_source gene 0 50 . + . gene_id "gene1"; transcript_id "transcript1"; +chr1 example_source exon 20 40 . + . gene_id "gene1"; transcript_id "transcript1"; +chr1 example_source gene 100 219 . + . gene_id "gene2"; transcript_id "transcript2"; +chr1 example_source exon 191 210 . + . gene_id "gene2"; transcript_id "transcript2"; +EOF + +cat > ref.cnt <<'EOF' +1 0 0 1 +0 0 0 +0 3 +0 1 +Inf 0 +EOF + +cat > ref.genes.results <<'EOF' +gene_id transcript_id(s) length effective_length expected_count TPM FPKM +gene1 transcript1 21.00 21.00 0.00 0.00 0.00 +gene2 transcript2 20.00 20.00 0.00 0.00 0.00 +EOF + +cat > ref.isoforms.results <<'EOF' +transcript_id gene_id length effective_length expected_count TPM FPKM IsoPct +transcript1 gene1 21 21.00 0.00 0.00 0.00 0.00 +transcript2 gene2 20 20.00 0.00 0.00 0.00 0.00 +EOF + + +echo "> Generate index" + +rsem-prepare-reference \ + --gtf "genes.gtf" \ + "genome.fasta" \ + "index" + +mkdir index +mv index.* index/ + +STAR \ + ${meta_cpus:+--runThreadN $meta_cpus} \ + --runMode genomeGenerate \ + --genomeDir "index/" \ + --genomeFastaFiles "genome.fasta" \ + --sjdbGTFfile "genes.gtf" \ + --genomeSAindexNbases 2 + +######################################################################################### + +echo ">>> Test 1: Paired-end reads using STAR to align reads" +"$meta_executable" \ + --star \ + --paired \ + --input "reads_R1.fastq;reads_R2.fastq" \ + --index index \ + --id test \ + --seed 1 \ + --quiet + +echo ">>> Checking whether output exists" +[ ! -f "test.genes.results" ] && echo "Gene level expression counts file does not exist!" && exit 1 +[ ! -s "test.genes.results" ] && echo "Gene level expression counts file is empty!" && exit 1 +[ ! -f "test.isoforms.results" ] && echo "Transcript level expression counts file does not exist!" && exit 1 +[ ! -s "test.isoforms.results" ] && echo "Transcript level expression counts file is empty!" && exit 1 +[ ! -d "test.stat" ] && echo "Stats file does not exist!" && exit 1 + +echo ">>> Check wheter output is correct" +diff ref.genes.results test.genes.results || { echo "Gene level expression counts file is incorrect!"; exit 1; } +diff ref.isoforms.results test.isoforms.results || { echo "Transcript level expression counts file is incorrect!"; exit 1; } +diff ref.cnt test.stat/test.cnt || { echo "Stats file is incorrect!"; exit 1; } + +##################################################################################################### + +echo "All tests succeeded!" +exit 0 diff --git a/src/salmon/salmon_quant/config.vsh.yaml b/src/salmon/salmon_quant/config.vsh.yaml index 1f96f0c9..5fa3d48f 100644 --- a/src/salmon/salmon_quant/config.vsh.yaml +++ b/src/salmon/salmon_quant/config.vsh.yaml @@ -24,7 +24,7 @@ argument_groups: description: | Format string describing the library. The library type string consists of three parts: - 1. Relative orientation of the reads: This part is only provided if the library is paired-end, THe possible options are + 1. Relative orientation of the reads: This part is only provided if the library is paired-end, The possible options are I = inward O = outward M = matching @@ -118,7 +118,7 @@ argument_groups: direction: output description: | Salmon quantification file. - required: true + required: false example: quant.sf - name: Basic options @@ -327,7 +327,7 @@ argument_groups: If this option is provided, then the selective-alignment results will be written out in SAM-compatible format. By default, output will be directed to stdout, but an alternative file name can be provided instead. - name: --mapping_sam type: file - description: Path to file that should output the selective-alignment results in SAM-compatible format. THis option must be provided while using --write_mappings + description: Path to file that should output the selective-alignment results in SAM-compatible format. This option must be provided while using --write_mappings required: false direction: output example: mappings.sam diff --git a/src/umi_tools/umi_tools_prepareforrsem/config.vsh.yaml b/src/umi_tools/umi_tools_prepareforrsem/config.vsh.yaml new file mode 100644 index 00000000..ceac2052 --- /dev/null +++ b/src/umi_tools/umi_tools_prepareforrsem/config.vsh.yaml @@ -0,0 +1,107 @@ +name: "umi_tools_prepareforrsem" +namespace: "umi_tools" +description: Make the output from umi-tools dedup or group compatible with RSEM +keywords: [umi_tools, rsem, bam, sam] +links: + homepage: https://umi-tools.readthedocs.io/en/latest/ + documentation: https://umi-tools.readthedocs.io/en/latest/reference/extract.html + repository: https://github.com/CGATOxford/UMI-tools +references: + doi: 10.1101/gr.209601.116 +license: MIT + +argument_groups: +- name: "Input" + arguments: + - name: "--input" + alternatives: ["-I", "--stdin"] + type: file + required: true + example: $id.transcriptome.bam + +- name: "Output" + arguments: + - name: "--output" + alternatives: ["-S", "--stdout"] + type: file + direction: output + example: $id.transcriptome_sorted.bam + - name: "--log" + alternatives: ["-L"] + type: file + direction: output + description: File with logging information [default = stdout]. + - name: "--error" + alternatives: ["-E"] + type: file + direction: output + description: File with error information [default = stderr]. + - name: "--log2stderr" + type: boolean_true + description: Send logging information to stderr [default = False]. + - name: "--temp_dir" + type: string + description: | + Directory for temporary files. If not set, the bash environmental variable + TMPDIR is used. + - name: "--compresslevel" + type: integer + description: | + Level of Gzip compression to use. Default (6) matchesGNU gzip rather than python + gzip default (which is 9). + +- name: "Options" + arguments: + - name: "--tags" + type: string + description: | + Comma-seperated list of tags to transfer from read1 to read2 (Default: 'UG,BX') + example: "UG,BX" + - name: "--sam" + type: boolean_true + description: Input and output SAM rather than BAM. + - name: "--timeit" + type: string + description: | + Store timeing information in file [none]. + - name: "--timeit_name" + type: string + description: | + Name in timing file for this class of jobs [all]. + - name: "--timeit_header" + type: boolean_true + description: Add header for timing information [none]. + - name: "--verbose" + alternatives: ["-v"] + type: integer + description: | + Loglevel [1]. The higher, the more output. + - name: "--random_seed" + type: integer + description: | + Random seed to initialize number generator with [none]. + + +resources: + - type: bash_script + path: script.sh + # copied from https://github.com/nf-core/rnaseq/blob/3.12.0/bin/prepare-for-rsem.py + - path: prepare-for-rsem.py +test_resources: + - type: bash_script + path: test.sh + - type: file + path: test_data + +engines: + - type: docker + image: quay.io/biocontainers/umi_tools:1.1.5--py38h0020b31_3 + setup: + - type: docker + run: | + umi_tools -v | sed 's/ version//g' > /var/software_versions.txt + + +runners: +- type: executable +- type: nextflow \ No newline at end of file diff --git a/src/umi_tools/umi_tools_prepareforrsem/help.txt b/src/umi_tools/umi_tools_prepareforrsem/help.txt new file mode 100644 index 00000000..efaf4de6 --- /dev/null +++ b/src/umi_tools/umi_tools_prepareforrsem/help.txt @@ -0,0 +1,54 @@ +``` +umi_tools prepare-for-rsem --help +``` + +prepare_for_rsem - make output from dedup or group compatible with RSEM + +Usage: umi_tools prepare_for_rsem [OPTIONS] [--stdin=IN_BAM] [--stdout=OUT_BAM] + + note: If --stdout is ommited, standard out is output. To + generate a valid BAM file on standard out, please + redirect log with --log=LOGFILE or --log2stderr + +For full UMI-tools documentation, see https://umi-tools.readthedocs.io/en/latest/ + +Options: + --version show program's version number and exit + + RSEM preparation specific options: + --tags=TAGS Comma-seperated list of tags to transfer from read1 to + read2 + --sam input and output SAM rather than BAM + + input/output options: + -I FILE, --stdin=FILE + file to read stdin from [default = stdin]. + -L FILE, --log=FILE + file with logging information [default = stdout]. + -E FILE, --error=FILE + file with error information [default = stderr]. + -S FILE, --stdout=FILE + file where output is to go [default = stdout]. + --temp-dir=FILE Directory for temporary files. If not set, the bash + environmental variable TMPDIR is used[default = None]. + --log2stderr send logging information to stderr [default = False]. + --compresslevel=COMPRESSLEVEL + Level of Gzip compression to use. Default (6) + matchesGNU gzip rather than python gzip default (which + is 9) + + profiling options: + --timeit=TIMEIT_FILE + store timeing information in file [none]. + --timeit-name=TIMEIT_NAME + name in timing file for this class of jobs [all]. + --timeit-header add header for timing information [none]. + + common options: + -v LOGLEVEL, --verbose=LOGLEVEL + loglevel [1]. The higher, the more output. + -h, --help output short help (command line options only). + --help-extended Output full documentation + --random-seed=RANDOM_SEED + random seed to initialize number generator with + [none]. \ No newline at end of file diff --git a/src/umi_tools/umi_tools_prepareforrsem/prepare-for-rsem.py b/src/umi_tools/umi_tools_prepareforrsem/prepare-for-rsem.py new file mode 100644 index 00000000..b53d30ac --- /dev/null +++ b/src/umi_tools/umi_tools_prepareforrsem/prepare-for-rsem.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 + +""" +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Credits +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This script is a clone of the "prepare-for-rsem.py" script written by +Ian Sudbury, Tom Smith and other contributors to the UMI-tools package: +https://github.com/CGATOxford/UMI-tools + +It has been included here to address problems encountered with +Salmon quant and RSEM as discussed in the issue below: +https://github.com/CGATOxford/UMI-tools/issues/465 + +When the "umi_tools prepare-for-rsem" command becomes available in an official +UMI-tools release this script will be replaced and deprecated. + +Commit: +https://github.com/CGATOxford/UMI-tools/blob/bf8608d6a172c5ca0dcf33c126b4e23429177a72/umi_tools/prepare-for-rsem.py + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +prepare_for_rsem - make the output from dedup or group compatible with RSEM +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The SAM format specification states that the mnext and mpos fields should point +to the primary alignment of a read's mate. However, not all aligners adhere to +this standard. In addition, the RSEM software requires that the mate of a read1 +appears directly after it in its input BAM. This requires that there is exactly +one read1 alignment for every read2 and vice versa. + +In general (except in a few edge cases) UMI tools outputs only the read2 to that +corresponds to the read specified in the mnext and mpos positions of a selected +read1, and only outputs this read once, even if multiple read1s point to it. +This makes UMI-tools outputs incompatible with RSEM. This script takes the output +from dedup or groups and ensures that each read1 has exactly one read2 (and vice +versa), that read2 always appears directly after read1,and that pairs point to +each other (note this is technically not valid SAM format). Copy any specified +tags from read1 to read2 if they are present (by default, UG and BX, the unique +group and correct UMI tags added by _group_) + +Input must to name sorted. + + +https://raw.githubusercontent.com/CGATOxford/UMI-tools/master/LICENSE + +""" + +from umi_tools import Utilities as U +from collections import defaultdict, Counter +import pysam +import sys + + +usage = """ +prepare_for_rsem - make output from dedup or group compatible with RSEM + +Usage: umi_tools prepare_for_rsem [OPTIONS] [--stdin=IN_BAM] [--stdout=OUT_BAM] + + note: If --stdout is omited, standard out is output. To + generate a valid BAM file on standard out, please + redirect log with --log=LOGFILE or --log2stderr """ + + +def chunk_bam(bamfile): + """Take in a iterator of pysam.AlignmentSegment entries and yield + lists of reads that all share the same name""" + + last_query_name = None + output_buffer = list() + + for read in bamfile: + if last_query_name is not None and last_query_name != read.query_name: + yield (output_buffer) + output_buffer = list() + + last_query_name = read.query_name + output_buffer.append(read) + + yield (output_buffer) + + +def copy_tags(tags, read1, read2): + """Given a list of tags, copies the values of these tags from read1 + to read2, if the tag is set""" + + for tag in tags: + try: + read1_tag = read1.get_tag(tag, with_value_type=True) + read2.set_tag(tag, value=read1_tag[0], value_type=read1_tag[1]) + except KeyError: + pass + + return read2 + + +def pick_mate(read, template_dict, mate_key): + """Find the mate of read in the template dict using key. It will retrieve + all reads at that key, and then scan to pick the one that refers to _read_ + as it's mate. If there is no such read, it picks a first one it comes to""" + + mate = None + + # get a list of secondary reads at the correct alignment position + potential_mates = template_dict[not read.is_read1][mate_key] + + # search through one at a time to find a read that points to the current read + # as its mate. + for candidate_mate in potential_mates: + if ( + candidate_mate.next_reference_name == read.reference_name + and candidate_mate.next_reference_start == read.pos + ): + mate = candidate_mate + + # if no such read is found, then pick any old secondary alignment at that position + # note: this happens when UMI-tools outputs the wrong read as something's pair. + if mate is None and len(potential_mates) > 0: + mate = potential_mates[0] + + return mate + + +def main(argv=None): + if argv is None: + argv = sys.argv + + # setup command line parser + parser = U.OptionParser(version="%prog version: $Id$", usage=usage, description=globals()["__doc__"]) + group = U.OptionGroup(parser, "RSEM preparation specific options") + + group.add_option( + "--tags", + dest="tags", + type="string", + default="UG,BX", + help="Comma-separated list of tags to transfer from read1 to read2", + ) + group.add_option( + "--sam", dest="sam", action="store_true", default=False, help="input and output SAM rather than BAM" + ) + + parser.add_option_group(group) + + # add common options (-h/--help, ...) and parse command line + (options, args) = U.Start( + parser, argv=argv, add_group_dedup_options=False, add_umi_grouping_options=False, add_sam_options=False + ) + + skipped_stats = Counter() + + if options.stdin != sys.stdin: + in_name = options.stdin.name + options.stdin.close() + else: + in_name = "-" + + if options.sam: + mode = "" + else: + mode = "b" + + inbam = pysam.AlignmentFile(in_name, "r" + mode) + + if options.stdout != sys.stdout: + out_name = options.stdout.name + options.stdout.close() + else: + out_name = "-" + + outbam = pysam.AlignmentFile(out_name, "w" + mode, template=inbam) + + options.tags = options.tags.split(",") + + for template in chunk_bam(inbam): + assert len(set(r.query_name for r in template)) == 1 + current_template = {True: defaultdict(list), False: defaultdict(list)} + + for read in template: + key = (read.reference_name, read.pos, not read.is_secondary) + current_template[read.is_read1][key].append(read) + + output = set() + + for read in template: + mate = None + + # if this read is a non_primary alignment, we first want to check if it has a mate + # with the non-primary alignment flag set. + + mate_key_primary = True + mate_key_secondary = (read.next_reference_name, read.next_reference_start, False) + + # First look for a read that has the same primary/secondary status + # as read (i.e. secondary mate for secondary read, and primary mate + # for primary read) + mate_key = (read.next_reference_name, read.next_reference_start, read.is_secondary) + mate = pick_mate(read, current_template, mate_key) + + # If none was found then look for the opposite (primary mate of secondary + # read or seconadary mate of primary read) + if mate is None: + mate_key = (read.next_reference_name, read.next_reference_start, not read.is_secondary) + mate = pick_mate(read, current_template, mate_key) + + # If we still don't have a mate, then their can't be one? + if mate is None: + skipped_stats["no_mate"] += 1 + U.warn( + "Alignment {} has no mate -- skipped".format( + "\t".join(map(str, [read.query_name, read.flag, read.reference_name, int(read.pos)])) + ) + ) + continue + + # because we might want to make changes to the read, but not have those changes reflected + # if we need the read again,we copy the read. This is only way I can find to do this. + read = pysam.AlignedSegment().from_dict(read.to_dict(), read.header) + mate = pysam.AlignedSegment().from_dict(mate.to_dict(), read.header) + + # Make it so that if our read is secondary, the mate is also secondary. We don't make the + # mate primary if the read is primary because we would otherwise end up with mulitple + # primary alignments. + if read.is_secondary: + mate.is_secondary = True + + # In a situation where there is already one mate for each read, then we will come across + # each pair twice - once when we scan read1 and once when we scan read2. Thus we need + # to make sure we don't output something already output. + if read.is_read1: + mate = copy_tags(options.tags, read, mate) + output_key = str(read) + str(mate) + + if output_key not in output: + output.add(output_key) + outbam.write(read) + outbam.write(mate) + skipped_stats["pairs_output"] += 1 + + elif read.is_read2: + read = copy_tags(options.tags, mate, read) + output_key = str(mate) + str(read) + + if output_key not in output: + output.add(output_key) + outbam.write(mate) + outbam.write(read) + skipped_stats["pairs_output"] += 1 + + else: + skipped_stats["skipped_not_read_12"] += 1 + U.warn( + "Alignment {} is neither read1 nor read2 -- skipped".format( + "\t".join(map(str, [read.query_name, read.flag, read.reference_name, int(read.pos)])) + ) + ) + continue + + if not out_name == "-": + outbam.close() + + U.info( + "Total pairs output: {}, Pairs skipped - no mates: {}," + " Pairs skipped - not read1 or 2: {}".format( + skipped_stats["pairs_output"], skipped_stats["no_mate"], skipped_stats["skipped_not_read12"] + ) + ) + U.Stop() + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/src/umi_tools/umi_tools_prepareforrsem/script.sh b/src/umi_tools/umi_tools_prepareforrsem/script.sh new file mode 100755 index 00000000..d6b3775f --- /dev/null +++ b/src/umi_tools/umi_tools_prepareforrsem/script.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -eo pipefail + +unset_if_false=( + par_sam + par_error + par_log2stderr + par_timeit_header ) + +for var in "${unset_if_false[@]}"; do + test_val="${!var}" + [[ "$test_val" == "false" ]] && unset $var +done + +umi_tools prepare-for-rsem \ + ${par_log:+--log "${par_log}"} \ + ${par_tags:+--tags "${par_tags}"} \ + ${par_sam:+--sam} \ + --stdin="${par_input}" \ + ${par_output:+--stdout "${par_output}"} \ + ${par_error:+--error "${par_error}"} \ + ${par_temp_dir:+--temp-dir "${par_temp_dir}"} \ + ${par_log2stderr:+--log2stderr} \ + ${par_verbose:+--verbose "${par_verbose}"} \ + ${par_random_seed:+--random-seed "${par_random_seed}"} \ + ${par_compresslevel:+--compresslevel "${par_compresslevel}"} + ${par_timeit:+--timeit "${par_timeit}"} \ + ${par_timeit_name:+--timeit-name "${par_timeit_name}"} \ + ${par_timeit_header:+--timeit-header} + + diff --git a/src/umi_tools/umi_tools_prepareforrsem/test.sh b/src/umi_tools/umi_tools_prepareforrsem/test.sh new file mode 100644 index 00000000..c94a202d --- /dev/null +++ b/src/umi_tools/umi_tools_prepareforrsem/test.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +test_dir="$meta_resources_dir/test_data" +apt-get -q update && apt-get -q install -y samtools + +################################################################################ +echo ">>> Test 1: with --sam:" + +"${meta_executable}" \ + --input "$test_dir/test_dedup.sam" \ + --output "$test_dir/test_output.sam" \ + --sam + +echo ">>> Check if output is present" +[[ ! -f "$test_dir/test_output.sam" ]] && echo "Output file not found" && exit 1 +[[ ! -s "$test_dir/test_output.sam" ]] && echo "Output file is empty" && exit 1 + +echo ">>> Check if output is correct" +# use diff but ignoring the header lines (which start with @) as they may differ slightly +diff <(grep -v "^@" "$test_dir/test_output.sam") <(grep -v "^@" "$test_dir/test.sam") && echo "Output is correct" || (echo "Output is incorrect" && exit 1) + +################################################################################ +echo ">>> Test 2: without --sam:" + +"${meta_executable}" \ + --input "$test_dir/test_dedup.bam" \ + --output "$test_dir/test_output.bam" + +echo ">>> Check if output is present" +[[ ! -f "$test_dir/test_output.bam" ]] && echo "Output file not found" && exit 1 +[[ ! -s "$test_dir/test_output.bam" ]] && echo "Output file is empty" && exit 1 + +echo ">>> Check if output is correct" +diff <(samtools view "$test_dir/test_output.bam") <(samtools view "$test_dir/test.bam") || (echo "Output is incorrect" && exit 1) + +################################################################################ +echo ">>> Test 3: with --log:" + +"${meta_executable}" \ + --log "$test_dir/test_log.log" \ + --input "$test_dir/test_dedup.sam" \ + --output "$test_dir/test_output.sam" \ + --sam + +echo ">>> Check if output is present" +[[ ! -f "$test_dir/test_output.sam" ]] && echo "Output file not found" && exit 1 +[[ ! -s "$test_dir/test_output.sam" ]] && echo "Output file is empty" && exit 1 +[[ ! -f "$test_dir/test_log.log" ]] && echo "Log file not found" && exit 1 +[[ ! -s "$test_dir/test_log.log" ]] && echo "Log file is empty" && exit 1 + +echo ">>> Check if log file is correct" +diff <(grep -v '^#' "$test_dir/test_log.log" | sed 's/^[0-9-]* [0-9:]*,[0-9]\{3\} //') <(grep -v '^#' "$test_dir/log.log" | sed 's/^[0-9-]* [0-9:]*,[0-9]\{3\} //') || (echo "Log file is incorrect" && exit 1) + +echo ">>> All test succeeded" +exit 0 \ No newline at end of file diff --git a/src/umi_tools/umi_tools_prepareforrsem/test_data/log.log b/src/umi_tools/umi_tools_prepareforrsem/test_data/log.log new file mode 100644 index 00000000..e4b56e57 --- /dev/null +++ b/src/umi_tools/umi_tools_prepareforrsem/test_data/log.log @@ -0,0 +1,103 @@ +# UMI-tools version: 1.1.5 +# output generated by prepare-for-rsem.py --log test_data/log.log --sam --stdin test_data/test_dedup.sam --stdout jnfgioeurg.sam +# job started at Tue Sep 10 06:43:30 2024 on 4855b4607095 -- 07ae7548-56e8-4772-9b48-7406710fd838 +# pid: 28, system: Linux 6.10.0-linuxkit #1 SMP PREEMPT_DYNAMIC Wed Jul 17 10:54:05 UTC 2024 x86_64 +# compresslevel : 6 +# log2stderr : False +# loglevel : 1 +# random_seed : None +# sam : True +# short_help : None +# stderr : <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> +# stdin : <_io.TextIOWrapper name='test_data/test_dedup.sam' mode='r' encoding='UTF-8'> +# stdlog : <_io.TextIOWrapper name='test_data/log.log' mode='a' encoding='UTF-8'> +# stdout : <_io.TextIOWrapper name='jnfgioeurg.sam' mode='w' encoding='UTF-8'> +# tags : UG,BX +# timeit_file : None +# timeit_header : None +# timeit_name : all +# tmpdir : None +2024-09-10 06:43:30,918 WARNING Alignment ERR5069949.114870 99 MT192765.1 642 has no mate -- skipped +2024-09-10 06:43:30,918 WARNING Alignment ERR5069949.147998 163 MT192765.1 673 has no mate -- skipped +2024-09-10 06:43:30,919 WARNING Alignment ERR5069949.114870 147 MT192765.1 747 has no mate -- skipped +2024-09-10 06:43:30,920 WARNING Alignment ERR5069949.147998 83 MT192765.1 918 has no mate -- skipped +2024-09-10 06:43:30,921 WARNING Alignment ERR5069949.184542 99 MT192765.1 1054 has no mate -- skipped +2024-09-10 06:43:30,921 WARNING Alignment ERR5069949.184542 147 MT192765.1 1254 has no mate -- skipped +2024-09-10 06:43:30,922 WARNING Alignment ERR5069949.376959 99 MT192765.1 4104 has no mate -- skipped +2024-09-10 06:43:30,924 WARNING Alignment ERR5069949.376959 147 MT192765.1 4189 has no mate -- skipped +2024-09-10 06:43:30,925 WARNING Alignment ERR5069949.532979 99 MT192765.1 5567 has no mate -- skipped +2024-09-10 06:43:30,926 WARNING Alignment ERR5069949.540529 163 MT192765.1 5569 has no mate -- skipped +2024-09-10 06:43:30,926 WARNING Alignment ERR5069949.532979 147 MT192765.1 5620 has no mate -- skipped +2024-09-10 06:43:30,927 WARNING Alignment ERR5069949.540529 83 MT192765.1 5658 has no mate -- skipped +2024-09-10 06:43:30,930 WARNING Alignment ERR5069949.856527 99 MT192765.1 10117 has no mate -- skipped +2024-09-10 06:43:30,931 WARNING Alignment ERR5069949.870926 99 MT192765.1 10117 has no mate -- skipped +2024-09-10 06:43:30,931 WARNING Alignment ERR5069949.856527 147 MT192765.1 10198 has no mate -- skipped +2024-09-10 06:43:30,931 WARNING Alignment ERR5069949.885966 99 MT192765.1 10229 has no mate -- skipped +2024-09-10 06:43:30,932 WARNING Alignment ERR5069949.870926 147 MT192765.1 10244 has no mate -- skipped +2024-09-10 06:43:30,932 WARNING Alignment ERR5069949.885966 147 MT192765.1 10276 has no mate -- skipped +2024-09-10 06:43:30,932 WARNING Alignment ERR5069949.937422 99 MT192765.1 10421 has no mate -- skipped +2024-09-10 06:43:30,933 WARNING Alignment ERR5069949.937422 147 MT192765.1 10590 has no mate -- skipped +2024-09-10 06:43:30,934 WARNING Alignment ERR5069949.1066259 99 MT192765.1 11336 has no mate -- skipped +2024-09-10 06:43:30,935 WARNING Alignment ERR5069949.1062611 163 MT192765.1 11426 has no mate -- skipped +2024-09-10 06:43:30,936 WARNING Alignment ERR5069949.1067032 163 MT192765.1 11433 has no mate -- skipped +2024-09-10 06:43:30,936 WARNING Alignment ERR5069949.1062611 83 MT192765.1 11453 has no mate -- skipped +2024-09-10 06:43:30,936 WARNING Alignment ERR5069949.1066259 147 MT192765.1 11479 has no mate -- skipped +2024-09-10 06:43:30,937 WARNING Alignment ERR5069949.1067032 83 MT192765.1 11480 has no mate -- skipped +2024-09-10 06:43:30,938 WARNING Alignment ERR5069949.1258508 163 MT192765.1 12424 has no mate -- skipped +2024-09-10 06:43:30,939 WARNING Alignment ERR5069949.1261808 99 MT192765.1 12592 has no mate -- skipped +2024-09-10 06:43:30,940 WARNING Alignment ERR5069949.1258508 83 MT192765.1 12637 has no mate -- skipped +2024-09-10 06:43:30,940 WARNING Alignment ERR5069949.1261808 147 MT192765.1 12653 has no mate -- skipped +2024-09-10 06:43:30,941 WARNING Alignment ERR5069949.1372331 163 MT192765.1 13010 has no mate -- skipped +2024-09-10 06:43:30,941 WARNING Alignment ERR5069949.1372331 83 MT192765.1 13131 has no mate -- skipped +2024-09-10 06:43:30,942 WARNING Alignment ERR5069949.1552198 99 MT192765.1 13943 has no mate -- skipped +2024-09-10 06:43:30,943 WARNING Alignment ERR5069949.1561137 163 MT192765.1 13990 has no mate -- skipped +2024-09-10 06:43:30,943 WARNING Alignment ERR5069949.1552198 147 MT192765.1 14026 has no mate -- skipped +2024-09-10 06:43:30,944 WARNING Alignment ERR5069949.1561137 83 MT192765.1 14080 has no mate -- skipped +2024-09-10 06:43:30,947 WARNING Alignment ERR5069949.2098070 99 MT192765.1 17114 has no mate -- skipped +2024-09-10 06:43:30,947 WARNING Alignment ERR5069949.2064910 99 MT192765.1 17122 has no mate -- skipped +2024-09-10 06:43:30,947 WARNING Alignment ERR5069949.2125592 99 MT192765.1 17179 has no mate -- skipped +2024-09-10 06:43:30,947 WARNING Alignment ERR5069949.2064910 147 MT192765.1 17179 has no mate -- skipped +2024-09-10 06:43:30,948 WARNING Alignment ERR5069949.2098070 147 MT192765.1 17269 has no mate -- skipped +2024-09-10 06:43:30,948 WARNING Alignment ERR5069949.2125592 147 MT192765.1 17288 has no mate -- skipped +2024-09-10 06:43:30,948 WARNING Alignment ERR5069949.2185111 163 MT192765.1 17405 has no mate -- skipped +2024-09-10 06:43:30,949 WARNING Alignment ERR5069949.2151832 163 MT192765.1 17415 has no mate -- skipped +2024-09-10 06:43:30,949 WARNING Alignment ERR5069949.2176303 99 MT192765.1 17441 has no mate -- skipped +2024-09-10 06:43:30,949 WARNING Alignment ERR5069949.2151832 83 MT192765.1 17452 has no mate -- skipped +2024-09-10 06:43:30,949 WARNING Alignment ERR5069949.2205229 99 MT192765.1 17475 has no mate -- skipped +2024-09-10 06:43:30,950 WARNING Alignment ERR5069949.2216307 163 MT192765.1 17503 has no mate -- skipped +2024-09-10 06:43:30,950 WARNING Alignment ERR5069949.2176303 147 MT192765.1 17518 has no mate -- skipped +2024-09-10 06:43:30,950 WARNING Alignment ERR5069949.2185111 83 MT192765.1 17536 has no mate -- skipped +2024-09-10 06:43:30,951 WARNING Alignment ERR5069949.2205229 147 MT192765.1 17584 has no mate -- skipped +2024-09-10 06:43:30,951 WARNING Alignment ERR5069949.2216307 83 MT192765.1 17600 has no mate -- skipped +2024-09-10 06:43:30,952 WARNING Alignment ERR5069949.2270078 163 MT192765.1 17969 has no mate -- skipped +2024-09-10 06:43:30,953 WARNING Alignment ERR5069949.2270078 83 MT192765.1 18102 has no mate -- skipped +2024-09-10 06:43:30,953 WARNING Alignment ERR5069949.2328704 163 MT192765.1 18285 has no mate -- skipped +2024-09-10 06:43:30,954 WARNING Alignment ERR5069949.2342766 99 MT192765.1 18396 has no mate -- skipped +2024-09-10 06:43:30,954 WARNING Alignment ERR5069949.2328704 83 MT192765.1 18411 has no mate -- skipped +2024-09-10 06:43:30,954 WARNING Alignment ERR5069949.2361683 99 MT192765.1 18425 has no mate -- skipped +2024-09-10 06:43:30,954 WARNING Alignment ERR5069949.2342766 147 MT192765.1 18468 has no mate -- skipped +2024-09-10 06:43:30,955 WARNING Alignment ERR5069949.2361683 147 MT192765.1 18512 has no mate -- skipped +2024-09-10 06:43:30,955 WARNING Alignment ERR5069949.2415814 99 MT192765.1 18597 has no mate -- skipped +2024-09-10 06:43:30,955 WARNING Alignment ERR5069949.2385514 99 MT192765.1 18602 has no mate -- skipped +2024-09-10 06:43:30,956 WARNING Alignment ERR5069949.2417063 99 MT192765.1 18648 has no mate -- skipped +2024-09-10 06:43:30,956 WARNING Alignment ERR5069949.2388984 99 MT192765.1 18653 has no mate -- skipped +2024-09-10 06:43:30,956 WARNING Alignment ERR5069949.2385514 147 MT192765.1 18684 has no mate -- skipped +2024-09-10 06:43:30,956 WARNING Alignment ERR5069949.2388984 147 MT192765.1 18693 has no mate -- skipped +2024-09-10 06:43:30,957 WARNING Alignment ERR5069949.2431709 99 MT192765.1 18748 has no mate -- skipped +2024-09-10 06:43:30,957 WARNING Alignment ERR5069949.2415814 147 MT192765.1 18764 has no mate -- skipped +2024-09-10 06:43:30,957 WARNING Alignment ERR5069949.2417063 147 MT192765.1 18765 has no mate -- skipped +2024-09-10 06:43:30,958 WARNING Alignment ERR5069949.2431709 147 MT192765.1 18776 has no mate -- skipped +2024-09-10 06:43:30,959 WARNING Alignment ERR5069949.2668880 99 MT192765.1 23124 has no mate -- skipped +2024-09-10 06:43:30,960 WARNING Alignment ERR5069949.2674295 163 MT192765.1 23133 has no mate -- skipped +2024-09-10 06:43:30,960 WARNING Alignment ERR5069949.2668880 147 MT192765.1 23145 has no mate -- skipped +2024-09-10 06:43:30,960 WARNING Alignment ERR5069949.2674295 83 MT192765.1 23203 has no mate -- skipped +2024-09-10 06:43:30,963 WARNING Alignment ERR5069949.2953930 99 MT192765.1 25344 has no mate -- skipped +2024-09-10 06:43:30,963 WARNING Alignment ERR5069949.2972968 163 MT192765.1 25425 has no mate -- skipped +2024-09-10 06:43:30,963 WARNING Alignment ERR5069949.2953930 147 MT192765.1 25464 has no mate -- skipped +2024-09-10 06:43:30,964 WARNING Alignment ERR5069949.2972968 83 MT192765.1 25518 has no mate -- skipped +2024-09-10 06:43:30,966 WARNING Alignment ERR5069949.3273002 163 MT192765.1 28442 has no mate -- skipped +2024-09-10 06:43:30,966 WARNING Alignment ERR5069949.3277445 99 MT192765.1 28508 has no mate -- skipped +2024-09-10 06:43:30,966 WARNING Alignment ERR5069949.3273002 83 MT192765.1 28543 has no mate -- skipped +2024-09-10 06:43:30,966 WARNING Alignment ERR5069949.3277445 147 MT192765.1 28573 has no mate -- skipped +2024-09-10 06:43:30,968 INFO Total pairs output: 56, Pairs skipped - no mates: 82, Pairs skipped - not read1 or 2: 0 +# job finished in 0 seconds at Tue Sep 10 06:43:30 2024 -- 4.44 0.25 0.00 0.00 -- 07ae7548-56e8-4772-9b48-7406710fd838 diff --git a/src/umi_tools/umi_tools_prepareforrsem/test_data/test.bam b/src/umi_tools/umi_tools_prepareforrsem/test_data/test.bam new file mode 100644 index 00000000..7793c7e3 Binary files /dev/null and b/src/umi_tools/umi_tools_prepareforrsem/test_data/test.bam differ diff --git a/src/umi_tools/umi_tools_prepareforrsem/test_data/test.sam b/src/umi_tools/umi_tools_prepareforrsem/test_data/test.sam new file mode 100644 index 00000000..6465827d --- /dev/null +++ b/src/umi_tools/umi_tools_prepareforrsem/test_data/test.sam @@ -0,0 +1,119 @@ +@HD VN:1.6 SO:coordinate +@SQ SN:MT192765.1 LN:29829 +@RG ID:1 LB:lib1 PL:ILLUMINA SM:test PU:barcode1 +@PG ID:minimap2 PN:minimap2 VN:2.17-r941 CL:minimap2 -ax sr tests/data/fasta/sarscov2/GCA_011545545.1_ASM1154554v1_genomic.fna tests/data/fastq/dna/sarscov2_1.fastq.gz tests/data/fastq/dna/sarscov2_2.fastq.gz +@PG ID:samtools PN:samtools PP:minimap2 VN:1.11 CL:samtools view -Sb sarscov2_aln.sam +@PG ID:samtools.1 PN:samtools PP:samtools VN:1.11 CL:samtools sort -o sarscov2_paired_aln.sorted.bam sarscov2_paired_aln.bam +@PG ID:samtools.2 PN:samtools PP:samtools.1 VN:1.20 CL:samtools view -h test_data/test_dedup.bam +ERR5069949.29668 83 MT192765.1 267 60 89M = 121 -235 CCTTGTCCCTGGTTACAACTAGAAACCACACGTCCAACTCAGTTTGCCTGTTTTACAGGTTCGCGACGTGCTCGTACGTGGCTTTGGAG E////6/E/EE/EE/<