diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c71bb4b..02bf7439 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,8 @@ * `sgedemux`: demultiplexing sequencing data generated on Singular Genomics' sequencing instruments (PR #166). +* `bases2fasta`: demultiplexing sequencing data generated by Element Biosciences instruments (PR #167). + ## BUG FIXES * `falco`: Fix a typo in the `--reverse_complement` argument (PR #157). diff --git a/src/bases2fastq/config.vsh.yaml b/src/bases2fastq/config.vsh.yaml new file mode 100644 index 00000000..b50e42fb --- /dev/null +++ b/src/bases2fastq/config.vsh.yaml @@ -0,0 +1,200 @@ +name: bases2fastq +description: | + Bases2Fastq demultiplexes sequencing data generated by Element Biosciences instruments and converts base calls into FASTQ files. +keywords: ["demultiplex", "fastq", "demux", "Element Biosciences"] +links: + documentation: https://docs.elembio.io/docs/bases2fastq/introduction/ +license: Proprietairy +requirements: + commands: [bases2fastq] +authors: + - __merge__: /src/_authors/dries_schaumont.yaml + roles: [ author, maintainer ] + +argument_groups: + - name: Input + arguments: + - name: "--analysis_directory" + type: file + description: Location of analysis directory + required: true + example: "input/" + - name: --run_manifest + alternatives: [-r] + type: file + description: Location of run manifest to use instead of default RunManifest.csv found in analysis directory + required: false + + - name: Output + arguments: + - name: "--output_directory" + alternatives: ["-o"] + type: file + direction: output + required: true + description: Location to save output fastqs + example: fastq_dir + - name: "--report" + type: file + required: false + direction: output + description: Output location for the HTML report + - name: "--logs" + type: file + direction: output + required: false + description: Directory containing log files + example: logs_dir + - name: Arguments + arguments: + - name: --chemistry_version + type: string + required: false + description: Run parameters override, chemistry version. + - name: "--demux_only" + alternatives: [-d] + type: boolean_true + description: | + Generate demux files and indexing stats without generating FASTQ + - name: "--detect_adapters" + type: boolean_true + description: | + Detect adapters sequences, overriding any sequences present in run manifest. + - name: "--error_on_missing" + type: boolean_true + description: | + Terminate execution for a missing file (by default, missing files are + skipped and execution continues). Also set by --strict. + - name: "--exclude_tile" + alternatives: [-e] + multiple: true + type: string + description: | + Regex matching tile names to exclude. This flag can be specified multiple times. (e.g. L1.*C0[23]S.) + - name: "--filter_mask" + type: string + description: | + Run parameters override, custom pass filter mask. + - name: "--flowcell_id" + type: string + description: | + Run parameters override, flowcell ID. + - name: "--force_index_orientation" + type: boolean_true + description: | + Do not attempt to find orientation for I1/I2 reads (reverse complement). + Use orientation given in run manifest. + - name: "--group_fastq" + type: boolean_true + description: | + Group all FASTQ/stats/metrics for a project are in the project folder. + - name: "--i1_cycles" + type: integer + min: 1 + description: | + Run parameters override, I1 cycles. + - name: "--i2_cycles" + type: integer + min: 1 + description: | + Run parameters override, I2 cycles + - name: "--include_tile" + alternatives: [-i] + type: string + multiple: true + description: | + Regex matching tile names to include. This flag + can be specified multiple times. (e.g. L1.*C0[23]S.) + - name: "--kit_configuration" + type: string + description: | + Run parameters override, kit configuration. + - name: "--legacy_fastq" + type: boolean_true + description: | + Legacy naming for FASTQ files (e.g. SampleName_S1_L001_R1_001.fastq.gz) + - name: "--log_level" + type: string + alternatives: [-l] + choices: ["DEBUG", "INFO", "WARNING", "ERROR"] + description: | + Severity level for logging. + example: INFO + - name: "--no_error_on_invalid" + type: boolean_true + description: | + Skip invalid files and continue execution. Overridden by --strict options + - name: "--no_projects" + type: boolean_true + description: | + Disable project directories + - name: "--num_unassigned" + type: integer + min: 0 + max: 1000 + example: 30 + description: | + Max Number of unassigned sequences to report. + - name: "--preparation_workflow" + type: string + description: | + Run parameters override, preparation workflow. + - name: --qc_only + type: boolean_true + description: | + Quickly generate run stats for single tile without generating FASTQ. + Use --include_tile/--exclude_tile to define custom tile set. + - name: --r1_cycles + type: integer + min: 1 + description: | + Run parameters override, R1 cycles. + - name: --r2_cycles + type: integer + min: 1 + description: | + Run parameters override, R2 cycles. + - name: "--split_lanes" + type: boolean_true + description: | + Split FASTQ files by lane. + - name: --strict + type: boolean_true + description: | + In strict mode any invalid or missing input file will terminate execution + (overrides no_error_on_invalid and sets --error_on_missing) + + # --help, -h Display this usage statement + # --input-remote, NAME Rclone remote name for remote ANALYSIS_DIRECTORY + # --num-threads, -p NUMBER Number of threads (default 1) + # --output-remote, NAME Rclone remote name for remote OUTPUT_DIRECTORY + # --settings SELECTION Run manifest settings override. This option may be specified multiple times. + # --version, -v Display bases2fastq version + # --skip-qc-report SELECTION Do not generate HTML QC report. + +resources: + - type: bash_script + path: script.sh + +test_resources: + - type: bash_script + path: test.sh + +engines: +- type: docker + image: elembio/bases2fastq:2.1.0 + setup: + - type: apt + packages: + - procps + - tree + - type: docker + run: | + echo "bases2fastq: $(bases2fastq --version | cut -d' ' -f3)" > /var/software_versions.txt + test_setup: + - type: apt + packages: curl + +runners: + - type: executable + - type: nextflow diff --git a/src/bases2fastq/help.txt b/src/bases2fastq/help.txt new file mode 100644 index 00000000..2fcadf9b --- /dev/null +++ b/src/bases2fastq/help.txt @@ -0,0 +1,40 @@ + +Usage: bases2fastq [OPTIONS] ANALYSIS_DIRECTORY OUTPUT_DIRECTORY + +positional arguments: + ANALYSIS_DIRECTORY Location of analysis directory + OUTPUT_DIRECTORY Location to save output + +optional arguments: + --chemistry-version VERSION Run parameters override, chemistry version. + --demux-only, -d Generate demux files and indexing stats without generating FASTQ + --detect-adapters Detect adapters sequences, overriding any sequences present in run manifest. + --error-on-missing Terminate execution for a missing file (by default, missing files are skipped and execution continues). Also set by --strict. + --exclude-tile, -e SELECTION Regex matching tile names to exclude. This flag can be specified multiple times. (e.g. L1.*C0[23]S.) + --filter-mask MASK Run parameters override, custom pass filter mask. + --flowcell-id FLOWCELL_ID Run parameters override, flowcell ID. + --force-index-orientation Do not attempt to find orientation for I1/I2 reads (reverse complement). Use orientation given in run manifest. + --group-fastq Group all FASTQ/stats/metrics for a project are in the project folder (default false) + --help, -h Display this usage statement + --i1-cycles NUM_CYCLES Run parameters override, I1 cycles. + --i2-cycles NUM_CYCLES Run parameters override, I2 cycles. + --include-tile, -i SELECTION Regex matching tile names to include. This flag can be specified multiple times. (e.g. L1.*C0[23]S.) + --input-remote, NAME Rclone remote name for remote ANALYSIS_DIRECTORY + --kit-configuration KIT_CONFIG Run parameters override, kit configuration. + --legacy-fastq Legacy naming for FASTQ files (e.g. SampleName_S1_L001_R1_001.fastq.gz) + --log-level, -l LEVEL Severity level for logging. i.e. DEBUG, INFO, WARNING, ERROR (default INFO) + --no-error-on-invalid Skip invalid files and continue execution (by default, execution is terminated for an invalid file). Overridden by --strict options. + --no-projects Disable project directories (default false) + --num-threads, -p NUMBER Number of threads (default 1) + --num-unassigned NUMBER Max Number of unassigned sequences to report. Must be <= 1000 (default 30) + --output-remote, NAME Rclone remote name for remote OUTPUT_DIRECTORY + --preparation-workflow WORKFLOW Run parameters override, preparation workflow. + --qc-only Quickly generate run stats for single tile without generating FASTQ. Use --include-tile/--exclude-tile to define custom tile set. + --r1-cycles NUM_CYCLES Run parameters override, R1 cycles. + --r2-cycles NUM_CYCLES Run parameters override, R2 cycles. + --run-manifest, -r PATH Location of run manifest to use instead of default RunManifest.csv found in analysis directory + --settings SELECTION Run manifest settings override. This option may be specified multiple times. + --skip-qc-report SELECTION Do not generate HTML QC report. + --split-lanes Split FASTQ files by lane + --strict, -s In strict mode any invalid or missing input file will terminate execution (overrides no-error-on-invalid and sets --error-on-missing) + --version, -v Display bases2fastq version \ No newline at end of file diff --git a/src/bases2fastq/script.sh b/src/bases2fastq/script.sh new file mode 100644 index 00000000..988acbc4 --- /dev/null +++ b/src/bases2fastq/script.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +## VIASH START +## VIASH END + +# Exit on error +set -eo pipefail + +# Unset parameters +unset_if_false=( + par_demux_only + par_detect_adapters + par_error_on_missing + par_group_fastq + par_legacy_fastq + par_no_error_on_invalid + par_no_projects + par_qc_only + par_split_lanes + par_skip_qc_report + par_strict + par_force_index_orientation +) + +for par in ${unset_if_false[@]}; do + test_val="${!par}" + [[ "$test_val" == "false" ]] && unset $par +done + +# NOTE: --preparation-workflow is bugged in bases2fastq +args=( + ${par_demux_only:+--demux-only} + ${par_detect_adapters:+--detect-adapters} + ${par_error_on_missing:+--error-on-missing} + ${par_group_fastq:+--group-fastq} + ${par_legacy_fastq:+--legacy-fastq} + ${par_no_error_on_invalid:+--no-error-on-invalid} + ${par_no_projects:+--no-projects} + ${par_split_lanes:+--split-lanes} + ${par_strict:+--strict} + ${par_force_index_orientation:+--force-index-orientation} + ${par_chemistry_version:+--chemistry-version "$par_chemistry_version"} + ${par_filter_mask:+--filter-mask "$par_filter_mask"} + ${par_flowcell_id:+--flowcell-id "$par_flowcell_id"} + ${par_i1_cycles:+--i1-cycles "$par_i1_cycles"} + ${par_i2_cycles:+--i2-cycles "$par_i2_cycles"} + ${par_r1_cycles:+--r1-cycles "$par_r1_cycles"} + ${par_r2_cycles:+--r2-cycles "$par_r2_cycles"} + ${par_kit_configuration:+--kit-configuration "$par_kit_configuration"} + ${par_log_level:+--log-level "$par_log_level"} + ${par_num_unassigned:+--num-unassigned "$par_num_unassigned"} + ${par_preparation_workflow:+--preparation-workflow "$par_preparation_workflow"} + ${meta_cpus:+--num-threads "$meta_cpus"} + ${par_run_manifest:+--run-manifest "$par_run_manifest"} +) + +# Create arrays for inputs that contain multiple arguments +IFS=";" read -ra exclude_tile <<< "$par_exclude_tile" +IFS=";" read -ra include_tile <<< "$par_include_tile" + +if [ -z "$par_report" ]; then + args+=( --skip-qc-report ) +fi + +for arg_value in "${exclude_tile[@]}"; do + args+=( "--exclude-tile" "$arg_value" ) +done + +for arg_value in "${include_tile[@]}"; do + args+=( "--include-tile" "$arg_value" ) +done + +echo "> Creating temporary directory." +# create temporary directory and clean up on exit +TMPDIR=$(mktemp -d "$meta_temp_dir/$meta_name-XXXXXX") +echo "> Created $TMPDIR" +function clean_up { + [[ -d "$TMPDIR" ]] && rm -rf "$TMPDIR" +} +trap clean_up EXIT + +args+=( "$par_analysis_directory" "$TMPDIR") +echo "> Running bases2fastq with arguments: ${args[@]}" +bases2fastq ${args[@]} +echo "> Done running sgdemux" + +echo "> Output folder:" +tree "$TMPDIR" + +echo "> Moving FASTQ files into final output directory" +mkdir -p "$par_output_directory/" +mv "$TMPDIR"/Samples/* --target-directory="$par_output_directory" + +if [ ! -z "$par_report" ]; then + echo "> Moving HTML report to the output ($par_report)" + mv "$TMPDIR"/*.html "$par_report" +else + echo " > Leaving reports alone" +fi + +# Logs is everything else +if [ ! -z "$par_logs" ]; then + mkdir -p "$par_logs" + echo "> Moving logs to their own location ($par_logs)" + mv "$TMPDIR/"* "$par_logs/" +else + echo "> Not moving logs" +fi + + + diff --git a/src/bases2fastq/test.sh b/src/bases2fastq/test.sh new file mode 100644 index 00000000..9334e343 --- /dev/null +++ b/src/bases2fastq/test.sh @@ -0,0 +1,200 @@ +#!/bin/bash + +set -eou pipefail + +# Helper functions +assert_file_exists() { + [ -f "$1" ] || { echo "File '$1' does not exist" && exit 1; } +} + +assert_file_not_exists() { + [ ! -f "$1" ] || { echo "File '$1' does not exist" && exit 1; } +} + +assert_directory_exists() { + [ -d "$1" ] || { echo "Directory '$1' does not exist" && exit 1; } +} + +assert_file_not_empty() { + [ -s "$1" ] || { echo "File '$1' is empty but shouldn't be" && exit 1; } +} +assert_file_contains() { + grep -q "$2" "$1" || { echo "File '$1' does not contain '$2'" && exit 1; } +} + +# Example output +# Note that the format of the fastq file names and organization into subfolders +# can differ based on the arguments provided to bases2fastq + +# |-- 20230404-Bases2Fastq-Sim_QC.html +# |-- IndexAssignment.csv +# |-- Metrics.csv +# |-- RunManifest.csv +# |-- RunManifest.json +# |-- RunParameters.json +# |-- RunStats.json +# |-- Samples +# | |-- DefaultProject +# | | |-- DefaultProject_IndexAssignment.csv +# | | |-- DefaultProject_Metrics.csv +# | | |-- DefaultProject_QC.html +# | | |-- DefaultProject_RunStats.json +# | | |-- sample_0 +# | | | |-- sample_0_L1_R1.fastq.gz +# | | | |-- sample_0_L1_R2.fastq.gz +# | | | |-- sample_0_L2_R1.fastq.gz +# | | | |-- sample_0_L2_R2.fastq.gz +# | | | `-- sample_0_stats.json +# | | |-- sample_1 +# | | | |-- sample_1_L1_R1.fastq.gz +# | | | |-- sample_1_L1_R2.fastq.gz +# | | | |-- sample_1_L2_R1.fastq.gz +# | | | |-- sample_1_L2_R2.fastq.gz +# | | | `-- sample_1_stats.json +# | | |-- sample_2 +# | | | |-- sample_2_L1_R1.fastq.gz +# | | | |-- sample_2_L1_R2.fastq.gz +# | | | |-- sample_2_L2_R1.fastq.gz +# | | | |-- sample_2_L2_R2.fastq.gz +# | | | `-- sample_2_stats.json +# | | |-- sample_3 +# | | | |-- sample_3_L1_R1.fastq.gz +# | | | |-- sample_3_L1_R2.fastq.gz +# | | | |-- sample_3_L2_R1.fastq.gz +# | | | |-- sample_3_L2_R2.fastq.gz +# | | | `-- sample_3_stats.json +# | | `-- sample_4 +# | | |-- sample_4_L1_R1.fastq.gz +# | | |-- sample_4_L1_R2.fastq.gz +# | | |-- sample_4_L2_R1.fastq.gz +# | | |-- sample_4_L2_R2.fastq.gz +# | | `-- sample_4_stats.json +# | `-- Unassigned +# | |-- Unassigned_L1_R1.fastq.gz +# | |-- Unassigned_L1_R2.fastq.gz +# | |-- Unassigned_L2_R1.fastq.gz +# | `-- Unassigned_L2_R2.fastq.gz +# |-- UnassignedSequences.csv +# `-- info +# |-- Bases2Fastq.log +# `-- RunManifestErrors.json + + +# create temporary directory and clean up on exit +TMPDIR=$(mktemp -d "$meta_temp_dir/$meta_name-XXXXXX") +function clean_up { + [[ -d "$TMPDIR" ]] && rm -rf "$TMPDIR" +} +trap clean_up EXIT + +# Unpack test input files +TAR_DIR="$TMPDIR/tar" +mkdir -p "$TAR_DIR" +curl http://element-public-data.s3.amazonaws.com/bases2fastq-share/bases2fastq-v2/20230404-bases2fastq-sim-151-151-9-9.tar.gz \ +-o "$TAR_DIR/20230404-bases2fastq-sim-151-151-9-9.tar.gz" + +BCL_DIR="$TMPDIR/bcl" +mkdir "$BCL_DIR" +tar -xvf "$TAR_DIR/20230404-bases2fastq-sim-151-151-9-9.tar.gz" -C "$BCL_DIR" + +mkdir "$TMPDIR/test1" && pushd "$TMPDIR/test1" > /dev/null +expected_out_dir="$TMPDIR/test1/out" +expected_report="$TMPDIR/report.html" +expected_logs="$TMPDIR/logs" +"$meta_executable" \ + --analysis_directory "$BCL_DIR/20230404-bases2fastq-sim-151-151-9-9" \ + --output_directory "$expected_out_dir" \ + --logs "$expected_logs" \ + --report "$expected_report" \ + --include_tile "L1R02C01S1;L2R21C01S1;L1R02C01S2;L2R21C01S2;L1R03C01S2;L2R20C01S2" \ + --exclude_tile "L1R04C01S1" \ + --chemistry_version 2 \ + --i1_cycles 10 \ + --i2_cycles 10 \ + --r1_cycles 152 \ + --r2_cycles 152 \ + --kit_configuration "300Cycles" \ + --detect_adapters \ + --error_on_missing \ + --flowcell_id foo \ + --force_index_orientation \ + --group_fastq \ + --legacy_fastq \ + --log_level DEBUG \ + --no_projects \ + --num_unassigned 30 \ + --strict \ + --run_manifest "$BCL_DIR/20230404-bases2fastq-sim-151-151-9-9/RunManifest.csv" + +assert_directory_exists "$expected_out_dir" +assert_directory_exists "$expected_logs" +assert_file_exists "$expected_report" +assert_file_not_empty "$expected_report" + +expected_samples=( + Undetermined_S0 + sample_0_S1 + sample_1_S2 + sample_2_S3 + sample_3_S4 + sample_4_S5 +) + +for sample in "${expected_samples[@]}"; do + for lane in "L001" "L002"; do + for orientation in "R1" "R2"; do + assert_file_exists "$expected_out_dir/${sample}_${lane}_${orientation}_001.fastq.gz" + done + done +done +popd > /dev/null + +mkdir "$TMPDIR/test3" && pushd "$TMPDIR/test3" > /dev/null +expected_out_dir="$TMPDIR/test3/out" +"$meta_executable" \ + --analysis_directory "$BCL_DIR/20230404-bases2fastq-sim-151-151-9-9" \ + --output_directory "$expected_out_dir" + +expected_samples=( + sample_0 + sample_1 + sample_2 + sample_3 + sample_4 +) +tree "$expected_out_dir" + +for sample in "${expected_samples[@]}"; do + for orientation in "R1" "R2"; do + assert_file_exists "$expected_out_dir/DefaultProject/${sample}/${sample}_${orientation}.fastq.gz" + done +done +assert_file_exists "$expected_out_dir/Unassigned/Unassigned_R1.fastq.gz" +assert_file_exists "$expected_out_dir/Unassigned/Unassigned_R2.fastq.gz" +popd > /dev/null + +mkdir "$TMPDIR/test4" && pushd "$TMPDIR/test4" > /dev/null +expected_out_dir="$TMPDIR/test4/out" +"$meta_executable" \ + --analysis_directory "$BCL_DIR/20230404-bases2fastq-sim-151-151-9-9" \ + --output_directory "$expected_out_dir" \ + --split_lanes + +expected_samples=( + "Unassigned/Unassigned" + DefaultProject/sample_0/sample_0 + DefaultProject/sample_1/sample_1 + DefaultProject/sample_2/sample_2 + DefaultProject/sample_3/sample_3 + DefaultProject/sample_4/sample_4 +) +tree "$expected_out_dir" + +for sample in "${expected_samples[@]}"; do + for lane in "L1" "L2"; do + for orientation in "R1" "R2"; do + assert_file_exists "$expected_out_dir/${sample}_${lane}_${orientation}.fastq.gz" + done + done +done +popd > /dev/null