Refactor pipeline filtering, create params flags for tools

CHANGELOG.md

@@ -243,3 +243,23 @@ Below are the list of changes to phx since is initial release. As fixes can take
  - [ARG-ANNOT](http://backup.mediterranee-infection.com/arkotheque/client/ihumed/_depot_arko/articles/2041/arg-annot-v4-aa-may2018_doc.fasta) hasn't changed since the last time the database was created and contains updates since version [NT v6 July 2019](https://www.mediterranee-infection.com/acces-ressources/base-de-donnees/arg-annot-2/) 
  - [ResFinder](https://bitbucket.org/genomicepidemiology/resfinder_db/src/master/) 
  - Includes until 2024-01-28 [commit 97d1fe0cd0a119172037f6bdb29f8a1c7c6e6019](https://bitbucket.org/genomicepidemiology/resfinder_db/commits/branch/master) 
+
+## [v3.1.0](https://github.com/CDCgov/phoenix/releases/tag/v3.1.0) (04/08/2024)
+**Implemented Enhancements**
+- refactors filtering failed samples for fairy
+- refactors ICA handling, terra handling
+- add a param flags in nextflow.config
+ - execution-based
+ - run_busco
+ - ncbi_excel_creation
+ - extended_qc
+ - run_srst2_mlst
+ - run_griphin
+ - feature-based
+ - save_trimmed_fail
+ - save_merged
+ - save_output_fastqs
+ - save_reads_assignment
+- moves parameter checks upstream to main.nf
+ - ICA
+ - TERRA

conf/modules.config

@@ -43,7 +43,7 @@ process {
  [
  path: { "${params.outdir}/${meta.id}/file_integrity" },
  mode: 'copy',
- pattern: "*{_summary.txt}"
+ pattern: "*{_summary_fairy.txt}"
  ]
  ]
  }
@@ -67,7 +67,7 @@ process {
  [
  path: { "${params.outdir}/${meta.id}/file_integrity" },
  mode: 'copy',
- pattern: "*{_summary.txt}"
+ pattern: "*{_summary_rawstats.txt}"
  ]
  ]
  }

main.nf

@@ -43,15 +43,15 @@ workflow PHOENIX {
  // Check input path parameters to see if they exist
  def checkPathParamList = [ params.input, params.multiqc_config, params.kraken2db] //removed , params.fasta to stop issue w/connecting to aws and igenomes not used
  for (param in checkPathParamList) { if (param) { file(param, checkIfExists: true) } }
-
- // Check mandatory parameters
+ if (params.ica != true && params.ica != false) {exit 1, "Please set params.ica to either \"true\" if running on ICA or \"false\" for all other methods."}
+ if (params.terra != true && params.terra != false) {exit 1, "Please set params.terra to either \"true\" if running on terra or \"false\" for all other methods."}
 
  //input on command line
  if (params.input) { ch_input = file(params.input) } else { exit 1, 'For -entry PHOENIX: Input samplesheet not specified!' }
  ch_versions = Channel.empty() // Used to collect the software versions
-
+ 
  main:
- PHOENIX_EXTERNAL ( ch_input, ch_versions, true )
+ PHOENIX_EXTERNAL ( ch_input, ch_versions, params.ncbi_excel_creation )
  emit:
  scaffolds = PHOENIX_EXTERNAL.out.scaffolds
  trimmed_reads = PHOENIX_EXTERNAL.out.trimmed_reads
@@ -60,9 +60,9 @@ workflow PHOENIX {
  gamma_ar = PHOENIX_EXTERNAL.out.gamma_ar
  phx_summary = PHOENIX_EXTERNAL.out.phx_summary
  //output for phylophoenix
- griphin_tsv = PHOENIX_EXTERNAL.out.griphin_tsv
- griphin_excel = PHOENIX_EXTERNAL.out.griphin_excel
- dir_samplesheet = PHOENIX_EXTERNAL.out.dir_samplesheet
+ griphin_tsv = params.run_griphin ? PHOENIX_EXTERNAL.out.griphin_tsv : null
+ griphin_excel = params.run_griphin ? PHOENIX_EXTERNAL.out.griphin_excel : null
+ dir_samplesheet = params.run_griphin ? PHOENIX_EXTERNAL.out.dir_samplesheet : null
  //output for ncbi upload 
  ncbi_sra_sheet = params.create_ncbi_sheet ? PHOENIX_EXTERNAL.out.ncbi_sra_sheet : null
  ncbi_biosample_sheet = params.create_ncbi_sheet ? PHOENIX_EXTERNAL.out.ncbi_biosample_sheet : null
@@ -83,6 +83,9 @@ workflow CDC_PHOENIX {
  if (params.input) { ch_input = file(params.input) } else { exit 1, 'For -entry CDC_PHOENIX: Input samplesheet not specified!' }
  ch_versions = Channel.empty() // Used to collect the software versions
 
+ // true is for -entry CDC_PHOENIX and CDC_SCAFFOLDS - used in SPADES
+ extended_qc=false
+
  main:
  PHOENIX_EXQC ( ch_input, ch_versions, true )
 

modules/local/bbduk.nf

@@ -13,10 +13,6 @@ process BBDUK {
  tuple val(meta), path('*.log') , emit: log
  path "versions.yml" , emit: versions
 
- when:
- //if the files are not corrupt and there are equal number of reads in each file then run bbduk
- "${fairy_outcome[0]}" == "PASSED: File ${meta.id}_R1 is not corrupt." && "${fairy_outcome[1]}" == "PASSED: File ${meta.id}_R2 is not corrupt." && "${fairy_outcome[2]}" == "PASSED: Read pairs for ${meta.id} are equal."
-
  script:
  def args = task.ext.args ?: ''
  def prefix = task.ext.prefix ?: "${meta.id}"

modules/local/check_mlst.nf

@@ -12,23 +12,18 @@ process CHECK_MLST {
  tuple val(meta), path("*_status.txt"), emit: status
  path("versions.yml") , emit: versions
 
- when:
- task.ext.when == null || task.ext.when
-
  script:
  // Adding if/else for if running on ICA it is a requirement to state where the script is, however, this causes CLI users to not run the pipeline from any directory.
- if (params.ica==false) { ica = "" } 
- else if (params.ica==true) { ica = "python ${workflow.launchDir}/bin/" }
- else { error "Please set params.ica to either \"true\" if running on ICA or \"false\" for all other methods." }
  def container_version = "base_v2.1.0"
  def container = task.container.toString() - "quay.io/jvhagey/phoenix@"
+ def script = params.ica ? "python ${params.ica_path}/fix_MLST2.py" : "fix_MLST2.py"
  """
- ${ica}fix_MLST2.py --input $mlst_file --taxonomy $taxonomy_file --mlst_database ${local_dbases}
+ ${script} --input $mlst_file --taxonomy $taxonomy_file --mlst_database ${local_dbases}
 
  cat <<-END_VERSIONS > versions.yml
  "${task.process}":
  python: \$(python --version | sed 's/Python //g')
- fix_MLST2.py: \$(${ica}fix_MLST2.py --version )
+ fix_MLST2.py: \$(${script} --version )
  phoenix_base_container_tag: ${container_version}
  phoenix_base_container: ${container}
  END_VERSIONS

modules/local/check_mlst_with_srst2.nf

@@ -17,26 +17,23 @@ process CHECK_MLST_WITH_SRST2 {
  task.ext.when == null || task.ext.when
 
  script:
- // Adding if/else for if running on ICA it is a requirement to state where the script is, however, this causes CLI users to not run the pipeline from any directory.
- if (params.ica==false) { ica = "" } 
- else if (params.ica==true) { ica = "python ${workflow.launchDir}/bin/" }
- else { error "Please set params.ica to either \"true\" if running on ICA or \"false\" for all other methods." }
  // define variables
  def container_version = "base_v2.1.0"
  def container = task.container.toString() - "quay.io/jvhagey/phoenix@"
+ def script = params.ica ? "python ${params.ica_path}/fix_MLST2.py" : "fix_MLST2.py"
  """
  if [[ "${status[0]}" == "True" ]]; then
- ${ica}fix_MLST2.py --input $mlst_file --srst2 $srst2_file --taxonomy $taxonomy_file --mlst_database $local_dbases
+ ${script} --input $mlst_file --srst2 $srst2_file --taxonomy $taxonomy_file --mlst_database $local_dbases
  elif [[ "${status[0]}" == "False" ]]; then
- ${ica}fix_MLST2.py --input $mlst_file --taxonomy $taxonomy_file --mlst_database $local_dbases
+ ${script} --input $mlst_file --taxonomy $taxonomy_file --mlst_database $local_dbases
  else 
  echo "Something went very wrong, please open an issue on Github for the PHoeNIx developers to address."
  fi
 
  cat <<-END_VERSIONS > versions.yml
  "${task.process}":
  python: \$(python --version | sed 's/Python //g')
- fix_MLST2.py: \$(${ica}fix_MLST2.py --version )
+ fix_MLST2.py: \$(${script} --version )
  phoenix_base_container_tag: ${container_version}
  phoenix_base_container: ${container}
  END_VERSIONS

modules/local/determine_taxa_id.nf

@@ -14,20 +14,17 @@ process DETERMINE_TAXA_ID {
  path("versions.yml") , emit: versions
 
  script: // This script is bundled with the pipeline, in cdcgov/phoenix/bin/
- // Adding if/else for if running on ICA it is a requirement to state where the script is, however, this causes CLI users to not run the pipeline from any directory.
- if (params.ica==false) { ica = "" } 
- else if (params.ica==true) { ica = "bash ${workflow.launchDir}/bin/" }
- else { error "Please set params.ica to either \"true\" if running on ICA or \"false\" for all other methods." }
  // define variables
  def prefix = task.ext.prefix ?: "${meta.id}"
  // -r needs to be last as in -entry SCAFFOLDS/CDC_SCAFFOLDS k2_bh_summary is not passed so its a blank argument
  def k2_bh_file = k2_bh_summary ? "-r $k2_bh_summary" : ""
  def container_version = "base_v2.1.0"
  def container = task.container.toString() - "quay.io/jvhagey/phoenix@"
+ def script = params.ica ? "${params.ica_path}/determine_taxID.sh" : "determine_taxID.sh"
  """
- ${ica}determine_taxID.sh -k $kraken_weighted -s $meta.id -f $formatted_ani_file -d $nodes_file -m $names_file $k2_bh_file
+ ${script} -k $kraken_weighted -s $meta.id -f $formatted_ani_file -d $nodes_file -m $names_file $k2_bh_file
 
- script_version=\$(${ica}determine_taxID.sh -V)
+ script_version=\$(${script} -V)
 
  cat <<-END_VERSIONS > versions.yml
  "${task.process}":

modules/local/determine_taxa_id_failure.nf

@@ -17,18 +17,15 @@ process DETERMINE_TAXA_ID_FAILURE {
  "${spades_outcome[0]}" == "run_failure" || "${spades_outcome[1]}" == "no_scaffolds" || "${spades_outcome[2]}" == "no_contigs"
 
  script: // This script is bundled with the pipeline, in cdcgov/phoenix/bin/
- // Adding if/else for if running on ICA it is a requirement to state where the script is, however, this causes CLI users to not run the pipeline from any directory.
- if (params.ica==false) { ica = "" } 
- else if (params.ica==true) { ica = "bash ${workflow.launchDir}/bin/" }
- else { error "Please set params.ica to either \"true\" if running on ICA or \"false\" for all other methods." }
  // define variables
  def prefix = task.ext.prefix ?: "${meta.id}"
  def container_version = "base_v2.1.0"
  def container = task.container.toString() - "quay.io/jvhagey/phoenix@"
+ def script = params.ica ? "bash ${params.ica_path}/determine_taxID.sh" : "determine_taxID.sh"
  """
- ${ica}determine_taxID.sh -r $k2_bh_summary -s $meta.id -d $nodes_file -m $names_file
+ ${script} -r $k2_bh_summary -s $meta.id -d $nodes_file -m $names_file
 
- script_version=\$(${ica}determine_taxID.sh -V)
+ script_version=\$(${script} -V)
 
  cat <<-END_VERSIONS > versions.yml
  "${task.process}":

modules/local/determine_top_mash_hits.nf

@@ -17,25 +17,19 @@ process DETERMINE_TOP_MASH_HITS {
  "${fairy_outcome[4]}" == "PASSED: More than 0 scaffolds in ${meta.id} after filtering."
 
  script: // This script is bundled with the pipeline, in cdcgov/phoenix/bin/
- // terra=true sets paths for bc/wget for terra container paths
- if (params.terra==false) { terra = ""} 
- else if (params.terra==true) { terra = "-t terra" }
- else { error "Please set params.terra to either \"true\" or \"false\"" }
- // Adding if/else for if running on ICA it is a requirement to state where the script is, however, this causes CLI users to not run the pipeline from any directory.
- if (params.ica==false) { ica = "" } 
- else if (params.ica==true) { ica = "bash ${workflow.launchDir}/bin/" }
- else { error "Please set params.ica to either \"true\" if running on ICA or \"false\" for all other methods." }
  // define variables
  def prefix = task.ext.prefix ?: "${meta.id}"
  def sample_name = "${mash_dists}" - ".txt" //get full sample name with REFSEQ_DATE
  def container_version = "base_v2.1.0"
  def container = task.container.toString() - "quay.io/jvhagey/phoenix@"
+ def script = params.ica ? "${params.ica_path}/sort_and_prep_dist.sh" : "sort_and_prep_dist.sh"
+ def terra = params.terra ? "-t terra" : ""
  """
  mkdir reference_dir
 
- ${ica}sort_and_prep_dist.sh -a $assembly_scaffolds -x $mash_dists -o reference_dir $terra
+ ${script} -a $assembly_scaffolds -x $mash_dists -o reference_dir $terra
 
- script_version=\$(${ica}sort_and_prep_dist.sh -V)
+ script_version=\$(${script} -V)
 
  if [[ ! -f ${sample_name}_best_MASH_hits.txt ]]; then
  echo "No MASH hit found" > ${sample_name}_best_MASH_hits.txt

modules/local/fairy_corruption_check.nf

@@ -9,35 +9,31 @@ process CORRUPTION_CHECK {
  val(busco_val)
 
  output:
- tuple val(meta), path('*_summary.txt'), emit: outcome
- tuple val(meta), path('*_summary_old.txt'), emit: outcome_to_edit
+ tuple val(meta), path('*_summary_fairy.txt'), emit: outcome
  path('*_summaryline.tsv'), optional:true, emit: summary_line
  tuple val(meta), path('*.synopsis'), optional:true, emit: synopsis
  path("versions.yml"), emit: versions
 
  script:
- // Adding if/else for if running on ICA it is a requirement to state where the script is, however, this causes CLI users to not run the pipeline from any directory.
- if (params.ica==false) { ica = "" } 
- else if (params.ica==true) { ica = "bash ${workflow.launchDir}/bin/" }
- else { error "Please set params.ica to either \"true\" if running on ICA or \"false\" for all other methods." }
  // define variables
  def prefix = task.ext.prefix ?: "${meta.id}"
  def num1 = "${reads[0]}".minus(".fastq.gz")
  def num2 = "${reads[1]}".minus(".fastq.gz")
  def busco_parameter = busco_val ? "-b" : ""
  def container_version = "base_v2.1.0"
  def container = task.container.toString() - "quay.io/jvhagey/phoenix@"
- """
+ def script = params.ica ? "python ${params.ica_path}/fairy_proc.sh" : "fairy_proc.sh"
+"""
  #set +e
  #check for file integrity and log errors
  #if there is a corruption problem the script will create a *_summaryline.tsv and *.synopsis file for the sample.
- ${ica}fairy_proc.sh -r ${reads[0]} -p ${prefix} ${busco_parameter}
- ${ica}fairy_proc.sh -r ${reads[1]} -p ${prefix} ${busco_parameter}
+ ${script} -r ${reads[0]} -p ${prefix} ${busco_parameter}
+ ${script} -r ${reads[1]} -p ${prefix} ${busco_parameter}
 
- script_version=\$(${ica}fairy_proc.sh -V)
+ script_version=\$(${script} -V)
 
  #making a copy of the summary file to pass to READ_COUNT_CHECKS to handle file names being the same
- cp ${prefix}_summary.txt ${prefix}_summary_old.txt
+ mv ${prefix}_summary.txt ${prefix}_summary_fairy.txt
 
  cat <<-END_VERSIONS > versions.yml
  "${task.process}":