Merge pull request #994 from broadinstitute/dp-dxwdl

dxWDL upgrade
broadinstitute · Oct 16, 2019 · 83a5fab · 83a5fab
2 parents 9f5b2c0 + a7c3595
commit 83a5fab
Show file tree

Hide file tree

Showing 13 changed files with 50 additions and 50 deletions.
diff --git a/pipes/WDL/dx-launcher/demux_launcher.yml b/pipes/WDL/dx-launcher/demux_launcher.yml
@@ -177,7 +177,7 @@ runSpec:
         fi
         for i in $(seq "$lane_count"); do
           folder2=$(printf "%s/%s/reads/L%d" "$folder" "$run_id" $i)
-          runcmd="dx run $demux_workflow_id -i stage-1.flowcell_tgz=$run_tarball -i illumina_demux.lane=$i -i illumina_demux.maxReadsInRamPerTile=$max_reads_in_ram_per_tile -i illumina_demux.maxRecordsInRam=$max_records_in_ram -i illumina_demux.minimumBaseQuality=$min_base_quality -i illumina_demux.threads=$demux_threads $sequencing_center_input --folder $folder2 --instance-type illumina_demux=$demux_instance_type --name demux:$run_id:L$i -y --brief"
+          runcmd="dx run $demux_workflow_id -i stage-0.flowcell_tgz=$run_tarball -i illumina_demux.lane=$i -i illumina_demux.maxReadsInRamPerTile=$max_reads_in_ram_per_tile -i illumina_demux.maxRecordsInRam=$max_records_in_ram -i illumina_demux.minimumBaseQuality=$min_base_quality -i illumina_demux.threads=$demux_threads $sequencing_center_input --folder $folder2 --instance-type illumina_demux=$demux_instance_type --name demux:$run_id:L$i -y --brief"
           echo "$runcmd"
           set +x
           if [ -n "$api_token" ]; then

diff --git a/pipes/WDL/workflows/tasks/tasks_assembly.wdl b/pipes/WDL/workflows/tasks/tasks_assembly.wdl
@@ -83,7 +83,7 @@ task assemble {
         docker: "quay.io/broadinstitute/viral-ngs"
         memory: "15 GB"
         cpu: 4
-        dx_instance_type: "mem1_ssd1_x8"
+        dx_instance_type: "mem1_ssd1_v2_x8"
     }
 
 }
@@ -167,7 +167,7 @@ task scaffold {
         docker: "quay.io/broadinstitute/viral-ngs"
         memory: "15 GB"
         cpu: 4
-        dx_instance_type: "mem1_ssd1_x8"
+        dx_instance_type: "mem1_ssd1_v2_x8"
     }
 }
 
@@ -224,7 +224,7 @@ task refine {
         docker: "quay.io/broadinstitute/viral-ngs"
         memory: "7 GB"
         cpu: 8
-        dx_instance_type: "mem1_ssd1_x8"
+        dx_instance_type: "mem1_ssd1_v2_x8"
     }
 }
 
@@ -365,7 +365,7 @@ task refine_2x_and_plot {
         docker: "quay.io/broadinstitute/viral-ngs"
         memory: "7 GB"
         cpu: 8
-        dx_instance_type: "mem1_ssd1_x8"
+        dx_instance_type: "mem1_ssd1_v2_x8"
     }
 }
 
diff --git a/pipes/WDL/workflows/tasks/tasks_demux.wdl b/pipes/WDL/workflows/tasks/tasks_demux.wdl
@@ -22,7 +22,7 @@
 #    docker: "quay.io/broadinstitute/viral-ngs"
 #    memory: "7 GB"
 #    cpu: 4
-#    dx_instance_type: "mem1_hdd2_x8"
+#    dx_instance_type: "mem2_hdd2_v2_x4"
 #    preemptible: 0  # this is the very first operation before scatter, so let's get it done quickly & reliably
 #  }
 #}
@@ -201,7 +201,7 @@ task illumina_demux {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "16 GB"
     cpu: 36
-    dx_instance_type: "mem1_ssd2_x36"
+    dx_instance_type: "mem1_ssd2_v2_x36"
     preemptible: 0  # this is the very first operation before scatter, so let's get it done quickly & reliably
   }
 }
@@ -238,7 +238,7 @@ task merge_and_reheader_bams {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "2000 MB"
     cpu: 2
-    dx_instance_type: "mem1_ssd2_x4"
+    dx_instance_type: "mem1_ssd2_v2_x4"
   }
 }
 

diff --git a/pipes/WDL/workflows/tasks/tasks_interhost.wdl b/pipes/WDL/workflows/tasks/tasks_interhost.wdl
@@ -31,7 +31,7 @@ task multi_align_mafft_ref {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "7 GB"
     cpu: 8
-    dx_instance_type: "mem1_ssd1_x8"
+    dx_instance_type: "mem1_ssd1_v2_x8"
   }
 }
 
@@ -66,7 +66,7 @@ task multi_align_mafft {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "7 GB"
     cpu: 8
-    dx_instance_type: "mem1_ssd1_x8"
+    dx_instance_type: "mem1_ssd1_v2_x8"
   }
 }
 

diff --git a/pipes/WDL/workflows/tasks/tasks_metagenomics.wdl b/pipes/WDL/workflows/tasks/tasks_metagenomics.wdl
@@ -68,7 +68,7 @@ task krakenuniq {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "200 GB"
     cpu: 32
-    dx_instance_type: "mem3_ssd1_x32"
+    dx_instance_type: "mem3_ssd1_v2_x32"
     preemptible: 0
   }
 }
@@ -108,7 +108,7 @@ task krona {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "4 GB"
     cpu: 1
-    dx_instance_type: "mem1_ssd2_x2"
+    dx_instance_type: "mem1_ssd2_v2_x2"
   }
 }
 
@@ -166,7 +166,7 @@ task filter_bam_to_taxa {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "4 GB"
     cpu: 1
-    dx_instance_type: "mem1_ssd2_x2"
+    dx_instance_type: "mem1_ssd2_v2_x2"
   }
 
 }
@@ -233,6 +233,6 @@ task kaiju {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "100 GB"
     cpu: 16
-    dx_instance_type: "mem3_ssd1_x16"
+    dx_instance_type: "mem3_ssd1_v2_x16"
   }
 }
diff --git a/pipes/WDL/workflows/tasks/tasks_ncbi.wdl b/pipes/WDL/workflows/tasks/tasks_ncbi.wdl
@@ -20,7 +20,7 @@ task download_fasta {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "3 GB"
     cpu: 2
-    dx_instance_type: "mem1_ssd1_x2"
+    dx_instance_type: "mem1_ssd1_v2_x2"
   }
 }
 
@@ -55,7 +55,7 @@ task download_annotations {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "3 GB"
     cpu: 2
-    dx_instance_type: "mem1_ssd1_x2"
+    dx_instance_type: "mem1_ssd1_v2_x2"
   }
 }
 
@@ -84,20 +84,20 @@ task annot_transfer {
   }
 
   output {
-    Array[File]+ transferred_feature_tables = glob("*.tbl")
+    Array[File]  transferred_feature_tables = glob("*.tbl")
     String       viralngs_version           = "viral-ngs_version_unknown"
   }
   runtime {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "3 GB"
     cpu: 2
-    dx_instance_type: "mem1_ssd1_x2"
+    dx_instance_type: "mem1_ssd1_v2_x2"
   }
 }
 
 task prepare_genbank {
   Array[File]+ assemblies_fasta
-  Array[File]+ annotations_tbl
+  Array[File]  annotations_tbl
   File         authors_sbt
   File         biosampleMap
   File         genbankSourceTable
@@ -140,7 +140,7 @@ task prepare_genbank {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "3 GB"
     cpu: 2
-    dx_instance_type: "mem1_ssd1_x2"
+    dx_instance_type: "mem1_ssd1_v2_x2"
   }
 }
 

diff --git a/pipes/WDL/workflows/tasks/tasks_read_utils.wdl b/pipes/WDL/workflows/tasks/tasks_read_utils.wdl
@@ -33,7 +33,7 @@ task downsample_bams {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "3 GB"
     cpu: 2
-    dx_instance_type: "mem1_ssd1_x4"
+    dx_instance_type: "mem1_ssd1_v2_x4"
   }
 }
 

diff --git a/pipes/WDL/workflows/tasks/tasks_reports.wdl b/pipes/WDL/workflows/tasks/tasks_reports.wdl
@@ -110,7 +110,7 @@ task plot_coverage {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "3500 MB"
     cpu: 4
-    dx_instance_type: "mem1_ssd1_x8"
+    dx_instance_type: "mem1_ssd1_v2_x8"
   }
 }
 
@@ -135,7 +135,7 @@ task coverage_report {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "2000 MB"
     cpu: 2
-    dx_instance_type: "mem1_ssd2_x4"
+    dx_instance_type: "mem1_ssd2_v2_x4"
   }
 }
 
@@ -159,7 +159,7 @@ task fastqc {
     memory: "2 GB"
     cpu: 1
     docker: "quay.io/broadinstitute/viral-ngs"
-    dx_instance_type: "mem1_ssd1_x4"
+    dx_instance_type: "mem1_ssd1_v2_x4"
   }
 }
 
@@ -197,7 +197,7 @@ task spikein_report {
     memory: "3 GB"
     cpu: 2
     docker: "quay.io/broadinstitute/viral-ngs"
-    dx_instance_type: "mem1_ssd1_x4"
+    dx_instance_type: "mem1_ssd1_v2_x4"
   }
 }
 
@@ -223,7 +223,7 @@ task spikein_summary {
     memory: "3 GB"
     cpu: 2
     docker: "quay.io/broadinstitute/viral-ngs"
-    dx_instance_type: "mem1_ssd1_x4"
+    dx_instance_type: "mem1_ssd1_v2_x4"
   }
 }
 
@@ -256,7 +256,7 @@ task aggregate_metagenomics_reports {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "4 GB"
     cpu: 1
-    dx_instance_type: "mem1_ssd2_x2"
+    dx_instance_type: "mem1_ssd2_v2_x2"
     preemptible: 0
   }
 }
diff --git a/pipes/WDL/workflows/tasks/tasks_taxon_filter.wdl b/pipes/WDL/workflows/tasks/tasks_taxon_filter.wdl
@@ -71,7 +71,7 @@ task deplete_taxa {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "14 GB"
     cpu: 8
-    dx_instance_type: "mem1_ssd1_x16"
+    dx_instance_type: "mem1_ssd1_v2_x16"
     preemptible: 0
   }
 }
@@ -130,7 +130,7 @@ task filter_to_taxon {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "14 GB"
     cpu: 16
-    dx_instance_type: "mem1_ssd1_x8"
+    dx_instance_type: "mem1_ssd1_v2_x8"
   }
 }
 
@@ -153,7 +153,7 @@ task build_lastal_db {
     docker: "quay.io/broadinstitute/viral-ngs"
     memory: "7 GB"
     cpu: 2
-    dx_instance_type: "mem1_ssd1_x4"
+    dx_instance_type: "mem1_ssd1_v2_x4"
   }
 }
 
@@ -194,7 +194,7 @@ task merge_one_per_sample {
     memory: "7 GB"
     cpu: 4
     docker: "quay.io/broadinstitute/viral-ngs"
-    dx_instance_type: "mem1_ssd2_x4"
+    dx_instance_type: "mem1_ssd1_v2_x16"
   }
 }
 

diff --git a/test/input/WDL/test_inputs-assemble_denovo_with_deplete-dnanexus.dx.json b/test/input/WDL/test_inputs-assemble_denovo_with_deplete-dnanexus.dx.json
@@ -1,13 +1,13 @@
 {
-	"stage-1.raw_reads_unmapped_bam": { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-F8F2kVQ09y3Q9Qj14fF806q2" } },
+	"stage-0.raw_reads_unmapped_bam": { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-F8F2kVQ09y3Q9Qj14fF806q2" } },
 
-	"stage-1.bmtaggerDbs": [
+	"stage-0.bmtaggerDbs": [
 		{ "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-BYF6g8Q0zjF77x79bGYgJ1Zb" } }
 	],
 
-	"stage-2.lastal_db_fasta": { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-F1zXZ9Q0pvkp7FkZ646YJX2y" } },
-	"stage-4.reference_genome_fasta": [ { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-F1zXZ9Q0pvkp7FkZ646YJX2y" } } ],
+	"stage-1.lastal_db_fasta": { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-F1zXZ9Q0pvkp7FkZ646YJX2y" } },
+	"stage-3.reference_genome_fasta": [ { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-F1zXZ9Q0pvkp7FkZ646YJX2y" } } ],
 
-	"stage-5.gatk_jar": { "$dnanexus_link": { "project": "project-F89kXY805X08BpXq7xjxbX9F", "id": "file-F89Vx8Q09y3VFxf1KKXbbVv4" } },
-	"stage-5.novocraft_license": { "$dnanexus_link": { "project": "project-F89kXY805X08BpXq7xjxbX9F", "id": "file-F1zkVGQ0jy1P68J88kyJ8zVF" } }
+	"stage-4.gatk_jar": { "$dnanexus_link": { "project": "project-F89kXY805X08BpXq7xjxbX9F", "id": "file-F89Vx8Q09y3VFxf1KKXbbVv4" } },
+	"stage-4.novocraft_license": { "$dnanexus_link": { "project": "project-F89kXY805X08BpXq7xjxbX9F", "id": "file-F1zkVGQ0jy1P68J88kyJ8zVF" } }
 }
diff --git a/test/input/WDL/test_inputs-demux_plus-dnanexus.dx.json b/test/input/WDL/test_inputs-demux_plus-dnanexus.dx.json
@@ -1,16 +1,16 @@
 {
-  "stage-0.bmtaggerDbs": [
+  "stage-common.bmtaggerDbs": [
   { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-BYF6g8Q0zjF77x79bGYgJ1Zb" } }
   ],
-  "stage-0.blastDbs": [
+  "stage-common.blastDbs": [
   { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-BzBpkK80x0z8GBQPfZ4kgPKp" } }
   ],
 
-  "stage-1.flowcell_tgz": { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-F86qY0j09y3Q5FbKKK3k4gzq" } },
-  "stage-1.samplesheet":  { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-F8F32gQ09y3VJ4kjBzvgZkKv" } },
-  "stage-1.maxMismatches": 0,
-  "stage-1.minimumQuality": 25,
+  "stage-0.flowcell_tgz": { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-F86qY0j09y3Q5FbKKK3k4gzq" } },
+  "stage-0.samplesheet":  { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-F8F32gQ09y3VJ4kjBzvgZkKv" } },
+  "stage-0.maxMismatches": 0,
+  "stage-0.minimumQuality": 25,
 
-  "stage-3.krakenuniq_db_tar_lz4": { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-FVYQqP006zFF064QBGf022X1" } },
-  "stage-3.krona_taxonomy_db_tgz": { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-F4z0fgj07FZ8jg8yP7yz0Qzb" } }
+  "stage-5.krakenuniq_db_tar_lz4": { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-FVYQqP006zFF064QBGf022X1" } },
+  "stage-5.krona_taxonomy_db_tgz": { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-F4z0fgj07FZ8jg8yP7yz0Qzb" } }
 }
diff --git a/travis/install-wdl.sh b/travis/install-wdl.sh
@@ -17,11 +17,11 @@ cached_fetch_jar_from_github () {
 	ln -s $CACHE_DIR/$_jar_fname $_tool_name.jar
 }
 
-cached_fetch_jar_from_github broadinstitute cromwell womtool 33.1
-cached_fetch_jar_from_github broadinstitute cromwell cromwell 33.1
-cached_fetch_jar_from_github dnanexus dxWDL dxWDL 0.72
+cached_fetch_jar_from_github broadinstitute cromwell womtool 47
+cached_fetch_jar_from_github broadinstitute cromwell cromwell 47
+cached_fetch_jar_from_github dnanexus dxWDL dxWDL v1.33
 
-TGZ=dx-toolkit-v0.285.0-ubuntu-16.04-amd64.tar.gz
+TGZ=dx-toolkit-v0.288.0-ubuntu-16.04-amd64.tar.gz
 if [ ! -f $CACHE_DIR/$TGZ ]; then
 	echo "Fetching $TGZ"
 	wget --quiet https://dnanexus-sdk.s3.amazonaws.com/$TGZ

diff --git a/travis/version-wdl-runtimes.sh b/travis/version-wdl-runtimes.sh
@@ -10,7 +10,7 @@ sed -i -- "s|$OLD_TAG|$NEW_TAG|g" pipes/WDL/workflows/tasks/*.wdl
 CURRENT_VERSION=`echo "$NEW_TAG" | cut -d\: -f2`
 VERSION_PLACEHOLDER="viral-ngs_version_unknown"
 
-echo Replacing $VERSION_PLACEHOLDER with $CURENT_VERSION in all task WDL files
+echo Replacing $VERSION_PLACEHOLDER with $CURRENT_VERSION in all task WDL files
 sed -i -- "s|$VERSION_PLACEHOLDER|$CURRENT_VERSION|g" pipes/WDL/workflows/tasks/*.wdl
 
 #echo "DEBUG: here are the modified runtime lines:"