Add resource labels to workflows.

CHANGELOG.md

@@ -1,3 +1,9 @@
+# demultiplex v0.1.2
+
+## Minor updates
+
+* Add resource labels to workflows (PR #21).
+
 # demultiplex v0.1.1
 
 ## Minor updates

src/config/labels.config

@@ -1,24 +1,79 @@
 process {
-  // Default resources for components that hardly do any processing
-  memory = { 2.GB * task.attempt }
-  cpus = 1
+  container = 'nextflow/bash:latest'
+
+  // default resources
+  memory = { 8.Gb * task.attempt }
+  cpus = 8
+  maxForks = 36
 
   // Retry for exit codes that have something to do with memory issues
   errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
   maxRetries = 3
-  maxMemory = null
+  maxMemory = 192.GB
 
   // Resource labels
-  withLabel: singlecpu { cpus = 1 }
-  withLabel: lowcpu { cpus = 4 }
-  withLabel: midcpu { cpus = 10 }
-  withLabel: highcpu { cpus = 20 }
+  withLabel: verylowcpu { cpus = 2 }
+  withLabel: lowcpu { cpus = 8 }
+  withLabel: midcpu { cpus = 16 }
+  withLabel: highcpu { cpus = 32 }
 
-  withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } }
-  withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } }
-  withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
-  withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
+  withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } }
+  withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
+  withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } }
+  withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } }
+
+}
+
+profiles {
+  // detect tempdir
+  tempDir = java.nio.file.Paths.get(
+    System.getenv('NXF_TEMP') ?:
+      System.getenv('VIASH_TEMP') ?: 
+      System.getenv('TEMPDIR') ?: 
+      System.getenv('TMPDIR') ?: 
+      '/tmp'
+  ).toAbsolutePath()
 
+  mount_temp {
+    docker.temp            = tempDir
+    podman.temp            = tempDir
+    charliecloud.temp      = tempDir
+  }
+
+  no_publish {
+    process {
+      withName: '.*' {
+        publishDir = [
+          enabled: false
+        ]
+      }
+    }
+  }
+
+  docker {
+    docker.fixOwnership    = true
+    docker.enabled         = true
+    // docker.userEmulation   = true
+    singularity.enabled    = false
+    podman.enabled         = false
+    shifter.enabled        = false
+    charliecloud.enabled   = false
+  }
+
+  local {
+    // This config is for local processing.
+    process {
+        maxMemory = 25.GB
+        withLabel: verylowcpu { cpus = 2 }
+        withLabel: lowcpu { cpus = 4 }
+        withLabel: midcpu { cpus = 6 }
+        withLabel: highcpu { cpus = 12 }
+
+        withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
+        withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } }
+        withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } }
+    }
+  }
 }
 
 def get_memory(to_compare) {

src/demultiplex/integration_tests.sh

@@ -10,8 +10,8 @@ viash ns build --setup cb
 
 nextflow run . \
   -main-script src/demultiplex/test.nf \
-  -profile docker,no_publish \
+  -profile docker,no_publish,local \
   -entry test_wf \
-  -c src/config/tests.config \
+  -c src/config/labels.config \
   --resources_test https://raw.githubusercontent.com/nf-core/test-datasets/demultiplex/testdata/NovaSeq6000/ \
   -resume

src/demultiplex/main.nf

@@ -6,6 +6,7 @@ workflow run_wf {
     samples_ch = input_ch
       // untar input if needed
       | untar.run(
+        directives: [label: ["lowmem", "lowcpu"]],
         runIf: {id, state ->
           def inputStr = state.input.toString()
           inputStr.endsWith(".tar.gz") || \
@@ -38,6 +39,7 @@ workflow run_wf {
       }
 
       | interop_summary_to_csv.run(
+        directives: [label: ["lowmem", "verylowcpu"]],
         fromState: [
           "input": "input", 
         ],
@@ -48,6 +50,7 @@ workflow run_wf {
       )
       // run bcl_convert
       | bcl_convert.run(
+        directives: [label: ["highmem", "midcpu"]],
         fromState: [
           "bcl_input_directory": "input",
           "sample_sheet": "sample_sheet",
@@ -89,6 +92,7 @@ workflow run_wf {
         ]
       )
       | falco.run(
+        directives: [label: ["lowcpu", "lowmem"]],
         fromState: {id, state ->
           reverse_fastqs_list = state.reverse_fastqs ? state.reverse_fastqs : []
           [
@@ -104,6 +108,7 @@ workflow run_wf {
         },
       )
       | multiqc.run(
+        directives: [label: ["lowcpu", "lowmem"]],
         fromState: {id, state ->
           [
             "input": [