From 33dcdd19bb8751622d9bcf5c3bcfbfecf4ff9f1a Mon Sep 17 00:00:00 2001 From: DriesSchaumont <5946712+DriesSchaumont@users.noreply.github.com> Date: Wed, 6 Nov 2024 16:03:08 +0000 Subject: [PATCH 1/2] Add resource labels to workflows. --- src/config/labels.config | 79 +++++++++++++++++++++++----- src/config/tests.config | 38 ------------- src/demultiplex/integration_tests.sh | 4 +- src/demultiplex/main.nf | 5 ++ 4 files changed, 74 insertions(+), 52 deletions(-) delete mode 100644 src/config/tests.config diff --git a/src/config/labels.config b/src/config/labels.config index 9ea88b1..e7ab7b8 100644 --- a/src/config/labels.config +++ b/src/config/labels.config @@ -1,24 +1,79 @@ process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 + container = 'nextflow/bash:latest' + + // default resources + memory = { 8.Gb * task.attempt } + cpus = 8 + maxForks = 36 // Retry for exit codes that have something to do with memory issues errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } maxRetries = 3 - maxMemory = null + maxMemory = 192.GB // Resource labels - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 8 } + withLabel: midcpu { cpus = 16 } + withLabel: highcpu { cpus = 32 } - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } } + +} + +profiles { + // detect tempdir + tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' + ).toAbsolutePath() + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + + docker { + docker.fixOwnership = true + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + + local { + // This config is for local processing. + process { + maxMemory = 25.GB + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 6 } + withLabel: highcpu { cpus = 12 } + + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } } + } + } } def get_memory(to_compare) { diff --git a/src/config/tests.config b/src/config/tests.config deleted file mode 100644 index 3473a3d..0000000 --- a/src/config/tests.config +++ /dev/null @@ -1,38 +0,0 @@ - -process.container = 'nextflow/bash:latest' -profiles { - // detect tempdir - tempDir = java.nio.file.Paths.get( - System.getenv('NXF_TEMP') ?: - System.getenv('VIASH_TEMP') ?: - System.getenv('TEMPDIR') ?: - System.getenv('TMPDIR') ?: - '/tmp' - ).toAbsolutePath() - - mount_temp { - docker.temp = tempDir - podman.temp = tempDir - charliecloud.temp = tempDir - } - - no_publish { - process { - withName: '.*' { - publishDir = [ - enabled: false - ] - } - } - } - - docker { - docker.fixOwnership = true - docker.enabled = true - // docker.userEmulation = true - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - } -} diff --git a/src/demultiplex/integration_tests.sh b/src/demultiplex/integration_tests.sh index 5d01dcd..c5f6901 100755 --- a/src/demultiplex/integration_tests.sh +++ b/src/demultiplex/integration_tests.sh @@ -10,8 +10,8 @@ viash ns build --setup cb nextflow run . \ -main-script src/demultiplex/test.nf \ - -profile docker,no_publish \ + -profile docker,no_publish,local \ -entry test_wf \ - -c src/config/tests.config \ + -c src/config/labels.config \ --resources_test https://raw.githubusercontent.com/nf-core/test-datasets/demultiplex/testdata/NovaSeq6000/ \ -resume diff --git a/src/demultiplex/main.nf b/src/demultiplex/main.nf index 4c401da..fa3c3ae 100644 --- a/src/demultiplex/main.nf +++ b/src/demultiplex/main.nf @@ -6,6 +6,7 @@ workflow run_wf { samples_ch = input_ch // untar input if needed | untar.run( + directives: [label: ["lowmem", "lowcpu"]], runIf: {id, state -> def inputStr = state.input.toString() inputStr.endsWith(".tar.gz") || \ @@ -38,6 +39,7 @@ workflow run_wf { } | interop_summary_to_csv.run( + directives: [label: ["lowmem", "verylowcpu"]], fromState: [ "input": "input", ], @@ -48,6 +50,7 @@ workflow run_wf { ) // run bcl_convert | bcl_convert.run( + directives: [label: ["highmem", "midcpu"]], fromState: [ "bcl_input_directory": "input", "sample_sheet": "sample_sheet", @@ -89,6 +92,7 @@ workflow run_wf { ] ) | falco.run( + directives: [label: ["lowcpu", "lowmem"]], fromState: {id, state -> reverse_fastqs_list = state.reverse_fastqs ? state.reverse_fastqs : [] [ @@ -104,6 +108,7 @@ workflow run_wf { }, ) | multiqc.run( + directives: [label: ["lowcpu", "lowmem"]], fromState: {id, state -> [ "input": [ From 07b7368d2a96ea63c6daaf481e099b5c7ea5565a Mon Sep 17 00:00:00 2001 From: DriesSchaumont <5946712+DriesSchaumont@users.noreply.github.com> Date: Wed, 6 Nov 2024 16:04:56 +0000 Subject: [PATCH 2/2] Add CHANGELOG entry --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d7f413..957a9f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# demultiplex v0.1.2 + +## Minor updates + +* Add resource labels to workflows (PR #21). + # demultiplex v0.1.1 ## Minor updates