stjude.config

/*
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Nextflow config file for St. Jude Children's Research Hospital's High Performance Research Cluster (HPCF)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Author: Haidong Yi
Mail: hyi@stjude.org
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*/

params {
    config_profile_contact     = "Haidong Yi (hyi@stjude.org)"
    config_profile_description = "St. Jude Children's Research Hospital HPC cluster (HPCF) profile"
    config_profile_url         = "https://www.stjude.org/"

    max_cpus                   = 64
    max_memory                 = 1024.GB
    max_time                   = 240.h
}

process {
    // use `resourceLimits` directive to control resource
    // Note: `resourceLimits` is a new feature starting from version 24.04.0.
    // To use it, `Nextflow` should be upgraded to 24.04.0.
    // see: https://github.com/nf-core/tools/issues/2923
    resourceLimits = [
        memory: 1024.GB,
        cpus: 32,
        time: 240.h
    ]
    executor       = 'lsf'
    scratch        = false
    cache          = 'lenient'

    maxRetries     = 3
    errorStrategy  = { task.exitStatus in [143, 137, 134, 139, 140] ? 'retry' : 'finish' }

    afterScript    = 'sleep 10'
    // to avoid fail when using storeDir for missing output
    beforeScript   = """
    module load singularity/4.1.1
    export SINGULARITY_TMPDIR="${System.getenv('TMPDIR')}" # The 'TMPDIR' environment variable will be set up by LSF after a job is submitted.
    """

    // queue selection based on task configs
    // if urgent, change default queue from 'standard' to 'priority'
    queue          = {
        if (task.accelerator) {
            'gpu'
        }
        else if (task.time < 30.min) {
            "short"
        }
        else if (task.memory > 512.GB) {
            "large_mem"
        }
        else {
            "standard"
        }
    }

    // clusterOptions for gpu task:
    // NOTE: We use GPU exclusively in each job
    clusterOptions = { task.accelerator ? "-gpu \"num=${task.accelerator.request}/host:mode=shared:j_exclusive=yes\"" : null }
}

singularity {
    envWhitelist = "SINGULARITY_TMPDIR,TMPDIR,CUDA_VISIBLE_DEVICES"
    // allow the tmp dir and GPU visible devices visible in the containers
    enabled      = true
    autoMounts   = true
    runOptions   = '-p -B "$TMPDIR"'
    pullTimeout  = "3 hours"
}

// clean the generated files in the working directory
cleanup = true

executor {
    name            = 'lsf'
    queueSize       = 100
    perTaskReserve  = true
    perJobMemLimit  = false
    submitRateLimit = "10/1sec"
    exitReadTimeout = "5 min"
    jobName         = {
        task.name
            .replace("[", "(")
            .replace("]", ")")
            .replace(" ", "_")
    }
}