diff --git a/.circleci/config.yml b/.circleci/config.yml index b219eb82..29c17a48 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,7 +22,7 @@ jobs: # a collection of steps - run: name: Run tests command: | - ./gradlew build test $GRADLE_ARGS + ./gradlew build check test allBoms $GRADLE_ARGS - save_cache: paths: - ~/.gradle/wrapper @@ -34,26 +34,42 @@ jobs: # a collection of steps - store_test_results: # Upload test results for display in Test Summary: https://circleci.com/docs/2.0/collect-test-data/ path: build/test-results/test - - store_artifacts: # Upload test results for display in Artifacts: https://circleci.com/docs/2.0/artifacts/ + - store_artifacts: + # Upload test results for display in Artifacts: https://circleci.com/docs/2.0/artifacts/ path: build/test-results/test when: always - - persist_to_workspace: # Pass artifact over to next job via workspace. - root: build/libs + - persist_to_workspace: + # Pass artifact over to next job via workspace. + root: build/ paths: - - "*.jar" - # See https://circleci.com/docs/2.0/deployment-integrations/ for deploy examples + - "libs/*.jar" + - "reports/cyclonedx/*" + - "reports/tests/test/*" + + # See https://circleci.com/docs/2.0/deployment-integrations/ for deploy examples publish-github-release: # https://circleci.com/blog/publishing-to-github-releases-via-circleci/ docker: - image: cibuilds/github:0.13 resource_class: small steps: - attach_workspace: - at: build/libs + at: build/ - run: name: "Publish Release on GitHub" command: | VERSION=$(ls ./build/libs/*.jar | sed -r 's/.*BatchEuphoria-([[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+(-.+)?).jar/\1/') - ghr -t "$GITHUB_TOKEN" -u "$CIRCLE_PROJECT_USERNAME" -r "$CIRCLE_PROJECT_REPONAME" -c "$CIRCLE_SHA1" -n "$VERSION" -delete "$VERSION" ./build/libs/*.jar + tar -C build/reports/tests/ -cvzf test-report.tgz test/ + tar -C build/reports/ -cvzf cyclonedx-sbom.tgz cyclonedx/ + ghr \ + -t "$GITHUB_TOKEN" \ + -u "$CIRCLE_PROJECT_USERNAME" \ + -r "$CIRCLE_PROJECT_REPONAME" \ + -c "$CIRCLE_SHA1" \ + -n "$VERSION" \ + -delete "$VERSION" \ + ./build/libs/*.jar \ + ./cyclonedx-sbom.tgz \ + ./test-report.tgz workflows: version: 2 diff --git a/README.md b/README.md index ed7acaf3..a0e6d7cc 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Currently this library supports the following job schedulers: * direct execution ### Dependencies + * [RoddyToolLib](https://github.com/TheRoddyWMS/RoddyToolLib) ## Build @@ -89,6 +90,15 @@ To start the integration tests, please fill in host and user settings (password # Change Logs +* 0.2.0 + - major: `CommandI` classes to model commands and code + - minor: `ApptainerCommandBuilder` to create a wrapping command to run code in containers + * Submission as container works for LSF and SLURM. PBS and SGE might work, but were not tested. It won't work for REST-based submission to LSF, though. + - patch: Library updates + - patch: Fix `--propagate=none` for SLURM to `--propagate=NONE` + - patch: Added Gradle task `:allBoms` for CycloneDX SBOM generation + - patch: Added cyclonedx SBOM and test-report tarballs to GitHub releases + * 0.1.4 - patch: Changed the conversion of EpochSeconds to ZonedDateTime for the value "0". * For eligibleTime it is set to null @@ -100,7 +110,7 @@ To start the integration tests, please fill in host and user settings (password * Reworked queryExtendedJobStateById to always execute both scontrol and sacct and combine the output * Some refactoring and added comments * Fixed time parsing for jobs longer than 24h - * Added --propagate=none" to getAdditionalCommandParameters + * Added `--propagate=none` to getAdditionalCommandParameters * Added tests * 0.1.2 diff --git a/build.gradle b/build.gradle index d38128df..b5da8b9d 100644 --- a/build.gradle +++ b/build.gradle @@ -4,20 +4,27 @@ * Distributed under the MIT License (license terms are at https://www.github.com/theroddywms/Roddy/LICENSE.txt). */ +import org.cyclonedx.gradle.CycloneDxTask + plugins { - id 'org.ajoberstar.grgit' version '1.7.2' + id 'org.ajoberstar.grgit' version '4.1.1' + id 'org.cyclonedx.bom' version '1.8.2' + id 'maven-publish' } apply plugin: 'groovy' apply plugin: 'idea' apply plugin: 'application' -apply plugin: 'maven' group = "com.github.theroddywms" mainClassName = 'de.dkfz.roddy.BEIntegrationTestStarter' -sourceCompatibility = '1.8' -targetCompatibility = '1.8' + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(8) + } +} /* * Gets the version name from the current Git tag. If the current commit is not tagged, @@ -30,10 +37,12 @@ def getVersionName() { rootProject.version = getVersionName() +compileJava.options.compilerArgs = ["-release", targetCompatibility] + // Set -Pchecked on the command line to change this compiler parameter. -if (!project.hasProperty("checked") || !project.checked) { - compileJava.options.compilerArgs = ["-Xlint:unchecked"] - compileGroovy.options.compilerArgs = ["-Xlint:unchecked"] +if (project.hasProperty("checked") && !project.checked) { + compileJava.options.compilerArgs += ["-Xlint:unchecked", "-Xlint:deprecation"] + compileGroovy.options.compilerArgs += ["-Xlint:unchecked", "-Xlint:deprecation"] } repositories { @@ -46,41 +55,49 @@ configurations.all { } dependencies { - compile 'com.fasterxml.jackson.core:jackson-core:2.9.7' - compile 'com.fasterxml.jackson.core:jackson-annotations:2.9.7' - compile 'com.fasterxml.jackson.core:jackson-databind:2.9.7' - compile 'org.codehaus.groovy:groovy-all:2.4.9' - testCompile 'junit:junit:4.12' - testCompile 'org.spockframework:spock-core:1.1-groovy-2.4' - testRuntime "net.bytebuddy:byte-buddy:1.6.5" - testRuntime "org.objenesis:objenesis:2.5.1" - testCompile 'com.google.guava:guava:23.0' - testCompile 'org.slf4j:slf4j-nop:1.7.25' - compile 'org.slf4j:slf4j-api:1.7.25' - compile group: 'org.apache.commons', name: 'commons-io', version: '1.3.2' - compile group: 'commons-codec', name: 'commons-codec', version: '1.9' - compile group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.3' - compile group: 'commons-cli', name: 'commons-cli', version: '1.2' - compile group: 'org.apache.commons', name: 'commons-text', version: '1.1' - compile 'com.google.guava:guava:23.0' - compile 'com.github.theroddywms:RoddyToolLib:2.4.0' + implementation 'com.fasterxml.jackson.core:jackson-core:2.9.7' + implementation 'com.fasterxml.jackson.core:jackson-annotations:2.9.7' + implementation 'com.fasterxml.jackson.core:jackson-databind:2.12.7.1' + implementation 'org.codehaus.groovy:groovy-all:2.4.21' + testImplementation 'junit:junit:4.13.1' + testImplementation 'org.spockframework:spock-core:1.1-groovy-2.4' + testRuntimeOnly "net.bytebuddy:byte-buddy:1.6.5" + testRuntimeOnly "org.objenesis:objenesis:2.5.1" + testImplementation 'org.slf4j:slf4j-nop:1.7.25' + implementation 'org.slf4j:slf4j-api:1.7.25' + implementation group: 'commons-io', name: 'commons-io', version: '2.11.0' + implementation group: 'commons-codec', name: 'commons-codec', version: '1.15' + implementation group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.13' + implementation group: 'commons-cli', name: 'commons-cli', version: '1.5.0' + implementation group: 'org.apache.commons', name: 'commons-text', version: '1.11.0' + implementation group: 'com.google.guava', name: 'guava', version: '33.2.1-jre' + implementation 'com.github.theroddywms:RoddyToolLib:2.5.0' + + // Runtime retention of @NotNull and annotations. + implementation 'com.github.bbottema:jetbrains-runtime-annotations:1.0.1' } -task writePom { - doLast { - pom { - project { - version rootProject.version - inceptionYear '2017' +model { + tasks.generatePomFileForMavenJavaPublication { + destination = file("$buildDir/pom.xml") + } +} + +publishing { + publications { + mavenJava(org.gradle.api.publish.maven.MavenPublication) { + from components.java + pom { + name = 'BatchEuphoria' + url = "https://www.github.com/theroddywms/BatchEuphoria" licenses { license { - name 'MIT License' - url 'https://raw.githubusercontent.com/eilslabs/BatchEuphoria/master/LICENSE' - distribution 'repo' + name = 'MIT License' + url = 'https://raw.githubusercontent.com/eilslabs/BatchEuphoria/master/LICENSE' } } } - }.writeTo("$buildDir/pom.xml") + } } } @@ -89,13 +106,12 @@ jar { manifest { attributes("Implementation-Vendor": "German Cancer Research Center (DKFZ)") - attributes("Implementation-Title": baseName) + attributes("Implementation-Title": archiveBaseName.get()) attributes 'Implementation-Version': rootProject.version } - dependsOn writePom into("META-INF/maven/$project.group/$project.name") { - from "$buildDir/pom.xml" + from generatePomFileForMavenJavaPublication } } @@ -109,7 +125,7 @@ task listJars { task uberjar(type: Jar, dependsOn:[':compileJava',':compileGroovy']) { from files(sourceSets.main.output.dirs) - from configurations.runtime.asFileTree.files.collect { zipTree(it) } + from configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) } version = getVersionName() @@ -119,3 +135,35 @@ task uberjar(type: Jar, dependsOn:[':compileJava',':compileGroovy']) { } exclude 'META-INF/*.RSA', 'META-INF/*.SF','META-INF/*.DSA' } + +task listConfigurations { + group = "reporting" + description = "List all configurations, as needed for discerning development and production configurations by the SBOM tasks." + doLast { + configurations.each { Configuration c -> + println c.name + } + } +} + +task allBoms() { + group = "reporting" + description = "A grouping task to execute the CycloneDxTask for each configuration" +} + +rootProject.configurations.each { Configuration configuration -> + String name = configuration.name + task "${name}Bom"(type: CycloneDxTask) { + group = "reporting" + description = "Creates a CycloneDX SBOM for the configuration ${name}." + + destination = file("${project.buildDir}/reports/cyclonedx") + outputName = name + includeConfigs = [name] + schemaVersion = "1.5" + outputFormat="json" + includeLicenseText = false + projectType = "application" + } + allBoms.dependsOn tasks.getByName("${name}Bom") +} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 457aad0d..7f93135c 100755 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 558870da..ac72c34e 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index af6708ff..0adc8e1a 100755 --- a/gradlew +++ b/gradlew @@ -1,78 +1,127 @@ -#!/usr/bin/env sh +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m"' +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -81,92 +130,120 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=$((i+1)) + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=$(save "$@") - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" fi +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat index 0f8d5937..6689b85b 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -1,4 +1,20 @@ -@if "%DEBUG%" == "" @echo off +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -9,19 +25,23 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init +if %ERRORLEVEL% equ 0 goto execute echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. @@ -35,7 +55,7 @@ goto fail set JAVA_HOME=%JAVA_HOME:"=% set JAVA_EXE=%JAVA_HOME%/bin/java.exe -if exist "%JAVA_EXE%" goto init +if exist "%JAVA_EXE%" goto execute echo. echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% @@ -45,38 +65,26 @@ echo location of your Java installation. goto fail -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - :execute @rem Setup the command line set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal diff --git a/src/main/groovy/de/dkfz/roddy/TestExecutionService.groovy b/src/main/groovy/de/dkfz/roddy/TestExecutionService.groovy index 09a7aa8b..739de300 100644 --- a/src/main/groovy/de/dkfz/roddy/TestExecutionService.groovy +++ b/src/main/groovy/de/dkfz/roddy/TestExecutionService.groovy @@ -8,8 +8,8 @@ package de.dkfz.roddy import de.dkfz.roddy.execution.BEExecutionService import de.dkfz.roddy.execution.io.LocalExecutionHelper -import de.dkfz.roddy.execution.jobs.Command import de.dkfz.roddy.execution.io.ExecutionResult +import de.dkfz.roddy.execution.jobs.Command import de.dkfz.roddy.tools.BashUtils import groovy.transform.CompileStatic diff --git a/src/main/groovy/de/dkfz/roddy/config/EmptyResourceSet.groovy b/src/main/groovy/de/dkfz/roddy/config/EmptyResourceSet.groovy new file mode 100644 index 00000000..90f9fe03 --- /dev/null +++ b/src/main/groovy/de/dkfz/roddy/config/EmptyResourceSet.groovy @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2023 German Cancer Research Center (Deutsches Krebsforschungszentrum, DKFZ). + * + * Distributed under the MIT License (license terms are at https://www.github.com/TheRoddyWMS/Roddy/LICENSE.txt). + */ + +package de.dkfz.roddy.config + + +import de.dkfz.roddy.tools.TimeUnit +import groovy.transform.CompileStatic + +@CompileStatic +class EmptyResourceSet extends ResourceSet { + EmptyResourceSet() { + super(null, + null, + null, + null, + null as TimeUnit, + null, + null, + null) + } +} diff --git a/src/main/groovy/de/dkfz/roddy/execution/ApptainerCommandBuilder.groovy b/src/main/groovy/de/dkfz/roddy/execution/ApptainerCommandBuilder.groovy new file mode 100644 index 00000000..5d27368a --- /dev/null +++ b/src/main/groovy/de/dkfz/roddy/execution/ApptainerCommandBuilder.groovy @@ -0,0 +1,357 @@ +package de.dkfz.roddy.execution + + +import com.google.common.base.Preconditions +import de.dkfz.roddy.tools.EscapableString +import groovy.transform.AutoClone +import groovy.transform.CompileStatic + +import org.jetbrains.annotations.NotNull +import org.jetbrains.annotations.Nullable + +import java.nio.file.Path +import java.nio.file.Paths + +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + + +/** A bind specification for mounting a host directory into a container. */ +@CompileStatic +class BindSpec implements Comparable { + + enum Mode { + RO("ro"), + RW("rw"); + + String stringValue + + private Mode(String value) { + stringValue = value + } + + String toString() { stringValue; } + + static Mode from(String value) { + value.toUpperCase() as Mode + } + + } + + /** The path of the mount as seen from the host-system (outside the container). */ + @NotNull final Path hostPath + /** The path of the mount as seen from inside the container. If null, the hostPath is used. */ + final Path containerPath + /** The mode of the mount, e.g. read-only or read-write. */ + @NotNull final Mode mode + + BindSpec(@NotNull Path hostPath, + Path containerPath = null, + @NotNull Mode opts = Mode.RO) { + Preconditions.checkNotNull(hostPath) + this.hostPath = hostPath + if (containerPath == null) { + this.containerPath = hostPath + } else { + this.containerPath = containerPath + } + Preconditions.checkNotNull(opts) + this.mode = opts + } + + String toBindOption() { + "$hostPath:$containerPath:$mode" + } + + String toString() { + toBindOption() + } + + @Override + int compareTo(BindSpec other) { + Integer result = this.hostPath.compareTo(other.hostPath) + if (result != 0) return result + + result = this.mode.compareTo(other.mode) + if (result != 0) return result + + result = this.containerPath.compareTo(other.containerPath) + if (result != 0) return result + + return 0 + } + +} + + +@AutoClone +@CompileStatic +class ApptainerCommandBuilder { + + /** See `apptainer --help`, where it is called `Command` but there are already too many + * different meanings of "Command" in Roddy and BE. */ + enum Mode { + exec, + run + } + + private @NotNull List bindSpecifications + + private @NotNull Path apptainerExecutable = Paths.get("apptainer") + + private @NotNull List engineArgs + + private @Nullable EscapableString imageId + + private @NotNull String mode + + private @Nullable Path workingDir + + private @NotNull List environmentVariablesToCopy + + private @NotNull Map setEnvironmentVariables + + /** Create a command that will be wrapped into a singularity/apptainer call. + * For this to work all remote paths (i.e. on the execution node) need to be + * bound into the container. This includes + * + * * input data dir + * * output data dir + * * plugin base directories + * + * The whole set of directories will be deduplicated, mounts specified as both RO and RW + * will be bound RW, and everything appended with -B options to the singularity/apptainer + * command in a topologically sorted order (i.e. super-directories first). + * + * @param mode The run mode, e.g., exec, run, etc. Defaults to exec + */ + private ApptainerCommandBuilder() { + /** Default values for all other variables. Use the with-methods to set these. */ + this.mode = Mode.exec + this.engineArgs = [] + this.bindSpecifications = [] + this.apptainerExecutable = Paths.get("apptainer") + this.imageId = null + this.workingDir = null + this.environmentVariablesToCopy = [] + this.setEnvironmentVariables = [:] + } + + /** Seems kind of superfluous, but this fits a bit better into the fluent-API of the builder */ + static ApptainerCommandBuilder create() { + new ApptainerCommandBuilder() + } + + // Methods to set the fields. More wordy, but also more structure than a long sequence + // of constructor parameters. Groovy's support for named arguments is rudimentary and + // clumsy (compared to Scala; e.g. no automatic name-checking). + + /** Set the execution mode ("command"; subcommand) of the apptainer call. run or exec. */ + ApptainerCommandBuilder withMode(@NotNull Mode mode) { + Preconditions.checkNotNull(mode, "mode cannot be null") + ApptainerCommandBuilder copy = clone() + copy.mode = mode + copy + } + + /** Add container engine arguments. See apptainer run|exec --help. */ + ApptainerCommandBuilder withAddedEngineArgs(@NotNull List newArgs) { + Preconditions.checkNotNull(newArgs, "newArgs cannot be null") + ApptainerCommandBuilder copy = clone() + copy.engineArgs += newArgs + copy + } + + /** Add bind mounts. Include all directories needed to run the workload task/job. */ + ApptainerCommandBuilder withAddedBindingSpecs(@NotNull List newSpecs) { + Preconditions.checkNotNull(newSpecs, "newSpecs cannot be null") + ApptainerCommandBuilder copy = clone() + copy.bindSpecifications += newSpecs + copy + } + + /** The path to the apptainer/singularity executable on the execution side (maybe cluster). */ + ApptainerCommandBuilder withApptainerExecutable(@NotNull Path newExec) { + Preconditions.checkNotNull(newExec, "newExec cannot be null") + ApptainerCommandBuilder copy = clone() + copy.apptainerExecutable = newExec + copy + } + + /** The path or URI of the container image to wrap the command in. */ + ApptainerCommandBuilder withImageId(EscapableString newId) { + Preconditions.checkArgument(newId == null || newId.size() > 0, + "newId cannot be null and must not be empty") + ApptainerCommandBuilder copy = clone() + copy.imageId = newId + copy + } + ApptainerCommandBuilder withImageId(String newId) { + withImageId(u(newId)) + } + + /** The working directory inside the container. */ + ApptainerCommandBuilder withWorkingDir(@Nullable Path workingDir) { + ApptainerCommandBuilder copy = clone() + copy.workingDir = workingDir + copy + } + + /** List of variables that will be copied from the calling environment of the apptainer/ + * singularity command. This will usually be e.g. LSB_JOBID or LSB_JOBNAME, but may include + * others to configure the program that you want to run in the container. + */ + ApptainerCommandBuilder withCopiedEnvironmentVariables(@NotNull List newNames) { + Preconditions.checkNotNull(newNames, "newNames cannot be null") + ApptainerCommandBuilder copy = clone() + copy.environmentVariablesToCopy += newNames + copy + } + + /** Environment variables names and values that will be set on the container. + * Note that this will not check the syntactic (bash) correctness of the variable names + * or values. Also no quoting will be done, such that you can e.g. do something like + * ["renamedVariableName": "originalVariableValue"] + */ + ApptainerCommandBuilder withAddedEnvironmentVariables( + @NotNull Map newNameValues) { + Preconditions.checkNotNull(newNameValues, "newNameValues must not be null") + ApptainerCommandBuilder copy = clone() + copy.setEnvironmentVariables += newNameValues + copy + } + + // Helpers for the build() method to create the actual Command object. + + private List getWorkingDirParameter() { + if (workingDir != null) { + [u("-W"), e(workingDir.toString())] as List + } else { + [] + } + } + + private Map getExportedEnvironmentVariables() { + this.environmentVariablesToCopy.collectEntries { variableName -> + [variableName, u("\$$variableName")] + } + } + + /** This is similar to `getExportedLocalEnvironmentVariables()`, but here we actually set + * variables to (possibly new) values. Note that the values are *not* quoted. We leave it to + * the caller, to quote values correctly, if necessary. + */ + private Map getExplicitlySetVariables() { + setEnvironmentVariables + } + + /** Combine the copied (calling environment) variables and the explicitly set variables. + * Variables are not returned in an explicitly fixed order. + */ + private List getEnvironmentExportParameters() { + (exportedEnvironmentVariables + explicitlySetVariables). + collect { Map.Entry kv -> + [u("--env"), u(kv.key) + u("=") + kv.value] + }. + flatten() as List + } + + private List getFinalEngineArg() { + environmentExportParameters + + workingDirParameter + + (engineArgs.collect { String it -> u(it) } as List) + } + + /** + * Ensure that if a bind-point is below another bind_point in the container, that the + * high bind-point appears first in the list. + * + * This does *intentionally* not preserve the input order of paths. Rather the paths should + * by ordered logically, with parent-paths appearing before their children. + * + * This only uses the apparent path, but does not account for symbolic links. + */ + private static List sortMounts(@NotNull List specs) { + specs.sort().unique() + } + + /** + * Ensure that the same target path is not used multiple times for different host paths. + * If a directory is listed multiple times, return it only once, with the more relaxed + * (RW > RO) access mode. + * + * This only uses the apparent paths, but does not account for symbolic links. + */ + private static List deduplicateAndCheckBindSpecs(@NotNull List specs) { + LinkedHashMap containerPathIndex = [:] + for (bindSpec in specs) { + Optional previousO = + Optional.ofNullable(containerPathIndex.get(bindSpec.containerPath)) + if (previousO.present) { + BindSpec previous = previousO.get() + if (previous.hostPath != bindSpec.hostPath) { + throw new IllegalArgumentException( + "Cannot bind different host paths to same container path: " + + "container = $bindSpec.containerPath, " + + "host = " + [previous.hostPath, bindSpec.hostPath].toString()) + } else if (previous.mode < bindSpec.mode) { + containerPathIndex[previous.containerPath] = bindSpec + } + } else { + containerPathIndex[bindSpec.containerPath] = bindSpec + } + } + containerPathIndex.values().toList() + } + + /** + * Ensure that target paths mounted below another target paths are listed later in the list. + * It is possible to mount a RW path below an RO path. + */ + private static List prepareBindSpecs(@NotNull List specs) { + sortMounts(deduplicateAndCheckBindSpecs(specs)) + } + + /** + * Translate the bind specifications into command-line parameters for apptainer/singularity. + * @return + */ + private List getBindOptions() { + if (bindSpecifications.size() > 0) { + prepareBindSpecs(bindSpecifications). + collect { [u("-B"), u(it.toBindOption())] }. + flatten() as List + } else { + [] + } + } + + /** Compose the information into a valid apptainer/singularity call. This fails, if the + * imageId is obviously invalid (null or zero length; no more checks). + * + * @param imageId Optional imageId. Fall back to the instance-level configured image, + * if it exists. + * @return + */ + Command build(@Nullable EscapableString imageId) { + EscapableString _imageId = this.imageId + if (imageId != null) { + _imageId = imageId + } + Preconditions.checkArgument(_imageId != null && _imageId.size() > 0, + "imageId cannot be null and must not be empty") + new Command( + new Executable(apptainerExecutable), + ([u("exec")] as List) + + bindOptions + + finalEngineArg + + ([_imageId] as List)) + } + Command build(@Nullable String imageId = null) { + if (imageId == null) { + build(null as EscapableString) + } else { + build(u(imageId)) + } + } + +} diff --git a/src/main/groovy/de/dkfz/roddy/execution/CommandI.groovy b/src/main/groovy/de/dkfz/roddy/execution/CommandI.groovy new file mode 100644 index 00000000..e8d96b80 --- /dev/null +++ b/src/main/groovy/de/dkfz/roddy/execution/CommandI.groovy @@ -0,0 +1,260 @@ +/* + * Copyright (c) 2023 + * + * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt). + */ + +package de.dkfz.roddy.execution + +import com.google.common.base.Preconditions +import com.google.common.collect.ImmutableList +import de.dkfz.roddy.tools.EscapableString +import groovy.transform.CompileDynamic +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import org.apache.commons.lang3.RandomStringUtils +import org.jetbrains.annotations.NotNull + +import java.nio.file.Path +import java.nio.file.Paths + +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + + +/** Types of executable code or command. These are used as arguments for BEJob. The + * class hierarchy represents and abstract data type (ADT). + */ +@CompileStatic +abstract class CommandI { + + abstract EscapableString toEscapableString() + +} + +/** A command reference is opposed to the command code. Both are executable, but the command + * reference only represents a short name for a piece of code, usually on the filesystem. + */ +@CompileStatic +abstract class CommandReferenceI extends CommandI { + + /** The path to the executable file that will be executed. */ + abstract Path getExecutablePath() + + /** Get a list with command components, e.g. for Bash. + * + * @return + */ + abstract List toCommandSegmentList() + + @Override + EscapableString toEscapableString() { + join(toCommandSegmentList(), " ") + } + +} + +/** A self-contained command that does not take any command-line parameters. This is mostly + * used for Roddy tools, which are entirely configured via environment variables. + */ +@CompileStatic +@EqualsAndHashCode(includeFields = true, + excludes = ["executablePath", "md5"]) // without this getMd5 will be used, but results in different Optional instances. +final class Executable extends CommandReferenceI { + + private final Path path + + /** An executable is a file, e.g. a script, for which there can be an MD5 sum. This is + * important in some cases (which is why it is allowed to be `null`) if a script is + * uploaded to the cluster. + */ + private final String md5 + + Executable(@NotNull Path path, + String md5 = null) { + Preconditions.checkArgument(path != null) + this.path = path + Preconditions.checkArgument(md5 == null || md5.size() == 32) + this.md5 = md5 + } + + @Override + Path getExecutablePath() { + this.path + } + + @Override + List toCommandSegmentList() { + [u(path.toString())] as List + } + + Optional getMd5() { + Optional.ofNullable(this.md5) + } + +} + +/** A command is an executable with 0 or more arguments. + */ +@CompileStatic +@EqualsAndHashCode(includeFields = true) +final class Command extends CommandReferenceI { + + private final Executable executable + + private final ImmutableList arguments + + /** Concerning quoting arguments: Provide arguments like they should be used at the call-site. + * + * @param executable The executable to be called. + * @param arguments The arguments to be passed to the executable. + **/ + Command(@NotNull Executable executable, + @NotNull List arguments = []) { + Preconditions.checkArgument(executable != null) + this.executable = executable + Preconditions.checkArgument(arguments != null) + arguments.forEach { + Preconditions.checkArgument(it != null, + "Command.arguments must not contain null for executable: " + + executable.executablePath.toString()) + } + this.arguments = ImmutableList.copyOf(arguments) + } + + Executable getExecutable() { + executable + } + + Path getExecutablePath() { + executable.getExecutablePath() + } + + List toCommandSegmentList() { + executable.toCommandSegmentList() + arguments + } + + /** Append the Code as script to the command. The result is a Code object, not a + * SimpleCommand, because the appending is done via a HERE document + * + * @param other + * @param interpreter The interpreter must be Bash. The executable must be `bash`. + * @param terminator_prefix + * @return + */ + @CompileDynamic + Code cliAppend(@NotNull Code other, + @NotNull CommandReferenceI interpreter, // = new Executable(Paths.get("/bin/bash")), + // This gives a "Bad type on operand stack" error + @NotNull String terminator_prefix = "batch_euphoria_", + @NotNull String terminator_random = + RandomStringUtils.random(10, true, true)) { + Preconditions.checkArgument(other != null) + Preconditions.checkArgument(interpreter != null) + Preconditions.checkArgument(interpreter.executablePath.getFileName().toString() == "bash") + Preconditions.checkArgument(terminator_prefix != null) + Preconditions.checkArgument(terminator_random != null) + String terminator = terminator_prefix + "_" + terminator_random + new Code(c([ + this.toEscapableString() + " <<$terminator" + "\n" + + "#!" + other.interpreter.toEscapableString() + "\n" + + other.code + "\n" + + terminator + "\n" // Termination with terminator + "\n" is important for Bash. + ]), + interpreter) + } + + /** Append the other Command to the command-line of this Command. Use this to + * e.g. executed one command in the context of another, like e.g. `strace $other`, + * `singularity run $other`. + * + * @param other + * @param appendAsString Whether to quote the appended Command. + * @return + */ + @CompileDynamic + Command cliAppend(@NotNull CommandReferenceI other, + boolean appendAsString = false) { + Preconditions.checkArgument(other != null) + if (!appendAsString) { + new Command( + executable, + this.arguments + + other.toCommandSegmentList()) + } else { + new Command( + executable, + (this.arguments + e(other.toEscapableString())) as List) + } + } + +} + + +/** Take actual code to be executed. */ +@CompileStatic +@EqualsAndHashCode(includeFields = true) +final class Code extends CommandI { + + /** Code will usually be a script, maybe with a shebang line. Code may or may not be provided + * to the job submission command (e.g. bsub) via the standard input instead of as file. + */ + @NotNull private final EscapableString code + + /** An interpreter for the code. This is bash by default, but could (probably) also be + * python3, perl, or whatever. It is also possible to use commands with arguments as + * interpreters, e.g. `/bin/bash -xe` + */ + @NotNull private final CommandReferenceI interpreter + + Code(@NotNull EscapableString code, + @NotNull CommandReferenceI interpreter) { + Preconditions.checkArgument(code != null) + Preconditions.checkArgument(code.size() > 0) + this.code = code + Preconditions.checkArgument(interpreter != null) + this.interpreter = interpreter + } + + // Convenience constructors + + Code(@NotNull String code, + @NotNull Path interpreter = Paths.get("/bin/bash")) { + this(u(code), new Executable(interpreter)) + } + + Code(@NotNull EscapableString code, + @NotNull Path interpreter = Paths.get("/bin/bash")) { + this(code, new Executable(interpreter)) + } + + Code(@NotNull String code, + @NotNull CommandReferenceI interpreter) { + this(u(code), interpreter) + } + + EscapableString getCode() { + code + } + + CommandReferenceI getInterpreter() { + interpreter + } + + /** Create a little script. Note that the escapable string does *not* have automatically an + * newline appended, because it depends on the client context whether this is useful. For + * instance, a newline may be unnecessary in many cases, but for a HERE document the terminator + * must be followed by newline. */ + EscapableString toEscapableString(boolean terminate) { + EscapableString result = u("#!") + interpreter.toEscapableString() + "\n" + code + if (terminate) { + result += "\n" + } + return result + } + + @Override + EscapableString toEscapableString() { + toEscapableString(false) + } + +} diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJob.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJob.groovy index cb14db87..3f68857f 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJob.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJob.groovy @@ -6,13 +6,23 @@ package de.dkfz.roddy.execution.jobs -import de.dkfz.roddy.BEException +import com.google.common.base.Preconditions +import de.dkfz.roddy.config.EmptyResourceSet import de.dkfz.roddy.config.JobLog import de.dkfz.roddy.config.ResourceSet +import de.dkfz.roddy.tools.EscapableString +import de.dkfz.roddy.tools.BashInterpreter +import de.dkfz.roddy.execution.Code +import de.dkfz.roddy.execution.CommandI +import de.dkfz.roddy.execution.CommandReferenceI import groovy.transform.CompileStatic +import org.jetbrains.annotations.NotNull +import org.jetbrains.annotations.Nullable import java.util.concurrent.atomic.AtomicLong +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + /** * The job class represents a generic and abstract form of cluster job which can be run using a job manager. * When a job is executed with the JM, the used Command and a BEJobResult object will be created and added to this @@ -33,40 +43,30 @@ class BEJob implements Comparable implements Comparable parameters + protected final Map parameters protected SortedSet parentJobs = new TreeSet() @@ -96,6 +96,7 @@ class BEJob implements Comparable implements Comparable parentJobs, - Map parameters, - BatchEuphoriaJobManager jobManager, - JobLog jobLog, - File workingDirectory, - String accountingName = null) { - this.jobID = Optional.ofNullable(jobID).orElse(new BEJobID()) - this.jobName = jobName + BEJob(BEJobID jobID, // can be null for FakeBEJob + BatchEuphoriaJobManager jobManager, // can be null for FakeBEJob + EscapableString jobNameEscapable = null, + CommandI commandObj = null, + @NotNull ResourceSet resourceSet = new EmptyResourceSet(), + @NotNull Collection parentJobs = [], + @NotNull Map parameters = [:], + @NotNull JobLog jobLog = JobLog.none(), + File workingDirectory = null, + EscapableString accountingName = null) { + this.jobID = Optional.ofNullable(jobID).orElse(BEJobID.getNewUnknown()) + this.jobNameEscapable = jobNameEscapable this.currentJobState = JobState.UNSTARTED - this.tool = tool - this.toolScript = toolScript - // FakeJobs have neither tool nor toolScript set. - if (tool && toolScript) - throw new BEException("A job must have exactly one of tool and toolScript set. Found: tool=${tool}, toolScript=${toolScript}") - this.toolMD5 = toolMD5 + this.commandObj = commandObj + Preconditions.checkArgument(resourceSet != null) this.resourceSet = resourceSet + Preconditions.checkArgument(parameters != null) this.parameters = parameters this.jobManager = jobManager - assert jobLog: "jobLog not set" + Preconditions.checkArgument(jobLog != null) this.jobLog = jobLog this.workingDirectory = workingDirectory this.accountingName = accountingName - this.addParentJobs(Optional.ofNullable(parentJobs).orElse([])) + Preconditions.checkArgument(parentJobs != null) + this.addParentJobs(parentJobs) } - BEJob(BEJobID jobID, BatchEuphoriaJobManager jobManager) { - this(jobID, null, null, null, null, null, [], - [:] as Map, jobManager, JobLog.none(), null, null) - } - - BEJob addParentJobs(Collection parentJobs) { - assert (null != parentJobs) + BEJob addParentJobs(@NotNull Collection parentJobs) { + Preconditions.checkArgument(parentJobs != null) this.parentJobs.addAll(parentJobs) return this } - BEJob addParentJobIDs(List parentJobIDs, BatchEuphoriaJobManager jobManager) { - assert (null != parentJobIDs) + BEJob addParentJobIDs(@NotNull List parentJobIDs, + @NotNull BatchEuphoriaJobManager jobManager) { + Preconditions.checkArgument(parentJobIDs != null) this.parentJobs.addAll(parentJobIDs.collect { new BEJob(it, jobManager) }) return this } @@ -160,7 +153,7 @@ class BEJob implements Comparable implements Comparable implements Comparable getParameters() { + Map getParameters() { return parameters } @@ -212,17 +205,17 @@ class BEJob implements Comparable implements Comparable implements Comparable implements Comparable implements Comparable getCommand() { + if (commandObj instanceof CommandReferenceI) { + (commandObj as CommandReferenceI).toCommandSegmentList() + } else { + null + } + } + + EscapableString getCode() { + if (commandObj instanceof Code) { + (commandObj as Code).toEscapableString() + } else { + null + } + } } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobID.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobID.groovy index 61cfea3b..bc8925c7 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobID.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobID.groovy @@ -23,22 +23,23 @@ class BEJobID implements Comparable { private final String id - private static AtomicLong unkownIdCounter = new AtomicLong(0L) - - BEJobID() { - id = nextUnknownID() - } + private static AtomicLong unknownIdCounter = new AtomicLong(0L) BEJobID(String id) { this.id = id.split(/\./)[0] } + static BEJobID getNewUnknown() { + new BEJobID(nextUnknownID()) + } + + protected static String nextUnknownID(String prefix = "UnkownJobID-") { - return prefix + unkownIdCounter.incrementAndGet() + return prefix + unknownIdCounter.incrementAndGet() } boolean isValidID() { - return !BEFakeJobID.isFakeJobID(this.getId()) && getId() != null && getId() != "none" + return !BEFakeJobID.isFakeJobID(id) && id != null && id != "none" } String getId() { diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobResult.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobResult.groovy index 548ac98c..a2625460 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobResult.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobResult.groovy @@ -1,13 +1,19 @@ /* - * Copyright (c) 2017 eilslabs. + * Copyright (c) 2023 eilslabs. * * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt). */ package de.dkfz.roddy.execution.jobs +import com.google.common.base.Preconditions +import de.dkfz.roddy.tools.EscapableString import de.dkfz.roddy.execution.io.ExecutionResult import groovy.transform.CompileStatic +import org.jetbrains.annotations.NotNull +import org.jetbrains.annotations.Nullable + +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* /** * Result of a job run. @@ -23,7 +29,7 @@ class BEJobResult implements Serializable { /** * The command which was used to create this result. */ - final Command command + final Command beCommand /** * The current job's id, i.e. qsub id. * Used for dependencies. @@ -33,14 +39,11 @@ class BEJobResult implements Serializable { * The execution result object containing additional details about the execution (exit code and output). */ final ExecutionResult executionResult - /** - * The tool which was run for this job. - */ - final File toolID + /** * Parameters for the job. */ - final Map jobParameters + final Map jobParameters /** * Parent jobs. */ @@ -51,13 +54,35 @@ class BEJobResult implements Serializable { } - BEJobResult(Command command, BEJob job, ExecutionResult executionResult, File toolID, - Map jobParameters, List parentJobs) { - this.command = command - assert (null != job) + /** The tool parameter should be removed. This is only kept for backwards compatibility. */ + @Deprecated + BEJobResult(@Nullable Command beCommand, + BEJob job, + @Nullable ExecutionResult executionResult, File tool, + @Nullable Map jobParameters, + @Nullable List parentJobs) { + this.beCommand = beCommand + Preconditions.checkArgument(job != null) + Preconditions.checkArgument(tool == job.executableFile) + this.job = job + this.executionResult = executionResult + this.jobParameters = + jobParameters?.collectEntries { k, v -> + [k, v != null ? e(v) : v] + } as Map + this.parentJobs = parentJobs + // NOTE: tool is not used anymore. + } + + BEJobResult(Command beCommand, + @NotNull BEJob job, + ExecutionResult executionResult, + Map jobParameters, + List parentJobs) { + this.beCommand = beCommand + Preconditions.checkArgument(job != null) this.job = job this.executionResult = executionResult - this.toolID = toolID this.jobParameters = jobParameters this.parentJobs = parentJobs } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy index a31668b5..1d439ac4 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy @@ -6,11 +6,13 @@ package de.dkfz.roddy.execution.jobs +import com.google.common.base.Preconditions import de.dkfz.roddy.BEException import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.execution.BEExecutionService import de.dkfz.roddy.execution.io.ExecutionResult import groovy.transform.CompileStatic +import org.jetbrains.annotations.Nullable import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -81,8 +83,9 @@ abstract class BatchEuphoriaJobManager { boolean holdJobsIsEnabled Duration commandTimeout - BatchEuphoriaJobManager(BEExecutionService executionService, JobManagerOptions parms) { - assert (executionService) + BatchEuphoriaJobManager(BEExecutionService executionService, + JobManagerOptions parms) { + Preconditions.checkArgument(executionService != null) this.executionService = executionService this.isTrackingOfUserJobsEnabled = parms.userIdForJobQueries as boolean @@ -124,7 +127,9 @@ abstract class BatchEuphoriaJobManager { BEJobResult submitJob(BEJob job) throws TimeoutException, BEException { if (forbidFurtherJobSubmission) { - throw new BEException("You are not allowed to submit further jobs. This happens, when you call waitForJobs().") + throw new BEException( + "You are not allowed to submit further jobs." + + "You called waitForJobs() before.") } Command command = createCommand(job) ExecutionResult executionResult = executionService.execute(command, commandTimeout) @@ -276,7 +281,7 @@ abstract class BatchEuphoriaJobManager { return holdJobsIsEnabled } - String getUserEmail() { + @Nullable String getUserEmail() { return userEmail } @@ -296,8 +301,6 @@ abstract class BatchEuphoriaJobManager { return false } - abstract String getSubmissionCommand() - abstract String getQueryJobStatesCommand() abstract String getExtendedQueryJobStatesCommand() @@ -333,21 +336,23 @@ abstract class BatchEuphoriaJobManager { BEJobID jobID = new BEJobID(exID) command.setJobID(jobID) job.resetJobID(jobID) - jobResult = new BEJobResult(command, job, res, job.tool, job.parameters, job.parentJobs as List) + jobResult = new BEJobResult(command, job, res, job.parameters, job.parentJobs as List) job.setRunResult(jobResult) synchronized (cacheStatesLock) { cachedStates.put(jobID, isHoldJobsEnabled() ? JobState.HOLD : JobState.QUEUED) } } else { def job = command.getJob() - jobResult = new BEJobResult(command, job, res, job.tool, job.parameters, job.parentJobs as List) + jobResult = new BEJobResult(command, job, res, job.parameters, job.parentJobs as List) job.setRunResult(jobResult) throw new BEException("Job ${job.jobName ?: "NA"} could not be started. ${res.toStatusLine()}") } return jobResult } - abstract protected Command createCommand(BEJob job) + // This needs to be public, because the RoddyCore's NativeWorkflow breaks the abstraction + // (not just by calling this, but by actually messing with the JobManager interna). + abstract Command createCommand(BEJob job) abstract protected String parseJobID(String commandOutput) diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/Command.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/Command.groovy index 2a5ef64e..625fe163 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/Command.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/Command.groovy @@ -6,10 +6,14 @@ package de.dkfz.roddy.execution.jobs +import com.google.common.base.Preconditions +import de.dkfz.roddy.tools.EscapableString import groovy.transform.CompileStatic +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + /** - * Base class for all types of commands. + * Base class for all types of cluster interaction commands. *

* PBSCommand extends this. Also SGECommand and so on. *

@@ -20,12 +24,12 @@ import groovy.transform.CompileStatic @CompileStatic abstract class Command { - protected static final String WORKING_DIRECTORY_DEFAULT = '$HOME' + protected static final EscapableString WORKING_DIRECTORY_DEFAULT = u('$HOME') /** * The job name of this command. */ - protected final String jobName + protected final EscapableString jobName /** * The id which was created upon execution by the job system. */ @@ -40,7 +44,7 @@ abstract class Command { * Environment variables to be passed with a specific value or as they are declared in the submission environment. * null-valued parameters correspond to environment variables to be forwarded as locally defined. */ - public final LinkedHashMap parameters = [:] + public final LinkedHashMap parameters = [:] protected final BatchEuphoriaJobManager parentJobManager @@ -51,9 +55,11 @@ abstract class Command { * @param job * @param jobName * @param environmentVariables - * @param commandTags */ - protected Command(BatchEuphoriaJobManager parentJobManager, BEJob job, String jobName, Map environmentVariables) { + protected Command(BatchEuphoriaJobManager parentJobManager, + BEJob job, + EscapableString jobName, + Map environmentVariables) { this.parentJobManager = parentJobManager this.parameters.putAll(environmentVariables ?: [:]) this.creatingJob = job @@ -61,6 +67,7 @@ abstract class Command { } final void setJobID(BEJobID id) { + Preconditions.checkArgument(id != null) this.jobID = id } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/DummyCommand.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/DummyCommand.groovy index 95e9c045..6c94ed2c 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/DummyCommand.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/DummyCommand.groovy @@ -8,6 +8,8 @@ package de.dkfz.roddy.execution.jobs import groovy.transform.CompileStatic +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + @CompileStatic class DummyCommand extends Command { @@ -23,7 +25,7 @@ class DummyCommand extends Command { private String jobName DummyCommand(BatchEuphoriaJobManager parentJobManager, BEJob job, String jobName, boolean isArray) { - super(parentJobManager, job, "dummy_" + getNextIDCountValue(), null) + super(parentJobManager, job, e("dummy_" + getNextIDCountValue()), null) this.jobName = jobName setJobID(new BEFakeJobID(BEFakeJobID.FakeJobReason.NOT_EXECUTED)) } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/FakeBEJob.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/FakeBEJob.groovy index f42b4af6..d3dcbb94 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/FakeBEJob.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/FakeBEJob.groovy @@ -6,10 +6,12 @@ package de.dkfz.roddy.execution.jobs -import de.dkfz.roddy.config.JobLog + import de.dkfz.roddy.core.InfoObject import groovy.transform.CompileStatic +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + /** * Created by heinold on 27.02.17. */ @@ -17,13 +19,11 @@ import groovy.transform.CompileStatic class FakeBEJob extends BEJob { FakeBEJob(BEFakeJobID jobID) { - super(jobID, 'Fakejob', null, null, "", null, [], - [:] as Map, null, JobLog.none(), null) + super(jobID, null, u('Fakejob')) } FakeBEJob() { - super(null, 'Fakejob', null, null, "", null, [], - [:] as Map, null, JobLog.none(), null) + super(null, null, u('Fakejob')) } FakeBEJob(InfoObject context) { diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/GenericJobInfo.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/GenericJobInfo.groovy index 16f0c244..4262dda1 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/GenericJobInfo.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/GenericJobInfo.groovy @@ -28,7 +28,8 @@ class GenericJobInfo { /** The date-time the job entered the queue. */ ZonedDateTime submitTime - /** The date-time the job became eligible to run when all conditions like job dependencies are met, i.e. in a queued state while residing in an execution queue. */ + /** The date-time the job became eligible to run when all conditions like job dependencies + * are met, i.e. in a queued state while residing in an execution queue. */ ZonedDateTime eligibleTime /** The date-time the job was started. */ ZonedDateTime startTime diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/ProcessingParameters.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/ProcessingParameters.groovy index 265f386c..bca8268c 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/ProcessingParameters.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/ProcessingParameters.groovy @@ -6,19 +6,23 @@ package de.dkfz.roddy.execution.jobs import com.google.common.collect.LinkedHashMultimap +import de.dkfz.roddy.tools.EscapableString +import de.dkfz.roddy.tools.BashInterpreter import groovy.transform.CompileStatic import java.text.ParseException import java.util.regex.Pattern +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + /** */ @CompileStatic class ProcessingParameters implements Serializable { - private LinkedHashMultimap parameters = null + private LinkedHashMultimap parameters = null - ProcessingParameters(LinkedHashMultimap parameters) { + ProcessingParameters(LinkedHashMultimap parameters) { assert (null != parameters) this.parameters = parameters } @@ -34,11 +38,13 @@ class ProcessingParameters implements Serializable { */ static ProcessingParameters fromString(String parameterString) { def pattern = Pattern.compile(/(?[^\s=]+)(?:[\s+=](?.+?\s*))?/) - LinkedHashMultimap parameters = LinkedHashMultimap.create() + LinkedHashMultimap parameters = LinkedHashMultimap.create() parameterString.split(/(^|\s+)(?=-)/).findAll { it != '' }.eachWithIndex { String option, int i -> def matcher = pattern.matcher(option) if (matcher.matches()) { - parameters.put(matcher.group("optionName"), matcher.group("optionValue")) + String key = matcher.group("optionName") + String value = matcher.group("optionValue") + parameters.put(key, value != null ? u(value) : null) matcher.reset() } else { matcher.reset() @@ -48,18 +54,22 @@ class ProcessingParameters implements Serializable { return new ProcessingParameters(parameters) } - String getProcessingCommandString() { - // Careful with this method. We had one problem where an Integer value was in one of the Collections inside the map. - // In this particular case, Groovy said there is something wrong with the Integer [1]. - (parameters.asMap() as Map>).collect { k, vs -> - vs.collect { v -> - k + ' ' + v - }.join(' ') - }.join(' ') + EscapableString getProcessingCommandString() { + // Careful with this method. We had one problem where an Integer value was in one of the + // Collections inside the map. In this particular case, Groovy said there is something + // wrong with the Integer [1]. + join(((parameters.asMap() as Map>).collect { + String k, Collection vs -> + join(vs.findAll { it != null }. + collect { + u(k) + u(' ') + it + } as List, + u(' ')) + } as List), u(' ')) } @Override String toString() { - return "" + processingCommandString + return BashInterpreter.instance.interpret(processingCommandString) } } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommand.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommand.groovy index 3899a28b..5ba86210 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommand.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommand.groovy @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 German Cancer Research Center (Deutsches Krebsforschungszentrum, DKFZ). + * Copyright (c) 2023 German Cancer Research Center (Deutsches Krebsforschungszentrum, DKFZ). * * Distributed under the MIT License (license terms are at https://www.github.com/TheRoddyWMS/Roddy/LICENSE.txt). */ @@ -7,10 +7,10 @@ package de.dkfz.roddy.execution.jobs import de.dkfz.roddy.StringConstants import de.dkfz.roddy.config.JobLog -import de.dkfz.roddy.tools.BashUtils +import de.dkfz.roddy.tools.EscapableString import groovy.transform.CompileStatic -import static de.dkfz.roddy.StringConstants.EMPTY +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* @CompileStatic abstract class SubmissionCommand extends Command { @@ -22,33 +22,30 @@ abstract class SubmissionCommand extends Command { */ Optional passEnvironment = Optional.empty() - /** - * The command which should be called - */ - protected String command - - protected List dependencyIDs - protected final List processingParameters + /** * A command to be executed on the cluster head node, in particular qsub, bsub, qstat, etc. * * @param parentJobManager * @param job * @param jobName + * @param processingParameters * @param environmentVariables * + * Unfortunately, *all* these parameters can be null. */ - protected SubmissionCommand(BatchEuphoriaJobManager parentJobManager, BEJob job, String jobName, + protected SubmissionCommand(BatchEuphoriaJobManager parentJobManager, + BEJob job, + EscapableString jobName, List processingParameters, - Map environmentVariables, List dependencyIDs, - String command) { + Map environmentVariables) { super(parentJobManager, job, jobName, environmentVariables) this.processingParameters = processingParameters - this.command = command - this.dependencyIDs = dependencyIDs ?: new LinkedList() } + protected abstract Boolean getQuoteCommand() + /** * Should the local environment be passed? * @@ -64,72 +61,62 @@ abstract class SubmissionCommand extends Command { passEnvironment.orElse(parentJobManager.passEnvironment) } - @Override - String toBashCommandString() { + abstract String getSubmissionExecutableName() + + private List collectParameters() { String email = parentJobManager.getUserEmail() String umask = parentJobManager.getUserMask() String groupList = parentJobManager.getUserGroup() boolean holdJobsOnStart = parentJobManager.isHoldJobsEnabled() // collect parameters for job submission - List parameters = [] + List parameters = [] parameters << assembleVariableExportParameters() parameters << getAccountNameParameter() parameters << getJobNameParameter() if (holdJobsOnStart) parameters << getHoldParameter() parameters << getWorkingDirectoryParameter() parameters << getLoggingParameter(job.jobLog) - parameters << getEmailParameter(email) - if (groupList && groupList != "UNDEFINED") parameters << getGroupListParameter(groupList) - parameters << getUmaskString(umask) + if (email) parameters << getEmailParameter(e(email)) + if (groupList && groupList != "UNDEFINED") parameters << getGroupListParameter(e(groupList)) + if (umask) parameters << getUmaskString(e(umask)) parameters << assembleProcessingCommands() parameters << assembleDependencyParameter(creatingJob.parentJobIDs) parameters << getAdditionalCommandParameters() - // create job submission command call - StringBuilder command = new StringBuilder(EMPTY) - - if (environmentString) { - command << "${environmentString} " - } - - if (job.toolScript) { - command << "echo " << BashUtils.strongQuote("#!/bin/bash " + System.lineSeparator() + job.toolScript) << " | " - } - - command << parentJobManager.submissionCommand - command << " ${parameters.join(" ")} " + parameters + } - if (job.tool) { - command << " " << job.tool.absolutePath - } + abstract protected String composeCommandString(List parameters) - return command + @Override + String toBashCommandString() { + return composeCommandString(collectParameters()) } - abstract protected String getJobNameParameter() + abstract protected EscapableString getJobNameParameter() - abstract protected String getHoldParameter() + abstract protected EscapableString getHoldParameter() - protected String getAccountNameParameter() { - return "" + protected EscapableString getAccountNameParameter() { + return c() } - abstract protected String getWorkingDirectoryParameter() + abstract protected EscapableString getWorkingDirectoryParameter() - abstract protected String getLoggingParameter(JobLog jobLog) + abstract protected EscapableString getLoggingParameter(JobLog jobLog) - abstract protected String getEmailParameter(String address) + abstract protected EscapableString getEmailParameter(EscapableString address) - abstract protected String getGroupListParameter(String groupList) + abstract protected EscapableString getGroupListParameter(EscapableString groupList) - abstract protected String getUmaskString(String umask) + abstract protected EscapableString getUmaskString(EscapableString umask) - abstract protected String assembleDependencyParameter(List jobIds) + abstract protected EscapableString assembleDependencyParameter(List jobIds) - abstract protected String getAdditionalCommandParameters() + abstract protected EscapableString getAdditionalCommandParameters() - abstract protected String getEnvironmentString() + abstract protected EscapableString getEnvironmentString() /** If passLocalEnvironment is true, all local variables will be forwarded to the execution host. * If passLocalEnvironment is false, no local variables will be forwarded by default. @@ -138,17 +125,15 @@ abstract class SubmissionCommand extends Command { * * @return A set of parameters for the submission command to achieve the requested variable exports. */ - abstract protected String assembleVariableExportParameters() - - String assembleProcessingCommands() { - StringBuilder commands = new StringBuilder() - for (ProcessingParameters pcmd in job.getListOfProcessingParameters()) { - if (!(pcmd instanceof ProcessingParameters)) continue - ProcessingParameters command = (ProcessingParameters) pcmd - if (command == null) - continue - commands << StringConstants.WHITESPACE << command.getProcessingCommandString() + abstract protected EscapableString assembleVariableExportParameters() + + EscapableString assembleProcessingCommands() { + EscapableString commands = c() + for (ProcessingParameters command in job.getListOfProcessingParameters()) { + if (command != null) + commands += u(StringConstants.WHITESPACE) + command.getProcessingCommandString() } - return commands.toString() + commands } + } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManager.groovy index 12cff5d1..7cd7a5b0 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManager.groovy @@ -6,9 +6,11 @@ package de.dkfz.roddy.execution.jobs.cluster +import com.google.common.base.Preconditions import com.google.common.collect.LinkedHashMultimap import de.dkfz.roddy.BEException import de.dkfz.roddy.config.ResourceSet +import de.dkfz.roddy.tools.EscapableString import de.dkfz.roddy.execution.BEExecutionService import de.dkfz.roddy.execution.jobs.BEJob import de.dkfz.roddy.execution.jobs.BatchEuphoriaJobManager @@ -65,9 +67,9 @@ abstract class ClusterJobManager extends BatchEuphoriaJobMana @Override ProcessingParameters convertResourceSet(BEJob job, ResourceSet resourceSet) { - assert resourceSet + Preconditions.checkArgument(resourceSet != null) - LinkedHashMultimap parameters = LinkedHashMultimap.create() + LinkedHashMultimap parameters = LinkedHashMultimap.create() createDefaultManagerParameters(parameters) @@ -92,15 +94,15 @@ abstract class ClusterJobManager extends BatchEuphoriaJobMana return new ProcessingParameters(parameters) } - abstract void createDefaultManagerParameters(LinkedHashMultimap parameters) + abstract void createDefaultManagerParameters(LinkedHashMultimap parameters) - abstract void createComputeParameter(ResourceSet resourceSet, LinkedHashMultimap parameters) + abstract void createComputeParameter(ResourceSet resourceSet, LinkedHashMultimap parameters) - abstract void createQueueParameter(LinkedHashMultimap parameters, String queue) + abstract void createQueueParameter(LinkedHashMultimap parameters, String queue) - abstract void createWalltimeParameter(LinkedHashMultimap parameters, ResourceSet resourceSet) + abstract void createWalltimeParameter(LinkedHashMultimap parameters, ResourceSet resourceSet) - abstract void createMemoryParameter(LinkedHashMultimap parameters, ResourceSet resourceSet) + abstract void createMemoryParameter(LinkedHashMultimap parameters, ResourceSet resourceSet) - abstract void createStorageParameters(LinkedHashMultimap parameters, ResourceSet resourceSet) + abstract void createStorageParameters(LinkedHashMultimap parameters, ResourceSet resourceSet) } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManager.groovy index d3663628..0f7b36d4 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManager.groovy @@ -9,6 +9,7 @@ package de.dkfz.roddy.execution.jobs.cluster import com.google.common.collect.LinkedHashMultimap import de.dkfz.roddy.BEException import de.dkfz.roddy.config.ResourceSet +import de.dkfz.roddy.tools.EscapableString import de.dkfz.roddy.execution.BEExecutionService import de.dkfz.roddy.execution.io.ExecutionResult import de.dkfz.roddy.execution.jobs.BEJobID @@ -44,11 +45,6 @@ abstract class GridEngineBasedJobManager extends ClusterJobMa @Override boolean getDefaultForHoldJobsEnabled() { return true } - @Override - String getSubmissionCommand() { - return "qsub" - } - protected int getColumnOfJobID() { return 0 } @@ -286,11 +282,11 @@ abstract class GridEngineBasedJobManager extends ClusterJobMa } else { return null } - new File(fileName.replace("\$${getJobIdVariable()}", jobId)) + new File(fileName.replace("\$$jobIdVariable", jobId)) } @Override - void createDefaultManagerParameters(LinkedHashMultimap parameters) { + void createDefaultManagerParameters(LinkedHashMultimap parameters) { } } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedSubmissionCommand.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedSubmissionCommand.groovy index d7e07c5c..ffab3787 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedSubmissionCommand.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedSubmissionCommand.groovy @@ -6,13 +6,15 @@ package de.dkfz.roddy.execution.jobs.cluster -import de.dkfz.roddy.execution.jobs.BEJob -import de.dkfz.roddy.execution.jobs.BEJobID -import de.dkfz.roddy.execution.jobs.BatchEuphoriaJobManager -import de.dkfz.roddy.execution.jobs.ProcessingParameters -import de.dkfz.roddy.execution.jobs.SubmissionCommand + +import de.dkfz.roddy.tools.EscapableString +import de.dkfz.roddy.tools.BashInterpreter +import de.dkfz.roddy.tools.ConcatenatedString +import de.dkfz.roddy.execution.jobs.* import groovy.transform.CompileStatic +import static de.dkfz.roddy.StringConstants.EMPTY +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* @CompileStatic abstract class GridEngineBasedSubmissionCommand extends SubmissionCommand { @@ -27,32 +29,35 @@ abstract class GridEngineBasedSubmissionCommand extends SubmissionCommand { * @param jobName * @param processingParameter s@param environmentVariables *@param dependencyIDs @param command */ - GridEngineBasedSubmissionCommand(BatchEuphoriaJobManager parentJobManager, BEJob job, String jobName, + GridEngineBasedSubmissionCommand(BatchEuphoriaJobManager parentJobManager, + BEJob job, + EscapableString jobName, List processingParameters, - Map environmentVariables, - List dependencyIDs, String command) { - super(parentJobManager, job, jobName, processingParameters, environmentVariables, dependencyIDs, command) + Map environmentVariables) { + super(parentJobManager, job, jobName, processingParameters, environmentVariables) } @Override - protected String assembleDependencyParameter(List jobIds) { - StringBuilder qsubCall = new StringBuilder("") - LinkedList tempDependencies = + protected EscapableString assembleDependencyParameter(List jobIds) { + EscapableString qsubCall = c() + LinkedList tempDependencies = jobIds.findAll { - it.getId() != "" && it.getId() != NONE && it.getId() != "-1" + it.id != "" && it.id != NONE && it.id != "-1" }.collect { - it.getId().split("\\.")[0] // Keep the command line short. GE accepts the job number for dependencies. - } as LinkedList + e(it.id.split("\\.")[0]) + // Keep the command line short. GE accepts the job number for dependencies. + } as LinkedList if (tempDependencies.size() > 0) { - qsubCall << - getDependsSuperParameter() << - getDependencyParameterName() << - getDependencyOptionSeparator() << - tempDependencies.join(getDependencyIDSeparator()) + qsubCall = join([qsubCall, + u(getDependsSuperParameter()), + u(getDependencyParameterName()), + u(getDependencyOptionSeparator()) + ], "") + + join(tempDependencies, getDependencyIDSeparator()) } - return qsubCall + qsubCall } protected abstract String getDependsSuperParameter() @@ -63,6 +68,45 @@ abstract class GridEngineBasedSubmissionCommand extends SubmissionCommand { protected abstract String getDependencyIDSeparator() + @Override + protected Boolean getQuoteCommand() { + true + } + @Override + String getSubmissionExecutableName() { + return "qsub" + } + @Override + protected String composeCommandString(List parameters) { + StringBuilder command = new StringBuilder(EMPTY) + + if (job.code) { + command << + "echo -ne " << + BashInterpreter.instance.interpret(e(job.code)) << + " | " + } + + command << getSubmissionExecutableName() + + command << " ${parameters.join(" ")} " + + if (job.command) { + // Commands that are appended to the submission command and its parameters, e.g., + // in `bsub ... command ...` need to be quoted to prevent that expressions and + // variables are evaluated on the submission site instead of the actual remote + // cluster node. + // This won't have any effect unless you have Bash special characters in your command. + ConcatenatedString commandToBeExecuted = join(job.command, " ") + if (quoteCommand) { + command += u(" ") + e(commandToBeExecuted) + } else { + command += u(" ") + commandToBeExecuted + } + } + + return command + } } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/AbstractLSFJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/AbstractLSFJobManager.groovy index e0c04d84..99517689 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/AbstractLSFJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/AbstractLSFJobManager.groovy @@ -8,6 +8,7 @@ package de.dkfz.roddy.execution.jobs.cluster.lsf import com.google.common.collect.LinkedHashMultimap import de.dkfz.roddy.config.ResourceSet +import de.dkfz.roddy.tools.EscapableString import de.dkfz.roddy.execution.BEExecutionService import de.dkfz.roddy.execution.jobs.JobManagerOptions import de.dkfz.roddy.execution.jobs.cluster.ClusterJobManager @@ -16,6 +17,8 @@ import groovy.transform.CompileStatic import java.time.Duration +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + @CompileStatic abstract class AbstractLSFJobManager extends ClusterJobManager { @@ -66,8 +69,8 @@ abstract class AbstractLSFJobManager extends ClusterJobManager parameters) { + void createDefaultManagerParameters(LinkedHashMultimap parameters) { } @Override - void createComputeParameter(ResourceSet resourceSet, LinkedHashMultimap parameters) { + void createComputeParameter(ResourceSet resourceSet, + LinkedHashMultimap parameters) { int nodes = resourceSet.isNodesSet() && resourceSet.getNodes() > 0 ? resourceSet.getNodes() : 1 int cores = resourceSet.isCoresSet() ? resourceSet.getCores() : 1 @@ -96,31 +100,43 @@ abstract class AbstractLSFJobManager extends ClusterJobManager parameters, String queue) { - parameters.put('-q', queue) + void createQueueParameter(LinkedHashMultimap parameters, + String queue) { + parameters.put('-q', e(queue)) } @Override - void createWalltimeParameter(LinkedHashMultimap parameters, ResourceSet resourceSet) { - parameters.put('-W', durationToLSFWallTime(resourceSet.getWalltime())) + void createWalltimeParameter(LinkedHashMultimap parameters, + ResourceSet resourceSet) { + parameters.put('-W', e(durationToLSFWallTime(resourceSet.getWalltime()).toString())) } @Override - void createMemoryParameter(LinkedHashMultimap parameters, ResourceSet resourceSet) { + void createMemoryParameter(LinkedHashMultimap parameters, + ResourceSet resourceSet) { // LSF does not like the buffer unit at the end and always takes MB - def memval = resourceSet.getMem().toResourceStringWithoutUnit(BufferUnit.M) - parameters.put("-M", "${memval} -R \"rusage[mem=${memval}]\"") + EscapableString memval = e(resourceSet.getMem().toResourceStringWithoutUnit(BufferUnit.M)) + parameters.put("-M", memval) + parameters.put("-R", e("rusage[mem=") + memval + e("]")) } @Override - void createStorageParameters(LinkedHashMultimap parameters, ResourceSet resourceSet) { + void createStorageParameters(LinkedHashMultimap parameters, + ResourceSet resourceSet) { } } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy index 30b7162e..d66b3fa2 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy @@ -18,6 +18,8 @@ import groovy.transform.CompileStatic import java.time.* import java.util.regex.Matcher +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + /** * Factory for the management of LSF cluster systems. * @@ -77,7 +79,7 @@ class LSFJobManager extends AbstractLSFJobManager { } day = Integer.parseInt(matcher.group("day")) month = MONTH_VALUE[matcher.group("month")] - year = saveGet("year", LocalDateTime.now().year) + year = saveGet("year", referenceDate.year) hour = Integer.parseInt(matcher.group("hour")) minute = Integer.parseInt(matcher.group("minute")) second = saveGet("second", 0) @@ -141,10 +143,9 @@ class LSFJobManager extends AbstractLSFJobManager { return js } - protected Command createCommand(BEJob job) { + LSFSubmissionCommand createCommand(BEJob job) { return new LSFSubmissionCommand( - this, job, job.jobName, [], job.parameters, job.parentJobIDs*.id, - job.tool?.absolutePath ?: job.toolScript) + this, job, u(job.jobName), [], job.parameters) } @Override @@ -165,7 +166,7 @@ class LSFJobManager extends AbstractLSFJobManager { Map> runBjobs(List jobIDs, boolean extended, Duration timeout = Duration.ZERO) { StringBuilder queryCommand = new StringBuilder() - queryCommand << "${environmentString} " + queryCommand << forBash(environmentString) + " " queryCommand << (extended ? LSF_COMMAND_QUERY_EXTENDED_STATES : LSF_COMMAND_QUERY_STATES) // user argument must be passed before the job IDs if (isTrackingOfUserJobsEnabled) @@ -183,7 +184,8 @@ class LSFJobManager extends AbstractLSFJobManager { throw new BEException(error) } - Map> result = convertBJobsJsonOutputToResultMap(resultLines.join("\n")) + Map> result = + convertBJobsJsonOutputToResultMap(resultLines.join("\n")) return result; //filterJobMapByAge(result, maxTrackingTimeForFinishedJobs) } @@ -341,7 +343,7 @@ class LSFJobManager extends AbstractLSFJobManager { } else if (executionService.execute( "LC_ALL=C stat -c %F ${BashUtils.strongQuote(path)} 2> /dev/null", commandTimeout).firstStdoutLine == "directory") { - return new File(path, "${jobID.getId()}.${fileTypeSuffix}") + return new File(path, "${jobID.id}.$fileTypeSuffix") } else { return new File(path) } @@ -349,13 +351,13 @@ class LSFJobManager extends AbstractLSFJobManager { @Override protected ExecutionResult executeKillJobs(List jobIDs) { - String command = "${getEnvironmentString()} ${LSF_COMMAND_DELETE_JOBS} ${jobIDs*.id.join(" ")}" + String command = forBash(environmentString) + " $LSF_COMMAND_DELETE_JOBS ${jobIDs*.id.join(" ")}" return executionService.execute(command, false, commandTimeout) } @Override protected ExecutionResult executeStartHeldJobs(List jobIDs) { - String command = "${getEnvironmentString()} bresume ${jobIDs*.id.join(" ")}" + String command = forBash(environmentString) + " bresume ${jobIDs*.id.join(" ")}" return executionService.execute(command, false, commandTimeout) } @@ -363,16 +365,11 @@ class LSFJobManager extends AbstractLSFJobManager { String parseJobID(String commandOutput) { String result = commandOutput.find(/<[0-9]+>/) if (result == null) - throw new BEException("Could not parse raw ID from: '${commandOutput}'") + throw new BEException("Could not parse raw ID from: '$commandOutput'") String exID = result.substring(1, result.length() - 1) return exID } - @Override - String getSubmissionCommand() { - return "bsub" - } - @Override String getQueryJobStatesCommand() { return null diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFSubmissionCommand.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFSubmissionCommand.groovy index a891bf2e..b9b48993 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFSubmissionCommand.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFSubmissionCommand.groovy @@ -6,16 +6,16 @@ package de.dkfz.roddy.execution.jobs.cluster.lsf + +import de.dkfz.roddy.config.JobLog +import de.dkfz.roddy.tools.EscapableString +import de.dkfz.roddy.tools.BashInterpreter +import de.dkfz.roddy.tools.ConcatenatedString +import de.dkfz.roddy.execution.jobs.* import groovy.transform.CompileStatic import static de.dkfz.roddy.StringConstants.EMPTY - -import de.dkfz.roddy.config.JobLog -import de.dkfz.roddy.execution.jobs.BEJob -import de.dkfz.roddy.execution.jobs.BEJobID -import de.dkfz.roddy.execution.jobs.BatchEuphoriaJobManager -import de.dkfz.roddy.execution.jobs.ProcessingParameters -import de.dkfz.roddy.execution.jobs.SubmissionCommand +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* /** * This class is used to create and execute bsub commands. @@ -26,84 +26,108 @@ class LSFSubmissionCommand extends SubmissionCommand { LSFSubmissionCommand( BatchEuphoriaJobManager parentJobManager, BEJob job, - String jobName, + EscapableString jobName, List processingParameters, - Map environmentVariables, - List dependencyIDs, - String command) { - super(parentJobManager, job, jobName, processingParameters, environmentVariables, dependencyIDs, command) + Map environmentVariables) { + super(parentJobManager, job, jobName, processingParameters, environmentVariables) } @Override - protected String getJobNameParameter() { - return "-J ${jobName}" as String + String getSubmissionExecutableName() { + return "bsub" } @Override - protected String getHoldParameter() { - return "-H" + protected EscapableString getJobNameParameter() { + u("-J ") + jobName } @Override - protected String getAccountNameParameter() { - return job.accountingName != null ? "-P \"${job.accountingName}\"" : "" + protected EscapableString getHoldParameter() { + u("-H") } @Override - protected String getWorkingDirectoryParameter() { - return "-cwd ${job.workingDirectory ?: WORKING_DIRECTORY_DEFAULT}" as String + protected EscapableString getAccountNameParameter() { + job.accountingName != null ? + u("-P ") + job.accountingName : + u("") } @Override - protected String getLoggingParameter(JobLog jobLog) { - return getLoggingParameters(jobLog) + protected EscapableString getWorkingDirectoryParameter() { + ConcatenatedString result = c(u("-cwd ")) + if (job.workingDirectory) { + // The workingDirectory is a File object. So no variables (such as $HOME) are supported. + result += e(job.workingDirectory.toString()) + } else { + // The $HOME will be quoted with double quotes, but not escaped. The variable should + // be expanded on the call-site. + result += u('"') + WORKING_DIRECTORY_DEFAULT + u('"') + } + return result } - static getLoggingParameters(JobLog jobLog) { + @Override + protected EscapableString getLoggingParameter(JobLog jobLog) { + getLoggingParameters(jobLog) + } + + @Override + protected Boolean getQuoteCommand() { + true + } + + static EscapableString getLoggingParameters(JobLog jobLog) { if (!jobLog.out && !jobLog.error) { - return "-o /dev/null" //always set logging because it interacts with mail options + u("-o /dev/null") //always set logging because it interacts with mail options } else if (jobLog.out == jobLog.error) { - return "-oo ${jobLog.out.replace(JobLog.JOB_ID, '%J')}" + u("-oo ") + e(jobLog.out.replace(JobLog.JOB_ID, '%J')) } else { - return "-oo ${jobLog.out.replace(JobLog.JOB_ID, '%J')} -eo ${jobLog.error.replace(JobLog.JOB_ID, '%J')}" + u( "-oo ") + e(jobLog.out.replace(JobLog.JOB_ID, '%J')) + + u(" -eo ") + e(jobLog.error.replace(JobLog.JOB_ID, '%J')) } } @Override - protected String getEmailParameter(String address) { - return address ? "-u ${address}" : "" + protected EscapableString getEmailParameter(EscapableString address) { + address ? u("-u ") + address : u("") } @Override - String getGroupListParameter(String groupList) { - return "-G" + groupList + EscapableString getGroupListParameter(EscapableString groupList) { + u("-G ") + groupList } @Override - protected String getUmaskString(String umask) { - return EMPTY + protected EscapableString getUmaskString(EscapableString umask) { + u(EMPTY) } @Override - protected String assembleDependencyParameter(List jobIds) { + protected EscapableString assembleDependencyParameter(List jobIds) { List validJobIds = BEJob.uniqueValidJobIDs(jobIds) if (validJobIds.size() > 0) { - String joinedParentJobs = validJobIds.collect { "done(${it})" }.join(" && ") + EscapableString joinedParentJobs = + join(validJobIds.collect { + u("done(${it})") + } as List, u(" && ")) + // -ti: Immediate orphan job termination for jobs with failed dependencies. - return "-ti -w \"${joinedParentJobs}\" " + u("-ti -w ") + e(joinedParentJobs) } else { - return EMPTY + c() } } @Override - protected String getAdditionalCommandParameters() { - return EMPTY + protected EscapableString getAdditionalCommandParameters() { + u(EMPTY) } @Override - protected String getEnvironmentString() { - return LSFJobManager.environmentString + protected EscapableString getEnvironmentString() { + LSFJobManager.environmentString } // TODO Code duplication with PBSCommand. Check also DirectSynchronousCommand. @@ -113,22 +137,53 @@ class LSFSubmissionCommand extends SubmissionCommand { * * @return a String of '{-env {"none", "{all|}(, varName[=varValue](, varName[=varValue])*|}"' */ @Override - String assembleVariableExportParameters() { + EscapableString assembleVariableExportParameters() { - List environmentStrings = parameters.collect { key, value -> + List environmentStrings = parameters.collect { key, value -> if (null == value) - "${key}" // returning just the variable name makes bsub take the value form the bsub-commands execution environment + // Returning just the variable name lets bsub take the value from the bsub-commands execution environment + u(key) else - "${key}=${value}" // sets value to value - } as List + // Set value to value + u(key) + u("=") + value + } as List if (passLocalEnvironment) { - environmentStrings = ["all"] + environmentStrings + environmentStrings = ([u("all")] as List) + environmentStrings } else if (parameters.isEmpty()) { - environmentStrings += "none" + environmentStrings = [u("none")] as List + } + + return u("-env") + " " + join(environmentStrings, e(", ")) + } + + @Override + protected String composeCommandString(List parameters) { + ConcatenatedString command = c() + + if (job.code) { + // LSF can just read the script to execute from the standard input. + command += u("echo -ne ") + e(job.code) + u(" | ") + } + + if (environmentString) { + command += environmentString + u(" ") + } + + command += u(submissionExecutableName) + + command += c(u(" "), join(parameters, u(" ")), u(" ")) + + if (job.command) { + ConcatenatedString commandToBeExecuted = join(job.command, " ") + if (quoteCommand) { + command += u(" ") + e(commandToBeExecuted) + } else { + command += u(" ") + commandToBeExecuted + } } - return "-env \"${environmentStrings.join(", ")}\"" + return BashInterpreter.instance.interpret(command) } } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy index ff3b4cdd..06c055d0 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy @@ -9,7 +9,9 @@ package de.dkfz.roddy.execution.jobs.cluster.lsf.rest import de.dkfz.roddy.BEException import de.dkfz.roddy.StringConstants import de.dkfz.roddy.config.ResourceSet +import de.dkfz.roddy.tools.EscapableString import de.dkfz.roddy.execution.BEExecutionService +import de.dkfz.roddy.tools.BashInterpreter import de.dkfz.roddy.execution.RestExecutionService import de.dkfz.roddy.execution.jobs.* import de.dkfz.roddy.execution.jobs.cluster.lsf.AbstractLSFJobManager @@ -29,6 +31,8 @@ import java.time.Duration import java.time.ZonedDateTime import java.time.format.DateTimeFormatter +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + /** * REST job manager for cluster systems. * @@ -93,7 +97,7 @@ class LSFRestJobManager extends AbstractLSFJobManager { "--bqJky99mlBWa-ZuqjC53mG6EzbmlxB--\r\n" */ @Override - protected RestSubmissionCommand createCommand(BEJob job) { + RestSubmissionCommand createCommand(BEJob job) { List

headers = [] headers << new BasicHeader("Accept", "text/xml,application/xml;") @@ -102,20 +106,21 @@ class LSFRestJobManager extends AbstractLSFJobManager { // --- Parameters Area --- List jobParts = [] - if (job.tool) { - jobParts << createJobPart("COMMAND", job.tool?.absolutePath as String, "COMMANDTORUN") + List command = job.command + if (command) { + jobParts << createJobPart("COMMAND", BashInterpreter.instance.interpret(join(command, " ")), "COMMANDTORUN") } else { jobParts << createJobPart("COMMAND", "${job.jobName},upload" as String, "COMMANDTORUN", "file") } if (job.getJobName()) { - jobParts << createJobPart("JOB_NAME", job.getJobName()) + jobParts << createJobPart("JOB_NAME", job.jobName) } jobParts << createJobPart("EXTRA_PARAMS", prepareExtraParams(job)) ContentWithHeaders jobPartsWithHeader = joinParts(jobParts) requestParts << createRequestPart("data", jobPartsWithHeader.content, jobPartsWithHeader.headers) - if (job.toolScript) { - requestParts << createRequestPart("f1", job.toolScript, [ + if (job.code) { + requestParts << createRequestPart("f1", BashInterpreter.instance.interpret(job.code), [ new BasicHeader(HTTP.CONTENT_TYPE, ContentType.APPLICATION_OCTET_STREAM.toString()), ] as List
, job.jobName) } @@ -436,13 +441,6 @@ class LSFRestJobManager extends AbstractLSFJobManager { return jobDetailsResult } - - @Override - String getSubmissionCommand() { - return null - } - - @Override String getQueryJobStatesCommand() { return null diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/RestSubmissionCommand.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/RestSubmissionCommand.groovy index a18bb106..a9319001 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/RestSubmissionCommand.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/RestSubmissionCommand.groovy @@ -6,7 +6,9 @@ package de.dkfz.roddy.execution.jobs.cluster.lsf.rest + import de.dkfz.roddy.config.JobLog +import de.dkfz.roddy.tools.EscapableString import de.dkfz.roddy.execution.jobs.BEJobID import de.dkfz.roddy.execution.jobs.SubmissionCommand import groovy.transform.CompileStatic @@ -36,70 +38,86 @@ class RestSubmissionCommand extends SubmissionCommand { * @param httpMethod - http Method POST or GET */ RestSubmissionCommand(String resource, String requestBody, List
requestHeaders, Enum httpMethod) { - super(null, null, null, null, null,null, null) + super(null, null, null, null, null) this.resource = resource this.requestBody = requestBody this.requestHeaders = requestHeaders this.httpMethod = httpMethod } + @Override + String getSubmissionExecutableName() { + return null + } + @Override String toBashCommandString() { throw new RuntimeException("This method should not be used!") } @Override - protected String assembleVariableExportParameters() { + protected EscapableString assembleVariableExportParameters() { throw new NotImplementedException() } @Override - protected String getAdditionalCommandParameters() { + protected EscapableString getAdditionalCommandParameters() { throw new NotImplementedException() } @Override - protected String getEnvironmentString() { + protected EscapableString getEnvironmentString() { return LSFRestJobManager.environmentString } @Override - protected String getWorkingDirectoryParameter() { + protected EscapableString getWorkingDirectoryParameter() { + throw new NotImplementedException() + } + + @Override + protected EscapableString assembleDependencyParameter(List jobIds) { throw new NotImplementedException() } @Override - protected String assembleDependencyParameter(List jobIds) { + protected EscapableString getUmaskString(EscapableString umask) { throw new NotImplementedException() } @Override - protected String getUmaskString(String umask) { + protected EscapableString getJobNameParameter() { throw new NotImplementedException() } @Override - protected String getJobNameParameter() { + protected EscapableString getGroupListParameter(EscapableString groupList) { throw new NotImplementedException() } @Override - protected String getGroupListParameter(String groupList) { + protected EscapableString getHoldParameter() { throw new NotImplementedException() } @Override - protected String getHoldParameter() { + protected EscapableString getEmailParameter(EscapableString address) { throw new NotImplementedException() } @Override - protected String getEmailParameter(String address) { + protected EscapableString getLoggingParameter(JobLog jobLog) { throw new NotImplementedException() } @Override - protected String getLoggingParameter(JobLog jobLog) { + protected Boolean getQuoteCommand() { + false + } + + + @Override + protected String composeCommandString(List parameters) { throw new NotImplementedException() } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobID.java b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobID.java deleted file mode 100644 index b29879fa..00000000 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobID.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2016 eilslabs. - * - * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt). - */ - -package de.dkfz.roddy.execution.jobs.cluster.pbs; - - -import de.dkfz.roddy.execution.jobs.BEJobID; - -import java.io.Serializable; - -/** - */ -public class PBSJobID extends BEJobID implements Serializable { - - public PBSJobID(String id) { - super(id); - } - - @Override - public boolean isValidID() { - return super.isValidID(); - } - - @Override - public String getShortID() { - return getId().split("[.]")[0]; - } - - @Override - public String toString() { - return getShortID(); - } -} diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManager.groovy index f31a868e..1664dcfd 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManager.groovy @@ -9,6 +9,7 @@ package de.dkfz.roddy.execution.jobs.cluster.pbs import com.google.common.collect.LinkedHashMultimap import de.dkfz.roddy.StringConstants import de.dkfz.roddy.config.ResourceSet +import de.dkfz.roddy.tools.EscapableString import de.dkfz.roddy.execution.BEExecutionService import de.dkfz.roddy.execution.jobs.BEJob import de.dkfz.roddy.execution.jobs.GenericJobInfo @@ -19,6 +20,8 @@ import de.dkfz.roddy.tools.BufferUnit import de.dkfz.roddy.tools.TimeUnit import groovy.transform.CompileStatic +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + @CompileStatic class PBSJobManager extends GridEngineBasedJobManager { @@ -27,9 +30,9 @@ class PBSJobManager extends GridEngineBasedJobManager { } @Override - protected PBSSubmissionCommand createCommand(BEJob job) { - return new PBSSubmissionCommand(this, job, job.jobName, [], job.parameters, job.parentJobIDs*.id, - job.tool?.getAbsolutePath() ?: job.getToolScript()) + PBSSubmissionCommand createCommand(BEJob job) { + return new PBSSubmissionCommand( + this, job, u(job.jobName), [], job.parameters) } @Override @@ -88,7 +91,8 @@ class PBSJobManager extends GridEngineBasedJobManager { } @Override - void createComputeParameter(ResourceSet resourceSet, LinkedHashMultimap parameters) { + void createComputeParameter(ResourceSet resourceSet, + LinkedHashMultimap parameters) { int nodes = resourceSet.isNodesSet() ? resourceSet.getNodes() : 1 int cores = resourceSet.isCoresSet() ? resourceSet.getCores() : 1 // Currently not active @@ -98,32 +102,38 @@ class PBSJobManager extends GridEngineBasedJobManager { if (resourceSet.isAdditionalNodeFlagSet()) { pVal << ':' << resourceSet.getAdditionalNodeFlag() } - parameters.put("-l", pVal) + parameters.put("-l", u(pVal)) } else { String[] nodesArr = enforceSubmissionNodes.split(StringConstants.SPLIT_SEMICOLON) nodesArr.each { String node -> - parameters.put('-l', 'nodes=' + node + ':ppn=' + resourceSet.getCores()) + parameters.put('-l', + u('nodes=') + node + + ':ppn=' + resourceSet.getCores().toString()) } } } - void createQueueParameter(LinkedHashMultimap parameters, String queue) { - parameters.put('-q', queue) + void createQueueParameter(LinkedHashMultimap parameters, + String queue) { + parameters.put('-q', e(queue)) } @Override - void createWalltimeParameter(LinkedHashMultimap parameters, ResourceSet resourceSet) { - parameters.put('-l', 'walltime=' + TimeUnit.fromDuration(resourceSet.walltime)) + void createWalltimeParameter(LinkedHashMultimap parameters, + ResourceSet resourceSet) { + parameters.put('-l', u('walltime=') + TimeUnit.fromDuration(resourceSet.walltime).toString()) } @Override - void createMemoryParameter(LinkedHashMultimap parameters, ResourceSet resourceSet) { - parameters.put('-l', 'mem=' + resourceSet.getMem().toString(BufferUnit.M)) + void createMemoryParameter(LinkedHashMultimap parameters, + ResourceSet resourceSet) { + parameters.put('-l', u('mem=') + resourceSet.getMem().toString(BufferUnit.M)) } @Override - void createStorageParameters(LinkedHashMultimap parameters, ResourceSet resourceSet) { + void createStorageParameters(LinkedHashMultimap parameters, + ResourceSet resourceSet) { } @Override diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSSubmissionCommand.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSSubmissionCommand.groovy index 89e09ad1..1d435c1d 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSSubmissionCommand.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSSubmissionCommand.groovy @@ -7,16 +7,15 @@ package de.dkfz.roddy.execution.jobs.cluster.pbs import de.dkfz.roddy.config.JobLog +import de.dkfz.roddy.tools.EscapableString import de.dkfz.roddy.execution.jobs.BEJob import de.dkfz.roddy.execution.jobs.BatchEuphoriaJobManager import de.dkfz.roddy.execution.jobs.ProcessingParameters import de.dkfz.roddy.execution.jobs.cluster.GridEngineBasedSubmissionCommand import groovy.transform.CompileStatic -import static de.dkfz.roddy.StringConstants.COLON -import static de.dkfz.roddy.StringConstants.COMMA -import static de.dkfz.roddy.StringConstants.EMPTY -import static de.dkfz.roddy.StringConstants.WHITESPACE +import static de.dkfz.roddy.StringConstants.* +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* /** * This class is used to create and execute qsub commands @@ -30,65 +29,71 @@ class PBSSubmissionCommand extends GridEngineBasedSubmissionCommand { public static final String AFTEROK = "afterok" public static final String PARM_DEPENDS = " -W depend=" - PBSSubmissionCommand(BatchEuphoriaJobManager parentJobManager, BEJob job, String jobName, - List processingParameters, Map environmentVariables, - List dependencyIDs, String command) { - super(parentJobManager, job, jobName, processingParameters, environmentVariables, dependencyIDs, command) + PBSSubmissionCommand(BatchEuphoriaJobManager parentJobManager, + BEJob job, EscapableString jobName, + List processingParameters, + Map environmentVariables) { + super(parentJobManager, job, jobName, processingParameters, environmentVariables) } @Override - protected String getJobNameParameter() { - return "-N ${jobName}" as String + protected EscapableString getJobNameParameter() { + u("-N ") + jobName } @Override - protected String getHoldParameter() { - return "-h" + protected EscapableString getHoldParameter() { + u("-h") } @Override - protected String getAccountNameParameter() { - return job.accountingName != null ? "-A \"${job.accountingName}\"" : "" + protected EscapableString getAccountNameParameter() { + job.accountingName != null ? u("-A ") + job.accountingName : c() } @Override - protected String getWorkingDirectoryParameter() { - return "-w ${job.getWorkingDirectory() ?: WORKING_DIRECTORY_DEFAULT}" as String + protected EscapableString getWorkingDirectoryParameter() { + u("-w ") + e(job.getWorkingDirectory().toString()) ?: WORKING_DIRECTORY_DEFAULT } @Override - protected String getLoggingParameter(JobLog jobLog) { + protected EscapableString getLoggingParameter(JobLog jobLog) { if (!jobLog.out && !jobLog.error) { - return "-k n" + u("-k n") } else if (jobLog.out == jobLog.error) { - return "${joinLogParameter} -o \"${jobLog.out.replace(JobLog.JOB_ID, '\\$PBS_JOBID')}\"" + joinLogParameter + u("-o ") + e(jobLog.out.replace(JobLog.JOB_ID, '\\$PBS_JOBID')) } else { - return "-o \"${jobLog.out.replace(JobLog.JOB_ID, '\\$PBS_JOBID')} -e ${jobLog.error.replace(JobLog.JOB_ID, '\\$PBS_JOBID')}\"" + join([ + u("-o"), + e(jobLog.out.replace(JobLog.JOB_ID, '\\$PBS_JOBID')), + u("-e"), + e(jobLog.error.replace(JobLog.JOB_ID, '\\$PBS_JOBID')) + ], WHITESPACE) } } @Override - protected String getEmailParameter(String address) { - return address ? " -M " + address : "" + protected EscapableString getEmailParameter(EscapableString address) { + return address ? u(" -M ") + address : c() } - protected String getJoinLogParameter() { - return "-j oe" + protected EscapableString getJoinLogParameter() { + u("-j oe") } @Override - protected String getGroupListParameter(String groupList) { - return " -W group_list=" + groupList + protected EscapableString getGroupListParameter(EscapableString groupList) { + u(" -W group_list=") + groupList } @Override - protected String getUmaskString(String umask) { - return umask ? "-W umask=" + umask : "" + protected EscapableString getUmaskString(EscapableString umask) { + umask ? u("-W umask=") + umask : c() } @Override String getDependencyParameterName() { - return AFTEROK + AFTEROK } /** @@ -107,33 +112,36 @@ class PBSSubmissionCommand extends GridEngineBasedSubmissionCommand { } @Override - protected String getAdditionalCommandParameters() { - return EMPTY + protected EscapableString getAdditionalCommandParameters() { + u(EMPTY) } @Override - protected String getEnvironmentString() { - return "" + protected EscapableString getEnvironmentString() { + c() } @Override - String assembleVariableExportParameters() { - List parameterStrings = [] + EscapableString assembleVariableExportParameters() { - if (passLocalEnvironment) - parameterStrings << "-V" - - List environmentStrings = parameters.collect { key, value -> + List environmentStrings = parameters.collect { key, value -> if (null == value) - "${key}" // returning just the variable name makes qsub take the value form the qsub-commands execution environment + // Returning just the variable name makes qsub take the value form the qsub-commands execution environment + u(key) else - "${key}=${value}" // sets value to value - } as List + // Sets value to value + u(key) + e("=") + value + } as List + + List parameterStrings = [] + + if (passLocalEnvironment) + parameterStrings << u("-V") if (!environmentStrings.empty) - parameterStrings << "-v \"${environmentStrings.join(COMMA)}\"".toString() + parameterStrings << u("-v ") + join(environmentStrings, COMMA) - return parameterStrings.join(WHITESPACE) + return join(parameterStrings, " ") } protected String getDependsSuperParameter() { diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManager.groovy index 64a41a1b..4065f861 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManager.groovy @@ -13,11 +13,14 @@ import de.dkfz.roddy.execution.BEExecutionService import de.dkfz.roddy.execution.jobs.* import de.dkfz.roddy.execution.jobs.cluster.GridEngineBasedJobManager import de.dkfz.roddy.tools.* +import groovy.transform.CompileStatic + +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* /** * @author michael */ -@groovy.transform.CompileStatic +@CompileStatic class SGEJobManager extends GridEngineBasedJobManager { SGEJobManager(BEExecutionService executionService, JobManagerOptions parms) { @@ -25,8 +28,9 @@ class SGEJobManager extends GridEngineBasedJobManager { } @Override - protected SGESubmissionCommand createCommand(BEJob job) { - return new SGESubmissionCommand(this, job, job.jobName, [], job.parameters, job.parentJobIDs*.id, job.tool?.getAbsolutePath() ?: job.getToolScript()) + SGESubmissionCommand createCommand(BEJob job) { + return new SGESubmissionCommand( + this, job, e(job.jobName), [], job.parameters) } @Override @@ -72,26 +76,31 @@ class SGEJobManager extends GridEngineBasedJobManager { } @Override - void createComputeParameter(ResourceSet resourceSet, LinkedHashMultimap parameters) { - parameters.put("-pe", "serial ${resourceSet.cores}") + void createComputeParameter(ResourceSet resourceSet, + LinkedHashMultimap parameters) { + parameters.put("-pe", e("serial ${resourceSet.cores}")) } - void createQueueParameter(LinkedHashMultimap parameters, String queue) { - parameters.put('-q', queue) + void createQueueParameter(LinkedHashMultimap parameters, + String queue) { + parameters.put('-q', e(queue)) } @Override - void createWalltimeParameter(LinkedHashMultimap parameters, ResourceSet resourceSet) { - parameters.put("-l", "h_rt=${TimeUnit.fromDuration(resourceSet.walltime).toHourString()}") + void createWalltimeParameter(LinkedHashMultimap parameters, + ResourceSet resourceSet) { + parameters.put("-l", e("h_rt=${TimeUnit.fromDuration(resourceSet.walltime).toHourString()}")) } @Override - void createMemoryParameter(LinkedHashMultimap parameters, ResourceSet resourceSet) { - parameters.put("-l", "h_rss=${resourceSet.getMem().toString(BufferUnit.M)}") + void createMemoryParameter(LinkedHashMultimap parameters, + ResourceSet resourceSet) { + parameters.put("-l", e("h_rss=${resourceSet.getMem().toString(BufferUnit.M)}")) } @Override - void createStorageParameters(LinkedHashMultimap parameters, ResourceSet resourceSet) { + void createStorageParameters(LinkedHashMultimap parameters, + ResourceSet resourceSet) { } @Override diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGESubmissionCommand.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGESubmissionCommand.groovy index 354048b3..4c46a93a 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGESubmissionCommand.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGESubmissionCommand.groovy @@ -6,17 +6,17 @@ package de.dkfz.roddy.execution.jobs.cluster.sge - import de.dkfz.roddy.config.JobLog -import de.dkfz.roddy.execution.jobs.BatchEuphoriaJobManager +import de.dkfz.roddy.tools.EscapableString import de.dkfz.roddy.execution.jobs.BEJob +import de.dkfz.roddy.execution.jobs.BatchEuphoriaJobManager import de.dkfz.roddy.execution.jobs.ProcessingParameters import de.dkfz.roddy.execution.jobs.cluster.GridEngineBasedSubmissionCommand import groovy.transform.CompileStatic import static de.dkfz.roddy.StringConstants.COMMA -import static de.dkfz.roddy.StringConstants.EMPTY import static de.dkfz.roddy.StringConstants.WHITESPACE +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* /** * Created by michael on 20.05.14. @@ -24,60 +24,66 @@ import static de.dkfz.roddy.StringConstants.WHITESPACE @CompileStatic class SGESubmissionCommand extends GridEngineBasedSubmissionCommand { - SGESubmissionCommand(BatchEuphoriaJobManager parentJobManager, BEJob job, String jobName, - List processingParameters, Map environmentVariables, - List dependencyIDs, String command) { - super(parentJobManager, job, jobName, processingParameters, environmentVariables, dependencyIDs, command) + SGESubmissionCommand(BatchEuphoriaJobManager parentJobManager, + BEJob job, EscapableString jobName, + List processingParameters, + Map environmentVariables) { + super(parentJobManager, job, jobName, processingParameters, environmentVariables) } @Override - protected String getJobNameParameter() { - return "-N ${jobName}" as String + protected EscapableString getJobNameParameter() { + u("-N ") + jobName } @Override - protected String getHoldParameter() { - return "-h" + protected EscapableString getHoldParameter() { + u("-h") } @Override - protected String getAccountNameParameter() { - return job.accountingName != null ? "-A \"${job.accountingName}\"" : "" + protected EscapableString getAccountNameParameter() { + job.accountingName != null ? u("-A ") + job.accountingName : c() } @Override - protected String getWorkingDirectoryParameter() { - return "-wd ${job.getWorkingDirectory() ?: WORKING_DIRECTORY_DEFAULT}" as String + protected EscapableString getWorkingDirectoryParameter() { + u("-wd ") + e(job.getWorkingDirectory().toString()) ?: WORKING_DIRECTORY_DEFAULT } @Override - protected String getLoggingParameter(JobLog jobLog) { + protected EscapableString getLoggingParameter(JobLog jobLog) { if (!jobLog.out && !jobLog.error) { - return "-o /dev/null -e /dev/null" + u("-o /dev/null -e /dev/null") } else if (jobLog.out == jobLog.error) { - return "${joinLogParameter} -o \"${jobLog.out.replace(JobLog.JOB_ID, '\\$JOB_ID')}\"" + joinLogParameter + u(" -o ") + e(jobLog.out.replace(JobLog.JOB_ID, '\\$JOB_ID')) } else { - return "-o \"${jobLog.out.replace(JobLog.JOB_ID, '\\$JOB_ID')} -e ${jobLog.error.replace(JobLog.JOB_ID, '\\$JOB_ID')}\"" + join([ + u("-o"), + e(jobLog.out.replace(JobLog.JOB_ID, '\\$JOB_ID')), + u("-e"), + e(jobLog.error.replace(JobLog.JOB_ID, '\\$JOB_ID')) + ], " ") } } @Override - protected String getEmailParameter(String address) { - return address ? " -M " + address : "" + protected EscapableString getEmailParameter(EscapableString address) { + address ? u(" -M ") + address : c() } - protected String getJoinLogParameter() { - return "-j y" + protected EscapableString getJoinLogParameter() { + u("-j y") } @Override - protected String getGroupListParameter(String groupList) { - return EMPTY + protected EscapableString getGroupListParameter(EscapableString groupList) { + c() } @Override - protected String getUmaskString(String umask) { - return WHITESPACE + protected EscapableString getUmaskString(EscapableString umask) { + u(WHITESPACE) } @Override @@ -96,36 +102,37 @@ class SGESubmissionCommand extends GridEngineBasedSubmissionCommand { } @Override - protected String getAdditionalCommandParameters() { - return " -S /bin/bash " + protected EscapableString getAdditionalCommandParameters() { + u(" -S /bin/bash ") } @Override - protected String getEnvironmentString() { - return "" + protected EscapableString getEnvironmentString() { + c() } @Override - protected String assembleVariableExportParameters() { - List parameterStrings = [] + protected EscapableString assembleVariableExportParameters() { + List parameterStrings = [] if (passLocalEnvironment) - parameterStrings << "-V" + parameterStrings += u("-V") List environmentStrings = parameters.collect { key, value -> if (null == value) - "${key}" // returning just the variable name makes qsub take the value form the qsub-commands execution environment + u(key) // returning just the variable name makes qsub take the value form the qsub-commands execution environment else - "${key}=${value}" // sets value to value + u("$key=") + value // sets value to value } as List if (!environmentStrings.empty) - parameterStrings << "-v \"${environmentStrings.join(",")}\"".toString() + parameterStrings += u("-v ") + e(environmentStrings.join(",")) - return parameterStrings.join(" ") + join(parameterStrings, " ") } protected String getDependsSuperParameter() { return WHITESPACE } + } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManager.groovy index 6a5048cb..bcfc6678 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManager.groovy @@ -21,6 +21,8 @@ import groovy.transform.CompileStatic import java.time.* import java.time.format.DateTimeFormatter +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + @CompileStatic class SlurmJobManager extends GridEngineBasedJobManager { @@ -43,9 +45,9 @@ class SlurmJobManager extends GridEngineBasedJobManager { } @Override - protected SlurmSubmissionCommand createCommand(BEJob job) { - SlurmSubmissionCommand ssc = new SlurmSubmissionCommand(this, job, job.jobName, [], job.parameters, job.parentJobIDs*.id, - job.tool?.getAbsolutePath() ?: job.getToolScript()) + SlurmSubmissionCommand createCommand(BEJob job) { + SlurmSubmissionCommand ssc = new SlurmSubmissionCommand( + this, job, e(job.jobName), [], job.parameters) return ssc } @@ -156,13 +158,14 @@ class SlurmJobManager extends GridEngineBasedJobManager { } @CompileDynamic - protected static GenericJobInfo fillFromSupplement(GenericJobInfo primary, GenericJobInfo supplement) { + protected static GenericJobInfo fillFromSupplement(GenericJobInfo primary, + GenericJobInfo supplement) { if (!supplement) { return primary } - primary.properties.findAll { k, v -> - if (!v && supplement."${k}") { - primary."${k}" = supplement."${k}" + primary.properties.findAll { key, value -> + if (!value && supplement."$key") { + primary."$key" = supplement."$key" } } return primary @@ -404,30 +407,36 @@ class SlurmJobManager extends GridEngineBasedJobManager { } @Override - void createComputeParameter(ResourceSet resourceSet, LinkedHashMultimap parameters) { + void createComputeParameter(ResourceSet resourceSet, + LinkedHashMultimap parameters) { int nodes = resourceSet.isNodesSet() ? resourceSet.getNodes() : 1 int cores = resourceSet.isCoresSet() ? resourceSet.getCores() : 1 String nVal = "--nodes=" + nodes String cVal = " --cores-per-socket=" + cores - parameters.put(nVal, cVal) + parameters.put(nVal, u(cVal)) } - void createQueueParameter(LinkedHashMultimap parameters, String queue) { - parameters.put('-p', queue) + void createQueueParameter(LinkedHashMultimap parameters, + String queue) { + parameters.put('-p', e(queue)) } @Override - void createWalltimeParameter(LinkedHashMultimap parameters, ResourceSet resourceSet) { - parameters.put('--time=' + TimeUnit.fromDuration(resourceSet.walltime).toHourString(), " ") + void createWalltimeParameter(LinkedHashMultimap parameters, + ResourceSet resourceSet) { + parameters.put('--time=' + TimeUnit.fromDuration(resourceSet.walltime).toHourString(), + u(" ")) // What a hack. } @Override - void createMemoryParameter(LinkedHashMultimap parameters, ResourceSet resourceSet) { - parameters.put('--mem=' + resourceSet.getMem().toString(BufferUnit.M), " ") + void createMemoryParameter(LinkedHashMultimap parameters, + ResourceSet resourceSet) { + parameters.put('--mem=' + resourceSet.getMem().toString(BufferUnit.M), u(" ")) } @Override - void createStorageParameters(LinkedHashMultimap parameters, ResourceSet resourceSet) { + void createStorageParameters(LinkedHashMultimap parameters, + ResourceSet resourceSet) { } @Override @@ -460,9 +469,4 @@ class SlurmJobManager extends GridEngineBasedJobManager { return "SLURM_JOB_PARTITION" } - @Override - String getSubmissionCommand() { - return "sbatch" - } - } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmSubmissionCommand.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmSubmissionCommand.groovy index 5404d8da..5484fd71 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmSubmissionCommand.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmSubmissionCommand.groovy @@ -6,14 +6,20 @@ package de.dkfz.roddy.execution.jobs.cluster.slurm + import de.dkfz.roddy.config.JobLog +import de.dkfz.roddy.tools.EscapableString +import de.dkfz.roddy.tools.BashInterpreter +import de.dkfz.roddy.tools.ConcatenatedString import de.dkfz.roddy.execution.jobs.BEJob import de.dkfz.roddy.execution.jobs.BatchEuphoriaJobManager import de.dkfz.roddy.execution.jobs.ProcessingParameters import de.dkfz.roddy.execution.jobs.cluster.GridEngineBasedSubmissionCommand import groovy.transform.CompileStatic -import static de.dkfz.roddy.StringConstants.* +import static de.dkfz.roddy.StringConstants.COLON +import static de.dkfz.roddy.StringConstants.COMMA +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* @CompileStatic class SlurmSubmissionCommand extends GridEngineBasedSubmissionCommand { @@ -22,60 +28,76 @@ class SlurmSubmissionCommand extends GridEngineBasedSubmissionCommand { public static final String AFTEROK = "afterok" public static final String PARM_DEPENDS = " --dependency=" - SlurmSubmissionCommand(BatchEuphoriaJobManager parentJobManager, BEJob job, String jobName, - List processingParameters, Map environmentVariables, - List dependencyIDs, String command) { - super(parentJobManager, job, jobName, processingParameters, environmentVariables, dependencyIDs, command) + SlurmSubmissionCommand(BatchEuphoriaJobManager parentJobManager, BEJob job, + EscapableString jobName, + List processingParameters, + Map environmentVariables) { + super(parentJobManager, job, jobName, processingParameters, environmentVariables) } @Override - protected String getJobNameParameter() { - return "--job-name ${jobName}" as String + String getSubmissionExecutableName() { + return "sbatch" } @Override - protected String getHoldParameter() { - return "--hold" + protected EscapableString getJobNameParameter() { + u("--job-name ") + jobName } @Override - protected String getAccountNameParameter() { - return job.accountingName != null ? "--account=\"${job.accountingName}\"" : "" + protected EscapableString getHoldParameter() { + u("--hold") } @Override - protected String getWorkingDirectoryParameter() { - return "--chdir ${job.getWorkingDirectory() ?: WORKING_DIRECTORY_DEFAULT}" as String + protected EscapableString getAccountNameParameter() { + job.accountingName != null ? u("--account=") + job.accountingName : c() } @Override - protected String getLoggingParameter(JobLog jobLog) { + protected EscapableString getWorkingDirectoryParameter() { + ConcatenatedString result = c(u("--chdir ")) + if (job.workingDirectory) { + // The workingDirectory is a File object. So no variables (such as $HOME) are supported. + result += e(job.workingDirectory.toString()) + } else { + // The $HOME will be enclosed in double quotes, but not escaped. The variable should + // be expanded on the call-site. + result += u('"') + WORKING_DIRECTORY_DEFAULT + u('"') + } + return result + } + + @Override + protected EscapableString getLoggingParameter(JobLog jobLog) { if (!jobLog.out && !jobLog.error) { - return "" + c() } else if (jobLog.out == jobLog.error) { - return "--output=${jobLog.out.replace(JobLog.JOB_ID, '%j')}" + u("--output=") + e(jobLog.out.replace(JobLog.JOB_ID, '%j')) } else { - return "--output=${jobLog.out.replace(JobLog.JOB_ID, '%j')} --error=${jobLog.error.replace(JobLog.JOB_ID, '%j')}" + u("--output=") + e(jobLog.out.replace(JobLog.JOB_ID, '%j')) + + u(" --error=") + e(jobLog.error.replace(JobLog.JOB_ID, '%j')) } } @Override - protected String getEmailParameter(String address) { - return address ? " --mail-user=" + address : "" + protected EscapableString getEmailParameter(EscapableString address) { + address ? u(" --mail-user=") + address : c() } - protected String getParsableParameter() { - return "--parsable" + protected EscapableString getParsableParameter() { + u("--parsable") } @Override - protected String getGroupListParameter(String groupList) { - return " --grid=" + groupList + protected EscapableString getGroupListParameter(EscapableString groupList) { + u(" --grid=") + groupList } @Override - protected String getUmaskString(String umask) { - return "" + protected EscapableString getUmaskString(EscapableString umask) { + c() } @Override @@ -99,36 +121,69 @@ class SlurmSubmissionCommand extends GridEngineBasedSubmissionCommand { } @Override - protected String getAdditionalCommandParameters() { - return "${getParsableParameter()} --kill-on-invalid-dep=yes --propagate=none" as String + protected EscapableString getAdditionalCommandParameters() { + u(parsableParameter) + u(" --kill-on-invalid-dep=yes --propagate=NONE") } @Override - protected String getEnvironmentString() { - return "" + protected EscapableString getEnvironmentString() { + c() } @Override - String assembleVariableExportParameters() { - List parameterStrings = [] + EscapableString assembleVariableExportParameters() { + ConcatenatedString parameterStrings = c() if (passLocalEnvironment) - parameterStrings << "--get-user-env " + parameterStrings += u("--get-user-env ") - List environmentStrings = parameters.collect { key, value -> - if (null == value) - "${key}" - else - "${key}=${value}" - } as List + List environmentStrings = + parameters.collect { key, value -> + if (null == value) + u(key) + else + u(key) + e("=") + value + } as List if (!environmentStrings.empty) - parameterStrings << "--export=\"${environmentStrings.join(COMMA)}\"".toString() - - return parameterStrings.join(WHITESPACE) + parameterStrings += u("--export=") + join(environmentStrings, u(COMMA)) + return parameterStrings } protected String getDependsSuperParameter() { PARM_DEPENDS } + + @Override + protected String composeCommandString(List parameters) { + ConcatenatedString command = c() + + if (job.code) { + // SLURM must have a shebang line for the job script. + // Note that we escape the code once. + command += u("echo -ne ") + e(job.code) + u(" | ") + } + + if (environmentString != null && environmentString != c()) { + command += environmentString + u(" ") + } + + command += u(submissionExecutableName) + + command += c(u(" "), join(parameters, u(" ")), u(" ")) + + if (job.command) { + ConcatenatedString commandToBeExecuted = join(job.command, " ") + if (quoteCommand) { + command += u(" ") + e(commandToBeExecuted) + } else { + command += u(" ") + commandToBeExecuted + } + } else if (job.code) { + // SLURM can only read scripts from files, not from stdin. + command += " /dev/stdin" + } + + return BashInterpreter.instance.interpret(command) + } } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectCommand.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectCommand.groovy index 8ea68ad0..ca00af04 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectCommand.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectCommand.groovy @@ -12,8 +12,10 @@ import de.dkfz.roddy.execution.jobs.Command import de.dkfz.roddy.execution.jobs.ProcessingParameters import groovy.transform.CompileStatic +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + /** - * Local commands run locally and, if the workflow requires and supports it, concurrent. + * Local commands run locally and, if the workflow requires and supports it, concurrently. * They are called in a local process with waitFor after each call. Dependencies are therefore automatically resolved. * Roddy waits for the processes to exit. */ @@ -21,14 +23,15 @@ import groovy.transform.CompileStatic class DirectCommand extends Command { private final List processingParameters - private final String command public static final String PARM_WRAPPED_SCRIPT = "WRAPPED_SCRIPT=" - DirectCommand(DirectSynchronousExecutionJobManager parentManager, BEJob job, List processingParameters, @Deprecated String command = null) { - super(parentManager, job, job.tool.name, job.parameters) + DirectCommand(DirectSynchronousExecutionJobManager parentManager, + BEJob job, + List processingParameters) { + super(parentManager, + job, e(job.executableFile.name), job.parameters) this.processingParameters = processingParameters - this.command = command ?: job.tool.absolutePath } /** @@ -54,19 +57,17 @@ class DirectCommand extends Command { parameterBuilder << parameters.collect { key, value -> "${key}=${value}" }.join(" ") - // Dependencies are ignored. Direct commands are executed in-sync. - - // Processing commands are ignored BE does not offer job scheduling on its own. - - //TODO email handling? Better not - - //TODO Grouplist is ignored + // Grouplist is ignored + // Umask is ignored - //TODO Umask is ignored + // Maybe there is a need for a local file system provider? - //TODO Command assembly should be part of the file system provider? Maybe there is a need for a local file system provider? - //This is very linux specific... - commandString << parameterBuilder.toString() << StringConstants.WHITESPACE << command << " &> ${job.jobLog.getOut(job.jobCreationCounter.toString())}"; + // This is very linux specific... + commandString << + parameterBuilder.toString() << + StringConstants.WHITESPACE << + job.command.join(StringConstants.WHITESPACE) << + " &> ${job.jobLog.getOut(job.jobCreationCounter.toString())}"; return commandString.toString() } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectSynchronousExecutionJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectSynchronousExecutionJobManager.groovy index 78d5d899..2a6e5f49 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectSynchronousExecutionJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectSynchronousExecutionJobManager.groovy @@ -21,7 +21,8 @@ import java.time.Duration class DirectSynchronousExecutionJobManager extends BatchEuphoriaJobManager { - DirectSynchronousExecutionJobManager(BEExecutionService executionService, JobManagerOptions parms) { + DirectSynchronousExecutionJobManager(BEExecutionService executionService, + JobManagerOptions parms) { super(executionService, parms) } @@ -121,7 +122,7 @@ class DirectSynchronousExecutionJobManager extends BatchEuphoriaJobManager) + jobResult = new BEJobResult(command, job, res, job.parameters, job.parentJobs as List) job.setRunResult(jobResult) return jobResult @@ -164,11 +166,6 @@ class DirectSynchronousExecutionJobManager extends BatchEuphoriaJobManager)) + then: + forBash(code1.toEscapableString(true)) == """\ + |#!/bin/bash -xe + |echo hallo; sleep 50; + |""".stripMargin() + } + +} diff --git a/src/test/groovy/de/dkfz/roddy/execution/CommandSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/CommandSpec.groovy new file mode 100644 index 00000000..dee5c1e3 --- /dev/null +++ b/src/test/groovy/de/dkfz/roddy/execution/CommandSpec.groovy @@ -0,0 +1,134 @@ +package de.dkfz.roddy.execution + +import de.dkfz.roddy.tools.EscapableString +import de.dkfz.roddy.tools.BashInterpreter +import spock.lang.Specification + +import java.nio.file.Paths +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + +class CommandSpec extends Specification { + + def "throw with null string argument"() { + when: + new Command(new Executable(Paths.get("somePath")), + [null as EscapableString]) + then: + final IllegalArgumentException exception = thrown() + } + + def "GetExecutablePath"() { + given: + Command command = new Command(new Executable(Paths.get("somePath")), []) + expect: + command.executablePath == Paths.get("somePath") + } + + def "CommandHashCode"() { + given: + Command command1 = new Command(new Executable(Paths.get("somePath")), []) + Command command2 = new Command(new Executable(Paths.get("somePath")), []) + Command command3 = new Command(new Executable(Paths.get("otherPath")), []) + Command command4 = new Command(new Executable( + Paths.get("somePath")), + [u("otherArg")]) + expect: + command1.hashCode() == command2.hashCode() + command2.hashCode() != command3.hashCode() + command1.hashCode() != command4.hashCode() + } + + def "CommandEquals"() { + given: + Command command1 = new Command(new Executable(Paths.get("somePath")), []) + Command command2 = new Command(new Executable(Paths.get("somePath")), []) + Command command3 = new Command(new Executable(Paths.get("otherPath")), []) + Command command4 = new Command(new Executable(Paths.get("somePath")), + [u("otherArg")]) + expect: + command1 == command2 + command2 != command3 + command1 != command4 + } + + def "throw with null Executable"() { + when: + new Command(null as Executable, []) + then: + final IllegalArgumentException exception = thrown() + } + + def "throw with null argument list"() { + when: + new Command(new Executable(Paths.get("somePath")), null as List) + then: + final IllegalArgumentException exception = thrown() + } + + def "GetExecutable"() { + given: + Command command = new Command(new Executable(Paths.get("somePath")), []) + expect: + command.executable == new Executable(Paths.get("somePath")) + } + + def "ToList"() { + given: + Command commandWithoutArgs = new Command(new Executable( + Paths.get("somePath")), []) + Command commandWithArgs = new Command(new Executable( + Paths.get("someOtherPath")), [u("a"), u("b"), u("c")]) + expect: + commandWithoutArgs.toCommandSegmentList() == + ["somePath"].collect { u(it) } + commandWithoutArgs.toCommandSegmentList() == + [Paths.get("somePath").toString()].collect { u(it) } + commandWithArgs.toCommandSegmentList() == + ["someOtherPath", "a", "b", "c"].collect { u(it) } + commandWithArgs.toCommandSegmentList() == + [Paths.get("someOtherPath").toString(), "a", "b", "c"].collect { u(it) } + } + + def "CliAppendCommandExecutable"() { + given: + Command command1 = new Command(new Executable(Paths.get("strace")), + ["stracearg1", "--"].collect { u(it) }) + Command command2 = new Command(new Executable(Paths.get("someTool")), + ["toolarg1"].collect { u(it) }) + Executable executable = new Executable(Paths.get("executableX")) + + expect: + command1.cliAppend(command2).toCommandSegmentList() == [ + "strace", "stracearg1", "--", "someTool", "toolarg1" + ].collect { u(it) } as List + + command1.cliAppend(command2, true).toCommandSegmentList() == [ + u("strace"), u("stracearg1"), u("--"), + c(e("someTool"), e(" "), e("toolarg1")) + ] as List + + command1.cliAppend(executable).toCommandSegmentList() == [ + "strace", "stracearg1", "--", "executableX" + ].collect { u(it)} as List + } + + def "CliAppendCode"() { + given: + Code code1 = new Code("echo hallo; sleep 50;") + EscapableString result = + new Command(new Executable(Paths.get("cat")), ["-"].collect { u(it) }). + cliAppend(code1, + new Executable(Paths.get("/bin/bash")), + "prefix", + "test").toEscapableString() + expect: + BashInterpreter.instance.interpret(result) == """\ + |#!/bin/bash + |cat - <, + logFile) jobTest(jobManager, [testJobWithPipedScript]) } @@ -89,18 +101,52 @@ class BEIntegrationTest { if (!(properties["${system}.host".toString()] != "" && properties["${system}.account".toString()] != "")) return BatchEuphoriaJobManager jobManager = createJobManagerFor(system) - BEJob testParent = new BEJob(null, "batchEuphoriaTestJob_Parent", null, testScript, null, resourceSet, null, ["a": "value"], jobManager, logFile, null) - BEJob testJobChild1 = new BEJob(null, "batchEuphoriaTestJob_Child1", null, testScript, null, resourceSet, [testParent], ["a": "value"], jobManager, logFile, null) - BEJob testJobChild2 = new BEJob(null, "batchEuphoriaTestJob_Child2", null, testScript, null, resourceSet, [testParent, testJobChild1], ["a": "value"], jobManager, logFile, null) + BEJob testParent = new BEJob( + null, + jobManager, + u("batchEuphoriaTestJob_Parent"), + new Code(u(testScript)), + resourceSet, + [], + ["a": u("value")] as Map, + logFile) + BEJob testJobChild1 = new BEJob( + null, + jobManager, + u("batchEuphoriaTestJob_Child1"), + new Code(u(testScript)), + resourceSet, + [testParent], + ["a": u("value")] as Map, + logFile) + BEJob testJobChild2 = new BEJob( + null, + jobManager, + u("batchEuphoriaTestJob_Child2"), + new Code(u(testScript)), + resourceSet, + [testParent, testJobChild1], + ["a": u("value")] as Map, + logFile) jobTest(jobManager, [testParent, testJobChild1, testJobChild2]) } void checkAndPossiblyRunJobWithFile(AvailableClusterSystems system) { - if (!(properties["${system}.host".toString()] != "" && properties["${system}.account".toString()] != "" && properties["remoteToolPath"] != "")) return + if (!(properties["${system}.host".toString()] != "" && + properties["${system}.account".toString()] != "" && + properties["remoteToolPath"] != "")) return prepareTestScript(system) BatchEuphoriaJobManager jobManager = createJobManagerFor(system) - BEJob testJobWithFile = new BEJob(null, "batchEuphoriaTestJob", batchEuphoriaTestScript, null, null, resourceSet, null, ["a": "value"], jobManager, logFile, null) + BEJob testJobWithFile = new BEJob( + null, + jobManager, + u("batchEuphoriaTestJob"), + new Executable(batchEuphoriaTestScript.toPath()), + resourceSet, + [], + ["a": u("value")] as Map, + logFile) jobTest(jobManager, [testJobWithFile]) } @@ -109,9 +155,33 @@ class BEIntegrationTest { prepareTestScript(system) BatchEuphoriaJobManager jobManager = createJobManagerFor(system) - BEJob testParent = new BEJob(null, "batchEuphoriaTestJob_Parent", batchEuphoriaTestScript, null, null, resourceSet, null, ["a": "value"], jobManager, logFile, null) - BEJob testJobChild1 = new BEJob(null, "batchEuphoriaTestJob_Child1", batchEuphoriaTestScript, null, null, resourceSet, [testParent], ["a": "value"], jobManager, logFile, null) - BEJob testJobChild2 = new BEJob(null, "batchEuphoriaTestJob_Child2", batchEuphoriaTestScript, null, null, resourceSet, [testParent, testJobChild1], ["a": "value"], jobManager, logFile, null) + BEJob testParent = new BEJob( + null, + jobManager, + u("batchEuphoriaTestJob_Parent"), + new Executable(batchEuphoriaTestScript.toPath()), + resourceSet, + [], + ["a": u("value")] as Map, + logFile) + BEJob testJobChild1 = new BEJob( + null, + jobManager, + u("batchEuphoriaTestJob_Child1"), + new Executable(batchEuphoriaTestScript.toPath()), + resourceSet, + [testParent], + ["a": u("value")] as Map, + logFile) + BEJob testJobChild2 = new BEJob( + null, + jobManager, + u("batchEuphoriaTestJob_Child2"), + new Executable(batchEuphoriaTestScript.toPath()), + resourceSet, + [testParent, testJobChild1], + ["a": u("value")] as Map, + logFile) jobTest(jobManager, [testParent, testJobChild1, testJobChild2]) } @@ -128,8 +198,9 @@ class BEIntegrationTest { ensureProperJobStates(maxSleep, testJobs, [JobState.HOLD], jobManager) jobManager.startHeldJobs(testJobs) - // Wait for some seconds and see, if the status changes from HOLD to queued or running and from queued to running - // The queued to running check can take a lot more time. Also the default update time for queries to the job system + // Wait for some seconds and see, if the status changes from HOLD to queued or running + // and from queued to running. The queued to running check can take a lot more time. + // Also the default update time for queries to the job system // is too long for tests. We force updates everytime we run queryJobStatus ensureProperJobStates(maxSleep, testJobs, [JobState.QUEUED, JobState.RUNNING], jobManager) } else { diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/ProcessingParametersTest.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/ProcessingParametersTest.groovy index 3158a86a..93248d12 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/ProcessingParametersTest.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/ProcessingParametersTest.groovy @@ -6,20 +6,28 @@ package de.dkfz.roddy.execution.jobs import com.google.common.collect.LinkedHashMultimap +import de.dkfz.roddy.tools.EscapableString import groovy.transform.CompileStatic import org.junit.Test import static org.junit.Assert.* +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + @CompileStatic class ProcessingParametersTest { @Test void fromString() throws Exception { List values = ["-l", "abc", "-l", "def", "-W", null, "-v", null] - LinkedHashMultimap expected = LinkedHashMultimap.create() - values.collate(2).each { String k, String v -> expected.put(k, v) } - assertEquals(new ProcessingParameters(expected).toString(), ProcessingParameters.fromString(values.findAll { it }.join(" ")).toString()) + LinkedHashMultimap expected = LinkedHashMultimap.create() + values.collate(2).each { String k, String v -> + expected.put(k, v != null ? u(v) : null as EscapableString) + } + assertEquals(new ProcessingParameters(expected).toString(), + ProcessingParameters.fromString( + values.findAll { it }.join(" ") + ).toString()) } } \ No newline at end of file diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommandTest.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommandTest.groovy index 87518265..77148063 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommandTest.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommandTest.groovy @@ -9,6 +9,7 @@ package de.dkfz.roddy.execution.jobs import de.dkfz.roddy.BEException import de.dkfz.roddy.config.JobLog import de.dkfz.roddy.config.ResourceSet +import de.dkfz.roddy.tools.EscapableString import de.dkfz.roddy.execution.io.ExecutionResult import spock.lang.Specification @@ -56,11 +57,6 @@ class SubmissionCommandTest extends Specification { return null } - @Override - String getSubmissionCommand() { - return null - } - @Override String getQueryJobStatesCommand() { return null @@ -82,7 +78,7 @@ class SubmissionCommandTest extends Specification { } @Override - protected Command createCommand(BEJob job) { + Command createCommand(BEJob job) { return null } @@ -124,65 +120,86 @@ class SubmissionCommandTest extends Specification { } def makeSubmissionCommand(final BatchEuphoriaJobManager jobManager, final Optional passEnvironment) { - return new SubmissionCommand(jobManager, null, null, [], [:] as Map, [], "") { + return new SubmissionCommand( + jobManager, + null, + null as EscapableString, + [], + [:] as Map + ) { { this.setPassEnvironment(passEnvironment) } @Override - protected String getJobNameParameter() { + String getSubmissionExecutableName() { return null } @Override - protected String getHoldParameter() { + protected Boolean getQuoteCommand() { + true + } + + @Override + protected EscapableString getJobNameParameter() { return null } @Override - protected String getWorkingDirectoryParameter() { + protected EscapableString getHoldParameter() { return null } @Override - protected String getLoggingParameter(JobLog jobLog) { + protected EscapableString getWorkingDirectoryParameter() { return null } @Override - protected String getEmailParameter(String address) { + protected EscapableString getLoggingParameter(JobLog jobLog) { return null } @Override - protected String getGroupListParameter(String groupList) { + protected EscapableString getEmailParameter(EscapableString address) { return null } @Override - protected String getUmaskString(String umask) { + protected EscapableString getGroupListParameter(EscapableString groupList) { return null } @Override - protected String assembleDependencyParameter(List jobIds) { + protected EscapableString getUmaskString(EscapableString umask) { return null } @Override - protected String assembleVariableExportParameters() throws BEException { + protected EscapableString assembleDependencyParameter(List jobIds) { return null } @Override - protected String getAdditionalCommandParameters() { + protected EscapableString assembleVariableExportParameters() throws BEException { return null } @Override - protected String getEnvironmentString() { - return "" + protected EscapableString getAdditionalCommandParameters() { + return null + } + + @Override + protected EscapableString getEnvironmentString() { + c() + } + + @Override + protected String composeCommandString(List parameters) { + return null } } } diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBaseJobManagerTest.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBaseJobManagerTest.groovy index ee622b0d..4a60ae21 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBaseJobManagerTest.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBaseJobManagerTest.groovy @@ -4,6 +4,8 @@ import com.google.common.collect.LinkedHashMultimap import de.dkfz.roddy.config.JobLog import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.config.ResourceSetSize +import de.dkfz.roddy.tools.EscapableString +import de.dkfz.roddy.execution.Executable import de.dkfz.roddy.execution.jobs.BEJob import de.dkfz.roddy.execution.jobs.Command import de.dkfz.roddy.execution.jobs.GenericJobInfo @@ -17,6 +19,10 @@ import groovy.transform.CompileStatic import org.junit.Before import org.junit.Test +import java.nio.file.Paths + +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + @CompileStatic class GridEngineBaseJobManagerTest { @@ -96,7 +102,7 @@ class GridEngineBaseJobManagerTest { } @Override - protected Command createCommand(BEJob job) { + Command createCommand(BEJob job) { return null } @@ -112,21 +118,45 @@ class GridEngineBaseJobManagerTest { } } - private BEJob makeJob(Map mapOfParameters) { - BEJob job = new BEJob(null, "Test", new File("/tmp/test.sh"), null, null, new ResourceSet(ResourceSetSize.l, new BufferValue(1, BufferUnit.G), 4, 1, new TimeUnit("1h"), null, null, null), [], mapOfParameters, jobManager, JobLog.none(), null) + private BEJob makeJob(Map mapOfParameters) { + BEJob job = new BEJob + (null, + jobManager, + u("Test"), + new Executable(Paths.get("/tmp/test.sh")), + new ResourceSet(ResourceSetSize.l, + new BufferValue(1, BufferUnit.G), + 4, + 1, + new TimeUnit("1h"), + null, + null, + null), + [], + mapOfParameters) job } @Test void testAssembleDependencyStringWithoutDependencies() throws Exception { - def mapOfVars = ["a": "a", "b": "b"] - GridEngineBasedSubmissionCommand cmd = new GridEngineBasedSubmissionCommand(jobManager, makeJob(mapOfVars), - "jobName", null, mapOfVars, null, "/tmp/test.sh") { + def mapOfVars = ["a": u("a"), "b": u("b")] as Map + GridEngineBasedSubmissionCommand cmd = + new GridEngineBasedSubmissionCommand( + jobManager, + makeJob(mapOfVars), + u("jobName"), + null, + mapOfVars) { @Override protected String getDependsSuperParameter() { return null } + @Override + protected Boolean getQuoteCommand() { + true + } + @Override protected String getDependencyParameterName() { return null @@ -143,56 +173,61 @@ class GridEngineBaseJobManagerTest { } @Override - protected String getJobNameParameter() { + protected EscapableString getJobNameParameter() { return null } @Override - protected String getHoldParameter() { + protected EscapableString getHoldParameter() { return null } @Override - protected String getWorkingDirectoryParameter() { + protected EscapableString getWorkingDirectoryParameter() { return null } @Override - protected String getLoggingParameter(JobLog jobLog) { + protected EscapableString getLoggingParameter(JobLog jobLog) { return null } @Override - protected String getEmailParameter(String address) { + protected EscapableString getEmailParameter(EscapableString address) { return null } @Override - protected String getGroupListParameter(String groupList) { + protected EscapableString getGroupListParameter(EscapableString groupList) { return null } @Override - protected String getUmaskString(String umask) { + protected EscapableString getUmaskString(EscapableString umask) { return null } @Override - protected String getAdditionalCommandParameters() { + protected EscapableString getAdditionalCommandParameters() { return null } @Override - protected String getEnvironmentString() { - return "" + protected EscapableString getEnvironmentString() { + return u("") + } + + @Override + protected EscapableString assembleVariableExportParameters() { + return null } @Override - protected String assembleVariableExportParameters() { + protected String composeCommandString(List parameters) { return null } } - assert cmd.assembleDependencyParameter([]) == "" + assert cmd.assembleDependencyParameter([]) == c() } } diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/AbstractLSFJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/AbstractLSFJobManagerSpec.groovy index c41d1c27..2cff3259 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/AbstractLSFJobManagerSpec.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/AbstractLSFJobManagerSpec.groovy @@ -2,7 +2,9 @@ package de.dkfz.roddy.execution.jobs.cluster.lsf import com.google.common.collect.LinkedHashMultimap import de.dkfz.roddy.config.ResourceSet +import de.dkfz.roddy.tools.EscapableString import de.dkfz.roddy.execution.BEExecutionService +import de.dkfz.roddy.tools.BashInterpreter import de.dkfz.roddy.execution.io.ExecutionResult import de.dkfz.roddy.execution.jobs.Command import de.dkfz.roddy.execution.jobs.JobManagerOptionsBuilder @@ -72,7 +74,8 @@ class AbstractLSFJobManagerSpec extends Specification { def "test conversion of core and node resources with createComputeParameter"(ResourceSet input, LinkedHashMap results) { when: - LinkedHashMultimap parameters = new LinkedHashMultimap<>(1, 1) + LinkedHashMultimap parameters = + LinkedHashMultimap.create(1, 1) jobManager.createComputeParameter(input, parameters) then: @@ -80,14 +83,14 @@ class AbstractLSFJobManagerSpec extends Specification { // You also cannot use Groovy methods on the class parameters.size() == results.size() // It is also incredibly hard to compare the LinkedHashMultimap with "normal" maps. - results["-n"].toString() == parameters.get("-n").toString() + results["-n"] == parameters.get("-n").collect { BashInterpreter.instance.interpret(it) } where: input | results - new ResourceSet(null, 4, -2, null, null, null, null) | [("-n"): ['4 -R "span[hosts=1]"']] - new ResourceSet(null, 4, null, null, null, null, null) | [("-n"): ['4 -R "span[hosts=1]"']] - new ResourceSet(null, 4, 0, null, null, null, null) | [("-n"): ['4 -R "span[hosts=1]"']] - new ResourceSet(null, 4, 1, null, null, null, null) | [("-n"): ['4 -R "span[hosts=1]"']] - new ResourceSet(null, 4, 2, null, null, null, null) | [("-n"): ['8 -R "span[ptile=4]"']] + new ResourceSet(null, 4, -2, null, null, null, null) | [("-n"): ['4'], ("-R"): ['span\\[hosts\\=1]']] + new ResourceSet(null, 4, null, null, null, null, null) | [("-n"): ['4'], ("-R"): ['span\\[hosts\\=1]']] + new ResourceSet(null, 4, 0, null, null, null, null) | [("-n"): ['4'], ("-R"): ['span\\[hosts\\=1]']] + new ResourceSet(null, 4, 1, null, null, null, null) | [("-n"): ['4'], ("-R"): ['span\\[hosts\\=1]']] + new ResourceSet(null, 4, 2, null, null, null, null) | [("-n"): ['8'], ("-R"): ['span\\[ptile\\=4]']] } } diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy index 289b7238..72766e39 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy @@ -253,6 +253,9 @@ class LSFJobManagerSpec extends Specification { equals(laterLastYear.truncatedTo(ChronoUnit.MINUTES)) } + @Shared + Integer currentYear = ZonedDateTime.now().year + @Unroll void "parseTime, parses all known formats (#month #day #hour:#minute:#second #year #suffix #expectedYear)"() { given: @@ -271,6 +274,9 @@ class LSFJobManagerSpec extends Specification { String timestamp = [month, day, time, year, suffix].findAll { it != "" }.join(" ") when: + // NOTE: parseTime is not referentially transparent. Different calls may yield different + // results, because it interprets the timestamp dependent on the comparison of the + // reference time and the **current** time, to resolve (guess) the year. ZonedDateTime result = manager.parseTime(timestamp, referenceDate) LocalDateTime resultTime = result.toLocalDateTime() @@ -283,14 +289,14 @@ class LSFJobManagerSpec extends Specification { resultTime.second == (second ? Integer.parseInt(second) : 0) where: - month | day | hour | minute | second | year | suffix | expectedYear | referenceDate - "Jan" | "01" | "01" | "02" | "" | "" | "" | 2022 | ZonedDateTime.of(2022, 1, 29, 19, 56, 0, 0, ZoneId.systemDefault()) - "Feb" | "02" | "01" | "02" | "" | "" | "L" | 2022 | ZonedDateTime.of(2022, 1, 29, 19, 56, 0, 0, ZoneId.systemDefault()) - "Feb" | "02" | "01" | "02" | "" | "" | "L" | 2022 | ZonedDateTime.of(2022, 2, 3, 19, 56, 0, 0, ZoneId.systemDefault()) - "Mar" | "03" | "01" | "02" | "" | "1000" | "" | 1000 | ZonedDateTime.of(2021, 1, 29, 19, 56, 0, 0, ZoneId.systemDefault()) - "Apr" | "04" | "01" | "02" | "" | "1001" | "L" | 1001 | ZonedDateTime.of(2021, 1, 29, 19, 56, 0, 0, ZoneId.systemDefault()) - "May" | "5" | "01" | "02" | "03" | "1002" | "" | 1002 | ZonedDateTime.of(2021, 1, 29, 19, 56, 0, 0, ZoneId.systemDefault()) - "Jun" | "6" | "01" | "02" | "03" | "1003" | "L" | 1003 | ZonedDateTime.of(2021, 1, 29, 19, 56, 0, 0, ZoneId.systemDefault()) + month | day | hour | minute | second | year | suffix | expectedYear | referenceDate + "Jan" | "01" | "01" | "02" | "" | "" | "" | currentYear | ZonedDateTime.of(currentYear, 1, 29, 19, 56, 0, 0, ZoneId.systemDefault()) + "Feb" | "02" | "01" | "02" | "" | "" | "L" | currentYear - 1 | ZonedDateTime.of(currentYear, 1, 29, 19, 56, 0, 0, ZoneId.systemDefault()) + "Feb" | "02" | "01" | "02" | "" | "" | "L" | currentYear | ZonedDateTime.of(currentYear, 2, 3, 19, 56, 0, 0, ZoneId.systemDefault()) + "Mar" | "03" | "01" | "02" | "" | "1000" | "" | 1000 | ZonedDateTime.of(2021, 1, 29, 19, 56, 0, 0, ZoneId.systemDefault()) + "Apr" | "04" | "01" | "02" | "" | "1001" | "L" | 1001 | ZonedDateTime.of(2021, 1, 29, 19, 56, 0, 0, ZoneId.systemDefault()) + "May" | "5" | "01" | "02" | "03" | "1002" | "" | 1002 | ZonedDateTime.of(2021, 1, 29, 19, 56, 0, 0, ZoneId.systemDefault()) + "Jun" | "6" | "01" | "02" | "03" | "1003" | "L" | 1003 | ZonedDateTime.of(2021, 1, 29, 19, 56, 0, 0, ZoneId.systemDefault()) } // void "test queryExtendedJobStateById with overdue date"() { @@ -355,7 +361,14 @@ class LSFJobManagerSpec extends Specification { jobInfo.jobID == new BEJobID("22005") // The year-parsing/inference is checked in another test. Here just take the parsed value. - ZonedDateTime testTime = ZonedDateTime.of(jobInfo.submitTime.year, 12, 28, 19, 56, 0, 0, ZoneId.systemDefault()) + ZonedDateTime testTime = ZonedDateTime.of(jobInfo.submitTime.year, + 12, + 28, + 19, + 56, + 0, + 0, + ZoneId.systemDefault()) jobInfo.submitTime == testTime jobInfo.eligibleTime == null jobInfo.startTime == testTime diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFSubmissionCommandSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFSubmissionCommandSpec.groovy index 103a90c9..a312a933 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFSubmissionCommandSpec.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFSubmissionCommandSpec.groovy @@ -6,10 +6,13 @@ package de.dkfz.roddy.execution.jobs.cluster.lsf +import de.dkfz.roddy.tools.EscapableString import de.dkfz.roddy.config.JobLog import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.config.ResourceSetSize +import de.dkfz.roddy.execution.* import de.dkfz.roddy.execution.jobs.BEJob +import de.dkfz.roddy.execution.jobs.BEJobID import de.dkfz.roddy.execution.jobs.JobManagerOptions import de.dkfz.roddy.execution.jobs.TestHelper import de.dkfz.roddy.tools.BufferUnit @@ -17,90 +20,158 @@ import de.dkfz.roddy.tools.BufferValue import de.dkfz.roddy.tools.TimeUnit import spock.lang.Specification +import java.nio.file.Paths + +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + class LSFSubmissionCommandSpec extends Specification { LSFJobManager jobManager = new LSFJobManager(TestHelper.makeExecutionService(), JobManagerOptions.create().build()) - private BEJob makeJob(Map mapOfParameters, String accountingProject = null) { - BEJob job = new BEJob(null, "Test", new File("/tmp/test.sh"), - null,null, - new ResourceSet(ResourceSetSize.l, new BufferValue(1, BufferUnit.G), 4, - 1, new TimeUnit("1h"), null, null, null), - [], mapOfParameters, jobManager, JobLog.none(), null, accountingProject) + private BEJob makeJob(Map mapOfParameters, + CommandI command = new Command(new Executable(Paths.get("/tmp/test.sh")), + [u("\$someRemoteVariable")]), + // Here, someRemoteVariable is *not* quoted. The Command should take the values, + // like they are supposed to be used at the call site. + EscapableString accountingProject = null) { + BEJob job = new BEJob( + null, + jobManager, + u("Test"), + command, + new ResourceSet(ResourceSetSize.l, + new BufferValue(1, BufferUnit.G), + 4, + 1, + new TimeUnit("1h"), + null, + null, + null), + [new BEJob(new BEJobID("a"), jobManager, u("a")), + new BEJob(new BEJobID("b"), jobManager, u("b"))], + mapOfParameters, + JobLog.none(), + null, + accountingProject) job } def "assemble dependency string without dependencies" () throws Exception { when: - def mapOfVars = ["a": "a", "b": "b"] + Map mapOfVars = ["a": u("a"), "b": u("b")] BEJob job = makeJob(mapOfVars) - LSFSubmissionCommand cmd = new LSFSubmissionCommand(jobManager, makeJob(mapOfVars), - "jobName", null, mapOfVars, - null, "/tmp/test.sh") + LSFSubmissionCommand cmd = new LSFSubmissionCommand( + jobManager, + makeJob(mapOfVars), + u("jobName"), + null, + mapOfVars) then: - cmd.assembleDependencyParameter([]) == "" + cmd.assembleDependencyParameter([]) == c() } def "assemble variable export parameters with no variables" () { when: - LSFSubmissionCommand cmd = new LSFSubmissionCommand(jobManager, makeJob([:]), - "jobName", null, [:], - null, "/tmp/test.sh") + LSFSubmissionCommand cmd = new LSFSubmissionCommand( + jobManager, + makeJob([:]), + u("jobName"), + null, + [:]) then: - cmd.assembleVariableExportParameters() == "-env \"none\"" + cmd.assembleVariableExportParameters() == + c(u("-env"), u(" "), u("none")) } def "assemble variable export parameters with only variables" () { when: - Map mapOfVars = ["a": "a", "b": null] as LinkedHashMap - LSFSubmissionCommand cmd = new LSFSubmissionCommand(jobManager, makeJob(mapOfVars), - "jobName", null, mapOfVars, - null, "/tmp/test.sh", ) + Map mapOfVars = + ["a": e("a"), "b": null] + LSFSubmissionCommand cmd = new LSFSubmissionCommand( + jobManager, + makeJob(mapOfVars), + u("jobName"), + null, + mapOfVars) then: - cmd.assembleVariableExportParameters() == "-env \"a=a, b\"" + cmd.assembleVariableExportParameters() == + c(u("-env"), u(" "), u("a"), u("="), e("a"), e(", "), u("b")) } def "assemble variable export parameters with 'all' variables" () { when: - LSFSubmissionCommand cmd = new LSFSubmissionCommand(jobManager, makeJob([:] as LinkedHashMap), - "jobName", null, [:], - null, "/tmp/test.sh") + LSFSubmissionCommand cmd = new LSFSubmissionCommand( + jobManager, + makeJob([:] as LinkedHashMap), + u("jobName"), + null, + [:]) cmd.passEnvironment = Optional.of(true) then: - cmd.assembleVariableExportParameters() == "-env \"all\"" + cmd.assembleVariableExportParameters() == c(u("-env"), u(" "), u("all")) } def "assemble variable export parameters with 'all' and explicit variables" () { when: - Map mapOfVars = ["a": "a", "b": null] as LinkedHashMap - LSFSubmissionCommand cmd = new LSFSubmissionCommand(jobManager, makeJob(mapOfVars as LinkedHashMap), - "jobName", null, mapOfVars, - null,"/tmp/test.sh") + Map mapOfVars = ["a": u("a"), "b": null] + LSFSubmissionCommand cmd = new LSFSubmissionCommand( + jobManager, + makeJob(mapOfVars), + u("jobName"), + null, + mapOfVars) cmd.passEnvironment = Optional.of(true) then: - cmd.assembleVariableExportParameters() == "-env \"all, a=a, b\"" + cmd.assembleVariableExportParameters() == + c(u("-env"), u(" "), u("all"), e(", "), u("a"), u("="), u("a"), e(", "), u("b")) } def "command without accounting name" () { when: - LSFSubmissionCommand cmd = new LSFSubmissionCommand(jobManager, makeJob([:]), "jobname", - null, [:], null, "/tmp/test.sh") + LSFSubmissionCommand cmd = new LSFSubmissionCommand( + jobManager, + makeJob([:]), + u("jobname"), + null, + [:]) then: - cmd.toBashCommandString() == 'LSB_NTRIES=5 bsub -env "none" -J jobname -H -cwd $HOME -o /dev/null -M 1024 -R "rusage[mem=1024]" -W 60 -n 4 -R "span[hosts=1]" /tmp/test.sh' + cmd.toBashCommandString() == 'LSB_NTRIES=5 bsub -env none -J jobname -H -cwd "$HOME" -o /dev/null -M 1024 -R rusage\\[mem\\=1024] -R span\\[hosts\\=1] -W 60 -n 4 -ti -w done\\(a\\)\\ \\&\\&\\ done\\(b\\) /tmp/test.sh\\ \\$someRemoteVariable' } def "command with accounting name" () { when: - LSFSubmissionCommand cmd = new LSFSubmissionCommand(jobManager, makeJob([:], "accountingProject"), - "jobname",null, [:], null, "/tmp/test.sh") + LSFSubmissionCommand cmd = new LSFSubmissionCommand( + jobManager, + makeJob([:], + new Command(new Executable(Paths.get("/tmp/test.sh")), + [u("\$someRemoteVariable")]), + u("accountingProject")), + u("jobname"), + null, + [:]) + + then: + cmd.toBashCommandString() == 'LSB_NTRIES=5 bsub -env none -P accountingProject -J jobname -H -cwd "$HOME" -o /dev/null -M 1024 -R rusage\\[mem\\=1024] -R span\\[hosts\\=1] -W 60 -n 4 -ti -w done\\(a\\)\\ \\&\\&\\ done\\(b\\) /tmp/test.sh\\ \\\$someRemoteVariable' + } + def "submitting a script as code"() { + given: + Code code = new Code("echo 'Hello World';\\n") + when: + LSFSubmissionCommand cmd = new LSFSubmissionCommand( + jobManager, + makeJob([:], + code), + u("jobname"), + null, + [:]) then: - cmd.toBashCommandString() == 'LSB_NTRIES=5 bsub -env "none" -P "accountingProject" -J jobname -H -cwd $HOME -o /dev/null -M 1024 -R "rusage[mem=1024]" -W 60 -n 4 -R "span[hosts=1]" /tmp/test.sh' + cmd.toBashCommandString() == "echo -ne \\#'!'/bin/bash\\\\necho\\ \\'Hello\\ World\\'\\;\\\\n | LSB_NTRIES=5 bsub -env none -J jobname -H -cwd \"\$HOME\" -o /dev/null -M 1024 -R rusage\\[mem\\=1024] -R span\\[hosts\\=1] -W 60 -n 4 -ti -w done\\(a\\)\\ \\&\\&\\ done\\(b\\) " } diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandTest.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandTest.groovy index a4a5794f..a8c64d7e 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandTest.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandTest.groovy @@ -9,6 +9,9 @@ package de.dkfz.roddy.execution.jobs.cluster.pbs import de.dkfz.roddy.config.JobLog import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.config.ResourceSetSize +import de.dkfz.roddy.tools.EscapableString +import de.dkfz.roddy.tools.BashInterpreter +import de.dkfz.roddy.execution.Executable import de.dkfz.roddy.execution.jobs.BEJob import de.dkfz.roddy.execution.jobs.JobManagerOptions import de.dkfz.roddy.execution.jobs.TestHelper @@ -19,6 +22,10 @@ import groovy.transform.CompileStatic import org.junit.Before import org.junit.Test +import java.nio.file.Paths + +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + /** * Created by heinold on 26.03.17. */ @@ -33,40 +40,77 @@ class PBSCommandTest { jobManager = new PBSJobManager(TestHelper.makeExecutionService(), JobManagerOptions.create().build()) } - private BEJob makeJob(Map mapOfParameters) { - BEJob job = new BEJob(null, "Test", new File("/tmp/test.sh"), null, null, new ResourceSet(ResourceSetSize.l, new BufferValue(1, BufferUnit.G), 4, 1, new TimeUnit("1h"), null, null, null), [], mapOfParameters, jobManager, JobLog.none(), null) + private BEJob makeJob(Map mapOfParameters) { + BEJob job = new BEJob( + null, + jobManager, + u("Test"), + new Executable(Paths.get("/tmp/test.sh")), + new ResourceSet( + ResourceSetSize.l, + new BufferValue(1, BufferUnit.G), + 4, + 1, + new TimeUnit("1h"), + null, + null, + null), + [], + mapOfParameters, + JobLog.none(), + null) job } @Test void testAssembleVariableExportParameters_nothing() { - PBSSubmissionCommand cmd = new PBSSubmissionCommand(jobManager, makeJob([:]), - "jobName", null, [:], null, "/tmp/test.sh") - assert cmd.assembleVariableExportParameters() == "" + PBSSubmissionCommand cmd = new PBSSubmissionCommand( + jobManager, + makeJob([:]), + u("jobName"), + null, + [:] as Map) + assert cmd.assembleVariableExportParameters() == c() } @Test void testAssembleVariableExportParameters_onlyVars() { - Map mapOfVars = ["a": "a", "b": null] as LinkedHashMap - PBSSubmissionCommand cmd = new PBSSubmissionCommand(jobManager, makeJob(mapOfVars), - "jobName", null, mapOfVars, null, "/tmp/test.sh") - assert cmd.assembleVariableExportParameters() == "-v \"a=a,b\"" + Map mapOfVars = + ["a": u("a"), "b": null] as LinkedHashMap + PBSSubmissionCommand cmd = new PBSSubmissionCommand( + jobManager, + makeJob(mapOfVars), + u("jobName"), + null, + mapOfVars) + assert cmd.assembleVariableExportParameters() == + c(u("-v "), u("a"), e("="), u("a"), u(","), u("b")) } @Test void testAssembleVariableExportParameters_allVars() { - PBSSubmissionCommand cmd = new PBSSubmissionCommand(jobManager, makeJob([:] as LinkedHashMap), - "jobName", null, [:], null, "/tmp/test.sh") + PBSSubmissionCommand cmd = new PBSSubmissionCommand( + jobManager, + makeJob([:] as LinkedHashMap), + u("jobName"), + null, + [:] as Map) cmd.passEnvironment = Optional.of(true) - assert cmd.assembleVariableExportParameters() == "-V" + assert cmd.assembleVariableExportParameters() == c(u("-V")) } @Test void testAssembleVariableExportParameters_allVarsAndExplicit() { - Map mapOfVars = ["a": "a", "b": null] as LinkedHashMap - PBSSubmissionCommand cmd = new PBSSubmissionCommand(jobManager, makeJob(mapOfVars as LinkedHashMap), - "jobName", null, mapOfVars, null, "/tmp/test.sh") + Map mapOfVars = + ["a": e("a!"), "b": null] as LinkedHashMap + PBSSubmissionCommand cmd = new PBSSubmissionCommand( + jobManager, + makeJob(mapOfVars), + u("jobName"), + null, + mapOfVars) cmd.passEnvironment = Optional.of(true) - assert cmd.assembleVariableExportParameters() == "-V -v \"a=a,b\"" + assert BashInterpreter.instance.interpret(cmd.assembleVariableExportParameters()) == + "-V -v a\\=a'!',b" } } diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmSubmissionCommandSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmSubmissionCommandSpec.groovy index 91797634..ad0582ad 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmSubmissionCommandSpec.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmSubmissionCommandSpec.groovy @@ -3,6 +3,11 @@ package de.dkfz.roddy.execution.jobs.cluster.slurm import de.dkfz.roddy.config.JobLog import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.config.ResourceSetSize +import de.dkfz.roddy.execution.jobs.BEJobID +import de.dkfz.roddy.tools.EscapableString +import de.dkfz.roddy.execution.Code +import de.dkfz.roddy.execution.CommandI +import de.dkfz.roddy.execution.Executable import de.dkfz.roddy.execution.jobs.BEJob import de.dkfz.roddy.execution.jobs.JobManagerOptions import de.dkfz.roddy.execution.jobs.TestHelper @@ -11,84 +16,158 @@ import de.dkfz.roddy.tools.BufferValue import de.dkfz.roddy.tools.TimeUnit import spock.lang.Specification +import java.nio.file.Paths + +import static de.dkfz.roddy.tools.EscapableString.Shortcuts.* + class SlurmSubmissionCommandSpec extends Specification { SlurmJobManager jobManager = new SlurmJobManager(TestHelper.makeExecutionService(), JobManagerOptions.create().build()) - private BEJob makeJob(Map mapOfParameters, String accountingProject = null) { - BEJob job = new BEJob(null, "Test", new File("/tmp/test.sh"), - null, null, - new ResourceSet(ResourceSetSize.l, new BufferValue(1, BufferUnit.G), 4, - 1, new TimeUnit("1h"), null, null, null), - [], mapOfParameters, jobManager, JobLog.none(), null, accountingProject) + private BEJob makeJob( + Map mapOfParameters, + CommandI command, + EscapableString accountingProject = null) { + BEJob job = new BEJob( + null, + jobManager, + u("Test"), + command, + new ResourceSet( + ResourceSetSize.l, + new BufferValue(1, BufferUnit.G), + 4, + 1, + new TimeUnit("1h"), + null, + null, + null), + [], + mapOfParameters, + JobLog.none(), + null, + accountingProject) job } def "assemble dependency string without dependencies"() throws Exception { when: - def mapOfVars = ["a": "a", "b": "b"] - BEJob job = makeJob(mapOfVars) - SlurmSubmissionCommand cmd = new SlurmSubmissionCommand(jobManager, makeJob(mapOfVars), - "jobName", null, mapOfVars, - null, "/tmp/test.sh") + Map mapOfVars = ["a": u("a"), "b": u("b")] + BEJob job = makeJob(mapOfVars, + new Executable(Paths.get("/tmp/test.sh"))) + SlurmSubmissionCommand cmd = new SlurmSubmissionCommand( + jobManager, + makeJob(mapOfVars, + new Executable(Paths.get("/tmp/test.sh"))), + u("jobName"), + null, + mapOfVars) then: - cmd.assembleDependencyParameter([]) == "" + cmd.assembleDependencyParameter([]) == c() + cmd.assembleDependencyParameter([new BEJobID("1"), new BEJobID("2")]) == + c([u(" --dependency="), u("afterok"), u(":"), e("1"), u(":"), e("2")]) } def "assemble variable export parameters with no variables"() { when: - SlurmSubmissionCommand cmd = new SlurmSubmissionCommand(jobManager, makeJob([:]), - "jobName", null, [:], - null, "/tmp/test.sh") + SlurmSubmissionCommand cmd = new SlurmSubmissionCommand( + jobManager, + makeJob([:] as Map, + new Executable(Paths.get("/tmp/test.sh"))), + u("jobName"), + null, + [:]) then: - cmd.assembleVariableExportParameters() == "" + cmd.assembleVariableExportParameters() == c() } def "assemble variable export parameters with only variables"() { when: - Map mapOfVars = ["a": "a", "b": null] as LinkedHashMap - SlurmSubmissionCommand cmd = new SlurmSubmissionCommand(jobManager, makeJob(mapOfVars), - "jobName", null, mapOfVars, - null, "/tmp/test.sh",) + Map mapOfVars = ["a": u("a"), "b": null] + SlurmSubmissionCommand cmd = new SlurmSubmissionCommand( + jobManager, + makeJob(mapOfVars, + new Executable(Paths.get("/tmp/test.sh"))), + u("jobName"), + null, + mapOfVars) then: - cmd.assembleVariableExportParameters() == "--export=\"a=a,b\"" + cmd.assembleVariableExportParameters() == + c(u("--export="), u("a"), e("="), u("a"), u(","), u("b")) } def "assemble variable export parameters with 'all' variables"() { when: - SlurmSubmissionCommand cmd = new SlurmSubmissionCommand(jobManager, makeJob([:] as LinkedHashMap), - "jobName", null, [:], - null, "/tmp/test.sh") + SlurmSubmissionCommand cmd = new SlurmSubmissionCommand( + jobManager, + makeJob([:] as LinkedHashMap, + new Executable(Paths.get("/tmp/test.sh"))), + u("jobName"), + null, + [:]) cmd.passEnvironment = Optional.of(true) then: - cmd.assembleVariableExportParameters() == "--get-user-env " + cmd.assembleVariableExportParameters() == c(u("--get-user-env ")) } def "assemble variable export parameters with 'all' and explicit variables"() { when: - Map mapOfVars = ["a": "a", "b": null] as LinkedHashMap - SlurmSubmissionCommand cmd = new SlurmSubmissionCommand(jobManager, makeJob(mapOfVars as LinkedHashMap), - "jobName", null, mapOfVars, - null, "/tmp/test.sh") + Map mapOfVars = ["a": u("a"), "b": null] + SlurmSubmissionCommand cmd = new SlurmSubmissionCommand( + jobManager, + makeJob(mapOfVars as LinkedHashMap, + new Executable(Paths.get("/tmp/test.sh"))), + u("jobName"), + null, + mapOfVars) cmd.passEnvironment = Optional.of(true) then: - cmd.assembleVariableExportParameters() == "--get-user-env --export=\"a=a,b\"" + cmd.assembleVariableExportParameters() == + c(u("--get-user-env "), u("--export="), u("a"), e("="), u("a"), u(","), u("b")) } def "command without accounting name"() { when: - SlurmSubmissionCommand cmd = new SlurmSubmissionCommand(jobManager, makeJob([:]), "jobname", - null, [:], null, "/tmp/test.sh") + SlurmSubmissionCommand cmd = new SlurmSubmissionCommand( + jobManager, + makeJob([:] as Map, + new Executable(Paths.get("/tmp/test.sh"))), + u("jobname"), + null, + [:]) then: - cmd.toBashCommandString() == 'sbatch --job-name jobname --hold --chdir $HOME --mem=1024M --time=1:00:00 --nodes=1 --cores-per-socket=4 --parsable --kill-on-invalid-dep=yes --propagate=none /tmp/test.sh' + cmd.toBashCommandString() == 'sbatch --job-name jobname --hold --chdir "$HOME" --mem=1024M --time=1:00:00 --nodes=1 --cores-per-socket=4 --parsable --kill-on-invalid-dep=yes --propagate=NONE /tmp/test.sh' } def "command with accounting name"() { when: - SlurmSubmissionCommand cmd = new SlurmSubmissionCommand(jobManager, makeJob([:], "accountingProject"), - "jobname", null, [:], null, "/tmp/test.sh") + SlurmSubmissionCommand cmd = new SlurmSubmissionCommand( + jobManager, + makeJob([:] as Map, + new Executable(Paths.get("/tmp/test.sh")), + u("accountingProject")), + u("jobname"), + null, + [:]) + then: + cmd.toBashCommandString() == 'sbatch --account=accountingProject --job-name jobname --hold --chdir "$HOME" --mem=1024M --time=1:00:00 --nodes=1 --cores-per-socket=4 --parsable --kill-on-invalid-dep=yes --propagate=NONE /tmp/test.sh' + } + + def "submit script as code"() { + given: + BEJob job = makeJob([:], + new Code("echo 'Hello World'\n")) + + when: + SlurmSubmissionCommand cmd = new SlurmSubmissionCommand( + jobManager, + job, + u("jobname"), + null, + [:]) then: - cmd.toBashCommandString() == 'sbatch --account="accountingProject" --job-name jobname --hold --chdir $HOME --mem=1024M --time=1:00:00 --nodes=1 --cores-per-socket=4 --parsable --kill-on-invalid-dep=yes --propagate=none /tmp/test.sh' + cmd.toBashCommandString() == "echo -ne \\#'!'/bin/bash\\\\necho\\ \\'Hello\\ World\\'\\\\n | sbatch --job-name jobname --hold --chdir \"\$HOME\" --mem=1024M --time=1:00:00 --nodes=1 --cores-per-socket=4 --parsable --kill-on-invalid-dep=yes --propagate=NONE /dev/stdin" + } }