From 6c004eb7934dc53d656010ec5d1889a28310c510 Mon Sep 17 00:00:00 2001 From: Kai Waldrant Date: Wed, 7 Aug 2024 10:50:22 +0200 Subject: [PATCH] remove executables --- create_component/.build.yaml | 0 create_component/.config.vsh.yaml | 221 ---- create_component/create_component | 1610 ------------------------- create_task_readme/create_task_readme | 1171 ------------------ create_task_readme/read_api_files.R | 432 ------- sync_resources/.build.yaml | 0 sync_resources/.config.vsh.yaml | 184 --- sync_resources/sync_resources | 1193 ------------------ 8 files changed, 4811 deletions(-) delete mode 100644 create_component/.build.yaml delete mode 100644 create_component/.config.vsh.yaml delete mode 100755 create_component/create_component delete mode 100755 create_task_readme/create_task_readme delete mode 100644 create_task_readme/read_api_files.R delete mode 100644 sync_resources/.build.yaml delete mode 100644 sync_resources/.config.vsh.yaml delete mode 100755 sync_resources/sync_resources diff --git a/create_component/.build.yaml b/create_component/.build.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/create_component/.config.vsh.yaml b/create_component/.config.vsh.yaml deleted file mode 100644 index 275d1f3..0000000 --- a/create_component/.config.vsh.yaml +++ /dev/null @@ -1,221 +0,0 @@ -name: "create_component" -namespace: "common" -version: "dev" -argument_groups: -- name: "Arguments" - arguments: - - type: "string" - name: "--task" - description: "Which task the component will be added to." - info: null - example: - - "denoising" - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--type" - description: "The type of component to create. Typically must be one of 'method',\ - \ 'control_method' or 'metric'." - info: null - example: - - "metric" - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--language" - description: "Which scripting language to use. Options are 'python', 'r'." - info: null - default: - - "python" - required: false - choices: - - "python" - - "r" - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--name" - description: "Name of the new method, formatted in snake case." - info: null - example: - - "new_comp" - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "file" - name: "--output" - description: "Path to the component directory. Suggested location is `src//s/`." - info: null - default: - - "src/tasks/${VIASH_PAR_TASK}/${VIASH_PAR_TYPE}s/${VIASH_PAR_NAME}" - must_exist: true - create_parent: true - required: false - direction: "output" - multiple: false - multiple_sep: ";" - - type: "file" - name: "--api_file" - description: "Which API file to use. Defaults to `src//api/comp_.yaml`.\n\ - In tasks with different subtypes of method, this location might not exist and\ - \ you might need\nto manually specify a different API file to inherit from.\n" - info: null - default: - - "src/tasks/${VIASH_PAR_TASK}/api/comp_${VIASH_PAR_TYPE}.yaml" - must_exist: false - create_parent: true - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "file" - name: "--viash_yaml" - description: "Path to the project config file. Needed for knowing the relative\ - \ location of a file to the project root.\n" - info: null - default: - - "_viash.yaml" - must_exist: true - create_parent: true - required: false - direction: "input" - multiple: false - multiple_sep: ";" -resources: -- type: "python_script" - path: "script.py" - is_executable: true -description: "Create a component Viash component.\n\nUsage:\n```\nbin/create_component\ - \ --task denoising --type method --language r --name foo\nbin/create_component --task\ - \ denoising --type metric --language python --name bar\n```\n" -test_resources: -- type: "python_script" - path: "test.py" - is_executable: true -info: null -status: "enabled" -license: "MIT" -links: - repository: "https://github.com/openproblems-bio/common_resources" - docker_registry: "ghcr.io" -runners: -- type: "executable" - id: "executable" - docker_setup_strategy: "ifneedbepullelsecachedbuild" -- type: "nextflow" - id: "nextflow" - directives: - tag: "$id" - auto: - simplifyInput: true - simplifyOutput: false - transcript: false - publish: false - config: - labels: - mem1gb: "memory = 1000000000.B" - mem2gb: "memory = 2000000000.B" - mem5gb: "memory = 5000000000.B" - mem10gb: "memory = 10000000000.B" - mem20gb: "memory = 20000000000.B" - mem50gb: "memory = 50000000000.B" - mem100gb: "memory = 100000000000.B" - mem200gb: "memory = 200000000000.B" - mem500gb: "memory = 500000000000.B" - mem1tb: "memory = 1000000000000.B" - mem2tb: "memory = 2000000000000.B" - mem5tb: "memory = 5000000000000.B" - mem10tb: "memory = 10000000000000.B" - mem20tb: "memory = 20000000000000.B" - mem50tb: "memory = 50000000000000.B" - mem100tb: "memory = 100000000000000.B" - mem200tb: "memory = 200000000000000.B" - mem500tb: "memory = 500000000000000.B" - mem1gib: "memory = 1073741824.B" - mem2gib: "memory = 2147483648.B" - mem4gib: "memory = 4294967296.B" - mem8gib: "memory = 8589934592.B" - mem16gib: "memory = 17179869184.B" - mem32gib: "memory = 34359738368.B" - mem64gib: "memory = 68719476736.B" - mem128gib: "memory = 137438953472.B" - mem256gib: "memory = 274877906944.B" - mem512gib: "memory = 549755813888.B" - mem1tib: "memory = 1099511627776.B" - mem2tib: "memory = 2199023255552.B" - mem4tib: "memory = 4398046511104.B" - mem8tib: "memory = 8796093022208.B" - mem16tib: "memory = 17592186044416.B" - mem32tib: "memory = 35184372088832.B" - mem64tib: "memory = 70368744177664.B" - mem128tib: "memory = 140737488355328.B" - mem256tib: "memory = 281474976710656.B" - mem512tib: "memory = 562949953421312.B" - cpu1: "cpus = 1" - cpu2: "cpus = 2" - cpu5: "cpus = 5" - cpu10: "cpus = 10" - cpu20: "cpus = 20" - cpu50: "cpus = 50" - cpu100: "cpus = 100" - cpu200: "cpus = 200" - cpu500: "cpus = 500" - cpu1000: "cpus = 1000" - debug: false - container: "docker" -engines: -- type: "docker" - id: "docker" - image: "python:3.10-slim" - namespace_separator: "/" - setup: - - type: "apt" - packages: - - "git" - interactive: false - - type: "python" - user: false - pypi: - - "ruamel.yaml" - - "git+https://github.com/openproblems-bio/core.git#egg=openproblems&subdirectory=packages/python/openproblems" - upgrade: true - test_setup: - - type: "docker" - run: - - "git clone https://github.com/openproblems-bio/task_template.git /tmp/task_template\n" - entrypoint: [] - cmd: null -build_info: - config: "src/create_component/config.vsh.yaml" - runner: "executable" - engine: "docker" - output: "create_component" - executable: "create_component/create_component" - viash_version: "0.9.0-RC6" - git_commit: "66a108d61ca7ef1a18e88ecf56afc23c616aa3cb" - git_remote: "git@github.com:openproblems-bio/common-resources.git" -package_config: - name: "common_resources" - version: "dev" - description: "Common resources for the OpenProblems project.\n" - info: null - viash_version: "0.9.0-RC6" - source: "src" - target: "" - keywords: - - "openproblems" - - "common-resources" - - "single-cell" - - "benchmark" - license: "MIT" - organization: "openproblems-bio" - links: - repository: "https://github.com/openproblems-bio/common_resources" - docker_registry: "ghcr.io" - issue_tracker: "https://github.com/openproblems-bio/common_resources/issues" diff --git a/create_component/create_component b/create_component/create_component deleted file mode 100755 index 4172d6f..0000000 --- a/create_component/create_component +++ /dev/null @@ -1,1610 +0,0 @@ -#!/usr/bin/env bash - -# create_component dev -# -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a -# derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from -# Data Intuitive. -# -# The component may contain files which fall under a different license. The -# authors of this component should specify the license in the header of such -# files, or include a separate license file detailing the licenses of all included -# files. - -set -e - -if [ -z "$VIASH_TEMP" ]; then - VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMPDIR} - VIASH_TEMP=${VIASH_TEMP:-$VIASH_TEMPDIR} - VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMP} - VIASH_TEMP=${VIASH_TEMP:-$TMPDIR} - VIASH_TEMP=${VIASH_TEMP:-$TMP} - VIASH_TEMP=${VIASH_TEMP:-$TEMPDIR} - VIASH_TEMP=${VIASH_TEMP:-$TEMP} - VIASH_TEMP=${VIASH_TEMP:-/tmp} -fi - -# define helper functions -# ViashQuote: put quotes around non flag values -# $1 : unquoted string -# return : possibly quoted string -# examples: -# ViashQuote --foo # returns --foo -# ViashQuote bar # returns 'bar' -# Viashquote --foo=bar # returns --foo='bar' -function ViashQuote { - if [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+=.+$ ]]; then - echo "$1" | sed "s#=\(.*\)#='\1'#" - elif [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+$ ]]; then - echo "$1" - else - echo "'$1'" - fi -} -# ViashRemoveFlags: Remove leading flag -# $1 : string with a possible leading flag -# return : string without possible leading flag -# examples: -# ViashRemoveFlags --foo=bar # returns bar -function ViashRemoveFlags { - echo "$1" | sed 's/^--*[a-zA-Z0-9_\-]*=//' -} -# ViashSourceDir: return the path of a bash file, following symlinks -# usage : ViashSourceDir ${BASH_SOURCE[0]} -# $1 : Should always be set to ${BASH_SOURCE[0]} -# returns : The absolute path of the bash file -function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" - done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd -} -# ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks -# usage : ViashFindTargetDir 'ScriptPath' -# $1 : The location from where to start the upward search -# returns : The absolute path of the '.build.yaml' file -function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} - done - echo $SOURCE -} -# see https://en.wikipedia.org/wiki/Syslog#Severity_level -VIASH_LOGCODE_EMERGENCY=0 -VIASH_LOGCODE_ALERT=1 -VIASH_LOGCODE_CRITICAL=2 -VIASH_LOGCODE_ERROR=3 -VIASH_LOGCODE_WARNING=4 -VIASH_LOGCODE_NOTICE=5 -VIASH_LOGCODE_INFO=6 -VIASH_LOGCODE_DEBUG=7 -VIASH_VERBOSITY=$VIASH_LOGCODE_NOTICE - -# ViashLog: Log events depending on the verbosity level -# usage: ViashLog 1 alert Oh no something went wrong! -# $1: required verbosity level -# $2: display tag -# $3+: messages to display -# stdout: Your input, prepended by '[$2] '. -function ViashLog { - local required_level="$1" - local display_tag="$2" - shift 2 - if [ $VIASH_VERBOSITY -ge $required_level ]; then - >&2 echo "[$display_tag]" "$@" - fi -} - -# ViashEmergency: log events when the system is unstable -# usage: ViashEmergency Oh no something went wrong. -# stdout: Your input, prepended by '[emergency] '. -function ViashEmergency { - ViashLog $VIASH_LOGCODE_EMERGENCY emergency "$@" -} - -# ViashAlert: log events when actions must be taken immediately (e.g. corrupted system database) -# usage: ViashAlert Oh no something went wrong. -# stdout: Your input, prepended by '[alert] '. -function ViashAlert { - ViashLog $VIASH_LOGCODE_ALERT alert "$@" -} - -# ViashCritical: log events when a critical condition occurs -# usage: ViashCritical Oh no something went wrong. -# stdout: Your input, prepended by '[critical] '. -function ViashCritical { - ViashLog $VIASH_LOGCODE_CRITICAL critical "$@" -} - -# ViashError: log events when an error condition occurs -# usage: ViashError Oh no something went wrong. -# stdout: Your input, prepended by '[error] '. -function ViashError { - ViashLog $VIASH_LOGCODE_ERROR error "$@" -} - -# ViashWarning: log potentially abnormal events -# usage: ViashWarning Something may have gone wrong. -# stdout: Your input, prepended by '[warning] '. -function ViashWarning { - ViashLog $VIASH_LOGCODE_WARNING warning "$@" -} - -# ViashNotice: log significant but normal events -# usage: ViashNotice This just happened. -# stdout: Your input, prepended by '[notice] '. -function ViashNotice { - ViashLog $VIASH_LOGCODE_NOTICE notice "$@" -} - -# ViashInfo: log normal events -# usage: ViashInfo This just happened. -# stdout: Your input, prepended by '[info] '. -function ViashInfo { - ViashLog $VIASH_LOGCODE_INFO info "$@" -} - -# ViashDebug: log all events, for debugging purposes -# usage: ViashDebug This just happened. -# stdout: Your input, prepended by '[debug] '. -function ViashDebug { - ViashLog $VIASH_LOGCODE_DEBUG debug "$@" -} - -# find source folder of this component -VIASH_META_RESOURCES_DIR=`ViashSourceDir ${BASH_SOURCE[0]}` - -# find the root of the built components & dependencies -VIASH_TARGET_DIR=`ViashFindTargetDir $VIASH_META_RESOURCES_DIR` - -# define meta fields -VIASH_META_NAME="create_component" -VIASH_META_FUNCTIONALITY_NAME="create_component" -VIASH_META_EXECUTABLE="$VIASH_META_RESOURCES_DIR/$VIASH_META_NAME" -VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" -VIASH_META_TEMP_DIR="$VIASH_TEMP" - - -# ViashHelp: Display helpful explanation about this executable -function ViashHelp { - echo "create_component dev" - echo "" - echo "Create a component Viash component." - echo "" - echo "Usage:" - echo "\`\`\`" - echo "bin/create_component --task denoising --type method --language r --name foo" - echo "bin/create_component --task denoising --type metric --language python --name bar" - echo "\`\`\`" - echo "" - echo "Arguments:" - echo " --task" - echo " type: string" - echo " example: denoising" - echo " Which task the component will be added to." - echo "" - echo " --type" - echo " type: string" - echo " example: metric" - echo " The type of component to create. Typically must be one of 'method'," - echo " 'control_method' or 'metric'." - echo "" - echo " --language" - echo " type: string" - echo " default: python" - echo " choices: [ python, r ]" - echo " Which scripting language to use. Options are 'python', 'r'." - echo "" - echo " --name" - echo " type: string" - echo " example: new_comp" - echo " Name of the new method, formatted in snake case." - echo "" - echo " --output" - echo " type: file, output, file must exist" - echo " default:" - echo "src/tasks/\${VIASH_PAR_TASK}/\${VIASH_PAR_TYPE}s/\${VIASH_PAR_NAME}" - echo " Path to the component directory. Suggested location is" - echo " \`src//s/\`." - echo "" - echo " --api_file" - echo " type: file" - echo " default: src/tasks/\${VIASH_PAR_TASK}/api/comp_\${VIASH_PAR_TYPE}.yaml" - echo " Which API file to use. Defaults to \`src//api/comp_.yaml\`." - echo " In tasks with different subtypes of method, this location might not" - echo " exist and you might need" - echo " to manually specify a different API file to inherit from." - echo "" - echo " --viash_yaml" - echo " type: file, file must exist" - echo " default: _viash.yaml" - echo " Path to the project config file. Needed for knowing the relative" - echo " location of a file to the project root." -} - -# initialise variables -VIASH_MODE='run' -VIASH_ENGINE_ID='docker' - -######## Helper functions for setting up Docker images for viash ######## -# expects: ViashDockerBuild - -# ViashDockerInstallationCheck: check whether Docker is installed correctly -# -# examples: -# ViashDockerInstallationCheck -function ViashDockerInstallationCheck { - ViashDebug "Checking whether Docker is installed" - if [ ! command -v docker &> /dev/null ]; then - ViashCritical "Docker doesn't seem to be installed. See 'https://docs.docker.com/get-docker/' for instructions." - exit 1 - fi - - ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? - [[ $save =~ e ]] && set -e - if [ $out -ne 0 ]; then - ViashCritical "Docker daemon does not seem to be running. Try one of the following:" - ViashCritical "- Try running 'dockerd' in the command line" - ViashCritical "- See https://docs.docker.com/config/daemon/" - exit 1 - fi -} - -# ViashDockerRemoteTagCheck: check whether a Docker image is available -# on a remote. Assumes `docker login` has been performed, if relevant. -# -# $1 : image identifier with format `[registry/]image[:tag]` -# exit code $? : whether or not the image was found -# examples: -# ViashDockerRemoteTagCheck python:latest -# echo $? # returns '0' -# ViashDockerRemoteTagCheck sdaizudceahifu -# echo $? # returns '1' -function ViashDockerRemoteTagCheck { - docker manifest inspect $1 > /dev/null 2> /dev/null -} - -# ViashDockerLocalTagCheck: check whether a Docker image is available locally -# -# $1 : image identifier with format `[registry/]image[:tag]` -# exit code $? : whether or not the image was found -# examples: -# docker pull python:latest -# ViashDockerLocalTagCheck python:latest -# echo $? # returns '0' -# ViashDockerLocalTagCheck sdaizudceahifu -# echo $? # returns '1' -function ViashDockerLocalTagCheck { - [ -n "$(docker images -q $1)" ] -} - -# ViashDockerPull: pull a Docker image -# -# $1 : image identifier with format `[registry/]image[:tag]` -# exit code $? : whether or not the image was found -# examples: -# ViashDockerPull python:latest -# echo $? # returns '0' -# ViashDockerPull sdaizudceahifu -# echo $? # returns '1' -function ViashDockerPull { - ViashNotice "Checking if Docker image is available at '$1'" - if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then - docker pull $1 && return 0 || return 1 - else - save=$-; set +e - docker pull $1 2> /dev/null > /dev/null - out=$? - [[ $save =~ e ]] && set -e - if [ $out -ne 0 ]; then - ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." - fi - return $out - fi -} - -# ViashDockerPush: push a Docker image -# -# $1 : image identifier with format `[registry/]image[:tag]` -# exit code $? : whether or not the image was found -# examples: -# ViashDockerPush python:latest -# echo $? # returns '0' -# ViashDockerPush sdaizudceahifu -# echo $? # returns '1' -function ViashDockerPush { - ViashNotice "Pushing image to '$1'" - save=$-; set +e - if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then - docker push $1 - out=$? - else - docker push $1 2> /dev/null > /dev/null - out=$? - fi - [[ $save =~ e ]] && set -e - if [ $out -eq 0 ]; then - ViashNotice "Container '$1' push succeeded." - else - ViashError "Container '$1' push errored. You might not be logged in or have the necessary permissions." - fi - return $out -} - -# ViashDockerPullElseBuild: pull a Docker image, else build it -# -# $1 : image identifier with format `[registry/]image[:tag]` -# ViashDockerBuild : a Bash function which builds a docker image, takes image identifier as argument. -# examples: -# ViashDockerPullElseBuild mynewcomponent -function ViashDockerPullElseBuild { - save=$-; set +e - ViashDockerPull $1 - out=$? - [[ $save =~ e ]] && set -e - if [ $out -ne 0 ]; then - ViashDockerBuild $@ - fi -} - -# ViashDockerSetup: create a Docker image, according to specified docker setup strategy -# -# $1 : image identifier with format `[registry/]image[:tag]` -# $2 : docker setup strategy, see DockerSetupStrategy.scala -# examples: -# ViashDockerSetup mynewcomponent alwaysbuild -function ViashDockerSetup { - local image_id="$1" - local setup_strategy="$2" - if [ "$setup_strategy" == "alwaysbuild" -o "$setup_strategy" == "build" -o "$setup_strategy" == "b" ]; then - ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") - elif [ "$setup_strategy" == "alwayspull" -o "$setup_strategy" == "pull" -o "$setup_strategy" == "p" ]; then - ViashDockerPull $image_id - elif [ "$setup_strategy" == "alwayspullelsebuild" -o "$setup_strategy" == "pullelsebuild" ]; then - ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") - elif [ "$setup_strategy" == "alwayspullelsecachedbuild" -o "$setup_strategy" == "pullelsecachedbuild" ]; then - ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id") - elif [ "$setup_strategy" == "alwayscachedbuild" -o "$setup_strategy" == "cachedbuild" -o "$setup_strategy" == "cb" ]; then - ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id") - elif [[ "$setup_strategy" =~ ^ifneedbe ]]; then - local save=$-; set +e - ViashDockerLocalTagCheck $image_id - local outCheck=$? - [[ $save =~ e ]] && set -e - if [ $outCheck -eq 0 ]; then - ViashInfo "Image $image_id already exists" - elif [ "$setup_strategy" == "ifneedbebuild" ]; then - ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") - elif [ "$setup_strategy" == "ifneedbecachedbuild" ]; then - ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id") - elif [ "$setup_strategy" == "ifneedbepull" ]; then - ViashDockerPull $image_id - elif [ "$setup_strategy" == "ifneedbepullelsebuild" ]; then - ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") - elif [ "$setup_strategy" == "ifneedbepullelsecachedbuild" ]; then - ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id") - else - ViashError "Unrecognised Docker strategy: $setup_strategy" - exit 1 - fi - elif [ "$setup_strategy" == "push" -o "$setup_strategy" == "forcepush" -o "$setup_strategy" == "alwayspush" ]; then - ViashDockerPush "$image_id" - elif [ "$setup_strategy" == "pushifnotpresent" -o "$setup_strategy" == "gentlepush" -o "$setup_strategy" == "maybepush" ]; then - local save=$-; set +e - ViashDockerRemoteTagCheck $image_id - local outCheck=$? - [[ $save =~ e ]] && set -e - if [ $outCheck -eq 0 ]; then - ViashNotice "Container '$image_id' exists, doing nothing." - else - ViashNotice "Container '$image_id' does not yet exist." - ViashDockerPush "$image_id" - fi - elif [ "$setup_strategy" == "donothing" -o "$setup_strategy" == "meh" ]; then - ViashNotice "Skipping setup." - else - ViashError "Unrecognised Docker strategy: $setup_strategy" - exit 1 - fi -} - -# ViashDockerCheckCommands: Check whether a docker container has the required commands -# -# $1 : image identifier with format `[registry/]image[:tag]` -# $@ : commands to verify being present -# examples: -# ViashDockerCheckCommands bash:4.0 bash ps foo -function ViashDockerCheckCommands { - local image_id="$1" - shift 1 - local commands="$@" - local save=$-; set +e - local missing # mark 'missing' as local in advance, otherwise the exit code of the command will be missing and always be '0' - missing=$(docker run --rm --entrypoint=sh "$image_id" -c "for command in $commands; do command -v \$command >/dev/null 2>&1; if [ \$? -ne 0 ]; then echo \$command; exit 1; fi; done") - local outCheck=$? - [[ $save =~ e ]] && set -e - if [ $outCheck -ne 0 ]; then - ViashError "Docker container '$image_id' does not contain command '$missing'." - exit 1 - fi -} - -# ViashDockerBuild: build a docker image -# $1 : image identifier with format `[registry/]image[:tag]` -# $... : additional arguments to pass to docker build -# $VIASH_META_TEMP_DIR : temporary directory to store dockerfile & optional resources in -# $VIASH_META_NAME : name of the component -# $VIASH_META_RESOURCES_DIR : directory containing the resources -# $VIASH_VERBOSITY : verbosity level -# exit code $? : whether or not the image was built successfully -function ViashDockerBuild { - local image_id="$1" - shift 1 - - # create temporary directory to store dockerfile & optional resources in - local tmpdir=$(mktemp -d "$VIASH_META_TEMP_DIR/dockerbuild-$VIASH_META_NAME-XXXXXX") - local dockerfile="$tmpdir/Dockerfile" - function clean_up { - rm -rf "$tmpdir" - } - trap clean_up EXIT - - # store dockerfile and resources - ViashDockerfile "$VIASH_ENGINE_ID" > "$dockerfile" - - # generate the build command - local docker_build_cmd="docker build -t '$image_id' $@ '$VIASH_META_RESOURCES_DIR' -f '$dockerfile'" - - # build the container - ViashNotice "Building container '$image_id' with Dockerfile" - ViashInfo "$docker_build_cmd" - local save=$-; set +e - if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then - eval $docker_build_cmd - else - eval $docker_build_cmd &> "$tmpdir/docker_build.log" - fi - - # check exit code - local out=$? - [[ $save =~ e ]] && set -e - if [ $out -ne 0 ]; then - ViashError "Error occurred while building container '$image_id'" - if [ $VIASH_VERBOSITY -lt $VIASH_LOGCODE_INFO ]; then - ViashError "Transcript: --------------------------------" - cat "$tmpdir/docker_build.log" - ViashError "End of transcript --------------------------" - fi - exit 1 - fi -} - -######## End of helper functions for setting up Docker images for viash ######## - -# ViashDockerFile: print the dockerfile to stdout -# $1 : engine identifier -# return : dockerfile required to run this component -# examples: -# ViashDockerFile -function ViashDockerfile { - local engine_id="$1" - - if [[ "$engine_id" == "docker" ]]; then - cat << 'VIASHDOCKER' -FROM python:3.10-slim -ENTRYPOINT [] -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y git && \ - rm -rf /var/lib/apt/lists/* - -RUN pip install --upgrade pip && \ - pip install --upgrade --no-cache-dir "ruamel.yaml" "git+https://github.com/openproblems-bio/core.git#egg=openproblems&subdirectory=packages/python/openproblems" - -LABEL org.opencontainers.image.description="Companion container for running component common create_component" -LABEL org.opencontainers.image.created="2024-07-10T21:24:27+02:00" -LABEL org.opencontainers.image.source="https://github.com/openproblems-bio/common_resources" -LABEL org.opencontainers.image.revision="66a108d61ca7ef1a18e88ecf56afc23c616aa3cb" -LABEL org.opencontainers.image.version="dev" - -VIASHDOCKER - fi -} - -# ViashDockerBuildArgs: return the arguments to pass to docker build -# $1 : engine identifier -# return : arguments to pass to docker build -function ViashDockerBuildArgs { - local engine_id="$1" - - if [[ "$engine_id" == "docker" ]]; then - echo "" - fi -} - -# ViashAbsolutePath: generate absolute path from relative path -# borrowed from https://stackoverflow.com/a/21951256 -# $1 : relative filename -# return : absolute path -# examples: -# ViashAbsolutePath some_file.txt # returns /path/to/some_file.txt -# ViashAbsolutePath /foo/bar/.. # returns /foo -function ViashAbsolutePath { - local thePath - if [[ ! "$1" =~ ^/ ]]; then - thePath="$PWD/$1" - else - thePath="$1" - fi - echo "$thePath" | ( - IFS=/ - read -a parr - declare -a outp - for i in "${parr[@]}"; do - case "$i" in - ''|.) continue ;; - ..) - len=${#outp[@]} - if ((len==0)); then - continue - else - unset outp[$((len-1))] - fi - ;; - *) - len=${#outp[@]} - outp[$len]="$i" - ;; - esac - done - echo /"${outp[*]}" - ) -} -# ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker -# examples: -# ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' -# ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' -function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") - if [ -d "$abs_path" ]; then - mount_source="$abs_path" - base_name="" - else - mount_source=`dirname "$abs_path"` - base_name=`basename "$abs_path"` - fi - mount_target="/viash_automount$mount_source" - if [ -z "$base_name" ]; then - echo "$mount_target" - else - echo "$mount_target/$base_name" - fi -} -function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") - if [ -d "$abs_path" ]; then - mount_source="$abs_path" - base_name="" - else - mount_source=`dirname "$abs_path"` - base_name=`basename "$abs_path"` - fi - mount_target="/viash_automount$mount_source" - ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" - echo "--volume=\"$mount_source:$mount_target\"" -} -function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" -} -# initialise variables -VIASH_DIRECTORY_MOUNTS=() - -# initialise docker variables -VIASH_DOCKER_RUN_ARGS=(-i --rm) - -# initialise array -VIASH_POSITIONAL_ARGS='' - -while [[ $# -gt 0 ]]; do - case "$1" in - -h|--help) - ViashHelp - exit - ;; - ---v|---verbose) - let "VIASH_VERBOSITY=VIASH_VERBOSITY+1" - shift 1 - ;; - ---verbosity) - VIASH_VERBOSITY="$2" - shift 2 - ;; - ---verbosity=*) - VIASH_VERBOSITY="$(ViashRemoveFlags "$1")" - shift 1 - ;; - --version) - echo "create_component dev" - exit - ;; - --task) - [ -n "$VIASH_PAR_TASK" ] && ViashError Bad arguments for option \'--task\': \'$VIASH_PAR_TASK\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_TASK="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to --task. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - --task=*) - [ -n "$VIASH_PAR_TASK" ] && ViashError Bad arguments for option \'--task=*\': \'$VIASH_PAR_TASK\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_TASK=$(ViashRemoveFlags "$1") - shift 1 - ;; - --type) - [ -n "$VIASH_PAR_TYPE" ] && ViashError Bad arguments for option \'--type\': \'$VIASH_PAR_TYPE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_TYPE="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to --type. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - --type=*) - [ -n "$VIASH_PAR_TYPE" ] && ViashError Bad arguments for option \'--type=*\': \'$VIASH_PAR_TYPE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_TYPE=$(ViashRemoveFlags "$1") - shift 1 - ;; - --language) - [ -n "$VIASH_PAR_LANGUAGE" ] && ViashError Bad arguments for option \'--language\': \'$VIASH_PAR_LANGUAGE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_LANGUAGE="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to --language. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - --language=*) - [ -n "$VIASH_PAR_LANGUAGE" ] && ViashError Bad arguments for option \'--language=*\': \'$VIASH_PAR_LANGUAGE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_LANGUAGE=$(ViashRemoveFlags "$1") - shift 1 - ;; - --name) - [ -n "$VIASH_PAR_NAME" ] && ViashError Bad arguments for option \'--name\': \'$VIASH_PAR_NAME\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_NAME="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to --name. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - --name=*) - [ -n "$VIASH_PAR_NAME" ] && ViashError Bad arguments for option \'--name=*\': \'$VIASH_PAR_NAME\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_NAME=$(ViashRemoveFlags "$1") - shift 1 - ;; - --output) - [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_OUTPUT="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to --output. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - --output=*) - [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output=*\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_OUTPUT=$(ViashRemoveFlags "$1") - shift 1 - ;; - --api_file) - [ -n "$VIASH_PAR_API_FILE" ] && ViashError Bad arguments for option \'--api_file\': \'$VIASH_PAR_API_FILE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_API_FILE="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to --api_file. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - --api_file=*) - [ -n "$VIASH_PAR_API_FILE" ] && ViashError Bad arguments for option \'--api_file=*\': \'$VIASH_PAR_API_FILE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_API_FILE=$(ViashRemoveFlags "$1") - shift 1 - ;; - --viash_yaml) - [ -n "$VIASH_PAR_VIASH_YAML" ] && ViashError Bad arguments for option \'--viash_yaml\': \'$VIASH_PAR_VIASH_YAML\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_VIASH_YAML="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to --viash_yaml. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - --viash_yaml=*) - [ -n "$VIASH_PAR_VIASH_YAML" ] && ViashError Bad arguments for option \'--viash_yaml=*\': \'$VIASH_PAR_VIASH_YAML\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_VIASH_YAML=$(ViashRemoveFlags "$1") - shift 1 - ;; - ---engine) - VIASH_ENGINE_ID="$2" - shift 2 - ;; - ---engine=*) - VIASH_ENGINE_ID="$(ViashRemoveFlags "$1")" - shift 1 - ;; - ---setup) - VIASH_MODE='setup' - VIASH_SETUP_STRATEGY="$2" - shift 2 - ;; - ---setup=*) - VIASH_MODE='setup' - VIASH_SETUP_STRATEGY="$(ViashRemoveFlags "$1")" - shift 1 - ;; - ---dockerfile) - VIASH_MODE='dockerfile' - shift 1 - ;; - ---debug) - VIASH_MODE='debug' - shift 1 - ;; - ---cpus) - [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_META_CPUS="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to ---cpus. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - ---cpus=*) - [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus=*\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_META_CPUS=$(ViashRemoveFlags "$1") - shift 1 - ;; - ---memory) - [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_META_MEMORY="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to ---memory. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - ---memory=*) - [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory=*\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_META_MEMORY=$(ViashRemoveFlags "$1") - shift 1 - ;; - *) # positional arg or unknown option - # since the positional args will be eval'd, can we always quote, instead of using ViashQuote - VIASH_POSITIONAL_ARGS="$VIASH_POSITIONAL_ARGS '$1'" - [[ $1 == -* ]] && ViashWarning $1 looks like a parameter but is not a defined parameter and will instead be treated as a positional argument. Use "--help" to get more information on the parameters. - shift # past argument - ;; - esac -done - -# parse positional parameters -eval set -- $VIASH_POSITIONAL_ARGS - - -if [ "$VIASH_ENGINE_ID" == "docker" ] ; then - VIASH_ENGINE_TYPE='docker' -else - ViashError "Engine '$VIASH_ENGINE_ID' is not recognized. Options are: docker." - exit 1 -fi - -if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then - # check if docker is installed properly - ViashDockerInstallationCheck - - # determine docker image id - if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='ghcr.io/openproblems-bio/common_resources/common/create_component:dev' - fi - - # print dockerfile - if [ "$VIASH_MODE" == "dockerfile" ]; then - ViashDockerfile "$VIASH_ENGINE_ID" - exit 0 - - # enter docker container - elif [[ "$VIASH_MODE" == "debug" ]]; then - VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} -v '$(pwd)':/pwd --workdir /pwd -t $VIASH_DOCKER_IMAGE_ID" - ViashNotice "+ $VIASH_CMD" - eval $VIASH_CMD - exit - - # build docker image - elif [ "$VIASH_MODE" == "setup" ]; then - ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" "$VIASH_SETUP_STRATEGY" - ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'bash' - exit 0 - fi - - # check if docker image exists - ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" ifneedbepullelsecachedbuild - ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'bash' -fi - -# setting computational defaults - -# helper function for parsing memory strings -function ViashMemoryAsBytes { - local memory=`echo "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]'` - local memory_regex='^([0-9]+)([kmgtp]i?b?|b)$' - if [[ $memory =~ $memory_regex ]]; then - local number=${memory/[^0-9]*/} - local symbol=${memory/*[0-9]/} - - case $symbol in - b) memory_b=$number ;; - kb|k) memory_b=$(( $number * 1000 )) ;; - mb|m) memory_b=$(( $number * 1000 * 1000 )) ;; - gb|g) memory_b=$(( $number * 1000 * 1000 * 1000 )) ;; - tb|t) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 )) ;; - pb|p) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 * 1000 )) ;; - kib|ki) memory_b=$(( $number * 1024 )) ;; - mib|mi) memory_b=$(( $number * 1024 * 1024 )) ;; - gib|gi) memory_b=$(( $number * 1024 * 1024 * 1024 )) ;; - tib|ti) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 )) ;; - pib|pi) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 * 1024 )) ;; - esac - echo "$memory_b" - fi -} -# compute memory in different units -if [ ! -z ${VIASH_META_MEMORY+x} ]; then - VIASH_META_MEMORY_B=`ViashMemoryAsBytes $VIASH_META_MEMORY` - # do not define other variables if memory_b is an empty string - if [ ! -z "$VIASH_META_MEMORY_B" ]; then - VIASH_META_MEMORY_KB=$(( ($VIASH_META_MEMORY_B+999) / 1000 )) - VIASH_META_MEMORY_MB=$(( ($VIASH_META_MEMORY_KB+999) / 1000 )) - VIASH_META_MEMORY_GB=$(( ($VIASH_META_MEMORY_MB+999) / 1000 )) - VIASH_META_MEMORY_TB=$(( ($VIASH_META_MEMORY_GB+999) / 1000 )) - VIASH_META_MEMORY_PB=$(( ($VIASH_META_MEMORY_TB+999) / 1000 )) - VIASH_META_MEMORY_KIB=$(( ($VIASH_META_MEMORY_B+1023) / 1024 )) - VIASH_META_MEMORY_MIB=$(( ($VIASH_META_MEMORY_KIB+1023) / 1024 )) - VIASH_META_MEMORY_GIB=$(( ($VIASH_META_MEMORY_MIB+1023) / 1024 )) - VIASH_META_MEMORY_TIB=$(( ($VIASH_META_MEMORY_GIB+1023) / 1024 )) - VIASH_META_MEMORY_PIB=$(( ($VIASH_META_MEMORY_TIB+1023) / 1024 )) - else - # unset memory if string is empty - unset $VIASH_META_MEMORY_B - fi -fi -# unset nproc if string is empty -if [ -z "$VIASH_META_CPUS" ]; then - unset $VIASH_META_CPUS -fi - - -# check whether required parameters exist -if [ -z ${VIASH_META_NAME+x} ]; then - ViashError 'name' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi -if [ -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then - ViashError 'functionality_name' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi -if [ -z ${VIASH_META_RESOURCES_DIR+x} ]; then - ViashError 'resources_dir' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi -if [ -z ${VIASH_META_EXECUTABLE+x} ]; then - ViashError 'executable' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi -if [ -z ${VIASH_META_CONFIG+x} ]; then - ViashError 'config' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi -if [ -z ${VIASH_META_TEMP_DIR+x} ]; then - ViashError 'temp_dir' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi - -# filling in defaults -if [ -z ${VIASH_PAR_LANGUAGE+x} ]; then - VIASH_PAR_LANGUAGE="python" -fi -if [ -z ${VIASH_PAR_OUTPUT+x} ]; then - VIASH_PAR_OUTPUT="src/tasks/${VIASH_PAR_TASK}/${VIASH_PAR_TYPE}s/${VIASH_PAR_NAME}" -fi -if [ -z ${VIASH_PAR_API_FILE+x} ]; then - VIASH_PAR_API_FILE="src/tasks/${VIASH_PAR_TASK}/api/comp_${VIASH_PAR_TYPE}.yaml" -fi -if [ -z ${VIASH_PAR_VIASH_YAML+x} ]; then - VIASH_PAR_VIASH_YAML="_viash.yaml" -fi - -# check whether required files exist -if [ ! -z "$VIASH_PAR_VIASH_YAML" ] && [ ! -e "$VIASH_PAR_VIASH_YAML" ]; then - ViashError "Input file '$VIASH_PAR_VIASH_YAML' does not exist." - exit 1 -fi - -# check whether parameters values are of the right type -if [[ -n "$VIASH_META_CPUS" ]]; then - if ! [[ "$VIASH_META_CPUS" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'cpus' has to be an integer. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_B" ]]; then - if ! [[ "$VIASH_META_MEMORY_B" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_b' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_KB" ]]; then - if ! [[ "$VIASH_META_MEMORY_KB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_kb' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_MB" ]]; then - if ! [[ "$VIASH_META_MEMORY_MB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_mb' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_GB" ]]; then - if ! [[ "$VIASH_META_MEMORY_GB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_gb' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_TB" ]]; then - if ! [[ "$VIASH_META_MEMORY_TB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_tb' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_PB" ]]; then - if ! [[ "$VIASH_META_MEMORY_PB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_pb' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_KIB" ]]; then - if ! [[ "$VIASH_META_MEMORY_KIB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_kib' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_MIB" ]]; then - if ! [[ "$VIASH_META_MEMORY_MIB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_mib' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_GIB" ]]; then - if ! [[ "$VIASH_META_MEMORY_GIB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_gib' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_TIB" ]]; then - if ! [[ "$VIASH_META_MEMORY_TIB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_tib' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_PIB" ]]; then - if ! [[ "$VIASH_META_MEMORY_PIB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_pib' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi - -# check whether value is belongs to a set of choices -if [ ! -z "$VIASH_PAR_LANGUAGE" ]; then - VIASH_PAR_LANGUAGE_CHOICES=("python;r") - IFS=';' - set -f - if ! [[ ";${VIASH_PAR_LANGUAGE_CHOICES[*]};" =~ ";$VIASH_PAR_LANGUAGE;" ]]; then - ViashError '--language' specified value of \'$VIASH_PAR_LANGUAGE\' is not in the list of allowed values. Use "--help" to get more information on the parameters. - exit 1 - fi - set +f - unset IFS -fi - -# create parent directories of output files, if so desired -if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -d "$(dirname "$VIASH_PAR_OUTPUT")" ]; then - mkdir -p "$(dirname "$VIASH_PAR_OUTPUT")" -fi - -if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then - # detect volumes from file arguments - VIASH_CHOWN_VARS=() -if [ ! -z "$VIASH_PAR_OUTPUT" ]; then - VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_OUTPUT")" ) - VIASH_PAR_OUTPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_OUTPUT") - VIASH_CHOWN_VARS+=( "$VIASH_PAR_OUTPUT" ) -fi -if [ ! -z "$VIASH_PAR_API_FILE" ]; then - VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_API_FILE")" ) - VIASH_PAR_API_FILE=$(ViashDockerAutodetectMount "$VIASH_PAR_API_FILE") -fi -if [ ! -z "$VIASH_PAR_VIASH_YAML" ]; then - VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_VIASH_YAML")" ) - VIASH_PAR_VIASH_YAML=$(ViashDockerAutodetectMount "$VIASH_PAR_VIASH_YAML") -fi -if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then - VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_RESOURCES_DIR")" ) - VIASH_META_RESOURCES_DIR=$(ViashDockerAutodetectMount "$VIASH_META_RESOURCES_DIR") -fi -if [ ! -z "$VIASH_META_EXECUTABLE" ]; then - VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_EXECUTABLE")" ) - VIASH_META_EXECUTABLE=$(ViashDockerAutodetectMount "$VIASH_META_EXECUTABLE") -fi -if [ ! -z "$VIASH_META_CONFIG" ]; then - VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_CONFIG")" ) - VIASH_META_CONFIG=$(ViashDockerAutodetectMount "$VIASH_META_CONFIG") -fi -if [ ! -z "$VIASH_META_TEMP_DIR" ]; then - VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_TEMP_DIR")" ) - VIASH_META_TEMP_DIR=$(ViashDockerAutodetectMount "$VIASH_META_TEMP_DIR") -fi - - # get unique mounts - VIASH_UNIQUE_MOUNTS=($(for val in "${VIASH_DIRECTORY_MOUNTS[@]}"; do echo "$val"; done | sort -u)) -fi - -if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then - # change file ownership - function ViashPerformChown { - if (( ${#VIASH_CHOWN_VARS[@]} )); then - set +e - VIASH_CMD="docker run --entrypoint=bash --rm ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID -c 'chown $(id -u):$(id -g) --silent --recursive ${VIASH_CHOWN_VARS[@]}'" - ViashDebug "+ $VIASH_CMD" - eval $VIASH_CMD - set -e - fi - } - trap ViashPerformChown EXIT -fi - -if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then - # helper function for filling in extra docker args - if [ ! -z "$VIASH_META_MEMORY_B" ]; then - VIASH_DOCKER_RUN_ARGS+=("--memory=${VIASH_META_MEMORY_B}") - fi - if [ ! -z "$VIASH_META_CPUS" ]; then - VIASH_DOCKER_RUN_ARGS+=("--cpus=${VIASH_META_CPUS}") - fi -fi - -if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then - VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID" -fi - - -# set dependency paths - - -ViashDebug "Running command: $(echo $VIASH_CMD)" -cat << VIASHEOF | eval $VIASH_CMD -set -e -tempscript=\$(mktemp "$VIASH_META_TEMP_DIR/viash-run-create_component-XXXXXX").py -function clean_up { - rm "\$tempscript" -} -function interrupt { - echo -e "\nCTRL-C Pressed..." - exit 1 -} -trap clean_up EXIT -trap interrupt INT SIGINT -cat > "\$tempscript" << 'VIASHMAIN' -from typing import Any -from pathlib import Path -import sys -import os -import re -from openproblems.utils import strip_margin -from openproblems.io import read_nested_yaml - -## VIASH START -# The following code has been auto-generated by Viash. -par = { - 'task': $( if [ ! -z ${VIASH_PAR_TASK+x} ]; then echo "r'${VIASH_PAR_TASK//\'/\'\"\'\"r\'}'"; else echo None; fi ), - 'type': $( if [ ! -z ${VIASH_PAR_TYPE+x} ]; then echo "r'${VIASH_PAR_TYPE//\'/\'\"\'\"r\'}'"; else echo None; fi ), - 'language': $( if [ ! -z ${VIASH_PAR_LANGUAGE+x} ]; then echo "r'${VIASH_PAR_LANGUAGE//\'/\'\"\'\"r\'}'"; else echo None; fi ), - 'name': $( if [ ! -z ${VIASH_PAR_NAME+x} ]; then echo "r'${VIASH_PAR_NAME//\'/\'\"\'\"r\'}'"; else echo None; fi ), - 'output': $( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "r'${VIASH_PAR_OUTPUT//\'/\'\"\'\"r\'}'"; else echo None; fi ), - 'api_file': $( if [ ! -z ${VIASH_PAR_API_FILE+x} ]; then echo "r'${VIASH_PAR_API_FILE//\'/\'\"\'\"r\'}'"; else echo None; fi ), - 'viash_yaml': $( if [ ! -z ${VIASH_PAR_VIASH_YAML+x} ]; then echo "r'${VIASH_PAR_VIASH_YAML//\'/\'\"\'\"r\'}'"; else echo None; fi ) -} -meta = { - 'name': $( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "r'${VIASH_META_NAME//\'/\'\"\'\"r\'}'"; else echo None; fi ), - 'functionality_name': $( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "r'${VIASH_META_FUNCTIONALITY_NAME//\'/\'\"\'\"r\'}'"; else echo None; fi ), - 'resources_dir': $( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "r'${VIASH_META_RESOURCES_DIR//\'/\'\"\'\"r\'}'"; else echo None; fi ), - 'executable': $( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "r'${VIASH_META_EXECUTABLE//\'/\'\"\'\"r\'}'"; else echo None; fi ), - 'config': $( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "r'${VIASH_META_CONFIG//\'/\'\"\'\"r\'}'"; else echo None; fi ), - 'temp_dir': $( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "r'${VIASH_META_TEMP_DIR//\'/\'\"\'\"r\'}'"; else echo None; fi ), - 'cpus': $( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "int(r'${VIASH_META_CPUS//\'/\'\"\'\"r\'}')"; else echo None; fi ), - 'memory_b': $( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "int(r'${VIASH_META_MEMORY_B//\'/\'\"\'\"r\'}')"; else echo None; fi ), - 'memory_kb': $( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KB//\'/\'\"\'\"r\'}')"; else echo None; fi ), - 'memory_mb': $( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MB//\'/\'\"\'\"r\'}')"; else echo None; fi ), - 'memory_gb': $( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GB//\'/\'\"\'\"r\'}')"; else echo None; fi ), - 'memory_tb': $( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TB//\'/\'\"\'\"r\'}')"; else echo None; fi ), - 'memory_pb': $( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PB//\'/\'\"\'\"r\'}')"; else echo None; fi ), - 'memory_kib': $( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KIB//\'/\'\"\'\"r\'}')"; else echo None; fi ), - 'memory_mib': $( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MIB//\'/\'\"\'\"r\'}')"; else echo None; fi ), - 'memory_gib': $( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GIB//\'/\'\"\'\"r\'}')"; else echo None; fi ), - 'memory_tib': $( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TIB//\'/\'\"\'\"r\'}')"; else echo None; fi ), - 'memory_pib': $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PIB//\'/\'\"\'\"r\'}')"; else echo None; fi ) -} -dep = { - -} - -## VIASH END - -def create_config(par, component_type, pretty_name, script_path) -> str: - info_str = generate_info(par, component_type, pretty_name) - resources_str = generate_resources(par, script_path) - docker_engine = generate_docker_engine(par) - - return strip_margin(f'''\\ - |# The API specifies which type of component this is. - |# It contains specifications for: - |# - The input/output files - |# - Common parameters - |# - A unit test - |__merge__: {os.path.relpath(par["api_file"], par["output"])} - | - | - |# A unique identifier for your component (required). - |# Can contain only lowercase letters or underscores. - |name: {par["name"]} - | - |# Metadata for your component - |info: - |{info_str} - |# Component-specific parameters (optional) - |# arguments: - |# - name: "--n_neighbors" - |# type: "integer" - |# default: 5 - |# description: Number of neighbors to use. - | - |# Resources required to run the component - |resources: - |{resources_str} - |engines: - | # Specifications for the Docker image for this component. - |{docker_engine} - |runners: - | # This platform allows running the component natively - | - type: executable - | # Allows turning the component into a Nextflow module / pipeline. - | - type: nextflow - | directives: - | label: [midtime,midmem,midcpu] - |''' - ) - -def generate_info(par, component_type, pretty_name) -> str: - """Generate the info for a component.""" - if component_type in ["method", "control_method"]: - str = strip_margin(f'''\\ - | # A relatively short label, used when rendering visualisations (required) - | label: {pretty_name} - | # A one sentence summary of how this method works (required). Used when - | # rendering summary tables. - | summary: "FILL IN: A one sentence summary of this method." - | # A multi-line description of how this component works (required). Used - | # when rendering reference documentation. - | description: | - | FILL IN: A (multi-line) description of how this method works. - | # Which normalisation method this component prefers to use (required). - | preferred_normalization: log_cp10k - |''') - if component_type == "method": - str += strip_margin(f'''\\ - | # A reference key from the bibtex library at src/common/library.bib (required). - | reference: bibtex_reference_key - | # URL to the documentation for this method (required). - | documentation_url: https://url.to/the/documentation - | # URL to the code repository for this method (required). - | repository_url: https://github.com/organisation/repository - |''') - return str - elif component_type == "metric": - return strip_margin(f'''\\ - | metrics: - | # A unique identifier for your metric (required). - | # Can contain only lowercase letters or underscores. - | name: {par["name"]} - | # A relatively short label, used when rendering visualisarions (required) - | label: {pretty_name} - | # A one sentence summary of how this metric works (required). Used when - | # rendering summary tables. - | summary: "FILL IN: A one sentence summary of this metric." - | # A multi-line description of how this component works (required). Used - | # when rendering reference documentation. - | description: | - | FILL IN: A (multi-line) description of how this metric works. - | # A reference key from the bibtex library at src/common/library.bib (required). - | reference: bibtex_reference_key - | # URL to the documentation for this metric (required). - | documentation_url: https://url.to/the/documentation - | # URL to the code repository for this metric (required). - | repository_url: https://github.com/organisation/repository - | # The minimum possible value for this metric (required) - | min: 0 - | # The maximum possible value for this metric (required) - | max: 1 - | # Whether a higher value represents a 'better' solution (required) - | maximize: true - |''') - - -def generate_resources(par, script_path) -> str: - """Add the script to the resources.""" - if par["language"] == "python": - type_str = "python_script" - elif par["language"] == "r": - type_str = "r_script" - - return strip_margin(f'''\\ - | # The script of your component (required) - | - type: {type_str} - | path: {script_path} - | # Additional resources your script needs (optional) - | # - type: file - | # path: weights.pt - |''') - -def generate_docker_engine(par) -> str: - """Set up the docker engine for Python.""" - if par["language"] == "python": - image_str = "ghcr.io/openproblems-bio/base_images/python:1.1.0" - setup_type = "python" - package_example = "scib==1.1.5" - elif par["language"] == "r": - image_str = "ghcr.io/openproblems-bio/base_images/r:1.1.0" - setup_type = "r" - package_example = "tidyverse" - return strip_margin(f'''\\ - | - type: docker - | image: {image_str} - | # Add custom dependencies here (optional). For more information, see - | # https://viash.io/reference/config/engines/docker/#setup . - | # setup: - | # - type: {setup_type} - | # packages: {package_example} - |''') - -def set_par_values(config) -> None: - """Adds values to each of the arguments in a config file.""" - args = config['arguments'] - for argi, arg in enumerate(args): - key = re.sub("^-*", "", arg['name']) - - # find value - if arg["type"] != "file": - value = arg.get("default", arg.get("example", "...")) - elif arg.get("direction", "input") == "input": - key_strip = key.replace("input_", "") - value = f'resources_test/{par["task"]}/pancreas/{key_strip}.h5ad' - else: - key_strip = key.replace("output_", "") - value = f'{key_strip}.h5ad' - - # store key and value - config['arguments'][argi]["key"] = key - config['arguments'][argi]["value"] = value - -def look_for_adata_arg(args, uns_field): - """Look for an argument that has a .uns[uns_field] in its info.slots.""" - for arg in args: - uns = arg.get("info", {}).get("slots", {}).get("uns", []) - for unval in uns: - if unval.get("name") == uns_field: - return arg["key"] - return "adata" - -def write_output_python(arg, copy_from_adata, is_metric): - """Create code for writing the output h5ad files.""" - slots = arg.get("info", {}).get("slots", {}) - outer = [] - for group_name, slots in slots.items(): - inner = [] - for slot in slots: - if group_name == "uns" and slot["name"] in ["dataset_id", "normalization_id"]: - value = f"{copy_from_adata}.uns['{slot['name']}']" - elif group_name == "uns" and slot["name"] == "method_id": - if is_metric: - value = f"{copy_from_adata}.uns['{slot['name']}']" - else: - value = "meta['name']" - else: - value = group_name + "_" + slot["name"] - inner.append(f"'{slot['name']}': {value}") - inner_values = ',\\n '.join(inner) - outer.append(f"{group_name}={{\\n {inner_values}\\n }}") - outer_values = ',\\n '.join(outer) - return strip_margin( - f'''\\ - |print("Write {arg["key"]} AnnData to file", flush=True) - |{arg["key"]} = ad.AnnData( - | {outer_values} - |) - |{arg["key"]}.write_h5ad(par['{arg["key"]}'], compression='gzip')''' - ) - -def write_output_r(arg, copy_from_adata, is_metric): - """Create code for writing the output h5ad files.""" - slots = arg.get("info", {}).get("slots", {}) - outer = [] - for group_name, slots in slots.items(): - inner = [] - for slot in slots: - if group_name == "uns" and slot["name"] in ["dataset_id", "normalization_id"]: - value = f"{copy_from_adata}\$uns[[\\"{slot['name']}\\"]]" - elif group_name == "uns" and slot["name"] == "method_id": - if is_metric: - value = f"{copy_from_adata}\$uns[[\\"{slot['name']}\\"]]" - else: - value = "meta[[\\"name\\"]]" - else: - value = group_name + "_" + slot["name"] - inner.append(f"{slot['name']} = {value}") - inner_values = ',\\n '.join(inner) - outer.append(f"{group_name} = list(\\n {inner_values}\\n )") - outer_values = ',\\n '.join(outer) - return strip_margin( - f'''\\ - |cat("Write {arg["key"]} AnnData to file\\\\n") - |{arg["key"]} <- anndata::AnnData( - | {outer_values} - |) - |{arg["key"]}\$write_h5ad(par[["{arg["key"]}"]], compression = "gzip")''' - ) - -def create_python_script(par, config, type): - args = config['arguments'] - - # create the arguments of the par string - par_string = ",\\n ".join(f"'{arg['key']}': '{arg['value']}'" for arg in args) - - # create code for reading the input h5ad file - read_h5ad_string = "\\n".join( - f"{arg['key']} = ad.read_h5ad(par['{arg['key']}'])" - for arg in args - if arg['type'] == "file" - and arg.get('direction', "input") == "input" - ) - - # determine which adata to copy from - copy_from_adata = look_for_adata_arg(args, "method_id" if type == "metric" else "dataset_id") - - # create code for writing the output h5ad files - write_h5ad_string = "\\n".join( - write_output_python(arg, copy_from_adata, type == "metric") - for arg in args - if arg["type"] == "file" - and arg.get("direction", "input") == "output" - ) - - if type == 'metric': - processing_string = strip_margin(f'''\\ - |print('Compute metrics', flush=True) - |# metric_ids and metric_values can have length > 1 - |# but should be of equal length - |uns_metric_ids = [ '{par['name']}' ] - |uns_metric_values = [ 0.5 ]''') - else: - processing_string = strip_margin(f'''\\ - |print('Preprocess data', flush=True) - |# ... preprocessing ... - | - |print('Train model', flush=True) - |# ... train model ... - | - |print('Generate predictions', flush=True) - |# ... generate predictions ...''') - - script = strip_margin(f'''\\ - |import anndata as ad - | - |## VIASH START - |# Note: this section is auto-generated by viash at runtime. To edit it, make changes - |# in config.vsh.yaml and then run \`viash config inject config.vsh.yaml\`. - |par = {{ - | {par_string} - |}} - |meta = {{ - | 'name': '{par["name"]}' - |}} - |## VIASH END - | - |print('Reading input files', flush=True) - |{read_h5ad_string} - | - |{processing_string} - | - |{write_h5ad_string} - |''') - - return script - -def create_r_script(par, api_spec, type): - args = api_spec['arguments'] - - # create the arguments of the par string - par_string = ",\\n ".join(f'{arg["key"]} = "{arg["value"]}"' for arg in args) - - # create helpers for reading the h5ad file - read_h5ad_string = "\\n".join( - f'{arg["key"]} <- anndata::read_h5ad(par[["{arg["key"]}"]])' - for arg in args - if arg['type'] == "file" - and arg.get("direction", "input") == "input" - ) - - # determine which adata to copy from - copy_from_adata = look_for_adata_arg(args, "method_id" if type == "metric" else "dataset_id") - - # create code for writing the output h5ad files - write_h5ad_string = "\\n".join( - write_output_r(arg, copy_from_adata, type == "metric") - for arg in args - if arg["type"] == "file" - and arg.get("direction", "input") == "output" - ) - - if type == 'metric': - processing_string = strip_margin(f'''\\ - |cat("Compute metrics\\\\n") - |# metric_ids and metric_values can have length > 1 - |# but should be of equal length - |uns_metric_ids <- c("{par['name']}") - |uns_metric_values <- c(0.5)''') - else: - processing_string = strip_margin(f'''\\ - |cat("Preprocess data\\\\n") - |# ... preprocessing ... - | - |cat("Train model\\\\n") - |# ... train model ... - | - |cat("Generate predictions\\\\n") - |# ... generate predictions ...''') - - script = strip_margin(f'''\\ - |library(anndata) - | - |## VIASH START - |par <- list( - | {par_string} - |) - |meta <- list( - | name = "{par["name"]}" - |) - |## VIASH END - | - |cat("Reading input files\\\\n") - |{read_h5ad_string} - | - |{processing_string} - | - |{write_h5ad_string} - |''') - - return script - -def main(par): - ####### CHECK INPUTS ####### - print("Check inputs", flush=True) - assert re.match("[a-z][a-z0-9_]*", par["name"]), "Name should match the regular expression '[a-z][a-z0-9_]*'. Example: 'my_component'." - assert len(par['name']) <= 50, "Method name should be at most 50 characters." - - pretty_name = re.sub("_", " ", par['name']).title() - - ####### CHECK LANGUAGE ####### - print("Check language", flush=True) - # check language and determine script path - if par["language"] == "python": - script_path = "script.py" - elif par["language"] == "r": - script_path = "script.R" - else: - sys.exit(f"Unrecognized language parameter '{par['language']}'.") - - ## CHECK API FILE - print("Check API file", flush=True) - api_file = Path(par["api_file"]) - viash_yaml = Path(par["viash_yaml"]) - project_dir = viash_yaml.parent - if not api_file.exists(): - comp_types = [x.with_suffix("").name.removeprefix("comp_") for x in api_file.parent.glob("**/comp_*.y*ml")] - list.sort(comp_types) - sys.exit(strip_margin(f"""\\ - |Error: Invalid --type argument. - | Reason: Could not find API file at '{api_file.relative_to(project_dir)}'. - | Possible values for --type: {', '.join(comp_types)}.""")) - - ## READ API FILE - print("Read API file", flush=True) - api = read_nested_yaml(api_file) - comp_type = api.get("info", {}).get("type", {}) - if not comp_type: - sys.exit(strip_margin(f"""\\ - |Error: API file is incorrectly formatted. - | Reason: Could not find component type at \`.info.type\`.' - | Please fix the formatting of the API file.""")) - - ####### CREATE OUTPUT DIR ####### - print("Create output dir", flush=True) - out_dir = Path(par["output"]) - out_dir.mkdir(exist_ok=True) - - ####### CREATE CONFIG ####### - print("Create config", flush=True) - config_file = out_dir / "config.vsh.yaml" - - # get config template - config_str = create_config(par, comp_type, pretty_name, script_path) - - with open(config_file, "w") as f: - f.write(config_str) - - ####### CREATE SCRIPT ####### - print("Create script", flush=True) - script_file = out_dir / script_path - - # set reasonable values - set_par_values(api) - - if par["language"] == "python": - script_out = create_python_script(par, api, comp_type) - - if par["language"] == "r": - script_out = create_r_script(par, api, comp_type) - - # write script - with open(script_file, "w") as f: - f.write(script_out) - - print("Done!", flush=True) - - -if __name__ == "__main__": - main(par) -VIASHMAIN -python -B "\$tempscript" & -wait "\$!" - -VIASHEOF - - -if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then - # strip viash automount from file paths - - if [ ! -z "$VIASH_PAR_OUTPUT" ]; then - VIASH_PAR_OUTPUT=$(ViashDockerStripAutomount "$VIASH_PAR_OUTPUT") - fi - if [ ! -z "$VIASH_PAR_API_FILE" ]; then - VIASH_PAR_API_FILE=$(ViashDockerStripAutomount "$VIASH_PAR_API_FILE") - fi - if [ ! -z "$VIASH_PAR_VIASH_YAML" ]; then - VIASH_PAR_VIASH_YAML=$(ViashDockerStripAutomount "$VIASH_PAR_VIASH_YAML") - fi - if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then - VIASH_META_RESOURCES_DIR=$(ViashDockerStripAutomount "$VIASH_META_RESOURCES_DIR") - fi - if [ ! -z "$VIASH_META_EXECUTABLE" ]; then - VIASH_META_EXECUTABLE=$(ViashDockerStripAutomount "$VIASH_META_EXECUTABLE") - fi - if [ ! -z "$VIASH_META_CONFIG" ]; then - VIASH_META_CONFIG=$(ViashDockerStripAutomount "$VIASH_META_CONFIG") - fi - if [ ! -z "$VIASH_META_TEMP_DIR" ]; then - VIASH_META_TEMP_DIR=$(ViashDockerStripAutomount "$VIASH_META_TEMP_DIR") - fi -fi - - -# check whether required files exist -if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -e "$VIASH_PAR_OUTPUT" ]; then - ViashError "Output file '$VIASH_PAR_OUTPUT' does not exist." - exit 1 -fi - - -exit 0 diff --git a/create_task_readme/create_task_readme b/create_task_readme/create_task_readme deleted file mode 100755 index e004753..0000000 --- a/create_task_readme/create_task_readme +++ /dev/null @@ -1,1171 +0,0 @@ -#!/usr/bin/env bash - -# create_task_readme dev -# -# This wrapper script is auto-generated by viash 0.8.0 and is thus a derivative -# work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data -# Intuitive. -# -# The component may contain files which fall under a different license. The -# authors of this component should specify the license in the header of such -# files, or include a separate license file detailing the licenses of all included -# files. - -set -e - -if [ -z "$VIASH_TEMP" ]; then - VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMPDIR} - VIASH_TEMP=${VIASH_TEMP:-$VIASH_TEMPDIR} - VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMP} - VIASH_TEMP=${VIASH_TEMP:-$TMPDIR} - VIASH_TEMP=${VIASH_TEMP:-$TMP} - VIASH_TEMP=${VIASH_TEMP:-$TEMPDIR} - VIASH_TEMP=${VIASH_TEMP:-$TEMP} - VIASH_TEMP=${VIASH_TEMP:-/tmp} -fi - -# define helper functions -# ViashQuote: put quotes around non flag values -# $1 : unquoted string -# return : possibly quoted string -# examples: -# ViashQuote --foo # returns --foo -# ViashQuote bar # returns 'bar' -# Viashquote --foo=bar # returns --foo='bar' -function ViashQuote { - if [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+=.+$ ]]; then - echo "$1" | sed "s#=\(.*\)#='\1'#" - elif [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+$ ]]; then - echo "$1" - else - echo "'$1'" - fi -} -# ViashRemoveFlags: Remove leading flag -# $1 : string with a possible leading flag -# return : string without possible leading flag -# examples: -# ViashRemoveFlags --foo=bar # returns bar -function ViashRemoveFlags { - echo "$1" | sed 's/^--*[a-zA-Z0-9_\-]*=//' -} -# ViashSourceDir: return the path of a bash file, following symlinks -# usage : ViashSourceDir ${BASH_SOURCE[0]} -# $1 : Should always be set to ${BASH_SOURCE[0]} -# returns : The absolute path of the bash file -function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" - done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd -} -# ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks -# usage : ViashFindTargetDir 'ScriptPath' -# $1 : The location from where to start the upward search -# returns : The absolute path of the '.build.yaml' file -function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} - done - echo $SOURCE -} -# see https://en.wikipedia.org/wiki/Syslog#Severity_level -VIASH_LOGCODE_EMERGENCY=0 -VIASH_LOGCODE_ALERT=1 -VIASH_LOGCODE_CRITICAL=2 -VIASH_LOGCODE_ERROR=3 -VIASH_LOGCODE_WARNING=4 -VIASH_LOGCODE_NOTICE=5 -VIASH_LOGCODE_INFO=6 -VIASH_LOGCODE_DEBUG=7 -VIASH_VERBOSITY=$VIASH_LOGCODE_NOTICE - -# ViashLog: Log events depending on the verbosity level -# usage: ViashLog 1 alert Oh no something went wrong! -# $1: required verbosity level -# $2: display tag -# $3+: messages to display -# stdout: Your input, prepended by '[$2] '. -function ViashLog { - local required_level="$1" - local display_tag="$2" - shift 2 - if [ $VIASH_VERBOSITY -ge $required_level ]; then - >&2 echo "[$display_tag]" "$@" - fi -} - -# ViashEmergency: log events when the system is unstable -# usage: ViashEmergency Oh no something went wrong. -# stdout: Your input, prepended by '[emergency] '. -function ViashEmergency { - ViashLog $VIASH_LOGCODE_EMERGENCY emergency "$@" -} - -# ViashAlert: log events when actions must be taken immediately (e.g. corrupted system database) -# usage: ViashAlert Oh no something went wrong. -# stdout: Your input, prepended by '[alert] '. -function ViashAlert { - ViashLog $VIASH_LOGCODE_ALERT alert "$@" -} - -# ViashCritical: log events when a critical condition occurs -# usage: ViashCritical Oh no something went wrong. -# stdout: Your input, prepended by '[critical] '. -function ViashCritical { - ViashLog $VIASH_LOGCODE_CRITICAL critical "$@" -} - -# ViashError: log events when an error condition occurs -# usage: ViashError Oh no something went wrong. -# stdout: Your input, prepended by '[error] '. -function ViashError { - ViashLog $VIASH_LOGCODE_ERROR error "$@" -} - -# ViashWarning: log potentially abnormal events -# usage: ViashWarning Something may have gone wrong. -# stdout: Your input, prepended by '[warning] '. -function ViashWarning { - ViashLog $VIASH_LOGCODE_WARNING warning "$@" -} - -# ViashNotice: log significant but normal events -# usage: ViashNotice This just happened. -# stdout: Your input, prepended by '[notice] '. -function ViashNotice { - ViashLog $VIASH_LOGCODE_NOTICE notice "$@" -} - -# ViashInfo: log normal events -# usage: ViashInfo This just happened. -# stdout: Your input, prepended by '[info] '. -function ViashInfo { - ViashLog $VIASH_LOGCODE_INFO info "$@" -} - -# ViashDebug: log all events, for debugging purposes -# usage: ViashDebug This just happened. -# stdout: Your input, prepended by '[debug] '. -function ViashDebug { - ViashLog $VIASH_LOGCODE_DEBUG debug "$@" -} - -# find source folder of this component -VIASH_META_RESOURCES_DIR=`ViashSourceDir ${BASH_SOURCE[0]}` - -# find the root of the built components & dependencies -VIASH_TARGET_DIR=`ViashFindTargetDir $VIASH_META_RESOURCES_DIR` - -# define meta fields -VIASH_META_FUNCTIONALITY_NAME="create_task_readme" -VIASH_META_EXECUTABLE="$VIASH_META_RESOURCES_DIR/$VIASH_META_FUNCTIONALITY_NAME" -VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" -VIASH_META_TEMP_DIR="$VIASH_TEMP" - - -# ViashHelp: Display helpful explanation about this executable -function ViashHelp { - echo "create_task_readme dev" - echo "" - echo "Create a README for the task." - echo "" - echo "Inputs:" - echo " --task" - echo " type: string" - echo " example: denoising" - echo " Which task the component will be added to." - echo "" - echo " --task_dir" - echo " type: file, file must exist" - echo " default: src/tasks/\${VIASH_PAR_TASK}" - echo " Path to the task directory." - echo "" - echo " --viash_yaml" - echo " type: file, file must exist" - echo " default: _viash.yaml" - echo " Path to the project config file. Needed for knowing the relative" - echo " location of a file to the project root." - echo "" - echo "Outputs:" - echo " --output" - echo " type: file, output, file must exist" - echo " default: src/tasks/\${VIASH_PAR_TASK}/README.md" - echo " Path to the component directory. Suggested location is" - echo " \`src/tasks//README.md\`." -} - -######## Helper functions for setting up Docker images for viash ######## -# expects: ViashDockerBuild - -# ViashDockerInstallationCheck: check whether Docker is installed correctly -# -# examples: -# ViashDockerInstallationCheck -function ViashDockerInstallationCheck { - ViashDebug "Checking whether Docker is installed" - if [ ! command -v docker &> /dev/null ]; then - ViashCritical "Docker doesn't seem to be installed. See 'https://docs.docker.com/get-docker/' for instructions." - exit 1 - fi - - ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? - [[ $save =~ e ]] && set -e - if [ $out -ne 0 ]; then - ViashCritical "Docker daemon does not seem to be running. Try one of the following:" - ViashCritical "- Try running 'dockerd' in the command line" - ViashCritical "- See https://docs.docker.com/config/daemon/" - exit 1 - fi -} - -# ViashDockerRemoteTagCheck: check whether a Docker image is available -# on a remote. Assumes `docker login` has been performed, if relevant. -# -# $1 : image identifier with format `[registry/]image[:tag]` -# exit code $? : whether or not the image was found -# examples: -# ViashDockerRemoteTagCheck python:latest -# echo $? # returns '0' -# ViashDockerRemoteTagCheck sdaizudceahifu -# echo $? # returns '1' -function ViashDockerRemoteTagCheck { - docker manifest inspect $1 > /dev/null 2> /dev/null -} - -# ViashDockerLocalTagCheck: check whether a Docker image is available locally -# -# $1 : image identifier with format `[registry/]image[:tag]` -# exit code $? : whether or not the image was found -# examples: -# docker pull python:latest -# ViashDockerLocalTagCheck python:latest -# echo $? # returns '0' -# ViashDockerLocalTagCheck sdaizudceahifu -# echo $? # returns '1' -function ViashDockerLocalTagCheck { - [ -n "$(docker images -q $1)" ] -} - -# ViashDockerPull: pull a Docker image -# -# $1 : image identifier with format `[registry/]image[:tag]` -# exit code $? : whether or not the image was found -# examples: -# ViashDockerPull python:latest -# echo $? # returns '0' -# ViashDockerPull sdaizudceahifu -# echo $? # returns '1' -function ViashDockerPull { - ViashNotice "Checking if Docker image is available at '$1'" - if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then - docker pull $1 && return 0 || return 1 - else - save=$-; set +e - docker pull $1 2> /dev/null > /dev/null - out=$? - [[ $save =~ e ]] && set -e - if [ $out -ne 0 ]; then - ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." - fi - return $out - fi -} - -# ViashDockerPush: push a Docker image -# -# $1 : image identifier with format `[registry/]image[:tag]` -# exit code $? : whether or not the image was found -# examples: -# ViashDockerPush python:latest -# echo $? # returns '0' -# ViashDockerPush sdaizudceahifu -# echo $? # returns '1' -function ViashDockerPush { - ViashNotice "Pushing image to '$1'" - save=$-; set +e - if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then - docker push $1 - out=$? - else - docker push $1 2> /dev/null > /dev/null - out=$? - fi - [[ $save =~ e ]] && set -e - if [ $out -eq 0 ]; then - ViashNotice "Container '$VSHD_ID' push succeeded." - else - ViashError "Container '$VSHD_ID' push errored. You might not be logged in or have the necessary permissions." - fi - return $out -} - -# ViashDockerPullElseBuild: pull a Docker image, else build it -# -# $1 : image identifier with format `[registry/]image[:tag]` -# ViashDockerBuild : a Bash function which builds a docker image, takes image identifier as argument. -# examples: -# ViashDockerPullElseBuild mynewcomponent -function ViashDockerPullElseBuild { - save=$-; set +e - ViashDockerPull $1 - out=$? - [[ $save =~ e ]] && set -e - if [ $out -ne 0 ]; then - ViashDockerBuild $@ - fi -} - -# ViashDockerSetup: create a Docker image, according to specified docker setup strategy -# -# $1 : image identifier with format `[registry/]image[:tag]` -# $2 : docker setup strategy, see DockerSetupStrategy.scala -# ViashDockerBuild : a Bash function which builds a docker image, takes image identifier as argument. -# examples: -# ViashDockerSetup mynewcomponent alwaysbuild -function ViashDockerSetup { - VSHD_ID="$1" - VSHD_STRAT="$2" - if [ "$VSHD_STRAT" == "alwaysbuild" -o "$VSHD_STRAT" == "build" -o "$VSHD_STRAT" == "b" ]; then - ViashDockerBuild $VSHD_ID --no-cache - elif [ "$VSHD_STRAT" == "alwayspull" -o "$VSHD_STRAT" == "pull" -o "$VSHD_STRAT" == "p" ]; then - ViashDockerPull $VSHD_ID - elif [ "$VSHD_STRAT" == "alwayspullelsebuild" -o "$VSHD_STRAT" == "pullelsebuild" ]; then - ViashDockerPullElseBuild $VSHD_ID --no-cache - elif [ "$VSHD_STRAT" == "alwayspullelsecachedbuild" -o "$VSHD_STRAT" == "pullelsecachedbuild" ]; then - ViashDockerPullElseBuild $VSHD_ID - elif [ "$VSHD_STRAT" == "alwayscachedbuild" -o "$VSHD_STRAT" == "cachedbuild" -o "$VSHD_STRAT" == "cb" ]; then - ViashDockerBuild $VSHD_ID - elif [[ "$VSHD_STRAT" =~ ^ifneedbe ]]; then - save=$-; set +e - ViashDockerLocalTagCheck $VSHD_ID - outCheck=$? - [[ $save =~ e ]] && set -e - if [ $outCheck -eq 0 ]; then - ViashInfo "Image $VSHD_ID already exists" - elif [ "$VSHD_STRAT" == "ifneedbebuild" ]; then - ViashDockerBuild $VSHD_ID --no-cache - elif [ "$VSHD_STRAT" == "ifneedbecachedbuild" ]; then - ViashDockerBuild $VSHD_ID - elif [ "$VSHD_STRAT" == "ifneedbepull" ]; then - ViashDockerPull $VSHD_ID - elif [ "$VSHD_STRAT" == "ifneedbepullelsebuild" ]; then - ViashDockerPullElseBuild $VSHD_ID --no-cache - elif [ "$VSHD_STRAT" == "ifneedbepullelsecachedbuild" ]; then - ViashDockerPullElseBuild $VSHD_ID - else - ViashError "Unrecognised Docker strategy: $VSHD_STRAT" - exit 1 - fi - elif [ "$VSHD_STRAT" == "push" -o "$VSHD_STRAT" == "forcepush" -o "$VSHD_STRAT" == "alwayspush" ]; then - ViashDockerPush "$VSHD_ID" - elif [ "$VSHD_STRAT" == "pushifnotpresent" -o "$VSHD_STRAT" == "gentlepush" -o "$VSHD_STRAT" == "maybepush" ]; then - save=$-; set +e - ViashDockerRemoteTagCheck $VSHD_ID - outCheck=$? - [[ $save =~ e ]] && set -e - if [ $outCheck -eq 0 ]; then - ViashNotice "Container '$VSHD_ID' exists, doing nothing." - else - ViashNotice "Container '$VSHD_ID' does not yet exist." - ViashDockerPush "$VSHD_ID" - fi - elif [ "$VSHD_STRAT" == "donothing" -o "$VSHD_STRAT" == "meh" ]; then - ViashNotice "Skipping setup." - else - ViashError "Unrecognised Docker strategy: $VSHD_STRAT" - exit 1 - fi -} - -# ViashDockerCheckCommands: Check whether a docker container has the required commands -# -# $1 : image identifier with format `[registry/]image[:tag]` -# $@ : commands to verify being present -# examples: -# ViashDockerCheckCommands bash:4.0 bash ps foo -function ViashDockerCheckCommands { - tag=$1 - shift 1 - commands="$@" - save=$-; set +e - missing=$(docker run --rm --entrypoint=sh $tag -c "for command in $commands; do command -v \$command >/dev/null 2>&1; if [ \$? -ne 0 ]; then echo \$command; exit 1; fi; done") - outCheck=$? - [[ $save =~ e ]] && set -e - if [ $outCheck -ne 0 ]; then - ViashError "Docker container '$tag' does not contain command '$missing'." - exit 1 - fi -} - - -######## End of helper functions for setting up Docker images for viash ######## - -# ViashDockerFile: print the dockerfile to stdout -# return : dockerfile required to run this component -# examples: -# ViashDockerFile -function ViashDockerfile { - cat << 'VIASHDOCKER' -FROM ghcr.io/openproblems-bio/base_r:1.0.4 - -ENTRYPOINT [] - - -RUN Rscript -e 'if (!requireNamespace("remotes", quietly = TRUE)) install.packages("remotes")' && \ - Rscript -e 'remotes::install_cran(c("dplyr", "purrr", "rlang", "glue", "yaml", "fs", "cli", "igraph", "rmarkdown", "processx"), repos = "https://cran.rstudio.com")' && \ - Rscript -e 'remotes::install_github(c("openproblems-bio/core/packages/r/openproblems"), repos = "https://cran.rstudio.com")' - -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y jq curl && \ - rm -rf /var/lib/apt/lists/* - -RUN release_info=$(curl -s https://api.github.com/repos/quarto-dev/quarto-cli/releases/latest) && \ - download_url=$(printf "%s" "$release_info" | jq -r '.assets[] | select(.name | test("quarto-.*-linux-amd64.deb")) | .browser_download_url') && \ - curl -sL "$download_url" -o /opt/quarto.deb && \ - dpkg -i /opt/quarto.deb && \ - rm /opt/quarto.deb - -LABEL org.opencontainers.image.description="Companion container for running component common create_task_readme" -LABEL org.opencontainers.image.created="2024-07-04T17:27:12+02:00" -LABEL org.opencontainers.image.source="https://github.com/openproblems-bio/openproblems-v2" -LABEL org.opencontainers.image.revision="780b5da28208002a0ab409315d42d6453d213635" -LABEL org.opencontainers.image.version="dev" - -VIASHDOCKER -} - -# ViashDockerBuild: build a docker container -# $1 : image identifier with format `[registry/]image[:tag]` -# exit code $? : whether or not the image was built -function ViashDockerBuild { - # create temporary directory to store dockerfile & optional resources in - tmpdir=$(mktemp -d "$VIASH_META_TEMP_DIR/dockerbuild-create_task_readme-XXXXXX") - dockerfile="$tmpdir/Dockerfile" - function clean_up { - rm -rf "$tmpdir" - } - trap clean_up EXIT - - # store dockerfile and resources - ViashDockerfile > $dockerfile - - # Build the container - ViashNotice "Building container '$1' with Dockerfile" - ViashInfo "Running 'docker build -t $@ $VIASH_META_RESOURCES_DIR -f $dockerfile'" - save=$-; set +e - if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then - docker build -t $@ $VIASH_META_RESOURCES_DIR -f $dockerfile - else - docker build -t $@ $VIASH_META_RESOURCES_DIR -f $dockerfile &> $tmpdir/docker_build.log - fi - out=$? - [[ $save =~ e ]] && set -e - if [ $out -ne 0 ]; then - ViashError "Error occurred while building container '$1'" - if [ $VIASH_VERBOSITY -lt $VIASH_LOGCODE_INFO ]; then - ViashError "Transcript: --------------------------------" - cat "$tmpdir/docker_build.log" - ViashError "End of transcript --------------------------" - fi - exit 1 - fi - ViashDockerCheckCommands "$1" 'bash' -} - -# ViashAbsolutePath: generate absolute path from relative path -# borrowed from https://stackoverflow.com/a/21951256 -# $1 : relative filename -# return : absolute path -# examples: -# ViashAbsolutePath some_file.txt # returns /path/to/some_file.txt -# ViashAbsolutePath /foo/bar/.. # returns /foo -function ViashAbsolutePath { - local thePath - if [[ ! "$1" =~ ^/ ]]; then - thePath="$PWD/$1" - else - thePath="$1" - fi - echo "$thePath" | ( - IFS=/ - read -a parr - declare -a outp - for i in "${parr[@]}"; do - case "$i" in - ''|.) continue ;; - ..) - len=${#outp[@]} - if ((len==0)); then - continue - else - unset outp[$((len-1))] - fi - ;; - *) - len=${#outp[@]} - outp[$len]="$i" - ;; - esac - done - echo /"${outp[*]}" - ) -} -# ViashAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_EXTRA_MOUNTS : Added another parameter to be passed to docker -# examples: -# ViashAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' -# ViashAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' -function ViashAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") - if [ -d "$abs_path" ]; then - mount_source="$abs_path" - base_name="" - else - mount_source=`dirname "$abs_path"` - base_name=`basename "$abs_path"` - fi - mount_target="/viash_automount$mount_source" - echo "$mount_target/$base_name" -} -function ViashAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") - if [ -d "$abs_path" ]; then - mount_source="$abs_path" - base_name="" - else - mount_source=`dirname "$abs_path"` - base_name=`basename "$abs_path"` - fi - mount_target="/viash_automount$mount_source" - ViashDebug "ViashAutodetectMountArg $1 -> $mount_source -> $mount_target" - echo "--volume=\"$mount_source:$mount_target\"" -} -function ViashStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" -} -# ViashExtractFlags: Retain leading flag -# $1 : string with a possible leading flag -# return : leading flag -# examples: -# ViashExtractFlags --foo=bar # returns --foo -function ViashExtractFlags { - echo $1 | sed 's/=.*//' -} -# initialise variables -VIASH_EXTRA_MOUNTS=() - -# initialise array -VIASH_POSITIONAL_ARGS='' -VIASH_MODE='run' - -while [[ $# -gt 0 ]]; do - case "$1" in - -h|--help) - ViashHelp - exit - ;; - ---v|---verbose) - let "VIASH_VERBOSITY=VIASH_VERBOSITY+1" - shift 1 - ;; - ---verbosity) - VIASH_VERBOSITY="$2" - shift 2 - ;; - ---verbosity=*) - VIASH_VERBOSITY="$(ViashRemoveFlags "$1")" - shift 1 - ;; - --version) - echo "create_task_readme dev" - exit - ;; - --task) - [ -n "$VIASH_PAR_TASK" ] && ViashError Bad arguments for option \'--task\': \'$VIASH_PAR_TASK\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_TASK="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to --task. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - --task=*) - [ -n "$VIASH_PAR_TASK" ] && ViashError Bad arguments for option \'--task=*\': \'$VIASH_PAR_TASK\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_TASK=$(ViashRemoveFlags "$1") - shift 1 - ;; - --task_dir) - [ -n "$VIASH_PAR_TASK_DIR" ] && ViashError Bad arguments for option \'--task_dir\': \'$VIASH_PAR_TASK_DIR\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_TASK_DIR="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to --task_dir. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - --task_dir=*) - [ -n "$VIASH_PAR_TASK_DIR" ] && ViashError Bad arguments for option \'--task_dir=*\': \'$VIASH_PAR_TASK_DIR\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_TASK_DIR=$(ViashRemoveFlags "$1") - shift 1 - ;; - --viash_yaml) - [ -n "$VIASH_PAR_VIASH_YAML" ] && ViashError Bad arguments for option \'--viash_yaml\': \'$VIASH_PAR_VIASH_YAML\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_VIASH_YAML="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to --viash_yaml. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - --viash_yaml=*) - [ -n "$VIASH_PAR_VIASH_YAML" ] && ViashError Bad arguments for option \'--viash_yaml=*\': \'$VIASH_PAR_VIASH_YAML\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_VIASH_YAML=$(ViashRemoveFlags "$1") - shift 1 - ;; - --output) - [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_OUTPUT="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to --output. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - --output=*) - [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output=*\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_OUTPUT=$(ViashRemoveFlags "$1") - shift 1 - ;; - ---setup) - VIASH_MODE='docker_setup' - VIASH_DOCKER_SETUP_STRATEGY="$2" - shift 1 - ;; - ---setup=*) - VIASH_MODE='docker_setup' - VIASH_DOCKER_SETUP_STRATEGY="$(ViashRemoveFlags "$1")" - shift 2 - ;; - ---dockerfile) - ViashDockerfile - exit 0 - ;; - ---v|---volume) - VIASH_EXTRA_MOUNTS+=("--volume='$2'") - shift 2 - ;; - ---volume=*) - VIASH_EXTRA_MOUNTS+=("--volume='$(ViashRemoveFlags "$2")'") - shift 1 - ;; - ---debug) - VIASH_MODE='docker_debug' - shift 1 - ;; - ---cpus) - [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_META_CPUS="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to ---cpus. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - ---cpus=*) - [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus=*\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_META_CPUS=$(ViashRemoveFlags "$1") - shift 1 - ;; - ---memory) - [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_META_MEMORY="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to ---memory. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - ---memory=*) - [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory=*\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_META_MEMORY=$(ViashRemoveFlags "$1") - shift 1 - ;; - *) # positional arg or unknown option - # since the positional args will be eval'd, can we always quote, instead of using ViashQuote - VIASH_POSITIONAL_ARGS="$VIASH_POSITIONAL_ARGS '$1'" - [[ $1 == -* ]] && ViashWarning $1 looks like a parameter but is not a defined parameter and will instead be treated as a positional argument. Use "--help" to get more information on the parameters. - shift # past argument - ;; - esac -done - -# parse positional parameters -eval set -- $VIASH_POSITIONAL_ARGS - - -ViashDockerInstallationCheck - -if [ $VIASH_MODE == "docker_setup" ]; then - ViashDockerSetup 'ghcr.io/openproblems-bio/common/create_task_readme:dev' "$VIASH_DOCKER_SETUP_STRATEGY" - exit 0 -fi -ViashDockerSetup 'ghcr.io/openproblems-bio/common/create_task_readme:dev' ifneedbepullelsecachedbuild - -if [ $VIASH_MODE == "docker_debug" ]; then - ViashNotice "+ docker run --entrypoint=bash -i --rm -v "$(pwd)":/pwd --workdir /pwd -t 'ghcr.io/openproblems-bio/common/create_task_readme:dev'" - docker run --entrypoint=bash -i --rm -v "$(pwd)":/pwd --workdir /pwd -t 'ghcr.io/openproblems-bio/common/create_task_readme:dev' - exit 0 -fi - -# setting computational defaults - -# helper function for parsing memory strings -function ViashMemoryAsBytes { - local memory=`echo "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]'` - local memory_regex='^([0-9]+)([kmgtp]b?|b)$' - if [[ $memory =~ $memory_regex ]]; then - local number=${memory/[^0-9]*/} - local symbol=${memory/*[0-9]/} - - case $symbol in - b) memory_b=$number ;; - kb|k) memory_b=$(( $number * 1024 )) ;; - mb|m) memory_b=$(( $number * 1024 * 1024 )) ;; - gb|g) memory_b=$(( $number * 1024 * 1024 * 1024 )) ;; - tb|t) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 )) ;; - pb|p) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 * 1024 )) ;; - esac - echo "$memory_b" - fi -} -# compute memory in different units -if [ ! -z ${VIASH_META_MEMORY+x} ]; then - VIASH_META_MEMORY_B=`ViashMemoryAsBytes $VIASH_META_MEMORY` - # do not define other variables if memory_b is an empty string - if [ ! -z "$VIASH_META_MEMORY_B" ]; then - VIASH_META_MEMORY_KB=$(( ($VIASH_META_MEMORY_B+1023) / 1024 )) - VIASH_META_MEMORY_MB=$(( ($VIASH_META_MEMORY_KB+1023) / 1024 )) - VIASH_META_MEMORY_GB=$(( ($VIASH_META_MEMORY_MB+1023) / 1024 )) - VIASH_META_MEMORY_TB=$(( ($VIASH_META_MEMORY_GB+1023) / 1024 )) - VIASH_META_MEMORY_PB=$(( ($VIASH_META_MEMORY_TB+1023) / 1024 )) - else - # unset memory if string is empty - unset $VIASH_META_MEMORY_B - fi -fi -# unset nproc if string is empty -if [ -z "$VIASH_META_CPUS" ]; then - unset $VIASH_META_CPUS -fi - - -# check whether required parameters exist -if [ -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then - ViashError 'functionality_name' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi -if [ -z ${VIASH_META_RESOURCES_DIR+x} ]; then - ViashError 'resources_dir' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi -if [ -z ${VIASH_META_EXECUTABLE+x} ]; then - ViashError 'executable' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi -if [ -z ${VIASH_META_CONFIG+x} ]; then - ViashError 'config' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi -if [ -z ${VIASH_META_TEMP_DIR+x} ]; then - ViashError 'temp_dir' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi - -# filling in defaults -if [ -z ${VIASH_PAR_TASK_DIR+x} ]; then - VIASH_PAR_TASK_DIR="src/tasks/${VIASH_PAR_TASK}" -fi -if [ -z ${VIASH_PAR_VIASH_YAML+x} ]; then - VIASH_PAR_VIASH_YAML="_viash.yaml" -fi -if [ -z ${VIASH_PAR_OUTPUT+x} ]; then - VIASH_PAR_OUTPUT="src/tasks/${VIASH_PAR_TASK}/README.md" -fi - -# check whether required files exist -if [ ! -z "$VIASH_PAR_TASK_DIR" ] && [ ! -e "$VIASH_PAR_TASK_DIR" ]; then - ViashError "Input file '$VIASH_PAR_TASK_DIR' does not exist." - exit 1 -fi -if [ ! -z "$VIASH_PAR_VIASH_YAML" ] && [ ! -e "$VIASH_PAR_VIASH_YAML" ]; then - ViashError "Input file '$VIASH_PAR_VIASH_YAML' does not exist." - exit 1 -fi - -# check whether parameters values are of the right type -if [[ -n "$VIASH_META_CPUS" ]]; then - if ! [[ "$VIASH_META_CPUS" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'cpus' has to be an integer. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_B" ]]; then - if ! [[ "$VIASH_META_MEMORY_B" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_b' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_KB" ]]; then - if ! [[ "$VIASH_META_MEMORY_KB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_kb' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_MB" ]]; then - if ! [[ "$VIASH_META_MEMORY_MB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_mb' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_GB" ]]; then - if ! [[ "$VIASH_META_MEMORY_GB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_gb' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_TB" ]]; then - if ! [[ "$VIASH_META_MEMORY_TB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_tb' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_PB" ]]; then - if ! [[ "$VIASH_META_MEMORY_PB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_pb' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi - -# create parent directories of output files, if so desired -if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -d "$(dirname "$VIASH_PAR_OUTPUT")" ]; then - mkdir -p "$(dirname "$VIASH_PAR_OUTPUT")" -fi - -# detect volumes from file arguments -VIASH_CHOWN_VARS=() -if [ ! -z "$VIASH_PAR_TASK_DIR" ]; then - VIASH_EXTRA_MOUNTS+=( "$(ViashAutodetectMountArg "$VIASH_PAR_TASK_DIR")" ) - VIASH_PAR_TASK_DIR=$(ViashAutodetectMount "$VIASH_PAR_TASK_DIR") -fi -if [ ! -z "$VIASH_PAR_VIASH_YAML" ]; then - VIASH_EXTRA_MOUNTS+=( "$(ViashAutodetectMountArg "$VIASH_PAR_VIASH_YAML")" ) - VIASH_PAR_VIASH_YAML=$(ViashAutodetectMount "$VIASH_PAR_VIASH_YAML") -fi -if [ ! -z "$VIASH_PAR_OUTPUT" ]; then - VIASH_EXTRA_MOUNTS+=( "$(ViashAutodetectMountArg "$VIASH_PAR_OUTPUT")" ) - VIASH_PAR_OUTPUT=$(ViashAutodetectMount "$VIASH_PAR_OUTPUT") - VIASH_CHOWN_VARS+=( "$VIASH_PAR_OUTPUT" ) -fi -if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then - VIASH_EXTRA_MOUNTS+=( "$(ViashAutodetectMountArg "$VIASH_META_RESOURCES_DIR")" ) - VIASH_META_RESOURCES_DIR=$(ViashAutodetectMount "$VIASH_META_RESOURCES_DIR") -fi -if [ ! -z "$VIASH_META_EXECUTABLE" ]; then - VIASH_EXTRA_MOUNTS+=( "$(ViashAutodetectMountArg "$VIASH_META_EXECUTABLE")" ) - VIASH_META_EXECUTABLE=$(ViashAutodetectMount "$VIASH_META_EXECUTABLE") -fi -if [ ! -z "$VIASH_META_CONFIG" ]; then - VIASH_EXTRA_MOUNTS+=( "$(ViashAutodetectMountArg "$VIASH_META_CONFIG")" ) - VIASH_META_CONFIG=$(ViashAutodetectMount "$VIASH_META_CONFIG") -fi -if [ ! -z "$VIASH_META_TEMP_DIR" ]; then - VIASH_EXTRA_MOUNTS+=( "$(ViashAutodetectMountArg "$VIASH_META_TEMP_DIR")" ) - VIASH_META_TEMP_DIR=$(ViashAutodetectMount "$VIASH_META_TEMP_DIR") -fi - -# get unique mounts -VIASH_UNIQUE_MOUNTS=($(for val in "${VIASH_EXTRA_MOUNTS[@]}"; do echo "$val"; done | sort -u)) - -# change file ownership -function ViashPerformChown { - if (( ${#VIASH_CHOWN_VARS[@]} )); then - set +e - eval docker run --entrypoint=chown -i --rm ${VIASH_UNIQUE_MOUNTS[@]} ghcr.io/openproblems-bio/common/create_task_readme:dev "$(id -u):$(id -g)" --silent --recursive ${VIASH_CHOWN_VARS[@]} - set -e - fi -} -trap ViashPerformChown EXIT - -# helper function for filling in extra docker args -VIASH_EXTRA_DOCKER_ARGS="" -if [ ! -z "$VIASH_META_MEMORY_MB" ]; then - VIASH_EXTRA_DOCKER_ARGS="$VIASH_EXTRA_DOCKER_ARGS --memory=${VIASH_META_MEMORY_MB}m" -fi -if [ ! -z "$VIASH_META_CPUS" ]; then - VIASH_EXTRA_DOCKER_ARGS="$VIASH_EXTRA_DOCKER_ARGS --cpus=${VIASH_META_CPUS}" -fi - - -# set dependency paths - - -ViashDebug "Running command: $(echo docker run --entrypoint=bash -i --rm ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_EXTRA_DOCKER_ARGS ghcr.io/openproblems-bio/common/create_task_readme:dev)" -cat << VIASHEOF | eval docker run --entrypoint=bash -i --rm ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_EXTRA_DOCKER_ARGS ghcr.io/openproblems-bio/common/create_task_readme:dev -set -e -tempscript=\$(mktemp "$VIASH_META_TEMP_DIR/viash-run-create_task_readme-XXXXXX").R -function clean_up { - rm "\$tempscript" -} -function interrupt { - echo -e "\nCTRL-C Pressed..." - exit 1 -} -trap clean_up EXIT -trap interrupt INT SIGINT -cat > "\$tempscript" << 'VIASHMAIN' -library(rlang, quietly = TRUE, warn.conflicts = FALSE) -library(purrr, quietly = TRUE, warn.conflicts = FALSE) -library(dplyr, quietly = TRUE, warn.conflicts = FALSE) -library(yaml, quietly = TRUE, warn.conflicts = FALSE) -library(openproblems) - -## VIASH START -# The following code has been auto-generated by Viash. -# treat warnings as errors -.viash_orig_warn <- options(warn = 2) - -par <- list( - "task" = $( if [ ! -z ${VIASH_PAR_TASK+x} ]; then echo -n "'"; echo -n "$VIASH_PAR_TASK" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi ), - "task_dir" = $( if [ ! -z ${VIASH_PAR_TASK_DIR+x} ]; then echo -n "'"; echo -n "$VIASH_PAR_TASK_DIR" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi ), - "viash_yaml" = $( if [ ! -z ${VIASH_PAR_VIASH_YAML+x} ]; then echo -n "'"; echo -n "$VIASH_PAR_VIASH_YAML" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi ), - "output" = $( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo -n "'"; echo -n "$VIASH_PAR_OUTPUT" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi ) -) -meta <- list( - "functionality_name" = $( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo -n "'"; echo -n "$VIASH_META_FUNCTIONALITY_NAME" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi ), - "resources_dir" = $( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo -n "'"; echo -n "$VIASH_META_RESOURCES_DIR" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi ), - "executable" = $( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo -n "'"; echo -n "$VIASH_META_EXECUTABLE" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi ), - "config" = $( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo -n "'"; echo -n "$VIASH_META_CONFIG" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi ), - "temp_dir" = $( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo -n "'"; echo -n "$VIASH_META_TEMP_DIR" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi ), - "cpus" = $( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo -n "as.integer('"; echo -n "$VIASH_META_CPUS" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi ), - "memory_b" = $( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo -n "bit64::as.integer64('"; echo -n "$VIASH_META_MEMORY_B" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi ), - "memory_kb" = $( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo -n "bit64::as.integer64('"; echo -n "$VIASH_META_MEMORY_KB" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi ), - "memory_mb" = $( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo -n "bit64::as.integer64('"; echo -n "$VIASH_META_MEMORY_MB" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi ), - "memory_gb" = $( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo -n "bit64::as.integer64('"; echo -n "$VIASH_META_MEMORY_GB" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi ), - "memory_tb" = $( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo -n "bit64::as.integer64('"; echo -n "$VIASH_META_MEMORY_TB" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi ), - "memory_pb" = $( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo -n "bit64::as.integer64('"; echo -n "$VIASH_META_MEMORY_PB" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi ) -) -dep <- list( - -) - - -# restore original warn setting -options(.viash_orig_warn) -rm(.viash_orig_warn) - -## VIASH END - -if (is.null(par\$task) && is.null(par\$task_dir)) { - stop("Either 'task' or 'task_dir' must be provided") -} -if (is.null(par\$viash_yaml)) { - stop("Argument 'viash_yaml' must be provided") -} -if (is.null(par\$output)) { - stop("Argument 'output' must be provided") -} - -# import helper function -source(paste0(meta["resources_dir"], "/read_api_files.R")) - -cat("Read task info\\n") -task_api <- read_task_api(par[["task_dir"]]) -viash_info <- yaml.load_file(par[["viash_yaml"]]) - -# determine ordering -root <- .task_graph_get_root(task_api) - -r_graph <- render_task_graph(task_api, root) - -cat("Render API details\\n") -order <- names(igraph::bfs(task_api\$task_graph, root)\$order) -r_details <- map_chr( - order, - function(file_name) { - if (file_name %in% names(task_api\$comp_specs)) { - render_component(task_api\$comp_specs[[file_name]]) - } else { - render_file(task_api\$file_specs[[file_name]]) - } - } -) - -cat("Render authors\\n") -authors_str <- - if (nrow(task_api\$authors) > 0) { - paste0( - "\\n## Authors & contributors\\n\\n", - task_api\$authors %>% knitr::kable() %>% paste(collapse = "\\n"), - "\\n" - ) - } else { - "" - } - -readme_base <- openproblems::strip_margin(glue::glue(" - §## Installation - § - §You need to have Docker, Java, and Viash installed. Follow - §[these instructions](https://openproblems.bio/documentation/fundamentals/requirements) - §to install the required dependencies. - § - §## Add a method - § - §To add a method to the repository, follow the instructions in the \`scripts/add_a_method.sh\` script. - § - §## Frequently used commands - § - §To get started, you can run the following commands: - § - §\`\`\`bash - §git clone git@github.com:openproblems-bio/{viash_info\$name}.git - § - §cd {viash_info\$name} - § - §# initialise submodule - §scripts/init_submodule.sh - § - §# download resources - §scripts/download_resources.sh - §\`\`\` - § - §To run the benchmark, you first need to build the components. Afterwards, you can run the benchmark: - § - §\`\`\`bash - §viash ns build --parallel --setup cachedbuild - § - §scripts/run_benchmark.sh - §\`\`\` - § - §After adding a component, it is recommended to run the tests to ensure that the component is working correctly: - § - §\`\`\`bash - §viash ns test --parallel - §\`\`\` - § - §Optionally, you can provide the \`--query\` argument to test only a subset of components: - § - §\`\`\`bash - §viash ns test --parallel --query 'component_name' - §\`\`\` -"), symbol = "§") - -readme_str <- - if (is.null(task_api\$task_info\$readme) || is.na(task_api\$task_info\$readme)) { - paste0( - "\\n## README\\n\\n", - readme_base, - "\\n" - ) - } else { - paste0( - "\\n## README\\n\\n", - task_api\$task_info\$readme, - "\\n" - ) - } - -# get relevant taks info - - - -cat("Generate qmd content\\n") -relative_path <- par[["task_dir"]] %>% - gsub(paste0(dirname(par[["viash_yaml"]]), "/*"), "", .) %>% - gsub("/*\$", "", .) -source_url <- paste0(viash_info\$links\$repository, "/", relative_path) -qmd_content <- openproblems::strip_margin(glue::glue(" - §--- - §title: \\"{task_api\$task_info\$label}\\" - §format: gfm - §--- - § - § - § - §{task_api\$task_info\$summary} - § - §Path to source: [\`{relative_path}\`]({source_url}) - § - §{readme_str} - § - §## Motivation - § - §{task_api\$task_info\$motivation} - § - §## Description - § - §{task_api\$task_info\$description} - §{authors_str} - §## API - § - §{r_graph} - § - §{paste(r_details, collapse = '\\n\\n')} - § - §"), symbol = "§") - -cat("Write README.qmd to file\\n") -qmd_file <- tempfile( - pattern = "README_", - fileext = ".qmd", - tmpdir = meta\$temp_dir -) - -if (!dir.exists(meta\$temp_dir)) { - dir.create(meta\$temp_dir, recursive = TRUE) -} -writeLines(qmd_content, qmd_file) - -cat("Render README.qmd to README.md\\n") -out <- processx::run( - command = "quarto", - args = c("render", qmd_file, "--output", "-"), - echo = TRUE -) - -writeLines(out\$stdout, par\$output) -VIASHMAIN -Rscript "\$tempscript" & -wait "\$!" - -VIASHEOF - - -# strip viash automount from file paths -if [ ! -z "$VIASH_PAR_TASK_DIR" ]; then - VIASH_PAR_TASK_DIR=$(ViashStripAutomount "$VIASH_PAR_TASK_DIR") -fi -if [ ! -z "$VIASH_PAR_VIASH_YAML" ]; then - VIASH_PAR_VIASH_YAML=$(ViashStripAutomount "$VIASH_PAR_VIASH_YAML") -fi -if [ ! -z "$VIASH_PAR_OUTPUT" ]; then - VIASH_PAR_OUTPUT=$(ViashStripAutomount "$VIASH_PAR_OUTPUT") -fi -if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then - VIASH_META_RESOURCES_DIR=$(ViashStripAutomount "$VIASH_META_RESOURCES_DIR") -fi -if [ ! -z "$VIASH_META_EXECUTABLE" ]; then - VIASH_META_EXECUTABLE=$(ViashStripAutomount "$VIASH_META_EXECUTABLE") -fi -if [ ! -z "$VIASH_META_CONFIG" ]; then - VIASH_META_CONFIG=$(ViashStripAutomount "$VIASH_META_CONFIG") -fi -if [ ! -z "$VIASH_META_TEMP_DIR" ]; then - VIASH_META_TEMP_DIR=$(ViashStripAutomount "$VIASH_META_TEMP_DIR") -fi - - -# check whether required files exist -if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -e "$VIASH_PAR_OUTPUT" ]; then - ViashError "Output file '$VIASH_PAR_OUTPUT' does not exist." - exit 1 -fi - - -exit 0 diff --git a/create_task_readme/read_api_files.R b/create_task_readme/read_api_files.R deleted file mode 100644 index 9abb351..0000000 --- a/create_task_readme/read_api_files.R +++ /dev/null @@ -1,432 +0,0 @@ - -anndata_struct_names <- c("obs", "var", "obsm", "obsp", "varm", "varp", "layers", "uns") - -read_anndata_spec <- function(path) { - spec <- openproblems::read_nested_yaml(path) - list( - info = read_anndata_info(spec, path), - slots = read_anndata_slots(spec, path) - ) -} -read_anndata_info <- function(spec, path) { - # TEMP: make it readable - spec$info$slots <- NULL - df <- list_as_tibble(spec) - if (list_contains_tibble(spec$info)) { - df <- dplyr::bind_cols(df, list_as_tibble(spec$info)) - } - df$file_name <- basename(path) %>% gsub("\\.yaml", "", .) - df$description <- df$description %||% NA_character_ %>% as.character - df$summary <- df$summary %||% NA_character_ %>% as.character - as_tibble(df) -} -read_anndata_slots <- function(spec, path) { - map_df( - anndata_struct_names, - function(struct_name, slot) { - slot <- spec$info$slots[[struct_name]] - if (is.null(slot)) return(NULL) - df <- map_df(slot, as.data.frame) - df$struct <- struct_name - df$file_name <- basename(path) %>% gsub("\\.yaml", "", .) - df$required <- df$required %||% TRUE %|% TRUE - df$multiple <- df$multiple %||% FALSE %|% FALSE - as_tibble(df) - } - ) -} - -format_slots <- function(spec) { - example <- spec$slots %>% - group_by(struct) %>% - summarise( - str = paste0(unique(struct), ": ", paste0("'", name, "'", collapse = ", ")) - ) %>% - arrange(match(struct, anndata_struct_names)) - - c(" AnnData object", paste0(" ", example$str)) -} - -format_slots_as_kable <- function(spec) { - if (nrow(spec$slots) == 0) return("") - spec$slots %>% - mutate( - tag_str = pmap_chr(lst(required), function(required) { - out <- c() - if (!required) { - out <- c(out, "Optional") - } - if (length(out) == 0) { - "" - } else { - paste0("(_", paste(out, collapse = ", "), "_) ") - } - }) - ) %>% - transmute( - Slot = paste0("`", struct, "[\"", name, "\"]`"), - Type = paste0("`", type, "`"), - Description = paste0( - tag_str, - description %>% gsub(" *\n *", " ", .) %>% gsub("\\. *$", "", .), - "." - ) - ) %>% - knitr::kable() -} - -list_contains_tibble <- function(li) { - is.list(li) && any(sapply(li, is.atomic)) -} - -list_as_tibble <- function(li) { - as.data.frame(li[sapply(li, is.atomic)], check.names = FALSE) -} - -read_comp_spec <- function(path) { - spec_yaml <- openproblems::read_nested_yaml(path) - list( - info = read_comp_info(spec_yaml, path), - args = read_comp_args(spec_yaml, path) - ) -} - -read_comp_info <- function(spec_yaml, path) { - # TEMP: make it readable - spec_yaml$arguments <- NULL - spec_yaml$argument_groups <- NULL - - df <- list_as_tibble(spec_yaml) - if (list_contains_tibble(spec_yaml$info)) { - df <- dplyr::bind_cols(df, list_as_tibble(spec_yaml$info)) - } - if (list_contains_tibble(spec_yaml$info$type_info)) { - df <- dplyr::bind_cols(df, list_as_tibble(spec_yaml$info$type_info)) - } - df$file_name <- basename(path) %>% gsub("\\.yaml", "", .) - as_tibble(df) -} - -read_comp_args <- function(spec_yaml, path) { - arguments <- spec_yaml$arguments - for (arg_group in spec_yaml$argument_groups) { - arguments <- c(arguments, arg_group$arguments) - } - map_df(arguments, function(arg) { - df <- list_as_tibble(arg) - if (list_contains_tibble(arg$info)) { - df <- dplyr::bind_cols(df, list_as_tibble(arg$info)) - } - df$file_name <- basename(path) %>% gsub("\\.yaml", "", .) - df$arg_name <- gsub("^-*", "", arg$name) - df$direction <- df$direction %||% "input" %|% "input" - df$parent <- df$`__merge__` %||% NA_character_ %>% basename() %>% gsub("\\.yaml", "", .) - df$required <- df$required %||% FALSE %|% FALSE - df$default <- df$default %||% NA_character_ %>% as.character - df$example <- df$example %||% NA_character_ %>% as.character - df$description <- df$description %||% NA_character_ %>% as.character - df$summary <- df$summary %||% NA_character_ %>% as.character - df - }) -} - -format_comp_args_as_tibble <- function(spec) { - if (nrow(spec$args) == 0) return("") - spec$args %>% - mutate( - tag_str = pmap_chr(lst(required, direction), function(required, direction) { - out <- c() - if (!required) { - out <- c(out, "Optional") - } - if (direction == "output") { - out <- c(out, "Output") - } - if (length(out) == 0) { - "" - } else { - paste0("(_", paste(out, collapse = ", "), "_) ") - } - }) - ) %>% - transmute( - Name = paste0("`--", arg_name, "`"), - Type = paste0("`", type, "`"), - Description = paste0( - tag_str, - (summary %|% description) %>% gsub(" *\n *", " ", .) %>% gsub("\\. *$", "", .), - ".", - ifelse(!is.na(default), paste0(" Default: `", default, "`."), "") - ) - ) %>% - knitr::kable() -} - -# path <- "src/datasets/api/comp_processor_knn.yaml" -render_component <- function(spec) { - if (is.character(spec)) { - spec <- read_comp_spec(spec) - } - - openproblems::strip_margin(glue::glue(" - §## Component type: {spec$info$label} - § - §Path: [`src/{spec$info$namespace}`](https://github.com/openproblems-bio/openproblems-v2/tree/main/src/{spec$info$namespace}) - § - §{spec$info$summary} - § - §Arguments: - § - §:::{{.small}} - §{paste(format_comp_args_as_tibble(spec), collapse = '\n')} - §::: - § - §"), symbol = "§") -} - -# path <- "src/datasets/api/file_pca.yaml" -render_file <- function(spec) { - if (is.character(spec)) { - spec <- read_anndata_spec(spec) - } - - if (!"label" %in% names(spec$info)) { - spec$info$label <- basename(spec$info$example) - } - - example <- - if (is.null(spec$info$example) || is.na(spec$info$example)) { - "" - } else { - paste0("Example file: `", spec$info$example, "`") - } - - description <- - if (is.null(spec$info$description) || is.na(spec$info$description)) { - "" - } else { - paste0("Description:\n\n", spec$info$description) - } - - openproblems::strip_margin(glue::glue(" - §## File format: {spec$info$label} - § - §{spec$info$summary %||% ''} - § - §{example} - § - §{description} - § - §Format: - § - §:::{{.small}} - §{paste(format_slots(spec), collapse = '\n')} - §::: - § - §Slot description: - § - §:::{{.small}} - §{paste(format_slots_as_kable(spec), collapse = '\n')} - §::: - § - §"), symbol = "§") -} - -# path <- "src/tasks/denoising" -read_task_api <- function(path) { - cli::cli_inform("Looking for project root") - project_path <- openproblems::find_project_root(path) - api_dir <- paste0(path, "/api") - - cli::cli_inform("Reading task info") - task_info_yaml <- list.files(api_dir, pattern = "task_info.ya?ml", full.names = TRUE) - assertthat::assert_that(length(task_info_yaml) == 1) - task_info <- openproblems::read_nested_yaml(task_info_yaml, project_path) - - cli::cli_inform("Reading task authors") - authors <- map_df(task_info$authors, function(aut) { - aut$roles <- paste(aut$roles, collapse = ", ") - list_as_tibble(aut) - }) - - cli::cli_inform("Reading component yamls") - comp_yamls <- list.files(api_dir, pattern = "comp_.*\\.ya?ml", full.names = TRUE) - comps <- map(comp_yamls, read_comp_spec) - comp_info <- map_df(comps, "info") - comp_args <- map_df(comps, "args") - names(comps) <- basename(comp_yamls) %>% gsub("\\..*$", "", .) - - cli::cli_inform("Reading file yamls") - file_yamls <- openproblems:::resolve_path( - path = na.omit(unique(comp_args$`__merge__`)), - project_path = project_path, - parent_path = api_dir - ) - files <- map(file_yamls, read_anndata_spec) - names(files) <- basename(file_yamls) %>% gsub("\\..*$", "", .) - file_info <- map_df(files, "info") - file_slots <- map_df(files, "slots") - - cli::cli_inform("Generating task graph") - task_graph <- create_task_graph(file_info, comp_info, comp_args) - - list( - task_info = task_info, - file_specs = files, - file_info = file_info, - file_slots = file_slots, - comp_specs = comps, - comp_info = comp_info, - comp_args = comp_args, - task_graph = task_graph, - authors = authors - ) -} - - -create_task_graph <- function(file_info, comp_info, comp_args) { - clean_id <- function(id) { - gsub("graph", "graaf", id) - } - nodes <- - bind_rows( - file_info %>% - mutate(id = file_name, label = label, is_comp = FALSE), - comp_info %>% - mutate(id = file_name, label = label, is_comp = TRUE) - ) %>% - select(id, label, everything()) %>% - mutate(str = paste0( - " ", - clean_id(id), - ifelse(is_comp, "[/\"", "(\""), - label, - ifelse(is_comp, "\"/]", "\")") - )) - edges <- bind_rows( - comp_args %>% - filter(type == "file", direction == "input") %>% - mutate( - from = parent, - to = file_name, - arrow = "---" - ), - comp_args %>% - filter(type == "file", direction == "output") %>% - mutate( - from = file_name, - to = parent, - arrow = "-->" - ) - ) %>% - select(from, to, everything()) %>% - mutate(str = paste0(" ", clean_id(from), arrow, clean_id(to))) - - igraph::graph_from_data_frame( - edges, - vertices = nodes, - directed = TRUE - ) -} - -.task_graph_get_root <- function(task_api) { - root <- names(which(igraph::degree(task_api$task_graph, mode = "in") == 0)) - if (length(root) > 1) { - warning( - "There should probably only be one node with in-degree equal to 0.\n", - " Nodes with in-degree == 0: ", paste(root, collapse = ", ") - ) - } - root[[1]] -} - -render_task_graph <- function(task_api, root = .task_graph_get_root(task_api)) { - order <- names(igraph::bfs(task_api$task_graph, root)$order) - - vdf <- igraph::as_data_frame(task_api$task_graph, "vertices") %>% - arrange(match(name, order)) - edf <- igraph::as_data_frame(task_api$task_graph, "edges") %>% - arrange(match(from, order), match(to, order)) - - openproblems::strip_margin(glue::glue(" - §```mermaid - §flowchart LR - §{paste(vdf$str, collapse = '\n')} - §{paste(edf$str, collapse = '\n')} - §``` - §"), symbol = "§") -} - - - -# Recursive function to process each property with indentation -.render_example_process_property <- function(prop, prop_name = NULL, indent_level = 0) { - if (is.null(prop_name)) { - prop_name <- "" - } - - out <- c() - - # define helper variables - indent_spaces <- strrep(" ", indent_level) - next_indent_spaces <- strrep(" ", indent_level + 2) - - # add comment if available - if ("description" %in% names(prop)) { - comment <- gsub("\n", paste0("\n", indent_spaces, "# "), stringr::str_trim(prop$description)) - out <- c(out, indent_spaces, "# ", comment, "\n") - } - - # add variable - out <- c(out, indent_spaces, prop_name, ": ") - - if (prop$type == "object" && "properties" %in% names(prop)) { - # Handle object with properties - prop_names <- setdiff(names(prop$properties), "additionalProperties") - sub_props <- unlist(lapply(prop_names, function(sub_prop_name) { - prop_out <- .render_example_process_property( - prop$properties[[sub_prop_name]], - sub_prop_name, - indent_level + 2 - ) - c(prop_out, "\n") - })) - c(out, "\n", sub_props[-length(sub_props)]) - } else if (prop$type == "array") { - if (is.list(prop$items) && "properties" %in% names(prop$items)) { - # Handle array of objects - array_items_yaml <- unlist(lapply(names(prop$items$properties), function(item_prop_name) { - prop_out <- .render_example_process_property( - prop$items$properties[[item_prop_name]], - item_prop_name, - indent_level + 4 - ) - c(prop_out, "\n") - })) - c(out, "\n", next_indent_spaces, "- ", array_items_yaml[-1]) - } else { - # Handle simple array - c(out, "[ ... ]") - } - } else { - c(out, "...") - } -} - -# Function for rendering an example yaml based on a JSON schema -render_example <- function(json_schema) { - if (!"properties" %in% names(json_schema)) { - return("") - } - text <- - unlist(lapply(names(json_schema$properties), function(prop_name) { - out <- .render_example_process_property( - json_schema$properties[[prop_name]], - prop_name, - 0 - ) - c(out, "\n") - })) - - paste(text, collapse = "") -} \ No newline at end of file diff --git a/sync_resources/.build.yaml b/sync_resources/.build.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/sync_resources/.config.vsh.yaml b/sync_resources/.config.vsh.yaml deleted file mode 100644 index 1e38d4f..0000000 --- a/sync_resources/.config.vsh.yaml +++ /dev/null @@ -1,184 +0,0 @@ -name: "sync_resources" -namespace: "common" -version: "dev" -argument_groups: -- name: "Arguments" - arguments: - - type: "file" - name: "--input" - alternatives: - - "-i" - description: "Path _viash.yaml file with s3 path to resources." - info: null - default: - - "_viash.yaml" - must_exist: true - create_parent: true - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "file" - name: "--output" - alternatives: - - "-o" - description: "Path to the test resource directory." - info: null - default: - - "/resources_test" - must_exist: true - create_parent: true - required: false - direction: "output" - multiple: false - multiple_sep: ";" - - type: "boolean_true" - name: "--quiet" - description: "Displays the operations that would be performed using the specified\ - \ command without actually running them." - info: null - direction: "input" - - type: "boolean_true" - name: "--dryrun" - description: "Does not display the operations performed from the specified command." - info: null - direction: "input" - - type: "boolean_true" - name: "--delete" - description: "Files that exist in the destination but not in the source are deleted\ - \ during sync." - info: null - direction: "input" - - type: "string" - name: "--exclude" - description: "Exclude all files or objects from the command that matches the specified\ - \ pattern." - info: null - required: false - direction: "input" - multiple: true - multiple_sep: ";" -resources: -- type: "bash_script" - path: "script.sh" - is_executable: true -description: "Synchronise the resources from s3 to resources_test" -usage: "sync_resources\nsync_resources --input _viash.yaml --output resources_test\n" -test_resources: -- type: "bash_script" - path: "run_test.sh" - is_executable: true -info: null -status: "enabled" -license: "MIT" -links: - repository: "https://github.com/openproblems-bio/common_resources" - docker_registry: "ghcr.io" -runners: -- type: "executable" - id: "executable" - docker_setup_strategy: "ifneedbepullelsecachedbuild" -- type: "nextflow" - id: "nextflow" - directives: - tag: "$id" - auto: - simplifyInput: true - simplifyOutput: false - transcript: false - publish: false - config: - labels: - mem1gb: "memory = 1000000000.B" - mem2gb: "memory = 2000000000.B" - mem5gb: "memory = 5000000000.B" - mem10gb: "memory = 10000000000.B" - mem20gb: "memory = 20000000000.B" - mem50gb: "memory = 50000000000.B" - mem100gb: "memory = 100000000000.B" - mem200gb: "memory = 200000000000.B" - mem500gb: "memory = 500000000000.B" - mem1tb: "memory = 1000000000000.B" - mem2tb: "memory = 2000000000000.B" - mem5tb: "memory = 5000000000000.B" - mem10tb: "memory = 10000000000000.B" - mem20tb: "memory = 20000000000000.B" - mem50tb: "memory = 50000000000000.B" - mem100tb: "memory = 100000000000000.B" - mem200tb: "memory = 200000000000000.B" - mem500tb: "memory = 500000000000000.B" - mem1gib: "memory = 1073741824.B" - mem2gib: "memory = 2147483648.B" - mem4gib: "memory = 4294967296.B" - mem8gib: "memory = 8589934592.B" - mem16gib: "memory = 17179869184.B" - mem32gib: "memory = 34359738368.B" - mem64gib: "memory = 68719476736.B" - mem128gib: "memory = 137438953472.B" - mem256gib: "memory = 274877906944.B" - mem512gib: "memory = 549755813888.B" - mem1tib: "memory = 1099511627776.B" - mem2tib: "memory = 2199023255552.B" - mem4tib: "memory = 4398046511104.B" - mem8tib: "memory = 8796093022208.B" - mem16tib: "memory = 17592186044416.B" - mem32tib: "memory = 35184372088832.B" - mem64tib: "memory = 70368744177664.B" - mem128tib: "memory = 140737488355328.B" - mem256tib: "memory = 281474976710656.B" - mem512tib: "memory = 562949953421312.B" - cpu1: "cpus = 1" - cpu2: "cpus = 2" - cpu5: "cpus = 5" - cpu10: "cpus = 10" - cpu20: "cpus = 20" - cpu50: "cpus = 50" - cpu100: "cpus = 100" - cpu200: "cpus = 200" - cpu500: "cpus = 500" - cpu1000: "cpus = 1000" - debug: false - container: "docker" -engines: -- type: "docker" - id: "docker" - image: "amazon/aws-cli:2.17.11" - namespace_separator: "/" - setup: - - type: "yum" - packages: - - "wget" - - type: "docker" - run: - - "wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64\ - \ -O /usr/bin/yq && \\\nchmod +x /usr/bin/yq\n" - entrypoint: [] - cmd: null -build_info: - config: "src/sync_resources/config.vsh.yaml" - runner: "executable" - engine: "docker" - output: "sync_resources" - executable: "sync_resources/sync_resources" - viash_version: "0.9.0-RC6" - git_commit: "7bd60870f551dd252f8316467e710ab0d9440bfa" - git_remote: "git@github.com:openproblems-bio/common-resources.git" -package_config: - name: "common_resources" - version: "dev" - description: "Common resources for the OpenProblems project.\n" - info: null - viash_version: "0.9.0-RC6" - source: "src" - target: "" - keywords: - - "openproblems" - - "common-resources" - - "single-cell" - - "benchmark" - license: "MIT" - organization: "openproblems-bio" - links: - repository: "https://github.com/openproblems-bio/common_resources" - docker_registry: "ghcr.io" - issue_tracker: "https://github.com/openproblems-bio/common_resources/issues" diff --git a/sync_resources/sync_resources b/sync_resources/sync_resources deleted file mode 100755 index 6cada5e..0000000 --- a/sync_resources/sync_resources +++ /dev/null @@ -1,1193 +0,0 @@ -#!/usr/bin/env bash - -# sync_resources dev -# -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a -# derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from -# Data Intuitive. -# -# The component may contain files which fall under a different license. The -# authors of this component should specify the license in the header of such -# files, or include a separate license file detailing the licenses of all included -# files. - -set -e - -if [ -z "$VIASH_TEMP" ]; then - VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMPDIR} - VIASH_TEMP=${VIASH_TEMP:-$VIASH_TEMPDIR} - VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMP} - VIASH_TEMP=${VIASH_TEMP:-$TMPDIR} - VIASH_TEMP=${VIASH_TEMP:-$TMP} - VIASH_TEMP=${VIASH_TEMP:-$TEMPDIR} - VIASH_TEMP=${VIASH_TEMP:-$TEMP} - VIASH_TEMP=${VIASH_TEMP:-/tmp} -fi - -# define helper functions -# ViashQuote: put quotes around non flag values -# $1 : unquoted string -# return : possibly quoted string -# examples: -# ViashQuote --foo # returns --foo -# ViashQuote bar # returns 'bar' -# Viashquote --foo=bar # returns --foo='bar' -function ViashQuote { - if [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+=.+$ ]]; then - echo "$1" | sed "s#=\(.*\)#='\1'#" - elif [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+$ ]]; then - echo "$1" - else - echo "'$1'" - fi -} -# ViashRemoveFlags: Remove leading flag -# $1 : string with a possible leading flag -# return : string without possible leading flag -# examples: -# ViashRemoveFlags --foo=bar # returns bar -function ViashRemoveFlags { - echo "$1" | sed 's/^--*[a-zA-Z0-9_\-]*=//' -} -# ViashSourceDir: return the path of a bash file, following symlinks -# usage : ViashSourceDir ${BASH_SOURCE[0]} -# $1 : Should always be set to ${BASH_SOURCE[0]} -# returns : The absolute path of the bash file -function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" - done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd -} -# ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks -# usage : ViashFindTargetDir 'ScriptPath' -# $1 : The location from where to start the upward search -# returns : The absolute path of the '.build.yaml' file -function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} - done - echo $SOURCE -} -# see https://en.wikipedia.org/wiki/Syslog#Severity_level -VIASH_LOGCODE_EMERGENCY=0 -VIASH_LOGCODE_ALERT=1 -VIASH_LOGCODE_CRITICAL=2 -VIASH_LOGCODE_ERROR=3 -VIASH_LOGCODE_WARNING=4 -VIASH_LOGCODE_NOTICE=5 -VIASH_LOGCODE_INFO=6 -VIASH_LOGCODE_DEBUG=7 -VIASH_VERBOSITY=$VIASH_LOGCODE_NOTICE - -# ViashLog: Log events depending on the verbosity level -# usage: ViashLog 1 alert Oh no something went wrong! -# $1: required verbosity level -# $2: display tag -# $3+: messages to display -# stdout: Your input, prepended by '[$2] '. -function ViashLog { - local required_level="$1" - local display_tag="$2" - shift 2 - if [ $VIASH_VERBOSITY -ge $required_level ]; then - >&2 echo "[$display_tag]" "$@" - fi -} - -# ViashEmergency: log events when the system is unstable -# usage: ViashEmergency Oh no something went wrong. -# stdout: Your input, prepended by '[emergency] '. -function ViashEmergency { - ViashLog $VIASH_LOGCODE_EMERGENCY emergency "$@" -} - -# ViashAlert: log events when actions must be taken immediately (e.g. corrupted system database) -# usage: ViashAlert Oh no something went wrong. -# stdout: Your input, prepended by '[alert] '. -function ViashAlert { - ViashLog $VIASH_LOGCODE_ALERT alert "$@" -} - -# ViashCritical: log events when a critical condition occurs -# usage: ViashCritical Oh no something went wrong. -# stdout: Your input, prepended by '[critical] '. -function ViashCritical { - ViashLog $VIASH_LOGCODE_CRITICAL critical "$@" -} - -# ViashError: log events when an error condition occurs -# usage: ViashError Oh no something went wrong. -# stdout: Your input, prepended by '[error] '. -function ViashError { - ViashLog $VIASH_LOGCODE_ERROR error "$@" -} - -# ViashWarning: log potentially abnormal events -# usage: ViashWarning Something may have gone wrong. -# stdout: Your input, prepended by '[warning] '. -function ViashWarning { - ViashLog $VIASH_LOGCODE_WARNING warning "$@" -} - -# ViashNotice: log significant but normal events -# usage: ViashNotice This just happened. -# stdout: Your input, prepended by '[notice] '. -function ViashNotice { - ViashLog $VIASH_LOGCODE_NOTICE notice "$@" -} - -# ViashInfo: log normal events -# usage: ViashInfo This just happened. -# stdout: Your input, prepended by '[info] '. -function ViashInfo { - ViashLog $VIASH_LOGCODE_INFO info "$@" -} - -# ViashDebug: log all events, for debugging purposes -# usage: ViashDebug This just happened. -# stdout: Your input, prepended by '[debug] '. -function ViashDebug { - ViashLog $VIASH_LOGCODE_DEBUG debug "$@" -} - -# find source folder of this component -VIASH_META_RESOURCES_DIR=`ViashSourceDir ${BASH_SOURCE[0]}` - -# find the root of the built components & dependencies -VIASH_TARGET_DIR=`ViashFindTargetDir $VIASH_META_RESOURCES_DIR` - -# define meta fields -VIASH_META_NAME="sync_resources" -VIASH_META_FUNCTIONALITY_NAME="sync_resources" -VIASH_META_EXECUTABLE="$VIASH_META_RESOURCES_DIR/$VIASH_META_NAME" -VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" -VIASH_META_TEMP_DIR="$VIASH_TEMP" - - -# ViashHelp: Display helpful explanation about this executable -function ViashHelp { - echo "sync_resources dev" - echo "" - echo "Synchronise the resources from s3 to resources_test" - echo "" - echo "Usage:" - echo "sync_resources" - echo "sync_resources --input _viash.yaml --output resources_test" - echo "" - echo "Arguments:" - echo " -i, --input" - echo " type: file, file must exist" - echo " default: _viash.yaml" - echo " Path _viash.yaml file with s3 path to resources." - echo "" - echo " -o, --output" - echo " type: file, output, file must exist" - echo " default: /resources_test" - echo " Path to the test resource directory." - echo "" - echo " --quiet" - echo " type: boolean_true" - echo " Displays the operations that would be performed using the specified" - echo " command without actually running them." - echo "" - echo " --dryrun" - echo " type: boolean_true" - echo " Does not display the operations performed from the specified command." - echo "" - echo " --delete" - echo " type: boolean_true" - echo " Files that exist in the destination but not in the source are deleted" - echo " during sync." - echo "" - echo " --exclude" - echo " type: string, multiple values allowed" - echo " Exclude all files or objects from the command that matches the specified" - echo " pattern." -} - -# initialise variables -VIASH_MODE='run' -VIASH_ENGINE_ID='docker' - -######## Helper functions for setting up Docker images for viash ######## -# expects: ViashDockerBuild - -# ViashDockerInstallationCheck: check whether Docker is installed correctly -# -# examples: -# ViashDockerInstallationCheck -function ViashDockerInstallationCheck { - ViashDebug "Checking whether Docker is installed" - if [ ! command -v docker &> /dev/null ]; then - ViashCritical "Docker doesn't seem to be installed. See 'https://docs.docker.com/get-docker/' for instructions." - exit 1 - fi - - ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? - [[ $save =~ e ]] && set -e - if [ $out -ne 0 ]; then - ViashCritical "Docker daemon does not seem to be running. Try one of the following:" - ViashCritical "- Try running 'dockerd' in the command line" - ViashCritical "- See https://docs.docker.com/config/daemon/" - exit 1 - fi -} - -# ViashDockerRemoteTagCheck: check whether a Docker image is available -# on a remote. Assumes `docker login` has been performed, if relevant. -# -# $1 : image identifier with format `[registry/]image[:tag]` -# exit code $? : whether or not the image was found -# examples: -# ViashDockerRemoteTagCheck python:latest -# echo $? # returns '0' -# ViashDockerRemoteTagCheck sdaizudceahifu -# echo $? # returns '1' -function ViashDockerRemoteTagCheck { - docker manifest inspect $1 > /dev/null 2> /dev/null -} - -# ViashDockerLocalTagCheck: check whether a Docker image is available locally -# -# $1 : image identifier with format `[registry/]image[:tag]` -# exit code $? : whether or not the image was found -# examples: -# docker pull python:latest -# ViashDockerLocalTagCheck python:latest -# echo $? # returns '0' -# ViashDockerLocalTagCheck sdaizudceahifu -# echo $? # returns '1' -function ViashDockerLocalTagCheck { - [ -n "$(docker images -q $1)" ] -} - -# ViashDockerPull: pull a Docker image -# -# $1 : image identifier with format `[registry/]image[:tag]` -# exit code $? : whether or not the image was found -# examples: -# ViashDockerPull python:latest -# echo $? # returns '0' -# ViashDockerPull sdaizudceahifu -# echo $? # returns '1' -function ViashDockerPull { - ViashNotice "Checking if Docker image is available at '$1'" - if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then - docker pull $1 && return 0 || return 1 - else - save=$-; set +e - docker pull $1 2> /dev/null > /dev/null - out=$? - [[ $save =~ e ]] && set -e - if [ $out -ne 0 ]; then - ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." - fi - return $out - fi -} - -# ViashDockerPush: push a Docker image -# -# $1 : image identifier with format `[registry/]image[:tag]` -# exit code $? : whether or not the image was found -# examples: -# ViashDockerPush python:latest -# echo $? # returns '0' -# ViashDockerPush sdaizudceahifu -# echo $? # returns '1' -function ViashDockerPush { - ViashNotice "Pushing image to '$1'" - save=$-; set +e - if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then - docker push $1 - out=$? - else - docker push $1 2> /dev/null > /dev/null - out=$? - fi - [[ $save =~ e ]] && set -e - if [ $out -eq 0 ]; then - ViashNotice "Container '$1' push succeeded." - else - ViashError "Container '$1' push errored. You might not be logged in or have the necessary permissions." - fi - return $out -} - -# ViashDockerPullElseBuild: pull a Docker image, else build it -# -# $1 : image identifier with format `[registry/]image[:tag]` -# ViashDockerBuild : a Bash function which builds a docker image, takes image identifier as argument. -# examples: -# ViashDockerPullElseBuild mynewcomponent -function ViashDockerPullElseBuild { - save=$-; set +e - ViashDockerPull $1 - out=$? - [[ $save =~ e ]] && set -e - if [ $out -ne 0 ]; then - ViashDockerBuild $@ - fi -} - -# ViashDockerSetup: create a Docker image, according to specified docker setup strategy -# -# $1 : image identifier with format `[registry/]image[:tag]` -# $2 : docker setup strategy, see DockerSetupStrategy.scala -# examples: -# ViashDockerSetup mynewcomponent alwaysbuild -function ViashDockerSetup { - local image_id="$1" - local setup_strategy="$2" - if [ "$setup_strategy" == "alwaysbuild" -o "$setup_strategy" == "build" -o "$setup_strategy" == "b" ]; then - ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") - elif [ "$setup_strategy" == "alwayspull" -o "$setup_strategy" == "pull" -o "$setup_strategy" == "p" ]; then - ViashDockerPull $image_id - elif [ "$setup_strategy" == "alwayspullelsebuild" -o "$setup_strategy" == "pullelsebuild" ]; then - ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") - elif [ "$setup_strategy" == "alwayspullelsecachedbuild" -o "$setup_strategy" == "pullelsecachedbuild" ]; then - ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id") - elif [ "$setup_strategy" == "alwayscachedbuild" -o "$setup_strategy" == "cachedbuild" -o "$setup_strategy" == "cb" ]; then - ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id") - elif [[ "$setup_strategy" =~ ^ifneedbe ]]; then - local save=$-; set +e - ViashDockerLocalTagCheck $image_id - local outCheck=$? - [[ $save =~ e ]] && set -e - if [ $outCheck -eq 0 ]; then - ViashInfo "Image $image_id already exists" - elif [ "$setup_strategy" == "ifneedbebuild" ]; then - ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") - elif [ "$setup_strategy" == "ifneedbecachedbuild" ]; then - ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id") - elif [ "$setup_strategy" == "ifneedbepull" ]; then - ViashDockerPull $image_id - elif [ "$setup_strategy" == "ifneedbepullelsebuild" ]; then - ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") - elif [ "$setup_strategy" == "ifneedbepullelsecachedbuild" ]; then - ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id") - else - ViashError "Unrecognised Docker strategy: $setup_strategy" - exit 1 - fi - elif [ "$setup_strategy" == "push" -o "$setup_strategy" == "forcepush" -o "$setup_strategy" == "alwayspush" ]; then - ViashDockerPush "$image_id" - elif [ "$setup_strategy" == "pushifnotpresent" -o "$setup_strategy" == "gentlepush" -o "$setup_strategy" == "maybepush" ]; then - local save=$-; set +e - ViashDockerRemoteTagCheck $image_id - local outCheck=$? - [[ $save =~ e ]] && set -e - if [ $outCheck -eq 0 ]; then - ViashNotice "Container '$image_id' exists, doing nothing." - else - ViashNotice "Container '$image_id' does not yet exist." - ViashDockerPush "$image_id" - fi - elif [ "$setup_strategy" == "donothing" -o "$setup_strategy" == "meh" ]; then - ViashNotice "Skipping setup." - else - ViashError "Unrecognised Docker strategy: $setup_strategy" - exit 1 - fi -} - -# ViashDockerCheckCommands: Check whether a docker container has the required commands -# -# $1 : image identifier with format `[registry/]image[:tag]` -# $@ : commands to verify being present -# examples: -# ViashDockerCheckCommands bash:4.0 bash ps foo -function ViashDockerCheckCommands { - local image_id="$1" - shift 1 - local commands="$@" - local save=$-; set +e - local missing # mark 'missing' as local in advance, otherwise the exit code of the command will be missing and always be '0' - missing=$(docker run --rm --entrypoint=sh "$image_id" -c "for command in $commands; do command -v \$command >/dev/null 2>&1; if [ \$? -ne 0 ]; then echo \$command; exit 1; fi; done") - local outCheck=$? - [[ $save =~ e ]] && set -e - if [ $outCheck -ne 0 ]; then - ViashError "Docker container '$image_id' does not contain command '$missing'." - exit 1 - fi -} - -# ViashDockerBuild: build a docker image -# $1 : image identifier with format `[registry/]image[:tag]` -# $... : additional arguments to pass to docker build -# $VIASH_META_TEMP_DIR : temporary directory to store dockerfile & optional resources in -# $VIASH_META_NAME : name of the component -# $VIASH_META_RESOURCES_DIR : directory containing the resources -# $VIASH_VERBOSITY : verbosity level -# exit code $? : whether or not the image was built successfully -function ViashDockerBuild { - local image_id="$1" - shift 1 - - # create temporary directory to store dockerfile & optional resources in - local tmpdir=$(mktemp -d "$VIASH_META_TEMP_DIR/dockerbuild-$VIASH_META_NAME-XXXXXX") - local dockerfile="$tmpdir/Dockerfile" - function clean_up { - rm -rf "$tmpdir" - } - trap clean_up EXIT - - # store dockerfile and resources - ViashDockerfile "$VIASH_ENGINE_ID" > "$dockerfile" - - # generate the build command - local docker_build_cmd="docker build -t '$image_id' $@ '$VIASH_META_RESOURCES_DIR' -f '$dockerfile'" - - # build the container - ViashNotice "Building container '$image_id' with Dockerfile" - ViashInfo "$docker_build_cmd" - local save=$-; set +e - if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then - eval $docker_build_cmd - else - eval $docker_build_cmd &> "$tmpdir/docker_build.log" - fi - - # check exit code - local out=$? - [[ $save =~ e ]] && set -e - if [ $out -ne 0 ]; then - ViashError "Error occurred while building container '$image_id'" - if [ $VIASH_VERBOSITY -lt $VIASH_LOGCODE_INFO ]; then - ViashError "Transcript: --------------------------------" - cat "$tmpdir/docker_build.log" - ViashError "End of transcript --------------------------" - fi - exit 1 - fi -} - -######## End of helper functions for setting up Docker images for viash ######## - -# ViashDockerFile: print the dockerfile to stdout -# $1 : engine identifier -# return : dockerfile required to run this component -# examples: -# ViashDockerFile -function ViashDockerfile { - local engine_id="$1" - - if [[ "$engine_id" == "docker" ]]; then - cat << 'VIASHDOCKER' -FROM amazon/aws-cli:2.17.11 -ENTRYPOINT [] -RUN yum install -y wget && \ - yum clean all && \ - rm -rf /var/cache/yum - -RUN wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && \ -chmod +x /usr/bin/yq - -LABEL org.opencontainers.image.description="Companion container for running component common sync_resources" -LABEL org.opencontainers.image.created="2024-07-11T22:25:47+02:00" -LABEL org.opencontainers.image.source="https://github.com/openproblems-bio/common_resources" -LABEL org.opencontainers.image.revision="7bd60870f551dd252f8316467e710ab0d9440bfa" -LABEL org.opencontainers.image.version="dev" - -VIASHDOCKER - fi -} - -# ViashDockerBuildArgs: return the arguments to pass to docker build -# $1 : engine identifier -# return : arguments to pass to docker build -function ViashDockerBuildArgs { - local engine_id="$1" - - if [[ "$engine_id" == "docker" ]]; then - echo "" - fi -} - -# ViashAbsolutePath: generate absolute path from relative path -# borrowed from https://stackoverflow.com/a/21951256 -# $1 : relative filename -# return : absolute path -# examples: -# ViashAbsolutePath some_file.txt # returns /path/to/some_file.txt -# ViashAbsolutePath /foo/bar/.. # returns /foo -function ViashAbsolutePath { - local thePath - if [[ ! "$1" =~ ^/ ]]; then - thePath="$PWD/$1" - else - thePath="$1" - fi - echo "$thePath" | ( - IFS=/ - read -a parr - declare -a outp - for i in "${parr[@]}"; do - case "$i" in - ''|.) continue ;; - ..) - len=${#outp[@]} - if ((len==0)); then - continue - else - unset outp[$((len-1))] - fi - ;; - *) - len=${#outp[@]} - outp[$len]="$i" - ;; - esac - done - echo /"${outp[*]}" - ) -} -# ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker -# examples: -# ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' -# ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' -function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") - if [ -d "$abs_path" ]; then - mount_source="$abs_path" - base_name="" - else - mount_source=`dirname "$abs_path"` - base_name=`basename "$abs_path"` - fi - mount_target="/viash_automount$mount_source" - if [ -z "$base_name" ]; then - echo "$mount_target" - else - echo "$mount_target/$base_name" - fi -} -function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") - if [ -d "$abs_path" ]; then - mount_source="$abs_path" - base_name="" - else - mount_source=`dirname "$abs_path"` - base_name=`basename "$abs_path"` - fi - mount_target="/viash_automount$mount_source" - ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" - echo "--volume=\"$mount_source:$mount_target\"" -} -function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" -} -# initialise variables -VIASH_DIRECTORY_MOUNTS=() - -# initialise docker variables -VIASH_DOCKER_RUN_ARGS=(-i --rm) - -# initialise array -VIASH_POSITIONAL_ARGS='' - -while [[ $# -gt 0 ]]; do - case "$1" in - -h|--help) - ViashHelp - exit - ;; - ---v|---verbose) - let "VIASH_VERBOSITY=VIASH_VERBOSITY+1" - shift 1 - ;; - ---verbosity) - VIASH_VERBOSITY="$2" - shift 2 - ;; - ---verbosity=*) - VIASH_VERBOSITY="$(ViashRemoveFlags "$1")" - shift 1 - ;; - --version) - echo "sync_resources dev" - exit - ;; - --input) - [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'--input\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_INPUT="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to --input. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - --input=*) - [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'--input=*\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_INPUT=$(ViashRemoveFlags "$1") - shift 1 - ;; - -i) - [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'-i\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_INPUT="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to -i. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - --output) - [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_OUTPUT="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to --output. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - --output=*) - [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output=*\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_OUTPUT=$(ViashRemoveFlags "$1") - shift 1 - ;; - -o) - [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'-o\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_OUTPUT="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to -o. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - --quiet) - [ -n "$VIASH_PAR_QUIET" ] && ViashError Bad arguments for option \'--quiet\': \'$VIASH_PAR_QUIET\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_QUIET=true - shift 1 - ;; - --dryrun) - [ -n "$VIASH_PAR_DRYRUN" ] && ViashError Bad arguments for option \'--dryrun\': \'$VIASH_PAR_DRYRUN\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_DRYRUN=true - shift 1 - ;; - --delete) - [ -n "$VIASH_PAR_DELETE" ] && ViashError Bad arguments for option \'--delete\': \'$VIASH_PAR_DELETE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_DELETE=true - shift 1 - ;; - --exclude) - if [ -z "$VIASH_PAR_EXCLUDE" ]; then - VIASH_PAR_EXCLUDE="$2" - else - VIASH_PAR_EXCLUDE="$VIASH_PAR_EXCLUDE;""$2" - fi - [ $# -lt 2 ] && ViashError Not enough arguments passed to --exclude. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - --exclude=*) - if [ -z "$VIASH_PAR_EXCLUDE" ]; then - VIASH_PAR_EXCLUDE=$(ViashRemoveFlags "$1") - else - VIASH_PAR_EXCLUDE="$VIASH_PAR_EXCLUDE;"$(ViashRemoveFlags "$1") - fi - shift 1 - ;; - ---engine) - VIASH_ENGINE_ID="$2" - shift 2 - ;; - ---engine=*) - VIASH_ENGINE_ID="$(ViashRemoveFlags "$1")" - shift 1 - ;; - ---setup) - VIASH_MODE='setup' - VIASH_SETUP_STRATEGY="$2" - shift 2 - ;; - ---setup=*) - VIASH_MODE='setup' - VIASH_SETUP_STRATEGY="$(ViashRemoveFlags "$1")" - shift 1 - ;; - ---dockerfile) - VIASH_MODE='dockerfile' - shift 1 - ;; - ---debug) - VIASH_MODE='debug' - shift 1 - ;; - ---cpus) - [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_META_CPUS="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to ---cpus. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - ---cpus=*) - [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus=*\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_META_CPUS=$(ViashRemoveFlags "$1") - shift 1 - ;; - ---memory) - [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_META_MEMORY="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to ---memory. Use "--help" to get more information on the parameters. && exit 1 - shift 2 - ;; - ---memory=*) - [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory=*\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_META_MEMORY=$(ViashRemoveFlags "$1") - shift 1 - ;; - *) # positional arg or unknown option - # since the positional args will be eval'd, can we always quote, instead of using ViashQuote - VIASH_POSITIONAL_ARGS="$VIASH_POSITIONAL_ARGS '$1'" - [[ $1 == -* ]] && ViashWarning $1 looks like a parameter but is not a defined parameter and will instead be treated as a positional argument. Use "--help" to get more information on the parameters. - shift # past argument - ;; - esac -done - -# parse positional parameters -eval set -- $VIASH_POSITIONAL_ARGS - - -if [ "$VIASH_ENGINE_ID" == "docker" ] ; then - VIASH_ENGINE_TYPE='docker' -else - ViashError "Engine '$VIASH_ENGINE_ID' is not recognized. Options are: docker." - exit 1 -fi - -if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then - # check if docker is installed properly - ViashDockerInstallationCheck - - # determine docker image id - if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='ghcr.io/openproblems-bio/common_resources/common/sync_resources:dev' - fi - - # print dockerfile - if [ "$VIASH_MODE" == "dockerfile" ]; then - ViashDockerfile "$VIASH_ENGINE_ID" - exit 0 - - # enter docker container - elif [[ "$VIASH_MODE" == "debug" ]]; then - VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} -v '$(pwd)':/pwd --workdir /pwd -t $VIASH_DOCKER_IMAGE_ID" - ViashNotice "+ $VIASH_CMD" - eval $VIASH_CMD - exit - - # build docker image - elif [ "$VIASH_MODE" == "setup" ]; then - ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" "$VIASH_SETUP_STRATEGY" - ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'bash' - exit 0 - fi - - # check if docker image exists - ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" ifneedbepullelsecachedbuild - ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'bash' -fi - -# setting computational defaults - -# helper function for parsing memory strings -function ViashMemoryAsBytes { - local memory=`echo "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]'` - local memory_regex='^([0-9]+)([kmgtp]i?b?|b)$' - if [[ $memory =~ $memory_regex ]]; then - local number=${memory/[^0-9]*/} - local symbol=${memory/*[0-9]/} - - case $symbol in - b) memory_b=$number ;; - kb|k) memory_b=$(( $number * 1000 )) ;; - mb|m) memory_b=$(( $number * 1000 * 1000 )) ;; - gb|g) memory_b=$(( $number * 1000 * 1000 * 1000 )) ;; - tb|t) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 )) ;; - pb|p) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 * 1000 )) ;; - kib|ki) memory_b=$(( $number * 1024 )) ;; - mib|mi) memory_b=$(( $number * 1024 * 1024 )) ;; - gib|gi) memory_b=$(( $number * 1024 * 1024 * 1024 )) ;; - tib|ti) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 )) ;; - pib|pi) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 * 1024 )) ;; - esac - echo "$memory_b" - fi -} -# compute memory in different units -if [ ! -z ${VIASH_META_MEMORY+x} ]; then - VIASH_META_MEMORY_B=`ViashMemoryAsBytes $VIASH_META_MEMORY` - # do not define other variables if memory_b is an empty string - if [ ! -z "$VIASH_META_MEMORY_B" ]; then - VIASH_META_MEMORY_KB=$(( ($VIASH_META_MEMORY_B+999) / 1000 )) - VIASH_META_MEMORY_MB=$(( ($VIASH_META_MEMORY_KB+999) / 1000 )) - VIASH_META_MEMORY_GB=$(( ($VIASH_META_MEMORY_MB+999) / 1000 )) - VIASH_META_MEMORY_TB=$(( ($VIASH_META_MEMORY_GB+999) / 1000 )) - VIASH_META_MEMORY_PB=$(( ($VIASH_META_MEMORY_TB+999) / 1000 )) - VIASH_META_MEMORY_KIB=$(( ($VIASH_META_MEMORY_B+1023) / 1024 )) - VIASH_META_MEMORY_MIB=$(( ($VIASH_META_MEMORY_KIB+1023) / 1024 )) - VIASH_META_MEMORY_GIB=$(( ($VIASH_META_MEMORY_MIB+1023) / 1024 )) - VIASH_META_MEMORY_TIB=$(( ($VIASH_META_MEMORY_GIB+1023) / 1024 )) - VIASH_META_MEMORY_PIB=$(( ($VIASH_META_MEMORY_TIB+1023) / 1024 )) - else - # unset memory if string is empty - unset $VIASH_META_MEMORY_B - fi -fi -# unset nproc if string is empty -if [ -z "$VIASH_META_CPUS" ]; then - unset $VIASH_META_CPUS -fi - - -# check whether required parameters exist -if [ -z ${VIASH_META_NAME+x} ]; then - ViashError 'name' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi -if [ -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then - ViashError 'functionality_name' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi -if [ -z ${VIASH_META_RESOURCES_DIR+x} ]; then - ViashError 'resources_dir' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi -if [ -z ${VIASH_META_EXECUTABLE+x} ]; then - ViashError 'executable' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi -if [ -z ${VIASH_META_CONFIG+x} ]; then - ViashError 'config' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi -if [ -z ${VIASH_META_TEMP_DIR+x} ]; then - ViashError 'temp_dir' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi - -# filling in defaults -if [ -z ${VIASH_PAR_INPUT+x} ]; then - VIASH_PAR_INPUT="_viash.yaml" -fi -if [ -z ${VIASH_PAR_OUTPUT+x} ]; then - VIASH_PAR_OUTPUT="/resources_test" -fi -if [ -z ${VIASH_PAR_QUIET+x} ]; then - VIASH_PAR_QUIET="false" -fi -if [ -z ${VIASH_PAR_DRYRUN+x} ]; then - VIASH_PAR_DRYRUN="false" -fi -if [ -z ${VIASH_PAR_DELETE+x} ]; then - VIASH_PAR_DELETE="false" -fi - -# check whether required files exist -if [ ! -z "$VIASH_PAR_INPUT" ] && [ ! -e "$VIASH_PAR_INPUT" ]; then - ViashError "Input file '$VIASH_PAR_INPUT' does not exist." - exit 1 -fi - -# check whether parameters values are of the right type -if [[ -n "$VIASH_PAR_QUIET" ]]; then - if ! [[ "$VIASH_PAR_QUIET" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then - ViashError '--quiet' has to be a boolean_true. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_PAR_DRYRUN" ]]; then - if ! [[ "$VIASH_PAR_DRYRUN" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then - ViashError '--dryrun' has to be a boolean_true. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_PAR_DELETE" ]]; then - if ! [[ "$VIASH_PAR_DELETE" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then - ViashError '--delete' has to be a boolean_true. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_CPUS" ]]; then - if ! [[ "$VIASH_META_CPUS" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'cpus' has to be an integer. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_B" ]]; then - if ! [[ "$VIASH_META_MEMORY_B" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_b' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_KB" ]]; then - if ! [[ "$VIASH_META_MEMORY_KB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_kb' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_MB" ]]; then - if ! [[ "$VIASH_META_MEMORY_MB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_mb' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_GB" ]]; then - if ! [[ "$VIASH_META_MEMORY_GB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_gb' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_TB" ]]; then - if ! [[ "$VIASH_META_MEMORY_TB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_tb' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_PB" ]]; then - if ! [[ "$VIASH_META_MEMORY_PB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_pb' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_KIB" ]]; then - if ! [[ "$VIASH_META_MEMORY_KIB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_kib' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_MIB" ]]; then - if ! [[ "$VIASH_META_MEMORY_MIB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_mib' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_GIB" ]]; then - if ! [[ "$VIASH_META_MEMORY_GIB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_gib' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_TIB" ]]; then - if ! [[ "$VIASH_META_MEMORY_TIB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_tib' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi -if [[ -n "$VIASH_META_MEMORY_PIB" ]]; then - if ! [[ "$VIASH_META_MEMORY_PIB" =~ ^[-+]?[0-9]+$ ]]; then - ViashError 'memory_pib' has to be a long. Use "--help" to get more information on the parameters. - exit 1 - fi -fi - -# create parent directories of output files, if so desired -if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -d "$(dirname "$VIASH_PAR_OUTPUT")" ]; then - mkdir -p "$(dirname "$VIASH_PAR_OUTPUT")" -fi - -if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then - # detect volumes from file arguments - VIASH_CHOWN_VARS=() -if [ ! -z "$VIASH_PAR_INPUT" ]; then - VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_INPUT")" ) - VIASH_PAR_INPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_INPUT") -fi -if [ ! -z "$VIASH_PAR_OUTPUT" ]; then - VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_OUTPUT")" ) - VIASH_PAR_OUTPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_OUTPUT") - VIASH_CHOWN_VARS+=( "$VIASH_PAR_OUTPUT" ) -fi -if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then - VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_RESOURCES_DIR")" ) - VIASH_META_RESOURCES_DIR=$(ViashDockerAutodetectMount "$VIASH_META_RESOURCES_DIR") -fi -if [ ! -z "$VIASH_META_EXECUTABLE" ]; then - VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_EXECUTABLE")" ) - VIASH_META_EXECUTABLE=$(ViashDockerAutodetectMount "$VIASH_META_EXECUTABLE") -fi -if [ ! -z "$VIASH_META_CONFIG" ]; then - VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_CONFIG")" ) - VIASH_META_CONFIG=$(ViashDockerAutodetectMount "$VIASH_META_CONFIG") -fi -if [ ! -z "$VIASH_META_TEMP_DIR" ]; then - VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_TEMP_DIR")" ) - VIASH_META_TEMP_DIR=$(ViashDockerAutodetectMount "$VIASH_META_TEMP_DIR") -fi - - # get unique mounts - VIASH_UNIQUE_MOUNTS=($(for val in "${VIASH_DIRECTORY_MOUNTS[@]}"; do echo "$val"; done | sort -u)) -fi - -if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then - # change file ownership - function ViashPerformChown { - if (( ${#VIASH_CHOWN_VARS[@]} )); then - set +e - VIASH_CMD="docker run --entrypoint=bash --rm ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID -c 'chown $(id -u):$(id -g) --silent --recursive ${VIASH_CHOWN_VARS[@]}'" - ViashDebug "+ $VIASH_CMD" - eval $VIASH_CMD - set -e - fi - } - trap ViashPerformChown EXIT -fi - -if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then - # helper function for filling in extra docker args - if [ ! -z "$VIASH_META_MEMORY_B" ]; then - VIASH_DOCKER_RUN_ARGS+=("--memory=${VIASH_META_MEMORY_B}") - fi - if [ ! -z "$VIASH_META_CPUS" ]; then - VIASH_DOCKER_RUN_ARGS+=("--cpus=${VIASH_META_CPUS}") - fi -fi - -if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then - VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID" -fi - - -# set dependency paths - - -ViashDebug "Running command: $(echo $VIASH_CMD)" -cat << VIASHEOF | eval $VIASH_CMD -set -e -tempscript=\$(mktemp "$VIASH_META_TEMP_DIR/viash-run-sync_resources-XXXXXX").sh -function clean_up { - rm "\$tempscript" -} -function interrupt { - echo -e "\nCTRL-C Pressed..." - exit 1 -} -trap clean_up EXIT -trap interrupt INT SIGINT -cat > "\$tempscript" << 'VIASHMAIN' -#!/bin/bash - -## VIASH START -# The following code has been auto-generated by Viash. -$( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "${VIASH_PAR_INPUT}" | sed "s#'#'\"'\"'#g;s#.*#par_input='&'#" ; else echo "# par_input="; fi ) -$( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "${VIASH_PAR_OUTPUT}" | sed "s#'#'\"'\"'#g;s#.*#par_output='&'#" ; else echo "# par_output="; fi ) -$( if [ ! -z ${VIASH_PAR_QUIET+x} ]; then echo "${VIASH_PAR_QUIET}" | sed "s#'#'\"'\"'#g;s#.*#par_quiet='&'#" ; else echo "# par_quiet="; fi ) -$( if [ ! -z ${VIASH_PAR_DRYRUN+x} ]; then echo "${VIASH_PAR_DRYRUN}" | sed "s#'#'\"'\"'#g;s#.*#par_dryrun='&'#" ; else echo "# par_dryrun="; fi ) -$( if [ ! -z ${VIASH_PAR_DELETE+x} ]; then echo "${VIASH_PAR_DELETE}" | sed "s#'#'\"'\"'#g;s#.*#par_delete='&'#" ; else echo "# par_delete="; fi ) -$( if [ ! -z ${VIASH_PAR_EXCLUDE+x} ]; then echo "${VIASH_PAR_EXCLUDE}" | sed "s#'#'\"'\"'#g;s#.*#par_exclude='&'#" ; else echo "# par_exclude="; fi ) -$( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "${VIASH_META_NAME}" | sed "s#'#'\"'\"'#g;s#.*#meta_name='&'#" ; else echo "# meta_name="; fi ) -$( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "${VIASH_META_FUNCTIONALITY_NAME}" | sed "s#'#'\"'\"'#g;s#.*#meta_functionality_name='&'#" ; else echo "# meta_functionality_name="; fi ) -$( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "${VIASH_META_RESOURCES_DIR}" | sed "s#'#'\"'\"'#g;s#.*#meta_resources_dir='&'#" ; else echo "# meta_resources_dir="; fi ) -$( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "${VIASH_META_EXECUTABLE}" | sed "s#'#'\"'\"'#g;s#.*#meta_executable='&'#" ; else echo "# meta_executable="; fi ) -$( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "${VIASH_META_CONFIG}" | sed "s#'#'\"'\"'#g;s#.*#meta_config='&'#" ; else echo "# meta_config="; fi ) -$( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "${VIASH_META_TEMP_DIR}" | sed "s#'#'\"'\"'#g;s#.*#meta_temp_dir='&'#" ; else echo "# meta_temp_dir="; fi ) -$( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "${VIASH_META_CPUS}" | sed "s#'#'\"'\"'#g;s#.*#meta_cpus='&'#" ; else echo "# meta_cpus="; fi ) -$( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "${VIASH_META_MEMORY_B}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_b='&'#" ; else echo "# meta_memory_b="; fi ) -$( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "${VIASH_META_MEMORY_KB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_kb='&'#" ; else echo "# meta_memory_kb="; fi ) -$( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "${VIASH_META_MEMORY_MB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_mb='&'#" ; else echo "# meta_memory_mb="; fi ) -$( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "${VIASH_META_MEMORY_GB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_gb='&'#" ; else echo "# meta_memory_gb="; fi ) -$( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "${VIASH_META_MEMORY_TB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_tb='&'#" ; else echo "# meta_memory_tb="; fi ) -$( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "${VIASH_META_MEMORY_PB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_pb='&'#" ; else echo "# meta_memory_pb="; fi ) -$( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "${VIASH_META_MEMORY_KIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_kib='&'#" ; else echo "# meta_memory_kib="; fi ) -$( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "${VIASH_META_MEMORY_MIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_mib='&'#" ; else echo "# meta_memory_mib="; fi ) -$( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "${VIASH_META_MEMORY_GIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_gib='&'#" ; else echo "# meta_memory_gib="; fi ) -$( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "${VIASH_META_MEMORY_TIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_tib='&'#" ; else echo "# meta_memory_tib="; fi ) -$( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_pib='&'#" ; else echo "# meta_memory_pib="; fi ) - -## VIASH END - -extra_params=( ) - -if [ "\$par_quiet" == "true" ]; then - extra_params+=( "--quiet" ) -fi -if [ "\$par_dryrun" == "true" ]; then - extra_params+=( "--dryrun" ) -fi -if [ "\$par_delete" == "true" ]; then - extra_params+=( "--delete" ) -fi - -if [ ! -z \${par_exclude+x} ]; then - IFS=":" - for var in \$par_exclude; do - unset IFS - extra_params+=( "--exclude" "\$var" ) - done -fi - - -# Disable the use of the Amazon EC2 instance metadata service (IMDS). -# see https://florian.ec/blog/github-actions-awscli-errors/ -# or https://github.com/aws/aws-cli/issues/5234#issuecomment-705831465 -export AWS_EC2_METADATA_DISABLED=true - - -function sync_s3() { - local s3_path="\$1" - local dest_path="\$2" - aws s3 sync \\ - "\$s3_path" \\ - "\$dest_path" \\ - --no-sign-request \\ - "\${extra_params[@]}" -} - -resources_detected=\$(yq e '.info | has("test_resources")' "\$par_input") -if [ "\$resources_detected" == "false" ]; then - echo "No test resources detected." - exit 0 -fi - -# reformat the test resources into a pseudo-json that can be read line-by-line by bash -nr_res=\$( yq e '.info.test_resources | length' "\$par_input") - -for ((i=0; i