From bbc5d73e0d9b19b1c7a016ddb0c2da6790669c29 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 18 Apr 2023 11:44:11 +0200 Subject: [PATCH 01/20] first version of bot/check-result.sh for software-layer - follows what has been implemented in the EESSI/eessi-bot-software-layer for checking the result of a job - re-uses code from a small script that was run manually on a bot instance to check a job's result - also re-uses code from the check-result.sh script in EESSI/compatibility-layer --- bot/check-result.sh | 195 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100755 bot/check-result.sh diff --git a/bot/check-result.sh b/bot/check-result.sh new file mode 100755 index 0000000000..38e76c3abc --- /dev/null +++ b/bot/check-result.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# +# Script to check the result of building the EESSI software layer. +# Intended use is that it is called by a (batch) job running on a compute +# node. +# +# This script is part of the EESSI compatibility layer, see +# https://github.com/EESSI/compatibility-layer.git +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + +# result cases + +# - SUCCESS (all of) +# - working directory contains slurm-JOBID.out file +# - working directory contains eessi*tar.gz +# - no message ERROR +# - no message FAILED +# - no message ' required modules missing:' +# - one or more of 'No missing modules!' +# - message regarding created tarball +# - FAILED (one of ... implemented as NOT SUCCESS) +# - no slurm-JOBID.out file +# - no tarball +# - message with ERROR +# - message with FAILED +# - message with ' required modules missing:' +# - no message regarding created tarball + +# stop as soon as something fails +# set -e + +TOPDIR=$(dirname $(realpath $0)) + +source ${TOPDIR}/scripts/utils.sh +source ${TOPDIR}/scripts/cfg_files.sh + +display_help() { + echo "usage: $0 [OPTIONS]" + echo " OPTIONS:" + echo " -h | --help - display this usage information [default: false]" + echo " -v | --verbose - display more information [default: false]" +} + +# set defaults for command line arguments +VERBOSE=0 + +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + display_help + exit 0 + ;; + -v|--verbose) + VERBOSE=1 + shift 1 + ;; + --) + shift + POSITIONAL_ARGS+=("$@") # save positional args + break + ;; + -*|--*) + fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" + ;; + *) # No more options + POSITIONAL_ARGS+=("$1") # save positional arg + shift + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" + +job_dir=${PWD} + +[[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}" + +GP_slurm_out="slurm-${SLURM_JOB_ID}.out" +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" +job_out=$(ls ${job_dir} | grep "${GP_slurm_out}") +[[ $? -eq 0 ]] && SLURM=1 || SLURM=0 +[[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" + +GP_error='ERROR: ' +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" +grep_out=$(grep "${GP_error}" ${job_dir}/${job_out}) +[[ $? -eq 0 ]] && ERROR=1 || ERROR=0 +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +GP_failed='FAILED: ' +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" +grep_out=$(grep "${GP_failed}" ${job_dir}/${job_out}) +[[ $? -eq 0 ]] && FAILED=1 || FAILED=0 +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +GP_req_missing=' required modules missing:' +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" +grep_out=$(grep "${GP_req_missing}" ${job_dir}/${job_out}) +[[ $? -eq 0 ]] && MISSING=1 || MISSING=0 +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +GP_no_missing='No missing modules!' +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" +grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out}) +[[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0 +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +GP_tgz_created="tar.gz created!" +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" +grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out}) +TARBALL= +if [[ $? -eq 0 ]]; then + TGZ=1 + TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@') +else + TGZ=0 +fi +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +echo "SUMMARY: ${job_dir}/${job_out}" +echo " test name : result (expected result)" +echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)" +echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" +echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" +echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" + +if [[ ${SLURM} -eq 1 ]] && \ + [[ ${ERROR} -eq 0 ]] && \ + [[ ${FAILED} -eq 0 ]] && \ + [[ ${MISSING} -eq 0 ]] && \ + [[ ${NO_MISSING} -eq 1 ]] && \ + [[ ${TGZ} -eq 1 ]] && \ + [[ ! -z ${TARBALL} ]]; then + # SUCCESS + echo "[RESULT]" > ${job_result_file} + echo "summary = :grin: SUCCESS" >> ${job_result_file} + echo "details =" >> ${job_result_file} +else + # FAILURE + echo "[RESULT]" > ${job_result_file} + echo "summary = :cry: FAILURE" >> ${job_result_file} + echo "details =" >> ${job_result_file} +fi + +if [[ ${SLURM} -eq 1 ]]; then + # need to indent by 4 spaces + echo " job output file ${job_out} (pattern: ${GP_slurm_out})" >> ${job_result_file} +else + echo " no job output file matching ${GP_slurm_out}" >> ${job_result_file} +fi + +if [[ ${ERROR} -eq 0 ]]; then + echo " job output lacks message matching ${GP_error}" >> ${job_result_file} +else + echo " job output contains message matching ${GP_error}" >> ${job_result_file} +fi + +if [[ ${FAILED} -eq 0 ]]; then + echo " job output lacks message matching ${GP_failed}" >> ${job_result_file} +else + echo " job output contains message matching ${GP_failed}" >> ${job_result_file} +fi + +if [[ ${MISSING} -eq 0 ]]; then + echo " job output lacks message matching ${GP_req_missing}" >> ${job_result_file} +else + echo " job output contains message matching ${GP_req_missing}" >> ${job_result_file} +fi + +if [[ ${NO_MISSING} -eq 1 ]]; then + echo " found message(s) matching ${GP_no_missing}" >> ${job_result_file} +else + echo " found no message matching ${GP_no_missing}" >> ${job_result_file} +fi + +if [[ ${TGZ} -eq 1 ]]; then + echo " found message matching ${GP_tgz_created}" >> ${job_result_file} +else + echo " found no message matching ${GP_tgz_created}" >> ${job_result_file} +fi + +echo "artefacts =" >> ${job_result_file} + +if [[ ! -z ${TARBALL} ]]; then + echo " ${TARBALL}" >> ${job_result_file} +fi + +exit 0 From fb04876141a464d98eb4cbb1aa3e34e4bc1a321c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 18 Apr 2023 14:34:27 +0200 Subject: [PATCH 02/20] fix path to helper scripts + define result file name --- bot/check-result.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 38e76c3abc..15356f10cf 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -35,8 +35,8 @@ TOPDIR=$(dirname $(realpath $0)) -source ${TOPDIR}/scripts/utils.sh -source ${TOPDIR}/scripts/cfg_files.sh +source ${TOPDIR}/../scripts/utils.sh +source ${TOPDIR}/../scripts/cfg_files.sh display_help() { echo "usage: $0 [OPTIONS]" @@ -131,6 +131,8 @@ echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +job_result_file=_bot_job${SLURM_JOB_ID}.result + if [[ ${SLURM} -eq 1 ]] && \ [[ ${ERROR} -eq 0 ]] && \ [[ ${FAILED} -eq 0 ]] && \ From 5ebc5d92014b97738b85102d175edf77d14e2799 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 18 Apr 2023 15:41:45 +0200 Subject: [PATCH 03/20] define functions for standardizing result output --- bot/check-result.sh | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 15356f10cf..dbc76e2dc9 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -151,41 +151,48 @@ else echo "details =" >> ${job_result_file} fi +function succeeded() { + echo "    :heavy_check_mark:${1}" +} + +function failed() { + echo "    :heavy_multiplication_x:${1}" +} + if [[ ${SLURM} -eq 1 ]]; then - # need to indent by 4 spaces - echo " job output file ${job_out} (pattern: ${GP_slurm_out})" >> ${job_result_file} + succeeded "job output file ${job_out}" >> ${job_result_file} else - echo " no job output file matching ${GP_slurm_out}" >> ${job_result_file} + failed "no job output file matching ${GP_slurm_out}" >> ${job_result_file} fi if [[ ${ERROR} -eq 0 ]]; then - echo " job output lacks message matching ${GP_error}" >> ${job_result_file} + succeeded "job output lacks message matching ${GP_error}" >> ${job_result_file} else - echo " job output contains message matching ${GP_error}" >> ${job_result_file} + failed "job output contains message matching ${GP_error}" >> ${job_result_file} fi if [[ ${FAILED} -eq 0 ]]; then - echo " job output lacks message matching ${GP_failed}" >> ${job_result_file} + succeeded "job output lacks message matching ${GP_failed}" >> ${job_result_file} else - echo " job output contains message matching ${GP_failed}" >> ${job_result_file} + failed "job output contains message matching ${GP_failed}" >> ${job_result_file} fi if [[ ${MISSING} -eq 0 ]]; then - echo " job output lacks message matching ${GP_req_missing}" >> ${job_result_file} + succeeded "job output lacks message matching ${GP_req_missing}" >> ${job_result_file} else - echo " job output contains message matching ${GP_req_missing}" >> ${job_result_file} + failed "job output contains message matching ${GP_req_missing}" >> ${job_result_file} fi if [[ ${NO_MISSING} -eq 1 ]]; then - echo " found message(s) matching ${GP_no_missing}" >> ${job_result_file} + succeeded "found message(s) matching ${GP_no_missing}" >> ${job_result_file} else - echo " found no message matching ${GP_no_missing}" >> ${job_result_file} + failed "found no message matching ${GP_no_missing}" >> ${job_result_file} fi if [[ ${TGZ} -eq 1 ]]; then - echo " found message matching ${GP_tgz_created}" >> ${job_result_file} + succeeded "found message matching ${GP_tgz_created}" >> ${job_result_file} else - echo " found no message matching ${GP_tgz_created}" >> ${job_result_file} + failed "found no message matching ${GP_tgz_created}" >> ${job_result_file} fi echo "artefacts =" >> ${job_result_file} From b8621545003d3d3e9bd482152a84f1aada2227ed Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 19 Apr 2023 11:29:05 +0200 Subject: [PATCH 04/20] polishing formating of status messages --- bot/check-result.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index dbc76e2dc9..9e34f81dd8 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -152,11 +152,11 @@ else fi function succeeded() { - echo "    :heavy_check_mark:${1}" + echo " :heavy_check_mark:${1}" } function failed() { - echo "    :heavy_multiplication_x:${1}" + echo " :heavy_multiplication_x:${1}" } if [[ ${SLURM} -eq 1 ]]; then From 95e8ff8e55f316852aeb1ec09ce94db5cacb1b62 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 19 Apr 2023 12:29:56 +0200 Subject: [PATCH 05/20] more polishing of comment details --- bot/check-result.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 9e34f81dd8..2d2f33be06 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -166,33 +166,33 @@ else fi if [[ ${ERROR} -eq 0 ]]; then - succeeded "job output lacks message matching ${GP_error}" >> ${job_result_file} + succeeded "no message matching ${GP_error}" >> ${job_result_file} else - failed "job output contains message matching ${GP_error}" >> ${job_result_file} + failed "found message matching ${GP_error}" >> ${job_result_file} fi if [[ ${FAILED} -eq 0 ]]; then - succeeded "job output lacks message matching ${GP_failed}" >> ${job_result_file} + succeeded "no message matching ${GP_failed}" >> ${job_result_file} else - failed "job output contains message matching ${GP_failed}" >> ${job_result_file} + failed "found message matching ${GP_failed}" >> ${job_result_file} fi if [[ ${MISSING} -eq 0 ]]; then - succeeded "job output lacks message matching ${GP_req_missing}" >> ${job_result_file} + succeeded "no message matching ${GP_req_missing}" >> ${job_result_file} else - failed "job output contains message matching ${GP_req_missing}" >> ${job_result_file} + failed "found message matching ${GP_req_missing}" >> ${job_result_file} fi if [[ ${NO_MISSING} -eq 1 ]]; then succeeded "found message(s) matching ${GP_no_missing}" >> ${job_result_file} else - failed "found no message matching ${GP_no_missing}" >> ${job_result_file} + failed "no message matching ${GP_no_missing}" >> ${job_result_file} fi if [[ ${TGZ} -eq 1 ]]; then succeeded "found message matching ${GP_tgz_created}" >> ${job_result_file} else - failed "found no message matching ${GP_tgz_created}" >> ${job_result_file} + failed "no message matching ${GP_tgz_created}" >> ${job_result_file} fi echo "artefacts =" >> ${job_result_file} From 6724b49b18dc0a4cabc87c5526e0e8741b148686 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 19 Apr 2023 13:06:15 +0200 Subject: [PATCH 06/20] minor polishing of job result messages --- bot/check-result.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 2d2f33be06..1598e9ca1b 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -152,11 +152,11 @@ else fi function succeeded() { - echo " :heavy_check_mark:${1}" + echo " :heavy_check_mark: ${1}" } function failed() { - echo " :heavy_multiplication_x:${1}" + echo " :heavy_multiplication_x: ${1}" } if [[ ${SLURM} -eq 1 ]]; then From b70c858d65eadba24366518d30f63a3fdca3b686 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 19 Apr 2023 13:36:15 +0200 Subject: [PATCH 07/20] only log if run with --verbose --- bot/check-result.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 1598e9ca1b..55276c405a 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -123,13 +123,13 @@ else fi [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" -echo "SUMMARY: ${job_dir}/${job_out}" -echo " test name : result (expected result)" -echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)" -echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" -echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" -echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" -echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +[[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}" +[[ ${VERBOSE} -ne 0 ]] && echo " test name : result (expected result)" +[[ ${VERBOSE} -ne 0 ]] && echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)" +[[ ${VERBOSE} -ne 0 ]] && echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" +[[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" +[[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +[[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" job_result_file=_bot_job${SLURM_JOB_ID}.result From 677c36296f1ab433964b8caa230ad8ef32c5d625 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 20 Apr 2023 15:22:40 +0200 Subject: [PATCH 08/20] fix small bug when checking for tarball msg --- bot/check-result.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 55276c405a..3f0c89ea21 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -112,9 +112,9 @@ grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out}) [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_tgz_created="tar.gz created!" +TARBALL= [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out}) -TARBALL= if [[ $? -eq 0 ]]; then TGZ=1 TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@') From e4932eefaff86adee6e91265efdfd5268efdf52a Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 20 Apr 2023 15:31:36 +0200 Subject: [PATCH 09/20] avoid creating false positives --- bot/check-result.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 3f0c89ea21..e28ae4e38e 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -82,38 +82,42 @@ job_dir=${PWD} [[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}" GP_slurm_out="slurm-${SLURM_JOB_ID}.out" -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" job_out=$(ls ${job_dir} | grep "${GP_slurm_out}") [[ $? -eq 0 ]] && SLURM=1 || SLURM=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" GP_error='ERROR: ' -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" grep_out=$(grep "${GP_error}" ${job_dir}/${job_out}) [[ $? -eq 0 ]] && ERROR=1 || ERROR=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_failed='FAILED: ' -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" grep_out=$(grep "${GP_failed}" ${job_dir}/${job_out}) [[ $? -eq 0 ]] && FAILED=1 || FAILED=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_req_missing=' required modules missing:' -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" grep_out=$(grep "${GP_req_missing}" ${job_dir}/${job_out}) [[ $? -eq 0 ]] && MISSING=1 || MISSING=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_no_missing='No missing modules!' -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out}) [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_tgz_created="tar.gz created!" TARBALL= -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out}) if [[ $? -eq 0 ]]; then TGZ=1 @@ -121,6 +125,8 @@ if [[ $? -eq 0 ]]; then else TGZ=0 fi +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" [[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}" From 9a0e220db5a21b6285785ab09683bc0bb62e3108 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 5 Jun 2023 18:48:43 +0200 Subject: [PATCH 10/20] also filter OpenSSL dependency in EasyBuild configuration --- configure_easybuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure_easybuild b/configure_easybuild index 245553f342..5d4e9b5a48 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -26,7 +26,7 @@ fi # note: filtering Bison may break some installations, like Qt5 (see https://github.com/EESSI/software-layer/issues/49) # filtering pkg-config breaks R-bundle-Bioconductor installation (see also https://github.com/easybuilders/easybuild-easyconfigs/pull/11104) # problems occur when filtering pkg-config with gnuplot too (picks up Lua 5.1 from $EPREFIX rather than from Lua 5.3 dependency) -DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,cURL,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,util-linux,XZ,zlib +DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,cURL,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,OpenSSL,util-linux,XZ,zlib # For aarch64 we need to also filter out Yasm. # See https://github.com/easybuilders/easybuild-easyconfigs/issues/11190 if [[ "$EESSI_CPU_FAMILY" == "aarch64" ]]; then From 074d7a1847ab2353e43045978dec6acd72a95cb1 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 18 Jun 2023 20:06:08 +0200 Subject: [PATCH 11/20] update check-result.sh to state tested in NESSI PR125 --- bot/check-result.sh | 429 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 351 insertions(+), 78 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index e28ae4e38e..04a23962e2 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -4,8 +4,8 @@ # Intended use is that it is called by a (batch) job running on a compute # node. # -# This script is part of the EESSI compatibility layer, see -# https://github.com/EESSI/compatibility-layer.git +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git # # author: Thomas Roeblitz (@trz42) # @@ -38,6 +38,20 @@ TOPDIR=$(dirname $(realpath $0)) source ${TOPDIR}/../scripts/utils.sh source ${TOPDIR}/../scripts/cfg_files.sh +# defaults +export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" + +# check if ${JOB_CFG_FILE} exists +if [[ ! -r "${JOB_CFG_FILE}" ]]; then + echo_red "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable" +else + echo "bot/check-result.sh: showing ${JOB_CFG_FILE} from software-layer side" + cat ${JOB_CFG_FILE} + + echo "bot/check-result.sh: obtaining configuration settings from '${JOB_CFG_FILE}'" + cfg_load ${JOB_CFG_FILE} +fi + display_help() { echo "usage: $0 [OPTIONS]" echo " OPTIONS:" @@ -88,46 +102,61 @@ job_out=$(ls ${job_dir} | grep "${GP_slurm_out}") [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" -GP_error='ERROR: ' -grep_out=$(grep "${GP_error}" ${job_dir}/${job_out}) -[[ $? -eq 0 ]] && ERROR=1 || ERROR=0 -# have to be careful to not add searched for pattern into slurm out file -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" -[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +ERROR=-1 +if [[ ${SLURM} -eq 1 ]]; then + GP_error='ERROR: ' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}") + [[ $? -eq 0 ]] && ERROR=1 || ERROR=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi -GP_failed='FAILED: ' -grep_out=$(grep "${GP_failed}" ${job_dir}/${job_out}) -[[ $? -eq 0 ]] && FAILED=1 || FAILED=0 -# have to be careful to not add searched for pattern into slurm out file -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" -[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +FAILED=-1 +if [[ ${SLURM} -eq 1 ]]; then + GP_failed='FAILED: ' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}") + [[ $? -eq 0 ]] && FAILED=1 || FAILED=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi -GP_req_missing=' required modules missing:' -grep_out=$(grep "${GP_req_missing}" ${job_dir}/${job_out}) -[[ $? -eq 0 ]] && MISSING=1 || MISSING=0 -# have to be careful to not add searched for pattern into slurm out file -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" -[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +MISSING=-1 +if [[ ${SLURM} -eq 1 ]]; then + GP_req_missing=' required modules missing:' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_req_missing}") + [[ $? -eq 0 ]] && MISSING=1 || MISSING=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi -GP_no_missing='No missing modules!' -grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out}) -[[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0 -# have to be careful to not add searched for pattern into slurm out file -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" -[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +NO_MISSING=-1 +if [[ ${SLURM} -eq 1 ]]; then + GP_no_missing='No missing installations' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_no_missing}") + [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi -GP_tgz_created="tar.gz created!" +TGZ=-1 TARBALL= -grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out}) -if [[ $? -eq 0 ]]; then - TGZ=1 - TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@') -else - TGZ=0 +if [[ ${SLURM} -eq 1 ]]; then + GP_tgz_created="\.tar\.gz created!" + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u) + if [[ $? -eq 0 ]]; then + TGZ=1 + TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@') + else + TGZ=0 + fi + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" fi -# have to be careful to not add searched for pattern into slurm out file -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" -[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" [[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}" [[ ${VERBOSE} -ne 0 ]] && echo " test name : result (expected result)" @@ -147,64 +176,308 @@ if [[ ${SLURM} -eq 1 ]] && \ [[ ${TGZ} -eq 1 ]] && \ [[ ! -z ${TARBALL} ]]; then # SUCCESS - echo "[RESULT]" > ${job_result_file} - echo "summary = :grin: SUCCESS" >> ${job_result_file} - echo "details =" >> ${job_result_file} + status="SUCCESS" + summary=":grin: SUCCESS" else # FAILURE - echo "[RESULT]" > ${job_result_file} - echo "summary = :cry: FAILURE" >> ${job_result_file} - echo "details =" >> ${job_result_file} + status="FAILURE" + summary=":cry: FAILURE" fi -function succeeded() { - echo " :heavy_check_mark: ${1}" +### Example details/descriptions +# Note, final string must not contain any line breaks. Below examples include +# line breaks for the sake of readability. +#
+# :cry: FAILURE _(click triangle for detailed information)_ +# Details:
+#     :heavy_check_mark: job output file slurm-470503.out
+#     :heavy_multiplication_x: found message matching ERROR:
+#     :heavy_multiplication_x: found message matching FAILED:
+#     :heavy_multiplication_x: found message matching required modules missing:
+#     :heavy_check_mark: found message(s) matching No missing installations
+#     :heavy_check_mark: found message matching tar.gz created!
+# Artefacts: +#
  • eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz
    • +#
    +# +#
    +# :grin: SUCCESS _(click triangle for detailed information)_ +# Details:
    +#     :heavy_check_mark: job output file slurm-470503.out
    +#     :heavy_check_mark: found message matching ERROR:
    +#     :heavy_check_mark: found message matching FAILED:
    +#     :heavy_check_mark: found message matching required modules missing:
    +#     :heavy_check_mark: found message(s) matching No missing installations
    +#     :heavy_check_mark: found message matching tar.gz created!
    +# Artefacts: +#
  • eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz
    • +#
    +# +#
    +# :grin: SUCCESS _(click triangle for detailed information)_ +#
    +#
    _Details_
    +#
    +# :heavy_check_mark: job output file slurm-4682.out
    +# :heavy_check_mark: no message matching ERROR:
    +# :heavy_check_mark: no message matching FAILED:
    +# :heavy_check_mark: no message matching required modules missing:
    +# :heavy_check_mark: found message(s) matching No missing installations
    +# :heavy_check_mark: found message matching tar.gz created!
    +#
    +#
    _Artefacts_
    +#
    +#
    +# eessi-2023.04-software-linux-x86_64-generic-1682696567.tar.gz +# size: 234 MiB (245366784 bytes)
    +# entries: 1234
    +# modules under _2023.04/software/linux/x86_64/intel/cascadelake/modules/all/_
    +# 
    +#           GCC/9.3.0.lua
    
+#           GCC/10.3.0.lua
    
+#           OpenSSL/1.1.lua
+#
    +# software under _2023.04/software/linux/x86_64/intel/cascadelake/software/_ +# 
    +#           GCC/9.3.0/
    
+#           CMake/3.20.1-GCCcore-10.3.0/
    
+#           OpenMPI/4.1.1-GCC-10.3.0/
+#
    +# other under _2023.04/software/linux/x86_64/intel/cascadelake/_ +# 
    +#           .lmod/cache/spiderT.lua
    
+#           .lmod/cache/spiderT.luac_5.1
    
+#           .lmod/cache/timestamp
+#
    +#
    +#
    +#
    +#
    +# +### + +# construct and write complete PR comment details: implements third alternative +comment_template="
    __SUMMARY_FMT__
    __DETAILS_FMT____ARTEFACTS_FMT__
    " +comment_summary_fmt="__SUMMARY__ _(click triangle for details)_" +comment_details_fmt="
    _Details_
    __DETAILS_LIST__
    " +comment_success_item_fmt=":heavy_check_mark: __ITEM__" +comment_failure_item_fmt=":heavy_multiplication_x: __ITEM__" +comment_artefacts_fmt="
    _Artefacts_
    __ARTEFACTS_LIST__
    " +comment_artefact_details_fmt="
    __ARTEFACT_SUMMARY____ARTEFACT_DETAILS__
    " + +function print_br_item() { + format="${1}" + item="${2}" + echo -n "${format//__ITEM__/${item}}
    " } -function failed() { - echo " :heavy_multiplication_x: ${1}" +function print_br_item2() { + format="${1}" + item="${2}" + item2="${3}" + format1="${format//__ITEM__/${item}}" + echo -n "${format1//__ITEM2__/${item2}}
    " } -if [[ ${SLURM} -eq 1 ]]; then - succeeded "job output file ${job_out}" >> ${job_result_file} -else - failed "no job output file matching ${GP_slurm_out}" >> ${job_result_file} -fi +function print_code_item() { + format="${1}" + item="${2}" + echo -n "${format//__ITEM__/${item}}" +} -if [[ ${ERROR} -eq 0 ]]; then - succeeded "no message matching ${GP_error}" >> ${job_result_file} -else - failed "found message matching ${GP_error}" >> ${job_result_file} -fi +function print_dd_item() { + format="${1}" + item="${2}" + echo -n "
    ${format//__ITEM__/${item}}
    " +} -if [[ ${FAILED} -eq 0 ]]; then - succeeded "no message matching ${GP_failed}" >> ${job_result_file} -else - failed "found message matching ${GP_failed}" >> ${job_result_file} -fi +function print_list_item() { + format="${1}" + item="${2}" + echo -n "
  • ${format//__ITEM__/${item}}
    • " +} -if [[ ${MISSING} -eq 0 ]]; then - succeeded "no message matching ${GP_req_missing}" >> ${job_result_file} -else - failed "found message matching ${GP_req_missing}" >> ${job_result_file} -fi +function print_pre_item() { + format="${1}" + item="${2}" + echo -n "
    ${format//__ITEM__/${item}}
    " +} -if [[ ${NO_MISSING} -eq 1 ]]; then - succeeded "found message(s) matching ${GP_no_missing}" >> ${job_result_file} -else - failed "no message matching ${GP_no_missing}" >> ${job_result_file} -fi +function success() { + format="${comment_success_item_fmt}" + item="$1" + print_br_item "${format}" "${item}" +} -if [[ ${TGZ} -eq 1 ]]; then - succeeded "found message matching ${GP_tgz_created}" >> ${job_result_file} -else - failed "no message matching ${GP_tgz_created}" >> ${job_result_file} -fi +function failure() { + format="${comment_failure_item_fmt}" + item="$1" + print_br_item "${format}" "${item}" +} + +function add_detail() { + actual=${1} + expected=${2} + success_msg="${3}" + failure_msg="${4}" + if [[ ${actual} -eq ${expected} ]]; then + success "${success_msg}" + else + failure "${failure_msg}" + fi +} + +echo "[RESULT]" > ${job_result_file} +echo -n "comment_description = " >> ${job_result_file} + +# construct values for placeholders in comment_template: +# - __SUMMARY_FMT__ -> variable $comment_summary +# - __DETAILS_FMT__ -> variable $comment_details +# - __ARTEFACTS_FMT__ -> variable $comment_artefacts -echo "artefacts =" >> ${job_result_file} +comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}" +# first construct comment_details_list, abbreviated CoDeList +# then use it to set comment_details +CoDeList="" + +success_msg="job output file ${job_out}" +failure_msg="no job output file matching ${GP_slurm_out}" +CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}") + +success_msg="no message matching ${GP_error}" +failure_msg="found message matching ${GP_error}" +CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") + +success_msg="no message matching ${GP_failed}" +failure_msg="found message matching ${GP_failed}" +CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") + +success_msg="no message matching ${GP_req_missing}" +failure_msg="found message matching ${GP_req_missing}" +CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}") + +success_msg="found message(s) matching ${GP_no_missing}" +failure_msg="no message matching ${GP_no_missing}" +CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") + +success_msg="found message matching ${GP_tgz_created}" +failure_msg="no message matching ${GP_tgz_created}" +CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") + +comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}" + + +# first construct comment_artefacts_list, abbreviated CoArList +# then use it to set comment_artefacts +CoArList="" + +# TARBALL should only contain a single tarball if [[ ! -z ${TARBALL} ]]; then - echo " ${TARBALL}" >> ${job_result_file} + # TODO add tarball details: size, num entries, modules, software pkgs, misc + #
    + #
    + # eessi-2023.04-software-linux-x86_64-generic-1682696567.tar.gz + # size: 234 MiB (245366784 bytes)
    + # entries: 1234
    + # modules under _2023.04/software/linux/x86_64/intel/cascadelake/modules/all/_
    + # 
    +    #       GCC/9.3.0.lua
    
+    #       GCC/10.3.0.lua
    
+    #       OpenSSL/1.1.lua
+    #
    + # software under _2023.04/software/linux/x86_64/intel/cascadelake/software/_ + # 
    +    #       GCC/9.3.0/
    
+    #       CMake/3.20.1-GCCcore-10.3.0/
    
+    #       OpenMPI/4.1.1-GCC-10.3.0/
+    #
    + # other under _2023.04/software/linux/x86_64/intel/cascadelake/_ + # 
    +    #       .lmod/cache/spiderT.lua
    
+    #       .lmod/cache/spiderT.luac_5.1
    
+    #       .lmod/cache/timestamp
+    #
    + #
    + #
    + size="$(stat --dereference --printf=%s ${TARBALL})" + size_mib=$((${size} >> 20)) + tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX) + tar tf ${TARBALL} > ${tmpfile} + entries=$(cat ${tmpfile} | wc -l) + # determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE + # 2023.04/software/linux/x86_64/intel/skylake_avx512 + # repo_version = 2022.11 + # software + # os_type = linux + # software_subdir = aarch64/generic + repo_version=$(cfg_get_value "repository" "repo_version") + os_type=$(cfg_get_value "architecture" "os_type") + software_subdir=$(cfg_get_value "architecture" "software_subdir") + prefix="${repo_version}/software/${os_type}/${software_subdir}" + modules_entries=$(grep "${prefix}/modules" ${tmpfile}) + software_entries=$(grep "${prefix}/software" ${tmpfile}) + lmod_entries=$(grep "${prefix}/.lmod/cache" ${tmpfile}) + other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software") + other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u) + modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u) + software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u) + lmod_shortened=$(echo "${lmod_entries}" | sed -e "s@${prefix}/@@") + + artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})" + CoArList="" + CoArList="${CoArList}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})" + CoArList="${CoArList}$(print_br_item 'entries: __ITEM__' ${entries})" + CoArList="${CoArList}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)" + CoArList="${CoArList}
    "
+    if [[ ! -z ${modules} ]]; then
+        while IFS= read -r mod ; do
+            CoArList="${CoArList}$(print_br_item '__ITEM__' ${mod})"
+        done <<< "${modules}"
+    else
+        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no module files in tarball')"
+    fi
+    CoArList="${CoArList}
    " + CoArList="${CoArList}$(print_br_item 'software under ___ITEM___' ${prefix}/software)" + CoArList="${CoArList}
    "
+    if [[ ! -z ${software_pkgs} ]]; then
+        while IFS= read -r sw_pkg ; do
+            CoArList="${CoArList}$(print_br_item '__ITEM__' ${sw_pkg})"
+        done <<< "${software_pkgs}"
+    else
+        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no software packages in tarball')"
+    fi
+    CoArList="${CoArList}
    " + CoArList="${CoArList}$(print_br_item 'other under ___ITEM___' ${prefix})" + CoArList="${CoArList}
    "
+    if [[ ! -z ${other_shortened} ]]; then
+        while IFS= read -r other ; do
+            CoArList="${CoArList}$(print_br_item '__ITEM__' ${other})"
+        done <<< "${other_shortened}"
+    else
+        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no other files in tarball')"
+    fi
+    CoArList="${CoArList}
    " +else + CoArList="${CoArList}$(print_dd_item 'No artefacts were created or found.' '')" fi +comment_artefacts_details="${comment_artefact_details_fmt/__ARTEFACT_SUMMARY__/${artefact_summary}}" +comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${CoArList}}" +comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}" + +# now put all pieces together creating comment_details from comment_template +comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}} +comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} +comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}} + +echo "${comment_description}" >> ${job_result_file} + +# add overall result: SUCCESS, FAILURE, UNKNOWN + artefacts +# - this should make use of subsequent steps such as deploying a tarball more +# efficient +echo "status = ${status}" >> ${job_result_file} +echo "artefacts = " >> ${job_result_file} +echo "${TARBALL}" | sed -e 's/^/ /g' >> ${job_result_file} + exit 0 From 3c43a92524f4f8f6d5db167c8a58cedc273d51a0 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 18 Jun 2023 20:09:00 +0200 Subject: [PATCH 12/20] check for missing installations requires a few updates for check-result.sh --- check_missing_installations.sh | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 926f475903..3627d1d0b5 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -1,18 +1,20 @@ #!/bin/bash # -# Script to check for missing installations in EESSI pilot software stack (version 2021.12) +# Script to check for missing installations in EESSI pilot software stack (version 2023.06) # # author: Kenneth Hoste (@boegel) +# author: Thomas Roeblitz (@trz42) # # license: GPLv2 # TOPDIR=$(dirname $(realpath $0)) -if [ -z ${EESSI_PILOT_VERSION} ]; then - echo "ERROR: \${EESSI_PILOT_VERSION} must be set to run $0!" >&2 +if [ $# -ne 1 ]; then + echo "ERROR: Usage: $0 " >&2 exit 1 fi +easystack=$1 LOCAL_TMPDIR=$(mktemp -d) @@ -21,12 +23,13 @@ source $TOPDIR/scripts/utils.sh source $TOPDIR/configure_easybuild echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." -ok_msg="No missing installations, party time!" -fail_msg="On no, some installations are still missing, how did that happen?!" eb_missing_out=$LOCAL_TMPDIR/eb_missing.out -# we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used -# PR #16531: Nextflow-22.10.1.eb -${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out} +${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out} +exit_code=${PIPESTATUS[0]} + +ok_msg="Command 'eb --missing ...' succeeded, analysing output..." +fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'" +check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}" # the above assesses the installed software for each easyconfig provided in # the easystack file and then print messages such as @@ -37,8 +40,11 @@ ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experim # output does not contain any line with ` required modules missing:` grep " required modules missing:" ${eb_missing_out} > /dev/null +exit_code=$? # if grep returns 1 (` required modules missing:` was NOT found), we set # MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1 -[[ $? -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1 +[[ ${exit_code} -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1 +ok_msg="No missing installations, party time!" +fail_msg="On no, some installations are still missing, how did that happen?!" check_exit_code ${MODULES_MISSING} "${ok_msg}" "${fail_msg}" From 5a0294e0bf1c553a108ab5ceba5dff07b90be8d6 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 18 Jun 2023 20:09:43 +0200 Subject: [PATCH 13/20] make sure experimental EasyBuild features can be used --- configure_easybuild | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configure_easybuild b/configure_easybuild index 19b2d7454b..8e79f3a35a 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -35,3 +35,6 @@ fi export EASYBUILD_FILTER_DEPS=$DEPS_TO_FILTER export EASYBUILD_MODULE_EXTENSIONS=1 + +# need to enable use of experimental features, since we're using easystack files +export EASYBUILD_EXPERIMENTAL=1 From 4110548e9280795f32a4c60ed979430217a50771 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 20 Jun 2023 10:35:44 +0200 Subject: [PATCH 14/20] removed OpenSSL from filter deps --- configure_easybuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure_easybuild b/configure_easybuild index ad34487902..23f3920154 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -26,7 +26,7 @@ fi # note: filtering Bison may break some installations, like Qt5 (see https://github.com/EESSI/software-layer/issues/49) # filtering pkg-config breaks R-bundle-Bioconductor installation (see also https://github.com/easybuilders/easybuild-easyconfigs/pull/11104) # problems occur when filtering pkg-config with gnuplot too (picks up Lua 5.1 from $EPREFIX rather than from Lua 5.3 dependency) -DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,cURL,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,OpenSSL,util-linux,XZ,zlib +DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,cURL,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,util-linux,XZ,zlib # For aarch64 we need to also filter out Yasm. # See https://github.com/easybuilders/easybuild-easyconfigs/issues/11190 if [[ "$EESSI_CPU_FAMILY" == "aarch64" ]]; then From f709a8a72d425acfa02dd9d3e8114af99b796b1f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 20 Jun 2023 10:43:56 +0200 Subject: [PATCH 15/20] let script's return value reflect SUCCESS (0) or FAILURE (1) --- bot/check-result.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/check-result.sh b/bot/check-result.sh index 04a23962e2..ec58fea557 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -480,4 +480,6 @@ echo "status = ${status}" >> ${job_result_file} echo "artefacts = " >> ${job_result_file} echo "${TARBALL}" | sed -e 's/^/ /g' >> ${job_result_file} +test "${status}" == "SUCCESS" +exit $? exit 0 From 5cfb4e1510cd8a0c93824af33ff5c4a805e3bd11 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 20 Jun 2023 10:52:06 +0200 Subject: [PATCH 16/20] use test to check if slurm-*.out exists + fix reporting message --- bot/check-result.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index ec58fea557..6c23cae680 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -96,11 +96,14 @@ job_dir=${PWD} [[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}" GP_slurm_out="slurm-${SLURM_JOB_ID}.out" -job_out=$(ls ${job_dir} | grep "${GP_slurm_out}") -[[ $? -eq 0 ]] && SLURM=1 || SLURM=0 -# have to be careful to not add searched for pattern into slurm out file [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" -[[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" +if [[ -f ${GP_slurm_out} ]]; then + SLURM=1 + [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${GP_slurm_out}"'" +else + SLURM=0 + [[ ${VERBOSE} -ne 0 ]] && echo " Slurm output file '"${GP_slurm_out}"' NOT found" +fi ERROR=-1 if [[ ${SLURM} -eq 1 ]]; then From 7478e7d65ab2836135e62fc6f866d84d27a7b0a7 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 20 Jun 2023 11:09:34 +0200 Subject: [PATCH 17/20] more polishing to address multiple comments/suggestions --- bot/check-result.sh | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 6c23cae680..33542d61ed 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -152,7 +152,7 @@ if [[ ${SLURM} -eq 1 ]]; then grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u) if [[ $? -eq 0 ]]; then TGZ=1 - TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@') + TARBALL=$(echo ${grep_out} | sed -e 's@^.*/\(eessi[^/ ]*\) .*$@\1@') else TGZ=0 fi @@ -162,7 +162,7 @@ if [[ ${SLURM} -eq 1 ]]; then fi [[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}" -[[ ${VERBOSE} -ne 0 ]] && echo " test name : result (expected result)" +[[ ${VERBOSE} -ne 0 ]] && echo " : ()" [[ ${VERBOSE} -ne 0 ]] && echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)" [[ ${VERBOSE} -ne 0 ]] && echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" [[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" @@ -377,25 +377,28 @@ CoArList="" # TARBALL should only contain a single tarball if [[ ! -z ${TARBALL} ]]; then - # TODO add tarball details: size, num entries, modules, software pkgs, misc + # Example of the detailed information for a tarball. The actual result MUST be a + # single line (no '\n') or it would break the structure of the markdown table + # that holds status updates of a bot job. + # #
    #
    - # eessi-2023.04-software-linux-x86_64-generic-1682696567.tar.gz + # eessi-2023.06-software-linux-x86_64-generic-1682696567.tar.gz # size: 234 MiB (245366784 bytes)
    # entries: 1234
    - # modules under _2023.04/software/linux/x86_64/intel/cascadelake/modules/all/_
    + # modules under _2023.06/software/linux/x86_64/intel/cascadelake/modules/all/_
    # 
         #       GCC/9.3.0.lua
    
     #       GCC/10.3.0.lua
    
     #       OpenSSL/1.1.lua
     #
    - # software under _2023.04/software/linux/x86_64/intel/cascadelake/software/_ + # software under _2023.06/software/linux/x86_64/intel/cascadelake/software/_ # 
         #       GCC/9.3.0/
    
     #       CMake/3.20.1-GCCcore-10.3.0/
    
     #       OpenMPI/4.1.1-GCC-10.3.0/
     #
    - # other under _2023.04/software/linux/x86_64/intel/cascadelake/_ + # other under _2023.06/software/linux/x86_64/intel/cascadelake/_ # 
         #       .lmod/cache/spiderT.lua
    
     #       .lmod/cache/spiderT.luac_5.1
    
@@ -409,23 +412,25 @@ if [[ ! -z ${TARBALL} ]]; then
     tar tf ${TARBALL} > ${tmpfile}
     entries=$(cat ${tmpfile} | wc -l)
     # determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE
-    # 2023.04/software/linux/x86_64/intel/skylake_avx512
-    # repo_version = 2022.11
-    # software
+    # e.g., 2023.06/software/linux/x86_64/intel/skylake_avx512
+    # cfg/job.cfg contains (only the attributes to be used are shown below):
+    # [repository]
+    # repo_version = 2023.06
+    # [architecture]
     # os_type = linux
-    # software_subdir = aarch64/generic
+    # software_subdir = x86_64/intel/skylake_avx512
     repo_version=$(cfg_get_value "repository" "repo_version")
     os_type=$(cfg_get_value "architecture" "os_type")
     software_subdir=$(cfg_get_value "architecture" "software_subdir")
     prefix="${repo_version}/software/${os_type}/${software_subdir}"
+
+    # extract directories/entries from tarball content
     modules_entries=$(grep "${prefix}/modules" ${tmpfile})
     software_entries=$(grep "${prefix}/software" ${tmpfile})
-    lmod_entries=$(grep "${prefix}/.lmod/cache" ${tmpfile})
     other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software")
     other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u)
     modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u)
     software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u)
-    lmod_shortened=$(echo "${lmod_entries}" | sed -e "s@${prefix}/@@")
 
     artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})"
     CoArList=""

From 827e686edfc1540d82a4bbf5d069ed9460822efc Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz 
Date: Tue, 20 Jun 2023 11:14:01 +0200
Subject: [PATCH 18/20] remove tmpfile + clarify/improve exit

---
 bot/check-result.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/bot/check-result.sh b/bot/check-result.sh
index 33542d61ed..2ffbcd6be6 100755
--- a/bot/check-result.sh
+++ b/bot/check-result.sh
@@ -488,6 +488,11 @@ echo "status = ${status}" >> ${job_result_file}
 echo "artefacts = " >> ${job_result_file}
 echo "${TARBALL}" | sed -e 's/^/    /g' >> ${job_result_file}
 
+# remove tmpfile
+if [[ -f ${tmpfile} ]]; then
+    rm ${tmpfile}
+fi
+
+# exit script with value that reflects overall job result: SUCCESS (0), FAILURE (1)
 test "${status}" == "SUCCESS"
 exit $?
-exit 0

From 9296ed75e52b680bf278aba9ff2fb82ec5318765 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz 
Date: Tue, 20 Jun 2023 11:47:19 +0200
Subject: [PATCH 19/20] polishing examples + comments

---
 bot/check-result.sh | 62 ++++++++++++++++++++-------------------------
 1 file changed, 28 insertions(+), 34 deletions(-)

diff --git a/bot/check-result.sh b/bot/check-result.sh
index 2ffbcd6be6..51e1777bab 100755
--- a/bot/check-result.sh
+++ b/bot/check-result.sh
@@ -20,7 +20,7 @@
 #    - no message ERROR
 #    - no message FAILED
 #    - no message ' required modules missing:'
-#    - one or more of 'No missing modules!'
+#    - one or more of 'No missing installations'
 #    - message regarding created tarball
 #  - FAILED (one of ... implemented as NOT SUCCESS)
 #    - no slurm-JOBID.out file
@@ -188,36 +188,12 @@ else
 fi
 
 ### Example details/descriptions
-# Note, final string must not contain any line breaks. Below examples include
-# line breaks for the sake of readability.
+# Note, final string must not contain any line breaks. Below example include
+# line breaks for the sake of readability. In case of FAILURE, the structure is
+# very similar (incl. information about Artefacts if any was produced), however,
+# under Details some lines will be marked with :heavy_multiplication_x:
 # 
    
-#   :cry: FAILURE _(click triangle for detailed information)_
-#   Details:
    
-#       :heavy_check_mark: job output file slurm-470503.out
    
-#       :heavy_multiplication_x: found message matching ERROR: 
    
-#       :heavy_multiplication_x: found message matching FAILED: 
    
-#       :heavy_multiplication_x: found message matching  required modules missing:
    
-#       :heavy_check_mark: found message(s) matching No missing installations
    
-#       :heavy_check_mark: found message matching tar.gz created!
    
-#   Artefacts:
-#   
  • eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz
    • 
-# 
    
-#
-# 
    
-#   :grin: SUCCESS _(click triangle for detailed information)_
-#   Details:
    
-#       :heavy_check_mark: job output file slurm-470503.out
    
-#       :heavy_check_mark: found message matching ERROR: 
    
-#       :heavy_check_mark: found message matching FAILED: 
    
-#       :heavy_check_mark: found message matching  required modules missing:
    
-#       :heavy_check_mark: found message(s) matching No missing installations
    
-#       :heavy_check_mark: found message matching tar.gz created!
    
-#   Artefacts:
-#   
  • eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz
    • 
-# 
    
-#
-# 
    
-#   :grin: SUCCESS _(click triangle for detailed information)_
+#   :grin: SUCCESS _(click triangle for details)_
 #   
    
 #     
    _Details_
    
 #     
    
@@ -231,22 +207,22 @@ fi
 #     
    _Artefacts_
    
 #     
    
 #       
    
-#         eessi-2023.04-software-linux-x86_64-generic-1682696567.tar.gz
+#         eessi-2023.06-software-linux-x86_64-generic-1682696567.tar.gz
 #         size: 234 MiB (245366784 bytes)
    
 #         entries: 1234
    
-#         modules under _2023.04/software/linux/x86_64/intel/cascadelake/modules/all/_
    
+#         modules under _2023.06/software/linux/x86_64/generic/modules/all/_
    
 #         
     #           GCC/9.3.0.lua
    
 #           GCC/10.3.0.lua
    
 #           OpenSSL/1.1.lua
 #         
    
-#         software under _2023.04/software/linux/x86_64/intel/cascadelake/software/_
+#         software under _2023.06/software/linux/x86_64/generic/software/_
 #         
     #           GCC/9.3.0/
    
 #           CMake/3.20.1-GCCcore-10.3.0/
    
 #           OpenMPI/4.1.1-GCC-10.3.0/
 #         
    
-#         other under _2023.04/software/linux/x86_64/intel/cascadelake/_
+#         other under _2023.06/software/linux/x86_64/generic/_
 #         
     #           .lmod/cache/spiderT.lua
    
 #           .lmod/cache/spiderT.luac_5.1
    
@@ -257,6 +233,24 @@ fi
 #   
    
 # 
    
 #
+# 
    
+#   :cry: FAILURE _(click triangle for details)_
+#   
    
+#     
    _Details_
    
+#     
    
+#       :heavy_check_mark: job output file slurm-4682.out
    
+#       :heavy_multiplication_x: no message matching ERROR: 
    
+#       :heavy_check_mark: no message matching FAILED: 
    
+#       :heavy_multiplication_x: no message matching  required modules missing:
    
+#       :heavy_check_mark: found message(s) matching No missing installations
    
+#       :heavy_check_mark: found message matching tar.gz created!
    
+#     
    
+#     
    _Artefacts_
    
+#     
    
+#       No artefacts were created or found.
+#     
    
+#   
    
+# 
    
 ###
 
 # construct and write complete PR comment details: implements third alternative

From 78a9dc8e9e67dc998c39e6f74f06b72117e0f22d Mon Sep 17 00:00:00 2001
From: Kenneth Hoste 
Date: Tue, 20 Jun 2023 13:50:37 +0200
Subject: [PATCH 20/20] make sure that $job_out is defined in
 bot/check-result.sh

---
 bot/check-result.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/bot/check-result.sh b/bot/check-result.sh
index 51e1777bab..182b8555a8 100755
--- a/bot/check-result.sh
+++ b/bot/check-result.sh
@@ -95,14 +95,14 @@ job_dir=${PWD}
 
 [[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}"
 
-GP_slurm_out="slurm-${SLURM_JOB_ID}.out"
-[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'"
-if  [[ -f ${GP_slurm_out} ]]; then
+job_out="slurm-${SLURM_JOB_ID}.out"
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"
+if  [[ -f ${job_out} ]]; then
     SLURM=1
-    [[ ${VERBOSE} -ne 0 ]] && echo "   found slurm output file '"${GP_slurm_out}"'"
+    [[ ${VERBOSE} -ne 0 ]] && echo "   found slurm output file '"${job_out}"'"
 else
     SLURM=0
-    [[ ${VERBOSE} -ne 0 ]] && echo "   Slurm output file '"${GP_slurm_out}"' NOT found"
+    [[ ${VERBOSE} -ne 0 ]] && echo "   Slurm output file '"${job_out}"' NOT found"
 fi
 
 ERROR=-1
@@ -339,7 +339,7 @@ comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
 CoDeList=""
 
 success_msg="job output file ${job_out}"
-failure_msg="no job output file matching ${GP_slurm_out}"
+failure_msg="no job output file ${job_out}"
 CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching ${GP_error}"