From 6cc2e3ed53708bc07da7932adc0326faac9c670f Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 14 Oct 2023 21:09:25 +0200
Subject: [PATCH 001/154] first version of bot/test.sh script

---
 bot/test.sh | 224 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 224 insertions(+)
 create mode 100755 bot/test.sh

diff --git a/bot/test.sh b/bot/test.sh
new file mode 100755
index 0000000000..6d217caf92
--- /dev/null
+++ b/bot/test.sh
@@ -0,0 +1,224 @@
+#!/usr/bin/env bash
+#
+# script to run tests or the test suite for the whole EESSI software layer or
+# just what has been built in a job. Intended use is that it is called
+# at the end of a (batch) job running on a compute node.
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
+
+# ASSUMPTIONs:
+# + assumption for the build step (as run through bot/build.sh which is provided
+#   in this repository too)
+#  - working directory has been prepared by the bot with a checkout of a
+#    pull request (OR by some other means)
+#  - the working directory contains a directory 'cfg' where the main config
+#    file 'job.cfg' has been deposited
+#  - the directory may contain any additional files referenced in job.cfg
+# + assumptions for the test step
+#  - temporary storage is still available
+#    example
+#    Using /localscratch/9640860/NESSI/eessi.x765Dd8mFh as tmp directory (to resume session add '--resume /localscratch/9640860/NESSI/eessi.x765Dd8mFh').
+#  - run test-suite.sh inside build container using tmp storage from build step
+#    plus possibly additional settings (repo, etc.)
+#  - needed setup steps may be similar to bot/inspect.sh (PR#317)
+
+# stop as soon as something fails
+set -e
+
+# source utils.sh and cfg_files.sh
+source scripts/utils.sh
+source scripts/cfg_files.sh
+
+# defaults
+export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+HOST_ARCH=$(uname -m)
+
+# check if ${JOB_CFG_FILE} exists
+if [[ ! -r "${JOB_CFG_FILE}" ]]; then
+    fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable"
+fi
+echo "bot/test.sh: showing ${JOB_CFG_FILE} from software-layer side"
+cat ${JOB_CFG_FILE}
+
+echo "bot/test.sh: obtaining configuration settings from '${JOB_CFG_FILE}'"
+cfg_load ${JOB_CFG_FILE}
+
+# if http_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $http_proxy
+HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy")
+HTTP_PROXY=${HTTP_PROXY:-${http_proxy}}
+echo "bot/test.sh: HTTP_PROXY='${HTTP_PROXY}'"
+
+# if https_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $https_proxy
+HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy")
+HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}}
+echo "bot/test.sh: HTTPS_PROXY='${HTTPS_PROXY}'"
+
+LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp")
+echo "bot/test.sh: LOCAL_TMP='${LOCAL_TMP}'"
+# TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
+
+# check if path to copy build logs to is specified, so we can copy build logs for failing builds there
+BUILD_LOGS_DIR=$(cfg_get_value "site_config" "build_logs_dir")
+echo "bot/test.sh: BUILD_LOGS_DIR='${BUILD_LOGS_DIR}'"
+# if $BUILD_LOGS_DIR is set, add it to $SINGULARITY_BIND so the path is available in the build container
+if [[ ! -z ${BUILD_LOGS_DIR} ]]; then
+    mkdir -p ${BUILD_LOGS_DIR}
+    if [[ -z ${SINGULARITY_BIND} ]]; then
+        export SINGULARITY_BIND="${BUILD_LOGS_DIR}"
+    else
+        export SINGULARITY_BIND="${SINGULARITY_BIND},${BUILD_LOGS_DIR}"
+    fi
+fi
+
+# check if path to directory on shared filesystem is specified,
+# and use it as location for source tarballs used by EasyBuild if so
+SHARED_FS_PATH=$(cfg_get_value "site_config" "shared_fs_path")
+echo "bot/test.sh: SHARED_FS_PATH='${SHARED_FS_PATH}'"
+# if $SHARED_FS_PATH is set, add it to $SINGULARITY_BIND so the path is available in the build container
+if [[ ! -z ${SHARED_FS_PATH} ]]; then
+    mkdir -p ${SHARED_FS_PATH}
+    if [[ -z ${SINGULARITY_BIND} ]]; then
+        export SINGULARITY_BIND="${SHARED_FS_PATH}"
+    else
+        export SINGULARITY_BIND="${SINGULARITY_BIND},${SHARED_FS_PATH}"
+    fi
+fi
+
+SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir")
+echo "bot/test.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
+if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
+    # make sure that separate directories are used for different CPU families
+    SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}/${HOST_ARCH}
+    export SINGULARITY_CACHEDIR
+fi
+
+# try to determine tmp directory from build job
+RESUME_DIR=$(grep 'Using .* as tmp directory' slurm-${SLURM_JOBID}.out | head -1 | awk '{print $2}')
+
+if [[ -z ${RESUME_DIR} ]]; then
+  echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> "
+  # replace any env variable in ${LOCAL_TMP} with its
+  #   current value (e.g., a value that is local to the job)
+  STORAGE=$(envsubst <<< ${LOCAL_TMP})
+  echo "'${STORAGE}'"
+
+  # make sure ${STORAGE} exists
+  mkdir -p ${STORAGE}
+
+  # make sure the base tmp storage is unique
+  JOB_STORAGE=$(mktemp --directory --tmpdir=${STORAGE} bot_job_tmp_XXX)
+  echo "bot/test.sh: created unique base tmp storage directory at ${JOB_STORAGE}"
+
+  RESUME_TGZ=${PWD}/previous_tmp/build_step/$(ls previous_tmp/build_step)
+  if [[ -z ${RESUME_TGZ} ]]; then
+    echo "bot/test.sh: no information about tmp directory and tarball of build step; --> giving up"
+    exit 2
+  fi
+fi
+
+# obtain list of modules to be loaded
+LOAD_MODULES=$(cfg_get_value "site_config" "load_modules")
+echo "bot/test.sh: LOAD_MODULES='${LOAD_MODULES}'"
+
+# singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND
+CONTAINER=$(cfg_get_value "repository" "container")
+export SINGULARITY_HOME="${PWD}:/eessi_bot_job"
+export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir"
+mkdir -p ${SINGULARITY_TMPDIR}
+
+# load modules if LOAD_MODULES is not empty
+if [[ ! -z ${LOAD_MODULES} ]]; then
+    for mod in $(echo ${LOAD_MODULES} | tr ',' '\n')
+    do
+        echo "bot/test.sh: loading module '${mod}'"
+        module load ${mod}
+    done
+else
+    echo "bot/test.sh: no modules to be loaded"
+fi
+
+# determine repository to be used from entry .repository in ${JOB_CFG_FILE}
+REPOSITORY=$(cfg_get_value "repository" "repo_id")
+EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir")
+export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg}
+echo "bot/test.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'"
+
+# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE}
+# here, just set & export EESSI_PILOT_VERSION_OVERRIDE
+# next script (eessi_container.sh) makes use of it via sourcing init scripts
+# (e.g., init/eessi_defaults or init/minimal_eessi_env)
+export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version")
+echo "bot/test.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'"
+
+# determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE}
+# here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
+# "source init/eessi_defaults" via sourcing init/minimal_eessi_env
+export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name")
+echo "bot/test.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'"
+
+# determine architecture to be used from entry .architecture in ${JOB_CFG_FILE}
+# fallbacks:
+#  - ${CPU_TARGET} handed over from bot
+#  - left empty to let downstream script(s) determine subdir to be used
+EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir")
+EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}}
+export EESSI_SOFTWARE_SUBDIR_OVERRIDE
+echo "bot/test.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'"
+
+# get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux)
+EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type")
+export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
+echo "bot/test.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'"
+
+# prepare arguments to eessi_container.sh common to build and tarball steps
+declare -a COMMON_ARGS=()
+COMMON_ARGS+=("--verbose")
+COMMON_ARGS+=("--access" "rw")
+COMMON_ARGS+=("--mode" "run")
+[[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}")
+[[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}")
+[[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
+[[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}")
+
+# make sure to use the same parent dir for storing tarballs of tmp
+PREVIOUS_TMP_DIR=${PWD}/previous_tmp
+
+# prepare directory to store tarball of tmp for build step
+TARBALL_TMP_TEST_STEP_DIR=${PREVIOUS_TMP_DIR}/test_step
+mkdir -p ${TARBALL_TMP_TEST_STEP_DIR}
+
+# prepare arguments to eessi_container.sh specific to test step
+declare -a TEST_STEP_ARGS=()
+TEST_STEP_ARGS+=("--save" "${TARBALL_TMP_TEST_STEP_DIR}")
+
+if [[ -z ${RESUME_DIR} ]]; then
+  TEST_STEP_ARGS+=("--storage" "${STORAGE}")
+  TEST_STEP_ARGS+=("--resume" "${RESUME_TGZ}")
+else
+  TEST_STEP_ARGS+=("--resume" "${RESUME_DIR}")
+fi
+
+# prepare arguments to test_suite.sh (specific to test step)
+declare -a TEST_SUITE_ARGS=()
+# if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
+#     TEST_SUITE_ARGS+=("--generic")
+# fi
+# [[ ! -z ${BUILD_LOGS_DIR} ]] && TEST_SUITE_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}")
+# [[ ! -z ${SHARED_FS_PATH} ]] && TEST_SUITE_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}")
+
+# create tmp file for output of build step
+test_outerr=$(mktemp test.outerr.XXXX)
+
+echo "Executing command to build software:"
+echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TEST_STEP_ARGS[@]}"
+echo "                     -- ./test_suite.sh \"${TEST_SUITE_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${test_outerr}"
+./eessi_container.sh "${COMMON_ARGS[@]}" "${TEST_STEP_ARGS[@]}" \
+                     -- ./test_suite.sh "${TEST_SUITE_ARGS[@]}" "$@" 2>&1 | tee -a ${test_outerr}
+
+exit 0

From 9a32dc629b34e4bbc91fda349198611a6a96a082 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 15 Oct 2023 20:29:14 +0200
Subject: [PATCH 002/154] scripts to run tests after a build job has finished

---
 run_tests.sh  |   4 ++
 test_suite.sh | 184 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 188 insertions(+)
 create mode 100644 run_tests.sh
 create mode 100644 test_suite.sh

diff --git a/run_tests.sh b/run_tests.sh
new file mode 100644
index 0000000000..ccbc751f22
--- /dev/null
+++ b/run_tests.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+base_dir=$(dirname $(realpath $0))
+source ${base_dir}/init/eessi_defaults
+./run_in_compat_layer_env.sh ./test_suite.sh "$@"
diff --git a/test_suite.sh b/test_suite.sh
new file mode 100644
index 0000000000..fcaaa339d2
--- /dev/null
+++ b/test_suite.sh
@@ -0,0 +1,184 @@
+#!/bin/bash
+#
+# Run sanity check for all requested modules (and built dependencies)?
+# get ec from diff
+# set up EasyBuild
+# run `eb --sanity-only ec`
+
+display_help() {
+  echo "usage: $0 [OPTIONS]"
+  echo "  -g | --generic         -  instructs script to test for generic architecture target"
+  echo "  -h | --help            -  display this usage information"
+  echo "  -x | --http-proxy URL  -  provides URL for the environment variable http_proxy"
+  echo "  -y | --https-proxy URL -  provides URL for the environment variable https_proxy"
+}
+
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -g|--generic)
+      EASYBUILD_OPTARCH="GENERIC"
+      shift
+      ;;
+    -h|--help)
+      display_help  # Call your function
+      # no shifting needed here, we're done.
+      exit 0
+      ;;
+    -x|--http-proxy)
+      export http_proxy="$2"
+      shift 2
+      ;;
+    -y|--https-proxy)
+      export https_proxy="$2"
+      shift 2
+      ;;
+    --build-logs-dir)
+      export build_logs_dir="${2}"
+      shift 2
+      ;;
+    --shared-fs-path)
+      export shared_fs_path="${2}"
+      shift 2
+      ;;
+    -*|--*)
+      echo "Error: Unknown option: $1" >&2
+      exit 1
+      ;;
+    *)  # No more options
+      POSITIONAL_ARGS+=("$1") # save positional arg
+      shift
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}"
+
+TOPDIR=$(dirname $(realpath $0))
+
+source $TOPDIR/scripts/utils.sh
+
+# honor $TMPDIR if it is already defined, use /tmp otherwise
+if [ -z $TMPDIR ]; then
+    export WORKDIR=/tmp/$USER
+else
+    export WORKDIR=$TMPDIR/$USER
+fi
+
+TMPDIR=$(mktemp -d)
+
+echo ">> Setting up environment..."
+
+source $TOPDIR/init/minimal_eessi_env
+
+if [ -d $EESSI_CVMFS_REPO ]; then
+    echo_green "$EESSI_CVMFS_REPO available, OK!"
+else
+    fatal_error "$EESSI_CVMFS_REPO is not available!"
+fi
+
+# make sure we're in Prefix environment by checking $SHELL
+if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then
+    echo_green ">> It looks like we're in a Gentoo Prefix environment, good!"
+else
+    fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!"
+fi
+
+# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
+export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
+
+DETECTION_PARAMETERS=''
+GENERIC=0
+EB='eb'
+if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then
+    echo_yellow ">> GENERIC build/test requested, taking appropriate measures!"
+    DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic"
+    GENERIC=1
+    EB='eb --optarch=GENERIC'
+fi
+
+echo ">> Determining software subdirectory to use for current build/test host..."
+if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
+  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
+  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
+else
+  echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
+fi
+
+# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE)
+# $EESSI_SILENT - don't print any messages
+# $EESSI_BASIC_ENV - give a basic set of environment variables
+EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables
+
+if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then
+    fatal_error "Failed to determine software subdirectory?!"
+elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then
+    fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!"
+else
+    echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!"
+fi
+
+echo ">> Initializing Lmod..."
+source $EPREFIX/usr/share/Lmod/init/bash
+ml_version_out=$TMPDIR/ml.out
+ml --version &> $ml_version_out
+if [[ $? -eq 0 ]]; then
+    echo_green ">> Found Lmod ${LMOD_VERSION}"
+else
+    fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}"
+fi
+
+echo ">> Configuring EasyBuild..."
+source $TOPDIR/configure_easybuild
+
+echo ">> Setting up \$MODULEPATH..."
+# make sure no modules are loaded
+module --force purge
+# ignore current $MODULEPATH entirely
+module unuse $MODULEPATH
+module use $EASYBUILD_INSTALLPATH/modules/all
+if [[ -z ${MODULEPATH} ]]; then
+    fatal_error "Failed to set up \$MODULEPATH?!"
+else
+    echo_green ">> MODULEPATH set up: ${MODULEPATH}"
+fi
+
+# assume there's only one diff file that corresponds to the PR patch file
+pr_diff=$(ls [0-9]*.diff | head -1)
+
+# "split" the file by prefixing each line belonging to the same file with the
+# same number
+split_file=$(awk '/^\+\+\+/{n++}{print n, "  ", $0 }' ${pr_diff})
+
+# determine which easystack files may have changed
+changed_es_files=$(echo "${split_file}" | grep '^[0-9 ]*+++ ./eessi.*.yml$' | egrep -v 'known-issues|missing')
+
+# process all changed easystackfiles
+for es_file_num in $(echo "${changed_es_files}" | cut -f1 -d' ')
+do
+    # determine added lines that do not contain a yaml comment only
+    added_lines=$(echo "${split_file}" | grep "${es_file_num}    + " | sed -e "s/^"${es_file_num}"    + //" | grep -v "^[ ]*#")
+    # determine easyconfigs
+    easyconfigs=$(echo "${added_lines}" | cut -f3 -d' ')
+    # get easystack file name
+    easystack_file=$(echo "${changed_es_files}" | grep "^${es_file_num}" | sed -e "s/^"${es_file_num}"    ... .\///")
+    echo -e "Processing easystack file ${easystack_file}...\n\n"
+
+    # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
+    eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
+
+    # load EasyBuild module
+    module load EasyBuild/${eb_version}
+
+    echo_green "All set, let's run sanity checks for installed packages..."
+
+    for easyconfig in ${easyconfigs};
+    do
+        echo "Running sanity check for '${easyconfig}'..."
+        eb --sanity-check-only ${easyconfig}
+    done
+done
+
+echo ">> Cleaning up ${TMPDIR}..."
+rm -r ${TMPDIR}

From 17cfd02779833bd5f2df96655f3dc19bb5f880e8 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 15 Oct 2023 20:30:44 +0200
Subject: [PATCH 003/154] small adjustments to test script

---
 bot/test.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/bot/test.sh b/bot/test.sh
index 6d217caf92..89c1a6a8bf 100755
--- a/bot/test.sh
+++ b/bot/test.sh
@@ -206,9 +206,9 @@ fi
 
 # prepare arguments to test_suite.sh (specific to test step)
 declare -a TEST_SUITE_ARGS=()
-# if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
-#     TEST_SUITE_ARGS+=("--generic")
-# fi
+if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
+    TEST_SUITE_ARGS+=("--generic")
+fi
 # [[ ! -z ${BUILD_LOGS_DIR} ]] && TEST_SUITE_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}")
 # [[ ! -z ${SHARED_FS_PATH} ]] && TEST_SUITE_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}")
 
@@ -217,8 +217,8 @@ test_outerr=$(mktemp test.outerr.XXXX)
 
 echo "Executing command to build software:"
 echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TEST_STEP_ARGS[@]}"
-echo "                     -- ./test_suite.sh \"${TEST_SUITE_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${test_outerr}"
+echo "                     -- ./run_tests.sh \"${TEST_SUITE_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${test_outerr}"
 ./eessi_container.sh "${COMMON_ARGS[@]}" "${TEST_STEP_ARGS[@]}" \
-                     -- ./test_suite.sh "${TEST_SUITE_ARGS[@]}" "$@" 2>&1 | tee -a ${test_outerr}
+                     -- ./run_tests.sh "${TEST_SUITE_ARGS[@]}" "$@" 2>&1 | tee -a ${test_outerr}
 
 exit 0

From ab2c260e4d9b9a1b70b7246b10977437ad0fcd15 Mon Sep 17 00:00:00 2001
From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com>
Date: Wed, 29 Nov 2023 10:38:57 +0100
Subject: [PATCH 004/154] {2023.06}[foss/2023a] OpenFOAM v10

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index e685785f16..b67e894825 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -12,3 +12,4 @@ easyconfigs:
       options:
         from-pr: 19270
   - SciPy-bundle-2023.07-gfbf-2023a.eb
+  - OpenFOAM-10-foss-2023a.eb

From 3439e8b07981dff466b9efb1af4215bedb603351 Mon Sep 17 00:00:00 2001
From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com>
Date: Fri, 1 Dec 2023 12:06:39 +0100
Subject: [PATCH 005/154] Include gnuplot fix from PR
 easybuilders/easyconfigs#19261

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index 4a302c409b..514ed0b86e 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -22,4 +22,7 @@ easyconfigs:
       options:
         from-pr: 19339
   - Qt5-5.15.10-GCCcore-12.3.0.eb
+  - gnuplot-5.4.8-GCCcore-12.3.0.eb: 
+      options:
+        from-pr: 19261
   - OpenFOAM-10-foss-2023a.eb

From 10751d25858239fe78465a5e523fc0077ac4b487 Mon Sep 17 00:00:00 2001
From: Neves-P <p.m.santos.neves@rug.nl>
Date: Fri, 15 Dec 2023 15:15:14 +0100
Subject: [PATCH 006/154] Add pre test hook to skip netCDF tests

---
 eb_hooks.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/eb_hooks.py b/eb_hooks.py
index 6fe92c7f7b..2081bfedab 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -366,6 +366,18 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs):
     if self.name == 'SciPy-bundle' and self.version in ['2021.10', '2023.07'] and cpu_target == CPU_TARGET_NEOVERSE_V1:
         self.cfg['testopts'] = "|| echo ignoring failing tests" 
 
+def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs):
+    """
+    Pre-test hook for SciPy-bundle: skip failing tests for selected netCDF versions on neoverse_v1
+    cfr. https://github.com/EESSI/software-layer/issues/325
+    The following tests are problematic:
+        163 - nc_test4_run_par_test (Timeout)
+        190 - h5_test_run_par_tests (Timeout)
+    A few other tests are skipped in the easyconfig and patches for similar issues, see above issue for details.
+    """
+    cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
+    if self.name == 'netCDF' and self.version == '4.9.2' and cpu_target == CPU_TARGET_NEOVERSE_V1:
+        self.cfg['testopts'] = "|| echo ignoring failing tests" 
 
 def pre_single_extension_hook(ext, *args, **kwargs):
     """Main pre-configure hook: trigger custom functions based on software name."""

From 94206acbcc02da084ba4d5d85219d9ae92f885ac Mon Sep 17 00:00:00 2001
From: Neves-P <p.m.santos.neves@rug.nl>
Date: Fri, 15 Dec 2023 15:16:05 +0100
Subject: [PATCH 007/154] Add correct issue reference in hook

---
 eb_hooks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 2081bfedab..3074eb8ee3 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -369,7 +369,7 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs):
 def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs):
     """
     Pre-test hook for SciPy-bundle: skip failing tests for selected netCDF versions on neoverse_v1
-    cfr. https://github.com/EESSI/software-layer/issues/325
+    cfr. https://github.com/EESSI/software-layer/issues/425
     The following tests are problematic:
         163 - nc_test4_run_par_test (Timeout)
         190 - h5_test_run_par_tests (Timeout)

From db150f4838c47d25e7a683ae239c44b25dd88a6d Mon Sep 17 00:00:00 2001
From: Neves-P <p.m.santos.neves@rug.nl>
Date: Fri, 15 Dec 2023 16:34:42 +0100
Subject: [PATCH 008/154] Fix hook

---
 eb_hooks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/eb_hooks.py b/eb_hooks.py
index 3074eb8ee3..bf5da0cc64 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -430,6 +430,7 @@ def pre_single_extension_isoband(ext, *args, **kwargs):
     'ESPResSo': pre_test_hook_ignore_failing_tests_ESPResSo,
     'FFTW.MPI': pre_test_hook_ignore_failing_tests_FFTWMPI,
     'SciPy-bundle': pre_test_hook_ignore_failing_tests_SciPybundle,
+    'netCDF': pre_test_hook_ignore_failing_tests_netCDF,
 }
 
 PRE_SINGLE_EXTENSION_HOOKS = {

From 98b51a90fa62e65ab9d87b096246b4744e8b96ab Mon Sep 17 00:00:00 2001
From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com>
Date: Wed, 3 Jan 2024 19:16:34 +0100
Subject: [PATCH 009/154] Use only half the cores to build OpenFOAM

---
 eb_hooks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 8e29d2c103..edff46221a 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -79,7 +79,7 @@ def post_ready_hook(self, *args, **kwargs):
     # 'parallel' easyconfig parameter is set via EasyBlock.set_parallel in ready step based on available cores.
     # here we reduce parallellism to only use half of that for selected software,
     # to avoid failing builds/tests due to out-of-memory problems
-    if self.name in ['TensorFlow']:
+    if self.name in ['TensorFlow'] or self.name in ['OpenFOAM']:
         parallel = self.cfg['parallel']
         if parallel > 1:
             self.cfg['parallel'] = parallel // 2

From 9c0ca0d408304ece1cdeed0aeabb42b9403bfd94 Mon Sep 17 00:00:00 2001
From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com>
Date: Mon, 8 Jan 2024 11:31:46 +0100
Subject: [PATCH 010/154] Revert to using all cores in OpenFOAM tests

Reducing in half did not solve issue with hanging/crashing in sanity checks
---
 eb_hooks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 3b52b54e9e..00aee51816 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -80,7 +80,7 @@ def post_ready_hook(self, *args, **kwargs):
     # 'parallel' easyconfig parameter is set via EasyBlock.set_parallel in ready step based on available cores.
     # here we reduce parallellism to only use half of that for selected software,
     # to avoid failing builds/tests due to out-of-memory problems
-    if self.name in ['TensorFlow'] or self.name in ['OpenFOAM']:
+    if self.name in ['TensorFlow']:
         parallel = self.cfg['parallel']
         if parallel > 1:
             self.cfg['parallel'] = parallel // 2

From 0f3646ef0eec6a89aa1a8a5479c8040264af5af3 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Mon, 5 Feb 2024 17:28:18 +0100
Subject: [PATCH 011/154] Second attempt at having the bot run the test suite,
 now based on Thomas' PR 366

---
 bot/check-test.sh | 66 ++++++++++++++++++++++++++++++++++
 run_tests.sh      | 10 +++++-
 test_suite.sh     | 92 +++++++++++++++++++++++++++++------------------
 3 files changed, 132 insertions(+), 36 deletions(-)
 create mode 100644 bot/check-test.sh

diff --git a/bot/check-test.sh b/bot/check-test.sh
new file mode 100644
index 0000000000..9fc783aa99
--- /dev/null
+++ b/bot/check-test.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+#
+# Dummy script that only creates test result file for the bot, without actually checking anything
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Kenneth Hoste (HPC-UGent)
+#
+# license: GPLv2
+#
+job_dir=${PWD}
+job_out="slurm-${SLURM_JOB_ID}.out"
+job_test_result_file="_bot_job${SLURM_JOB_ID}.test"
+
+# ReFrame prints e.g.
+#[----------] start processing checks
+#[ RUN      ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default
+#[ RUN      ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:genoa+default
+#[ RUN      ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=1_cpn_2_nodes %module_name=GROMACS/2021.3-foss-2021a /f4194106 @snellius:genoa+default
+#[     FAIL ] (1/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:genoa+default
+#==> test failed during 'sanity': test staged in '/scratch-shared/casparl/reframe_output/staging/snellius/genoa/default/GROMACS_EESSI_d597cff4'
+#[       OK ] (2/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default
+#P: perf: 8.441 ns/day (r:0, l:None, u:None)
+#[     FAIL ] (3/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=1_cpn_2_nodes %module_name=GROMACS/2021.3-foss-2021a /f4194106 @snellius:genoa+default
+#==> test failed during 'sanity': test staged in '/scratch-shared/casparl/reframe_output/staging/snellius/genoa/default/GROMACS_EESSI_f4194106'
+#[----------] all spawned checks have finished
+#[  FAILED  ] Ran 3/3 test case(s) from 2 check(s) (2 failure(s), 0 skipped, 0 aborted)
+
+# We will grep for the last and final line, since this reflects the overall result
+# Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails
+FAILED=-1
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_failed='\[\s*FAILED\s*\]'
+  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
+  [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
+
+
+# Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step
+# I.e. this is an error in execution of the run_tests.sh itself, NOT in running the actual tests
+ERROR=-1
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_error='ERROR: '
+  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}")
+  [[ $? -eq 0 ]] && ERROR=1 || ERROR=0
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
+
+echo "[TEST]" > ${job_test_result_file}
+if [[ ${ERROR} -eq 1 ]]; then
+    echo "comment_description = Failure to execute test step" >> ${job_test_result_file}
+    echo "status = FAILURE" >> ${job_test_result_file}
+elif [[ ${FAILED} -eq 1 ]]; then
+    echo "comment_description = EESSI test suite produced failures" >> ${job_test_result_file}
+else
+    echo "comment_description = Test step run succesfully" >> ${job_test_result_file}
+    echo "status = SUCCESS" >> ${job_test_result_file}
+fi
+
+exit 0
diff --git a/run_tests.sh b/run_tests.sh
index ccbc751f22..69672f18f3 100644
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -1,4 +1,12 @@
 #!/bin/bash
 base_dir=$(dirname $(realpath $0))
 source ${base_dir}/init/eessi_defaults
-./run_in_compat_layer_env.sh ./test_suite.sh "$@"
+
+# Note: for these tests, we _don't_ run in the compat layer env
+# These tests should mimic what users do, and they are typically not in a prefix environment
+
+# Run eb --sanity-check-only on changed easyconfigs
+# TODO: in the future we may implement this as a light first check.
+
+# Run the test suite
+./test_suite.sh "$@"
diff --git a/test_suite.sh b/test_suite.sh
index fcaaa339d2..36262a6cbf 100644
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -144,41 +144,63 @@ else
     echo_green ">> MODULEPATH set up: ${MODULEPATH}"
 fi
 
-# assume there's only one diff file that corresponds to the PR patch file
-pr_diff=$(ls [0-9]*.diff | head -1)
-
-# "split" the file by prefixing each line belonging to the same file with the
-# same number
-split_file=$(awk '/^\+\+\+/{n++}{print n, "  ", $0 }' ${pr_diff})
-
-# determine which easystack files may have changed
-changed_es_files=$(echo "${split_file}" | grep '^[0-9 ]*+++ ./eessi.*.yml$' | egrep -v 'known-issues|missing')
-
-# process all changed easystackfiles
-for es_file_num in $(echo "${changed_es_files}" | cut -f1 -d' ')
-do
-    # determine added lines that do not contain a yaml comment only
-    added_lines=$(echo "${split_file}" | grep "${es_file_num}    + " | sed -e "s/^"${es_file_num}"    + //" | grep -v "^[ ]*#")
-    # determine easyconfigs
-    easyconfigs=$(echo "${added_lines}" | cut -f3 -d' ')
-    # get easystack file name
-    easystack_file=$(echo "${changed_es_files}" | grep "^${es_file_num}" | sed -e "s/^"${es_file_num}"    ... .\///")
-    echo -e "Processing easystack file ${easystack_file}...\n\n"
-
-    # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
-    eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
-
-    # load EasyBuild module
-    module load EasyBuild/${eb_version}
-
-    echo_green "All set, let's run sanity checks for installed packages..."
-
-    for easyconfig in ${easyconfigs};
-    do
-        echo "Running sanity check for '${easyconfig}'..."
-        eb --sanity-check-only ${easyconfig}
-    done
-done
+# TODO: this should not be hardcoded. Ideally, we put some logic in place to discover the newest version
+# of the ReFrame module available in the current environment
+module load ReFrame/4.3.3
+if [[ $? -eq 0 ]]; then
+    echo_green ">> Loaded ReFrame/4.3.3"
+else
+    fatal_error "Failed to load the ReFrame module"
+fi
+
+# Check ReFrame came with the hpctestlib and we can import it
+python3 -c 'import hpctestlib.sciapps.gromacs'
+if [[ $? -eq 0 ]]; then
+    echo_green "Succesfully found and imported hpctestlib.sciapps.gromas"
+else
+    fatal_error "Failed to load hpctestlib"
+fi
+
+# Clone the EESSI test suite
+git clone https://github.com/EESSI/test-suite EESSI-test-suite
+export TESTSUITEPREFIX=$PWD/EESSI-test-suite
+export PYTHONPATH=$TESTSUITEPREFIX:$PYTHONPATH
+
+# Check that we can import from the testsuite
+python3 -c 'import eessi.testsuite'
+if [[ $? -eq 0 ]]; then
+    echo_green "Succesfully found and imported eessi.testsuite"
+else
+    fatal_error "FAILED to import from eessi.testsuite in Python"
+fi
+
+# Configure ReFrame
+export RFM_CONFIG_FILES=$TESTSUITEPREFIX/config/github_actions.py
+export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests
+export RFM_CHECK_SEARCH_RECURSIVE=1
+export RFM_PREFIX=$PWD/reframe_runs
+
+# Check we can run reframe
+reframe --version
+if [[ $? -eq 0 ]]; then
+    echo_green "Succesfully ran reframe --version"
+else
+    fatal_error "Failed to run ReFrame --version"
+fi
+
+# List the tests we want to run
+export REFRAME_ARGS='--tag CI --tag 1_nodes'
+reframe "${REFRAME_ARGS}" --list
+if [[ $? -eq 0 ]]; then
+    echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list"
+else
+    fatal_error "Failed to list ReFrame tests with command: reframe ${REFRAME_ARGS} --list"
+fi
+
+# Run all tests
+reframe "${REFRAME_ARGS}" --run
 
 echo ">> Cleaning up ${TMPDIR}..."
 rm -r ${TMPDIR}
+
+exit 0

From 34fca15420fb37eeb136da55542b44b0a474a63d Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Mon, 5 Feb 2024 17:35:29 +0100
Subject: [PATCH 012/154] Make a very simple change to an easystack file so we
 are able to trigger the bot build and test procedures and see if this PR
 actually works

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index 82190071ab..4c73b5887a 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -15,3 +15,4 @@ easyconfigs:
       options:
         from-pr: 19573
   - scikit-learn-1.3.1-gfbf-2023a.eb
+  - patchelf-0.18.0-GCCcore-12.3.0.eb

From 3faae9de814b96cd07896bd4042ae7b5b4b50f63 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 10:00:25 +0100
Subject: [PATCH 013/154] Make scripts executable

---
 bot/check-test.sh | 0
 run_tests.sh      | 0
 test_suite.sh     | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 bot/check-test.sh
 mode change 100644 => 100755 run_tests.sh
 mode change 100644 => 100755 test_suite.sh

diff --git a/bot/check-test.sh b/bot/check-test.sh
old mode 100644
new mode 100755
diff --git a/run_tests.sh b/run_tests.sh
old mode 100644
new mode 100755
diff --git a/test_suite.sh b/test_suite.sh
old mode 100644
new mode 100755

From 081e2b195b1dd948ae86d4f8e1634d368db3b467 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 10:30:25 +0100
Subject: [PATCH 014/154] Check for existence of SLURM output first

---
 bot/check-test.sh | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 9fc783aa99..9308a87e8e 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -13,6 +13,17 @@ job_dir=${PWD}
 job_out="slurm-${SLURM_JOB_ID}.out"
 job_test_result_file="_bot_job${SLURM_JOB_ID}.test"
 
+# Check that job output file is found
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"
+if  [[ -f ${job_out} ]]; then
+    SLURM=1
+    [[ ${VERBOSE} -ne 0 ]] && echo "   found slurm output file '"${job_out}"'"
+else
+    SLURM=0
+    [[ ${VERBOSE} -ne 0 ]] && echo "   Slurm output file '"${job_out}"' NOT found"
+fi
+
+
 # ReFrame prints e.g.
 #[----------] start processing checks
 #[ RUN      ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default
@@ -39,7 +50,6 @@ if [[ ${SLURM} -eq 1 ]]; then
   [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
 fi
 
-
 # Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step
 # I.e. this is an error in execution of the run_tests.sh itself, NOT in running the actual tests
 ERROR=-1
@@ -53,11 +63,13 @@ if [[ ${SLURM} -eq 1 ]]; then
 fi
 
 echo "[TEST]" > ${job_test_result_file}
-if [[ ${ERROR} -eq 1 ]]; then
-    echo "comment_description = Failure to execute test step" >> ${job_test_result_file}
+if [[ ${SLURM} -eq 0 ]]; then
+    echo "comment_description = FAILED (job output file not found)" >> ${job_test_result_file}
+elif [[ ${ERROR} -eq 1 ]]; then
+    echo "comment_description = FAILED (test step failed to execute)" >> ${job_test_result_file}
     echo "status = FAILURE" >> ${job_test_result_file}
 elif [[ ${FAILED} -eq 1 ]]; then
-    echo "comment_description = EESSI test suite produced failures" >> ${job_test_result_file}
+    echo "comment_description = FAILED (EESSI test suite produced failures)" >> ${job_test_result_file}
 else
     echo "comment_description = Test step run succesfully" >> ${job_test_result_file}
     echo "status = SUCCESS" >> ${job_test_result_file}

From 9b6fa731d6ac55e3e57538eda01c9c47d73300b1 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 10:43:20 +0100
Subject: [PATCH 015/154] Clarify return messages from the bot

---
 bot/check-test.sh |  8 ++++----
 test_suite.sh     | 20 --------------------
 2 files changed, 4 insertions(+), 24 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 9308a87e8e..05c7589640 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -64,14 +64,14 @@ fi
 
 echo "[TEST]" > ${job_test_result_file}
 if [[ ${SLURM} -eq 0 ]]; then
-    echo "comment_description = FAILED (job output file not found)" >> ${job_test_result_file}
+    echo "comment_description = :cry: FAILED (job output file not found, cannot check test results)" >> ${job_test_result_file}
 elif [[ ${ERROR} -eq 1 ]]; then
-    echo "comment_description = FAILED (test step failed to execute)" >> ${job_test_result_file}
+    echo "comment_description = :cry: FAILED (EESSI test suite was not run, test step itself failed to execute)" >> ${job_test_result_file}
     echo "status = FAILURE" >> ${job_test_result_file}
 elif [[ ${FAILED} -eq 1 ]]; then
-    echo "comment_description = FAILED (EESSI test suite produced failures)" >> ${job_test_result_file}
+    echo "comment_description = :cry: FAILED (EESSI test suite produced failures)" >> ${job_test_result_file}
 else
-    echo "comment_description = Test step run succesfully" >> ${job_test_result_file}
+    echo "comment_description = :grin: SUCCESS" >> ${job_test_result_file}
     echo "status = SUCCESS" >> ${job_test_result_file}
 fi
 
diff --git a/test_suite.sh b/test_suite.sh
index 36262a6cbf..7995f8aa3a 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -78,26 +78,9 @@ else
     fatal_error "$EESSI_CVMFS_REPO is not available!"
 fi
 
-# make sure we're in Prefix environment by checking $SHELL
-if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then
-    echo_green ">> It looks like we're in a Gentoo Prefix environment, good!"
-else
-    fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!"
-fi
-
 # avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
 export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
 
-DETECTION_PARAMETERS=''
-GENERIC=0
-EB='eb'
-if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then
-    echo_yellow ">> GENERIC build/test requested, taking appropriate measures!"
-    DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic"
-    GENERIC=1
-    EB='eb --optarch=GENERIC'
-fi
-
 echo ">> Determining software subdirectory to use for current build/test host..."
 if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
   export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
@@ -129,9 +112,6 @@ else
     fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}"
 fi
 
-echo ">> Configuring EasyBuild..."
-source $TOPDIR/configure_easybuild
-
 echo ">> Setting up \$MODULEPATH..."
 # make sure no modules are loaded
 module --force purge

From cb48b36e590f1d36d7a8d08df22eb538b716c70b Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 11:32:15 +0100
Subject: [PATCH 016/154] Use EESSI_SOFTWARE_PATH instead of
 EASYBUILD_INSTALLPATH to set the modulepath. It should be the same, and we no
 longer configure EasyBuild, since we don't use it when running the test suite

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 7995f8aa3a..1464a3a5d4 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -117,7 +117,7 @@ echo ">> Setting up \$MODULEPATH..."
 module --force purge
 # ignore current $MODULEPATH entirely
 module unuse $MODULEPATH
-module use $EASYBUILD_INSTALLPATH/modules/all
+module use ${EESSI_SOFTWARE_PATH}/modules/all
 if [[ -z ${MODULEPATH} ]]; then
     fatal_error "Failed to set up \$MODULEPATH?!"
 else

From d1a0219486fe4a4c1ad94039b4b74b41ebf7da1f Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 12:02:12 +0100
Subject: [PATCH 017/154] Do git clone in a seperate script, so that that can
 be run in the prefix layer

---
 clone_eessi_test_suite.sh | 1 +
 run_tests.sh              | 3 ++-
 test_suite.sh             | 9 +++++++--
 3 files changed, 10 insertions(+), 3 deletions(-)
 create mode 100755 clone_eessi_test_suite.sh

diff --git a/clone_eessi_test_suite.sh b/clone_eessi_test_suite.sh
new file mode 100755
index 0000000000..f7684c8cfd
--- /dev/null
+++ b/clone_eessi_test_suite.sh
@@ -0,0 +1 @@
+git clone https://github.com/EESSI/test-suite EESSI-test-suite
diff --git a/run_tests.sh b/run_tests.sh
index 69672f18f3..90185fea17 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -9,4 +9,5 @@ source ${base_dir}/init/eessi_defaults
 # TODO: in the future we may implement this as a light first check.
 
 # Run the test suite
-./test_suite.sh "$@"
+./run_in_compat_layer_env.sh clone_eessi_test_suite.sh
+./test_suite.sh
diff --git a/test_suite.sh b/test_suite.sh
index 1464a3a5d4..6177f0a253 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -141,9 +141,14 @@ else
     fatal_error "Failed to load hpctestlib"
 fi
 
-# Clone the EESSI test suite
-git clone https://github.com/EESSI/test-suite EESSI-test-suite
+# Cloning should already be done by clone_eessi_test_suite.sh, which runs in compat layer to have 'git' available
+# git clone https://github.com/EESSI/test-suite EESSI-test-suite
 export TESTSUITEPREFIX=$PWD/EESSI-test-suite
+if [ -d $TESTSUITEPREFIX ]; then
+    echo_green "Clone of the test suite $TESTSUITEPREFIX available, OK!"
+else
+    fatal_error "Clone of the test suite $TESTSUITEPREFIX is not available!"
+fi
 export PYTHONPATH=$TESTSUITEPREFIX:$PYTHONPATH
 
 # Check that we can import from the testsuite

From 86d5d3d47d076fbdf0a61c57ca83fee8a1c87123 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 12:06:09 +0100
Subject: [PATCH 018/154] Should use the one from current dir

---
 run_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/run_tests.sh b/run_tests.sh
index 90185fea17..bcca1ee417 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -9,5 +9,5 @@ source ${base_dir}/init/eessi_defaults
 # TODO: in the future we may implement this as a light first check.
 
 # Run the test suite
-./run_in_compat_layer_env.sh clone_eessi_test_suite.sh
+./run_in_compat_layer_env.sh ./clone_eessi_test_suite.sh
 ./test_suite.sh

From 7861aec18c0081c8ff24e7ed46d34f9ea52c2dae Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 12:10:56 +0100
Subject: [PATCH 019/154] Remove quotes, see if that helps

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 6177f0a253..1ac82a5e66 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -175,7 +175,7 @@ fi
 
 # List the tests we want to run
 export REFRAME_ARGS='--tag CI --tag 1_nodes'
-reframe "${REFRAME_ARGS}" --list
+reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list"
 else

From a7b5ee13c841e940ab136ee2d014b4426a23baa7 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 12:52:41 +0100
Subject: [PATCH 020/154] Apparently, we need to remove curly braces too

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 1ac82a5e66..46c3493f41 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -175,7 +175,7 @@ fi
 
 # List the tests we want to run
 export REFRAME_ARGS='--tag CI --tag 1_nodes'
-reframe ${REFRAME_ARGS} --list
+reframe $REFRAME_ARGS --list
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list"
 else

From f50e463e9d0d0f5d91b161805279b6a8a764b990 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 13:03:31 +0100
Subject: [PATCH 021/154] Make sure this actually gets reported as failure, as
 the ReFrame runtime fails to run the test suite (it is not an indication of
 tests failing themselves).

---
 test_suite.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 46c3493f41..8a69bdee8d 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -175,7 +175,7 @@ fi
 
 # List the tests we want to run
 export REFRAME_ARGS='--tag CI --tag 1_nodes'
-reframe $REFRAME_ARGS --list
+reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list"
 else
@@ -184,6 +184,12 @@ fi
 
 # Run all tests
 reframe "${REFRAME_ARGS}" --run
+if [[ $? -eq 0 ]]; then
+    echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run."
+else
+    fatal_error "ReFrame runtime failed to run with command: reframe ${REFRAME_ARGS} --run."
+fi
+
 
 echo ">> Cleaning up ${TMPDIR}..."
 rm -r ${TMPDIR}

From f67df9bc1319053901aa506668533cbb8b99716f Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 13:04:49 +0100
Subject: [PATCH 022/154] Fix the actual issue: make sure that two arguments
 are interpreted seperately by not quoting them

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 8a69bdee8d..633333890c 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -183,7 +183,7 @@ else
 fi
 
 # Run all tests
-reframe "${REFRAME_ARGS}" --run
+reframe ${REFRAME_ARGS} --run
 if [[ $? -eq 0 ]]; then
     echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run."
 else

From 3394851c5347a045431d7826f579ad5f35f01edf Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 13:32:27 +0100
Subject: [PATCH 023/154] See if we can make check-test formatting more fancy

---
 bot/check-test.sh | 84 +++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 78 insertions(+), 6 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 05c7589640..3fa7c1d694 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -64,15 +64,87 @@ fi
 
 echo "[TEST]" > ${job_test_result_file}
 if [[ ${SLURM} -eq 0 ]]; then
-    echo "comment_description = :cry: FAILED (job output file not found, cannot check test results)" >> ${job_test_result_file}
+    summary=":cry: FAILURE"
+    summary_details="(job output file not found, cannot check test results)"# >> ${job_test_result_file}
+    status="FAILURE"
 elif [[ ${ERROR} -eq 1 ]]; then
-    echo "comment_description = :cry: FAILED (EESSI test suite was not run, test step itself failed to execute)" >> ${job_test_result_file}
-    echo "status = FAILURE" >> ${job_test_result_file}
+    summary=":cry: FAILURE"
+    summary_details="(EESSI test suite was not run, test step itself failed to execute)"# >> ${job_test_result_file}
+    status="FAILURE"
+#    echo "status = FAILURE" >> ${job_test_result_file}
 elif [[ ${FAILED} -eq 1 ]]; then
-    echo "comment_description = :cry: FAILED (EESSI test suite produced failures)" >> ${job_test_result_file}
+    summary=":cry: FAILURE"
+    summary_details="(EESSI test suite produced failures)"# >> ${job_test_result_file}
+    status="FAILURE"
 else
-    echo "comment_description = :grin: SUCCESS" >> ${job_test_result_file}
-    echo "status = SUCCESS" >> ${job_test_result_file}
+    summary=":grin: SUCCESS"# >> ${job_test_result_file}
+    summary_details=""
+    status="SUCCESS"
+#    echo "status = SUCCESS" >> ${job_test_result_file}
 fi
 
+function add_detail() {
+    actual=${1}
+    expected=${2}
+    success_msg="${3}"
+    failure_msg="${4}"
+    if [[ ${actual} -eq ${expected} ]]; then
+        success "${success_msg}"
+    else
+        failure "${failure_msg}"
+    fi
+}
+
+echo "[TEST]" > ${job_result_file}
+echo -n "comment_description = " >> ${job_result_file}
+
+# Use template for writing PR comment with details
+# construct and write complete PR comment details: implements third alternative
+comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT____ARTEFACTS_FMT__</dl></details>"
+comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
+comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
+comment_success_item_fmt=":white_check_mark: __ITEM__"
+comment_failure_item_fmt=":x: __ITEM__"
+comment_artefacts_fmt="<dt>_Artefacts_</dt><dd>__ARTEFACTS_LIST__</dd>"
+comment_artefact_details_fmt="<details>__ARTEFACT_SUMMARY____ARTEFACT_DETAILS__</details>"
+
+comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
+
+# first construct comment_details_list, abbreviated CoDeList
+# then use it to set comment_details
+CoDeList=""
+
+success_msg="job output file <code>${job_out}</code>"
+failure_msg="no job output file <code>${job_out}</code>"
+CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
+
+success_msg="no message matching <code>${GP_error}</code>"
+failure_msg="found message matching <code>${GP_error}</code>"
+CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
+
+success_msg="no message matching <code>${GP_failed}</code>"
+failure_msg="found message matching <code>${GP_failed}</code>"
+CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
+
+# Should not be needed for testing, I think? Maybe for loading ReFrame module...
+# success_msg="no message matching <code>${GP_req_missing}</code>"
+# failure_msg="found message matching <code>${GP_req_missing}</code>"
+# CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
+# 
+# success_msg="found message(s) matching <code>${GP_no_missing}</code>"
+# failure_msg="no message matching <code>${GP_no_missing}</code>"
+# CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
+# 
+# success_msg="found message matching <code>${GP_tgz_created}</code>"
+# failure_msg="no message matching <code>${GP_tgz_created}</code>"
+# CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
+
+comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
+
+comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}
+comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
+
+# Actually writing the comment description to the result file
+echo "${comment_description}" >> ${job_result_file}
+
 exit 0

From 8e5682a47a589a2e251852ed674b71f16980ffc7 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 14:51:46 +0100
Subject: [PATCH 024/154] Replace the env var for the result file

---
 bot/check-test.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 3fa7c1d694..89d3d6a3c6 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -95,8 +95,8 @@ function add_detail() {
     fi
 }
 
-echo "[TEST]" > ${job_result_file}
-echo -n "comment_description = " >> ${job_result_file}
+echo "[TEST]" > ${job_test_result_file}
+echo -n "comment_description = " >> ${job_test_result_file}
 
 # Use template for writing PR comment with details
 # construct and write complete PR comment details: implements third alternative
@@ -145,6 +145,6 @@ comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}
 comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
 
 # Actually writing the comment description to the result file
-echo "${comment_description}" >> ${job_result_file}
+echo "${comment_description}" >> ${job_test_result_file}
 
 exit 0

From 6cf0cf959844081b004595bd3a6c21329ccf256d Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 14:53:21 +0100
Subject: [PATCH 025/154] Add missing functions

---
 bot/check-test.sh | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 89d3d6a3c6..87c79b3b30 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -83,6 +83,24 @@ else
 #    echo "status = SUCCESS" >> ${job_test_result_file}
 fi
 
+function print_br_item() {
+    format="${1}"
+    item="${2}"
+    echo -n "${format//__ITEM__/${item}}<br/>"
+}
+
+function success() {
+    format="${comment_success_item_fmt}"
+    item="$1"
+    print_br_item "${format}" "${item}"
+}
+
+function failure() {
+    format="${comment_failure_item_fmt}"
+    item="$1"
+    print_br_item "${format}" "${item}"
+}
+
 function add_detail() {
     actual=${1}
     expected=${2}

From 7fe24e13cdef3f703b51930423a2296e3dc07d59 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 15:14:50 +0100
Subject: [PATCH 026/154] Remove artefacts from reporting

---
 bot/check-test.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 87c79b3b30..da06cd630b 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -118,13 +118,11 @@ echo -n "comment_description = " >> ${job_test_result_file}
 
 # Use template for writing PR comment with details
 # construct and write complete PR comment details: implements third alternative
-comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT____ARTEFACTS_FMT__</dl></details>"
+comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT__</dl></details>"
 comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
 comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
 comment_success_item_fmt=":white_check_mark: __ITEM__"
 comment_failure_item_fmt=":x: __ITEM__"
-comment_artefacts_fmt="<dt>_Artefacts_</dt><dd>__ARTEFACTS_LIST__</dd>"
-comment_artefact_details_fmt="<details>__ARTEFACT_SUMMARY____ARTEFACT_DETAILS__</details>"
 
 comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
 

From afc309a168048e54b625ec857a5d09a14ca1d585 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 15:25:33 +0100
Subject: [PATCH 027/154] Add a reason for the failure

---
 bot/check-test.sh | 42 ++++++++++++++++++++++--------------------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index da06cd630b..38e86efccd 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -65,24 +65,37 @@ fi
 echo "[TEST]" > ${job_test_result_file}
 if [[ ${SLURM} -eq 0 ]]; then
     summary=":cry: FAILURE"
-    summary_details="(job output file not found, cannot check test results)"# >> ${job_test_result_file}
+    summary_details="Reason: job output file not found, cannot check test results."
     status="FAILURE"
 elif [[ ${ERROR} -eq 1 ]]; then
     summary=":cry: FAILURE"
-    summary_details="(EESSI test suite was not run, test step itself failed to execute)"# >> ${job_test_result_file}
+    summary_details="Reason: EESSI test suite was not run, test step itself failed to execute."
     status="FAILURE"
-#    echo "status = FAILURE" >> ${job_test_result_file}
 elif [[ ${FAILED} -eq 1 ]]; then
     summary=":cry: FAILURE"
-    summary_details="(EESSI test suite produced failures)"# >> ${job_test_result_file}
+    summary_details="Reason: EESSI test suite produced failures."
     status="FAILURE"
 else
-    summary=":grin: SUCCESS"# >> ${job_test_result_file}
+    summary=":grin: SUCCESS"
     summary_details=""
     status="SUCCESS"
-#    echo "status = SUCCESS" >> ${job_test_result_file}
 fi
 
+
+echo "[TEST]" > ${job_test_result_file}
+echo -n "comment_description = " >> ${job_test_result_file}
+
+# Use template for writing PR comment with details
+# construct and write complete PR comment details: implements third alternative
+comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT__</dl></details>"
+comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
+comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
+comment_success_item_fmt=":white_check_mark: __ITEM__"
+comment_failure_item_fmt=":x: __ITEM__"
+
+comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
+
+# Declare functions
 function print_br_item() {
     format="${1}"
     item="${2}"
@@ -113,22 +126,11 @@ function add_detail() {
     fi
 }
 
-echo "[TEST]" > ${job_test_result_file}
-echo -n "comment_description = " >> ${job_test_result_file}
-
-# Use template for writing PR comment with details
-# construct and write complete PR comment details: implements third alternative
-comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT__</dl></details>"
-comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
-comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
-comment_success_item_fmt=":white_check_mark: __ITEM__"
-comment_failure_item_fmt=":x: __ITEM__"
-
-comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
-
 # first construct comment_details_list, abbreviated CoDeList
 # then use it to set comment_details
-CoDeList=""
+
+# Initialize with summary_details, which elaborates on the reason for failure
+CoDeList=$(print_br_item "__ITEM__" "${summary_details}"
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"

From b84e48755dfb582a5a4bae3480e8865df179331e Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 15:32:07 +0100
Subject: [PATCH 028/154] Fixed missing bracket

---
 bot/check-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 38e86efccd..e340907bb3 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -130,7 +130,7 @@ function add_detail() {
 # then use it to set comment_details
 
 # Initialize with summary_details, which elaborates on the reason for failure
-CoDeList=$(print_br_item "__ITEM__" "${summary_details}"
+CoDeList=$(print_br_item "__ITEM__" "${summary_details}")
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"

From ef7bc0142f0ff518e82121371403c4da9bc0444d Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 15:48:44 +0100
Subject: [PATCH 029/154] Add reporting of the ReFrame result

---
 bot/check-test.sh | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index e340907bb3..78c153e5dc 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -43,11 +43,11 @@ fi
 FAILED=-1
 if [[ ${SLURM} -eq 1 ]]; then
   GP_failed='\[\s*FAILED\s*\]'
-  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
+  grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'"
-  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}"
 fi
 
 # Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step
@@ -62,6 +62,15 @@ if [[ ${SLURM} -eq 1 ]]; then
   [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
 fi
 
+# Grep for the success pattern, so we can report the amount of tests run
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_success='\[\s*PASSED\s*\]'
+  grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}"
+fi
+
 echo "[TEST]" > ${job_test_result_file}
 if [[ ${SLURM} -eq 0 ]]; then
     summary=":cry: FAILURE"
@@ -132,6 +141,11 @@ function add_detail() {
 # Initialize with summary_details, which elaborates on the reason for failure
 CoDeList=$(print_br_item "__ITEM__" "${summary_details}")
 
+# Add final ReFrame line as line
+if [[ ! -z ${grep_reframe_result} ]]; then
+    CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}"
+fi
+
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
 CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
@@ -140,8 +154,8 @@ success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"
 CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
 
-success_msg="no message matching <code>${GP_failed}</code>"
-failure_msg="found message matching <code>${GP_failed}</code>"
+success_msg="no message matching <code>""${GP_failed}""</code>"
+failure_msg="found message matching <code>""${GP_failed}""</code>"
 CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 
 # Should not be needed for testing, I think? Maybe for loading ReFrame module...

From 867681a2ed7413b8ecdbf646d05a20d7d2553adc Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 16:01:40 +0100
Subject: [PATCH 030/154] Add missing bracket. Again

---
 bot/check-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 78c153e5dc..7ff5f8a3ea 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -143,7 +143,7 @@ CoDeList=$(print_br_item "__ITEM__" "${summary_details}")
 
 # Add final ReFrame line as line
 if [[ ! -z ${grep_reframe_result} ]]; then
-    CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}"
+    CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}")
 fi
 
 success_msg="job output file <code>${job_out}</code>"

From 47e30cac825362f85138e1d6cb3fe8ba02f13c6e Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 16:17:25 +0100
Subject: [PATCH 031/154] Add some echo for debugging

---
 bot/check-test.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 7ff5f8a3ea..1a2d8b5d5d 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -171,6 +171,9 @@ CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 # failure_msg="no message matching <code>${GP_tgz_created}</code>"
 # CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
 
+echo "Finale CoDeList is:"
+echo ${CoDeList}
+
 comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
 
 comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}

From 1bdadf884e8f18ec911f8961a894125a6c5cccce Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 16:24:34 +0100
Subject: [PATCH 032/154] more debugging output

---
 bot/check-test.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 1a2d8b5d5d..841b8ed01b 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -137,14 +137,20 @@ function add_detail() {
 
 # first construct comment_details_list, abbreviated CoDeList
 # then use it to set comment_details
+CoDeList=""
 
 # Initialize with summary_details, which elaborates on the reason for failure
-CoDeList=$(print_br_item "__ITEM__" "${summary_details}")
+if [[ ! -z ${summary_details} ]]; then
+    CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${summary_details}")
+fi
 
 # Add final ReFrame line as line
 if [[ ! -z ${grep_reframe_result} ]]; then
     CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}")
 fi
+echo "CoDeList up here is"
+echo ${CoDeList}
+
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"

From 5152f5e47380c462c234566f5fc3c29922694ef1 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 16:54:02 +0100
Subject: [PATCH 033/154] Lets try some nicer formatting with headers etc. See
 if this actually prints something.

---
 bot/check-test.sh | 62 ++++++++++++++++++++++++++++++++---------------
 1 file changed, 42 insertions(+), 20 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 841b8ed01b..8f8eb66f5e 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -74,19 +74,19 @@ fi
 echo "[TEST]" > ${job_test_result_file}
 if [[ ${SLURM} -eq 0 ]]; then
     summary=":cry: FAILURE"
-    summary_details="Reason: job output file not found, cannot check test results."
+    reason="Reason: job output file not found, cannot check test results."
     status="FAILURE"
 elif [[ ${ERROR} -eq 1 ]]; then
     summary=":cry: FAILURE"
-    summary_details="Reason: EESSI test suite was not run, test step itself failed to execute."
+    reason="Reason: EESSI test suite was not run, test step itself failed to execute."
     status="FAILURE"
 elif [[ ${FAILED} -eq 1 ]]; then
     summary=":cry: FAILURE"
-    summary_details="Reason: EESSI test suite produced failures."
+    reason="Reason: EESSI test suite produced failures."
     status="FAILURE"
 else
     summary=":grin: SUCCESS"
-    summary_details=""
+    reason=""
     status="SUCCESS"
 fi
 
@@ -96,13 +96,36 @@ echo -n "comment_description = " >> ${job_test_result_file}
 
 # Use template for writing PR comment with details
 # construct and write complete PR comment details: implements third alternative
-comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT__</dl></details>"
-comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
-comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
+comment_template="<details>__SUMMARY_FMT__<dl>__REASON_FMT____REFRAME_FMT____DETAILS_FMT__</dl></details>"
 comment_success_item_fmt=":white_check_mark: __ITEM__"
 comment_failure_item_fmt=":x: __ITEM__"
 
+# Initialize comment_description
+comment_description=${comment_template}
+
+# Now, start replacing template items one by one
+comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
 comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
+comment_description=${comment_description/__SUMMARY_FMT__/${comment_summary}}
+
+
+# Omit this if there is no reason (e.g. because it was succesful)
+if [[ -z ${reason} ]]; then
+    comment_reason_fmt="<dt>_Reason_</dt><dd>__REASONS__</dd>"
+    reason_details="${comment_reason_fmt/__REASONS__/${reason}}"
+    comment_description=${comment_description/__REASON_FMT__/${reason_details}}
+else
+    comment_description=${comment_description/__REASON_FMT__/""}
+fi
+
+# Omit this if there is no reframe summary (i.e. the workflow didn't run succesfully)
+if [[ -z ${grep_reframe_result} ]]; then
+    comment_reframe_fmt="<dt>_ReFrame Summary_</dt><dd>__REFRAME_SUMMARY__</dd>"
+    reframe_summary=${comment_reframe_ftm/__REFRAME_SUMMARY__/${grep_reframe_result}}
+    comment_description=${comment_description/__REFRAME_FMT__/${reframe_summary}}
+else
+    comment_description=${comment_description/__REFRAME_FMT__/""}
+fi
 
 # Declare functions
 function print_br_item() {
@@ -139,17 +162,17 @@ function add_detail() {
 # then use it to set comment_details
 CoDeList=""
 
-# Initialize with summary_details, which elaborates on the reason for failure
-if [[ ! -z ${summary_details} ]]; then
-    CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${summary_details}")
-fi
-
-# Add final ReFrame line as line
-if [[ ! -z ${grep_reframe_result} ]]; then
-    CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}")
-fi
-echo "CoDeList up here is"
-echo ${CoDeList}
+# # Initialize with summary_details, which elaborates on the reason for failure
+# if [[ ! -z ${summary_details} ]]; then
+#     CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${summary_details}")
+# fi
+# 
+# # Add final ReFrame line as line
+# if [[ ! -z ${grep_reframe_result} ]]; then
+#     CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}")
+# fi
+# echo "CoDeList up here is"
+# echo ${CoDeList}
 
 
 success_msg="job output file <code>${job_out}</code>"
@@ -180,9 +203,8 @@ CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 echo "Finale CoDeList is:"
 echo ${CoDeList}
 
+comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
 comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
-
-comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}
 comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
 
 # Actually writing the comment description to the result file

From ea38c66114af601071e1edc626cf9952219a7a18 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 17:07:28 +0100
Subject: [PATCH 034/154] Inverse check, it was incorrect

---
 bot/check-test.sh | 21 ++++-----------------
 1 file changed, 4 insertions(+), 17 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 8f8eb66f5e..01769bbac3 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -109,8 +109,8 @@ comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
 comment_description=${comment_description/__SUMMARY_FMT__/${comment_summary}}
 
 
-# Omit this if there is no reason (e.g. because it was succesful)
-if [[ -z ${reason} ]]; then
+# Only add if there is a reason (e.g. no reason for successful runs)
+if [[ ! -z ${reason} ]]; then
     comment_reason_fmt="<dt>_Reason_</dt><dd>__REASONS__</dd>"
     reason_details="${comment_reason_fmt/__REASONS__/${reason}}"
     comment_description=${comment_description/__REASON_FMT__/${reason_details}}
@@ -118,8 +118,8 @@ else
     comment_description=${comment_description/__REASON_FMT__/""}
 fi
 
-# Omit this if there is no reframe summary (i.e. the workflow didn't run succesfully)
-if [[ -z ${grep_reframe_result} ]]; then
+# Only add if there is a reframe summary (e.g. no reframe summary if reframe wasn't launched succesfully)
+if [[ ! -z ${grep_reframe_result} ]]; then
     comment_reframe_fmt="<dt>_ReFrame Summary_</dt><dd>__REFRAME_SUMMARY__</dd>"
     reframe_summary=${comment_reframe_ftm/__REFRAME_SUMMARY__/${grep_reframe_result}}
     comment_description=${comment_description/__REFRAME_FMT__/${reframe_summary}}
@@ -162,19 +162,6 @@ function add_detail() {
 # then use it to set comment_details
 CoDeList=""
 
-# # Initialize with summary_details, which elaborates on the reason for failure
-# if [[ ! -z ${summary_details} ]]; then
-#     CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${summary_details}")
-# fi
-# 
-# # Add final ReFrame line as line
-# if [[ ! -z ${grep_reframe_result} ]]; then
-#     CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}")
-# fi
-# echo "CoDeList up here is"
-# echo ${CoDeList}
-
-
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
 CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")

From 8edb6fffb401f0e74e0b975b5585d3f9257f88f8 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 17:11:41 +0100
Subject: [PATCH 035/154] Remove debugging output

---
 bot/check-test.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 01769bbac3..7a663f35d1 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -187,9 +187,6 @@ CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 # failure_msg="no message matching <code>${GP_tgz_created}</code>"
 # CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
 
-echo "Finale CoDeList is:"
-echo ${CoDeList}
-
 comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
 comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
 comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}

From 26bd2cb04af381af580805b094f69c096ea40410 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 17:33:12 +0100
Subject: [PATCH 036/154] Correct the regex pattern to account for the fact
 that the slurm output file contains special characters in that pattern that
 are NOT whitespace

---
 bot/check-test.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 7a663f35d1..aebd56d333 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -42,7 +42,7 @@ fi
 # Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails
 FAILED=-1
 if [[ ${SLURM} -eq 1 ]]; then
-  GP_failed='\[\s*FAILED\s*\]'
+  GP_failed='\[.*FAILED.*\].*Ran .* test case'
   grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
   # have to be careful to not add searched for pattern into slurm out file
@@ -64,7 +64,7 @@ fi
 
 # Grep for the success pattern, so we can report the amount of tests run
 if [[ ${SLURM} -eq 1 ]]; then
-  GP_success='\[\s*PASSED\s*\]'
+  GP_success='\[.*PASSED.*\].*Ran .* test case'
   grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'"
@@ -119,6 +119,8 @@ else
 fi
 
 # Only add if there is a reframe summary (e.g. no reframe summary if reframe wasn't launched succesfully)
+echo "ReFrame result:"
+echo "${grep_reframe_result}"
 if [[ ! -z ${grep_reframe_result} ]]; then
     comment_reframe_fmt="<dt>_ReFrame Summary_</dt><dd>__REFRAME_SUMMARY__</dd>"
     reframe_summary=${comment_reframe_ftm/__REFRAME_SUMMARY__/${grep_reframe_result}}

From 416b72d3cb18d6150e0ef8760cd5b66867e29496 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 17:45:30 +0100
Subject: [PATCH 037/154] Corrected typo

---
 bot/check-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index aebd56d333..702add848d 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -123,7 +123,7 @@ echo "ReFrame result:"
 echo "${grep_reframe_result}"
 if [[ ! -z ${grep_reframe_result} ]]; then
     comment_reframe_fmt="<dt>_ReFrame Summary_</dt><dd>__REFRAME_SUMMARY__</dd>"
-    reframe_summary=${comment_reframe_ftm/__REFRAME_SUMMARY__/${grep_reframe_result}}
+    reframe_summary=${comment_reframe_fmt/__REFRAME_SUMMARY__/${grep_reframe_result}}
     comment_description=${comment_description/__REFRAME_FMT__/${reframe_summary}}
 else
     comment_description=${comment_description/__REFRAME_FMT__/""}

From 19900937361e16259dce714c9e4acda4b09e0a7b Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 17:51:27 +0100
Subject: [PATCH 038/154] Trigger failure in the workflow on purpose, to show
 what that looks like

---
 test_suite.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index 633333890c..4439821d56 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -166,7 +166,7 @@ export RFM_CHECK_SEARCH_RECURSIVE=1
 export RFM_PREFIX=$PWD/reframe_runs
 
 # Check we can run reframe
-reframe --version
+reframe --version BLABLABLA
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully ran reframe --version"
 else
@@ -174,7 +174,7 @@ else
 fi
 
 # List the tests we want to run
-export REFRAME_ARGS='--tag CI --tag 1_nodes'
+export REFRAME_ARGS='--tag CI --tag 1_node'
 reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list"

From 4b9885612fa13d2d1f45522ca1aa04ec333863b8 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 18:11:30 +0100
Subject: [PATCH 039/154] Change order of checking error codes. If the ReFrame
 tests ran and we find that pattern, that should take priority. Other errors
 might have come from errors in the build phase...

---
 bot/check-test.sh | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 702add848d..7b7ada98e7 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -62,10 +62,12 @@ if [[ ${SLURM} -eq 1 ]]; then
   [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
 fi
 
+SUCCESS=-1
 # Grep for the success pattern, so we can report the amount of tests run
 if [[ ${SLURM} -eq 1 ]]; then
   GP_success='\[.*PASSED.*\].*Ran .* test case'
   grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
+  [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'"
   [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}"
@@ -74,19 +76,27 @@ fi
 echo "[TEST]" > ${job_test_result_file}
 if [[ ${SLURM} -eq 0 ]]; then
     summary=":cry: FAILURE"
-    reason="Reason: job output file not found, cannot check test results."
+    reason="Job output file not found, cannot check test results."
     status="FAILURE"
-elif [[ ${ERROR} -eq 1 ]]; then
+# Should come before general errors: if SUCCESS==1, it indicates the test suite ran succesfully
+# regardless of other things that might have gone wrong
+elif [[ ${SUCCESS} -eq 1 ]]; then
+    summary=":grin: SUCCESS"
+    reason=""
+    status="SUCCESS"
+# Should come before general errors: if FAILED==1, it indicates the test suite ran
+# otherwise the pattern wouldn't have been there
+elif [[ ${FAILED} -eq 1 ]]; then
     summary=":cry: FAILURE"
-    reason="Reason: EESSI test suite was not run, test step itself failed to execute."
+    reason="EESSI test suite produced failures."
     status="FAILURE"
-elif [[ ${FAILED} -eq 1 ]]; then
+elif [[ ${ERROR} -eq 1 ]]; then
     summary=":cry: FAILURE"
-    reason="Reason: EESSI test suite produced failures."
+    reason="EESSI test suite was not run, test step itself failed to execute."
     status="FAILURE"
 else
-    summary=":grin: SUCCESS"
-    reason=""
+    summary=":grin: FAILURE"
+    reason="Failed for unknown reason"
     status="SUCCESS"
 fi
 

From 0f261420106862049091494d6a640a6c46de5dd8 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 12:11:25 +0100
Subject: [PATCH 040/154] Make sure we don't overwrite the grepped result from
 failures

---
 bot/check-test.sh | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 7b7ada98e7..8af45d26a6 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -43,11 +43,11 @@ fi
 FAILED=-1
 if [[ ${SLURM} -eq 1 ]]; then
   GP_failed='\[.*FAILED.*\].*Ran .* test case'
-  grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
+  grep_reframe_failed=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'"
-  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_failed}"
 fi
 
 # Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step
@@ -66,11 +66,17 @@ SUCCESS=-1
 # Grep for the success pattern, so we can report the amount of tests run
 if [[ ${SLURM} -eq 1 ]]; then
   GP_success='\[.*PASSED.*\].*Ran .* test case'
-  grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
+  grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
   [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'"
-  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_success}"
+fi
+
+if [[ ! -z ${grep_reframe_failed} ]]; then
+    grep_reframe_results=${grep_reframe_failed}
+else
+    grep_reframe_results=${grep_reframe_success}
 fi
 
 echo "[TEST]" > ${job_test_result_file}

From 94e0bb5e3b886192508d95485b221b3ec7a6ff5e Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 13:09:31 +0100
Subject: [PATCH 041/154] Fix plural/singular typo

---
 bot/check-test.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 8af45d26a6..39901f0691 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -74,9 +74,9 @@ if [[ ${SLURM} -eq 1 ]]; then
 fi
 
 if [[ ! -z ${grep_reframe_failed} ]]; then
-    grep_reframe_results=${grep_reframe_failed}
+    grep_reframe_result=${grep_reframe_failed}
 else
-    grep_reframe_results=${grep_reframe_success}
+    grep_reframe_result=${grep_reframe_success}
 fi
 
 echo "[TEST]" > ${job_test_result_file}

From 9e7ac1c7c4f2f64250ef504810d057e0cb3e5225 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 13:25:45 +0100
Subject: [PATCH 042/154] Be a bit more expressive about what we are doing

---
 test_suite.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index 4439821d56..1610d87738 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -165,8 +165,11 @@ export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests
 export RFM_CHECK_SEARCH_RECURSIVE=1
 export RFM_PREFIX=$PWD/reframe_runs
 
+echo "Configured reframe with the following environment variables:"
+env | grep "RFM_"
+
 # Check we can run reframe
-reframe --version BLABLABLA
+reframe --version
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully ran reframe --version"
 else
@@ -175,6 +178,7 @@ fi
 
 # List the tests we want to run
 export REFRAME_ARGS='--tag CI --tag 1_node'
+echo "Listing tests: reframe ${REFRAME_ARGS} --list"
 reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list"
@@ -183,6 +187,7 @@ else
 fi
 
 # Run all tests
+echo "Running tests: reframe ${REFRAME_ARGS} --run"
 reframe ${REFRAME_ARGS} --run
 if [[ $? -eq 0 ]]; then
     echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run."
@@ -190,7 +195,6 @@ else
     fatal_error "ReFrame runtime failed to run with command: reframe ${REFRAME_ARGS} --run."
 fi
 
-
 echo ">> Cleaning up ${TMPDIR}..."
 rm -r ${TMPDIR}
 

From 3a52bde23e5de6cfe4d5da170060c07d8dfc0009 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 14:28:08 +0100
Subject: [PATCH 043/154] Change to dedicated config file for testing by the
 bot in the build job

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 1610d87738..d6699878f9 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -160,7 +160,7 @@ else
 fi
 
 # Configure ReFrame
-export RFM_CONFIG_FILES=$TESTSUITEPREFIX/config/github_actions.py
+export RFM_CONFIG_FILES=$TESTSUITEPREFIX/config/software_layer_bot.py
 export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests
 export RFM_CHECK_SEARCH_RECURSIVE=1
 export RFM_PREFIX=$PWD/reframe_runs

From 159e384bdef117265e09a0a420cce3fa6d8842c8 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 14:47:54 +0100
Subject: [PATCH 044/154] Make it verbose, see if we can see why autodetect
 fails

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index d6699878f9..a8e1b2d97a 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -177,7 +177,7 @@ else
 fi
 
 # List the tests we want to run
-export REFRAME_ARGS='--tag CI --tag 1_node'
+export REFRAME_ARGS='--tag CI --tag 1_node -vvv'
 echo "Listing tests: reframe ${REFRAME_ARGS} --list"
 reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then

From b92044e5f913ce0a6c7082c0ebfb4e238d47f95d Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 16:23:53 +0100
Subject: [PATCH 045/154] Add ReFrame config template. Not that four template
 variables need to be replaced for this to be a valid ReFrame config file

---
 reframe_config_bot.py | 58 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 reframe_config_bot.py

diff --git a/reframe_config_bot.py b/reframe_config_bot.py
new file mode 100644
index 0000000000..65ccf9f43e
--- /dev/null
+++ b/reframe_config_bot.py
@@ -0,0 +1,58 @@
+# WARNING: this file is intended as template and the __X__ template variables need to be replaced
+# before it can act as a configuration file
+# Once replaced, this is a config file for running tests after the build phase, by the bot
+
+from eessi.testsuite.common_config import common_logging_config
+from eessi.testsuite.constants import *  # noqa: F403
+
+
+site_configuration = {
+    'systems': [
+        {
+            'name': 'Testing in bot Build jobs for EESSI software Layer',
+            'descr': 'Software-layer bot',
+            'hostnames': ['.*'],
+            'modules_system': 'lmod',
+            'partitions': [
+                {
+                    'name': 'default',
+                    'scheduler': 'local',
+                    'launcher': 'mpirun',
+                    'environs': ['default'],
+                    'features': [FEATURES[CPU]],
+                    'processor': {
+                        'num_cpus': __NUM_CPUS__,
+                        'num_sockets': __NUM_SOCKETS__,
+                        'num_cpus_per_core': __NUM_CPUS_PER_CORE__,
+                        'num_cpus_per_socket': __NUM_CPUS_PER_SOCKET__,
+                    }
+                    'resources': [
+                        {
+                            'name': 'memory',
+                            'options': ['--mem={size}'],
+                        }
+                    ],
+                    'max_jobs': 1
+                    }
+                ]
+            }
+        ],
+    'environments': [
+        {
+            'name': 'default',
+            'cc': 'cc',
+            'cxx': '',
+            'ftn': ''
+            }
+        ],
+    'general': [
+        {
+            'purge_environment': True,
+            'resolve_module_conflicts': False,  # avoid loading the module before submitting the job
+            # Enable automatic detection of CPU architecture
+            # See https://reframe-hpc.readthedocs.io/en/stable/configure.html#auto-detecting-processor-information
+            'remote_detect': True,
+        }
+    ],
+    'logging': common_logging_config(),
+}

From 02b0e3115dfc81387554f0cb9fe2c292e60098ab Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 16:30:18 +0100
Subject: [PATCH 046/154] Use template config file and make replacements based
 on lscpu

---
 test_suite.sh | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index a8e1b2d97a..f5cc7fabfd 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -156,11 +156,11 @@ python3 -c 'import eessi.testsuite'
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully found and imported eessi.testsuite"
 else
-    fatal_error "FAILED to import from eessi.testsuite in Python"
+    fatal_error "Failed to import from eessi.testsuite in Python"
 fi
 
 # Configure ReFrame
-export RFM_CONFIG_FILES=$TESTSUITEPREFIX/config/software_layer_bot.py
+export RFM_CONFIG_FILES=$TOPDIR/reframe_config_bot.py
 export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests
 export RFM_CHECK_SEARCH_RECURSIVE=1
 export RFM_PREFIX=$PWD/reframe_runs
@@ -168,6 +168,33 @@ export RFM_PREFIX=$PWD/reframe_runs
 echo "Configured reframe with the following environment variables:"
 env | grep "RFM_"
 
+# Inject correct CPU properties into the ReFrame config file
+cpuinfo=$(lscpu)
+if [[ "${cpuinfo}" =~ CPU\(s\):[^0-9]*([0-9]+) ]]; then
+    cpu_count=${BASH_REMATCH[1]}
+else
+    fatal_error "Failed to get the number of CPUs for the current test hardware with lscpu."
+fi
+if [[ "${text}" =~ Socket\(s\):[^0-9]*([0-9]+) ]]; then
+    socket_count=${BASH_REMATCH[1]}
+else
+    fatal_error "Failed to get the number of sockets for the current test hardware with lscpu."
+fi
+if [[ "${text}" =~ (Thread\(s\) per core:[^0-9]*([0-9]+)) ]]; then
+    threads_per_core=${BASH_REMATCH[2]}
+else
+    fatal_error "Failed to get the number of threads per core for the current test hardware with lscpu."
+fi
+if [[ "${text}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then
+    cores_per_socket=${BASH_REMATCH[2]}
+else
+    fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu."
+fi
+sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_CPUS_PER_SOCKETS__/${cores_per_socket}/g" $RFM_CONFIG_FILES
+
 # Check we can run reframe
 reframe --version
 if [[ $? -eq 0 ]]; then

From 58c4cfa772454405c456503c29a6fc8d59956910 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 16:39:46 +0100
Subject: [PATCH 047/154] Fix incorrect variable name

---
 test_suite.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index f5cc7fabfd..7ba5282327 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -175,17 +175,17 @@ if [[ "${cpuinfo}" =~ CPU\(s\):[^0-9]*([0-9]+) ]]; then
 else
     fatal_error "Failed to get the number of CPUs for the current test hardware with lscpu."
 fi
-if [[ "${text}" =~ Socket\(s\):[^0-9]*([0-9]+) ]]; then
+if [[ "${cpuinfo}" =~ Socket\(s\):[^0-9]*([0-9]+) ]]; then
     socket_count=${BASH_REMATCH[1]}
 else
     fatal_error "Failed to get the number of sockets for the current test hardware with lscpu."
 fi
-if [[ "${text}" =~ (Thread\(s\) per core:[^0-9]*([0-9]+)) ]]; then
+if [[ "${cpuinfo}" =~ (Thread\(s\) per core:[^0-9]*([0-9]+)) ]]; then
     threads_per_core=${BASH_REMATCH[2]}
 else
     fatal_error "Failed to get the number of threads per core for the current test hardware with lscpu."
 fi
-if [[ "${text}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then
+if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then
     cores_per_socket=${BASH_REMATCH[2]}
 else
     fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu."

From 2fc121e86399b7c7c3cab6092b609a91a88db43f Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 16:57:48 +0100
Subject: [PATCH 048/154] Missing comma

---
 reframe_config_bot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reframe_config_bot.py b/reframe_config_bot.py
index 65ccf9f43e..bbe8a7e2df 100644
--- a/reframe_config_bot.py
+++ b/reframe_config_bot.py
@@ -25,7 +25,7 @@
                         'num_sockets': __NUM_SOCKETS__,
                         'num_cpus_per_core': __NUM_CPUS_PER_CORE__,
                         'num_cpus_per_socket': __NUM_CPUS_PER_SOCKET__,
-                    }
+                    },
                     'resources': [
                         {
                             'name': 'memory',

From db08f4f39a618450c70f1d52f43c46f244011545 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 21:30:43 +0100
Subject: [PATCH 049/154] Corrected typo

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 7ba5282327..9bd95c6749 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -193,7 +193,7 @@ fi
 sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES
 sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES
 sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES
-sed -i "s/__NUM_CPUS_PER_SOCKETS__/${cores_per_socket}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES
 
 # Check we can run reframe
 reframe --version

From 6b3a6ccc985314e5c66c8097e8eb26e1c1fb3546 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 21:47:43 +0100
Subject: [PATCH 050/154] Remove verbose flags

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 9bd95c6749..99643c42b8 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -204,7 +204,7 @@ else
 fi
 
 # List the tests we want to run
-export REFRAME_ARGS='--tag CI --tag 1_node -vvv'
+export REFRAME_ARGS='--tag CI --tag 1_node'
 echo "Listing tests: reframe ${REFRAME_ARGS} --list"
 reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then

From a70b4f132e36f078cb3b4f0d802f96a877699050 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 21:55:16 +0100
Subject: [PATCH 051/154] Pick name without spaces, to avoid issues with
 directories

---
 reframe_config_bot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reframe_config_bot.py b/reframe_config_bot.py
index bbe8a7e2df..ad66a72041 100644
--- a/reframe_config_bot.py
+++ b/reframe_config_bot.py
@@ -9,7 +9,7 @@
 site_configuration = {
     'systems': [
         {
-            'name': 'Testing in bot Build jobs for EESSI software Layer',
+            'name': 'BotBuildTests',
             'descr': 'Software-layer bot',
             'hostnames': ['.*'],
             'modules_system': 'lmod',

From ce24e6d6de196ca191371b186b449b7a9d4aa5c9 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 17:50:39 +0100
Subject: [PATCH 052/154] Pick more meaningful name for SLURM variable, namely
 SLURM_OUTPUT_FOUND

---
 bot/check-build.sh | 22 +++++++++++-----------
 bot/check-test.sh  | 18 +++++++++---------
 2 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/bot/check-build.sh b/bot/check-build.sh
index ec1ca56bba..42dca46a08 100755
--- a/bot/check-build.sh
+++ b/bot/check-build.sh
@@ -95,18 +95,18 @@ job_dir=${PWD}
 
 [[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}"
 
-job_out="slurm-${SLURM_JOB_ID}.out"
+job_out="slurm-${SLURM_OUTPUT_FOUND_JOB_ID}.out"
 [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"
 if  [[ -f ${job_out} ]]; then
-    SLURM=1
+    SLURM_OUTPUT_FOUND=1
     [[ ${VERBOSE} -ne 0 ]] && echo "   found slurm output file '"${job_out}"'"
 else
-    SLURM=0
+    SLURM_OUTPUT_FOUND=0
     [[ ${VERBOSE} -ne 0 ]] && echo "   Slurm output file '"${job_out}"' NOT found"
 fi
 
 ERROR=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_error='ERROR: '
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}")
   [[ $? -eq 0 ]] && ERROR=1 || ERROR=0
@@ -116,7 +116,7 @@ if [[ ${SLURM} -eq 1 ]]; then
 fi
 
 FAILED=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_failed='FAILED: '
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
@@ -126,7 +126,7 @@ if [[ ${SLURM} -eq 1 ]]; then
 fi
 
 MISSING=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_req_missing=' required modules missing:'
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_req_missing}")
   [[ $? -eq 0 ]] && MISSING=1 || MISSING=0
@@ -136,7 +136,7 @@ if [[ ${SLURM} -eq 1 ]]; then
 fi
 
 NO_MISSING=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_no_missing='No missing installations'
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_no_missing}")
   [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0
@@ -147,7 +147,7 @@ fi
 
 TGZ=-1
 TARBALL=
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_tgz_created="\.tar\.gz created!"
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u)
   if [[ $? -eq 0 ]]; then
@@ -169,9 +169,9 @@ fi
 [[ ${VERBOSE} -ne 0 ]] && echo "  NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
 [[ ${VERBOSE} -ne 0 ]] && echo "  TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
 
-job_result_file=_bot_job${SLURM_JOB_ID}.result
+job_result_file=_bot_job${SLURM_OUTPUT_FOUND_JOB_ID}.result
 
-if [[ ${SLURM} -eq 1 ]] && \
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \
    [[ ${ERROR} -eq 0 ]] && \
    [[ ${FAILED} -eq 0 ]] && \
    [[ ${MISSING} -eq 0 ]] && \
@@ -340,7 +340,7 @@ CoDeList=""
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
-CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
+CoDeList=${CoDeList}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"
diff --git a/bot/check-test.sh b/bot/check-test.sh
index 39901f0691..549afd2f65 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -10,16 +10,16 @@
 # license: GPLv2
 #
 job_dir=${PWD}
-job_out="slurm-${SLURM_JOB_ID}.out"
-job_test_result_file="_bot_job${SLURM_JOB_ID}.test"
+job_out="slurm-${SLURM_OUTPUT_FOUND_JOB_ID}.out"
+job_test_result_file="_bot_job${SLURM_OUTPUT_FOUND_JOB_ID}.test"
 
 # Check that job output file is found
 [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"
 if  [[ -f ${job_out} ]]; then
-    SLURM=1
+    SLURM_OUTPUT_FOUND=1
     [[ ${VERBOSE} -ne 0 ]] && echo "   found slurm output file '"${job_out}"'"
 else
-    SLURM=0
+    SLURM_OUTPUT_FOUND=0
     [[ ${VERBOSE} -ne 0 ]] && echo "   Slurm output file '"${job_out}"' NOT found"
 fi
 
@@ -41,7 +41,7 @@ fi
 # We will grep for the last and final line, since this reflects the overall result
 # Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails
 FAILED=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_failed='\[.*FAILED.*\].*Ran .* test case'
   grep_reframe_failed=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
@@ -53,7 +53,7 @@ fi
 # Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step
 # I.e. this is an error in execution of the run_tests.sh itself, NOT in running the actual tests
 ERROR=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_error='ERROR: '
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}")
   [[ $? -eq 0 ]] && ERROR=1 || ERROR=0
@@ -64,7 +64,7 @@ fi
 
 SUCCESS=-1
 # Grep for the success pattern, so we can report the amount of tests run
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_success='\[.*PASSED.*\].*Ran .* test case'
   grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
   [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0
@@ -80,7 +80,7 @@ else
 fi
 
 echo "[TEST]" > ${job_test_result_file}
-if [[ ${SLURM} -eq 0 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 0 ]]; then
     summary=":cry: FAILURE"
     reason="Job output file not found, cannot check test results."
     status="FAILURE"
@@ -182,7 +182,7 @@ CoDeList=""
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
-CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
+CoDeList=${CoDeList}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"

From 9881c2ad5f93c617c075e8df81d770033ec2a121 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 17:51:36 +0100
Subject: [PATCH 053/154] Correct erroneous status

---
 bot/check-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 549afd2f65..9724f45832 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -103,7 +103,7 @@ elif [[ ${ERROR} -eq 1 ]]; then
 else
     summary=":grin: FAILURE"
     reason="Failed for unknown reason"
-    status="SUCCESS"
+    status="FAILURE"
 fi
 
 

From 1446055cb5cd4306d61809c67b5dcda4ce930c8b Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 17:53:09 +0100
Subject: [PATCH 054/154] Rename CoDeList variable to comment_details_list

---
 bot/check-build.sh | 18 +++++++++---------
 bot/check-test.sh  | 18 +++++++++---------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/bot/check-build.sh b/bot/check-build.sh
index 42dca46a08..584d4ff828 100755
--- a/bot/check-build.sh
+++ b/bot/check-build.sh
@@ -334,35 +334,35 @@ echo -n "comment_description = " >> ${job_result_file}
 
 comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
 
-# first construct comment_details_list, abbreviated CoDeList
+# first construct comment_details_list, abbreviated comment_details_list
 # then use it to set comment_details
-CoDeList=""
+comment_details_list=""
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
-CoDeList=${CoDeList}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"
-CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_failed}</code>"
 failure_msg="found message matching <code>${GP_failed}</code>"
-CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_req_missing}</code>"
 failure_msg="found message matching <code>${GP_req_missing}</code>"
-CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
 
 success_msg="found message(s) matching <code>${GP_no_missing}</code>"
 failure_msg="no message matching <code>${GP_no_missing}</code>"
-CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="found message matching <code>${GP_tgz_created}</code>"
 failure_msg="no message matching <code>${GP_tgz_created}</code>"
-CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
 
-comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
+comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}"
 
 
 # first construct comment_artefacts_list, abbreviated CoArList
diff --git a/bot/check-test.sh b/bot/check-test.sh
index 9724f45832..fe1fcfe85d 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -176,37 +176,37 @@ function add_detail() {
     fi
 }
 
-# first construct comment_details_list, abbreviated CoDeList
+# first construct comment_details_list, abbreviated comment_details_list
 # then use it to set comment_details
-CoDeList=""
+comment_details_list=""
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
-CoDeList=${CoDeList}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"
-CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>""${GP_failed}""</code>"
 failure_msg="found message matching <code>""${GP_failed}""</code>"
-CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 
 # Should not be needed for testing, I think? Maybe for loading ReFrame module...
 # success_msg="no message matching <code>${GP_req_missing}</code>"
 # failure_msg="found message matching <code>${GP_req_missing}</code>"
-# CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
+# comment_details_list=${comment_details_list}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
 # 
 # success_msg="found message(s) matching <code>${GP_no_missing}</code>"
 # failure_msg="no message matching <code>${GP_no_missing}</code>"
-# CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
+# comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
 # 
 # success_msg="found message matching <code>${GP_tgz_created}</code>"
 # failure_msg="no message matching <code>${GP_tgz_created}</code>"
-# CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
+# comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
 
 comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
-comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
+comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}"
 comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
 
 # Actually writing the comment description to the result file

From f80c0eb42f891c686cd1818ccbfc353e8bb29f05 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 17:54:00 +0100
Subject: [PATCH 055/154] Remove commented section, it is not relevant

---
 bot/check-test.sh | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index fe1fcfe85d..9e7a524607 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -192,19 +192,6 @@ success_msg="no message matching <code>""${GP_failed}""</code>"
 failure_msg="found message matching <code>""${GP_failed}""</code>"
 comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 
-# Should not be needed for testing, I think? Maybe for loading ReFrame module...
-# success_msg="no message matching <code>${GP_req_missing}</code>"
-# failure_msg="found message matching <code>${GP_req_missing}</code>"
-# comment_details_list=${comment_details_list}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
-# 
-# success_msg="found message(s) matching <code>${GP_no_missing}</code>"
-# failure_msg="no message matching <code>${GP_no_missing}</code>"
-# comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
-# 
-# success_msg="found message matching <code>${GP_tgz_created}</code>"
-# failure_msg="no message matching <code>${GP_tgz_created}</code>"
-# comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
-
 comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
 comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}"
 comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}

From ff5baace136c589becaf42adab32b4a0eed7afc6 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com>
Date: Thu, 8 Feb 2024 17:57:36 +0100
Subject: [PATCH 056/154] Update run_tests.sh

Avoid a one-line script (if possible)

Co-authored-by: Kenneth Hoste <kenneth.hoste@ugent.be>
---
 run_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/run_tests.sh b/run_tests.sh
index bcca1ee417..5e9fcb543d 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -9,5 +9,5 @@ source ${base_dir}/init/eessi_defaults
 # TODO: in the future we may implement this as a light first check.
 
 # Run the test suite
-./run_in_compat_layer_env.sh ./clone_eessi_test_suite.sh
+./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite"
 ./test_suite.sh

From 4adae1838edd1146d1a4a8c4322bffeb5e49db64 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 18:42:47 +0100
Subject: [PATCH 057/154] Fix mistake in replacing SLURM with
 SLURM_OUTPUT_FOUND

---
 bot/check-build.sh | 4 ++--
 bot/check-test.sh  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/bot/check-build.sh b/bot/check-build.sh
index 584d4ff828..1b46652049 100755
--- a/bot/check-build.sh
+++ b/bot/check-build.sh
@@ -95,7 +95,7 @@ job_dir=${PWD}
 
 [[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}"
 
-job_out="slurm-${SLURM_OUTPUT_FOUND_JOB_ID}.out"
+job_out="slurm-${SLURM_JOB_ID}.out"
 [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"
 if  [[ -f ${job_out} ]]; then
     SLURM_OUTPUT_FOUND=1
@@ -169,7 +169,7 @@ fi
 [[ ${VERBOSE} -ne 0 ]] && echo "  NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
 [[ ${VERBOSE} -ne 0 ]] && echo "  TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
 
-job_result_file=_bot_job${SLURM_OUTPUT_FOUND_JOB_ID}.result
+job_result_file=_bot_job${SLURM_JOB_ID}.result
 
 if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \
    [[ ${ERROR} -eq 0 ]] && \
diff --git a/bot/check-test.sh b/bot/check-test.sh
index 9e7a524607..faf6e440fa 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -10,8 +10,8 @@
 # license: GPLv2
 #
 job_dir=${PWD}
-job_out="slurm-${SLURM_OUTPUT_FOUND_JOB_ID}.out"
-job_test_result_file="_bot_job${SLURM_OUTPUT_FOUND_JOB_ID}.test"
+job_out="slurm-${SLURM_JOB_ID}.out"
+job_test_result_file="_bot_job${SLURM__JOB_ID}.test"
 
 # Check that job output file is found
 [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"

From 9e6955f53e70d3848ca2199756150993c36212ec Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 18:52:44 +0100
Subject: [PATCH 058/154] Change name of the template for the ReFrame config
 file to make clear it is a template. Then, alter the sed commands so that
 they output to a new file: the actual config file

---
 reframe_config_bot.py => reframe_config_bot.py.tmpl | 0
 test_suite.sh                                       | 9 +++++----
 2 files changed, 5 insertions(+), 4 deletions(-)
 rename reframe_config_bot.py => reframe_config_bot.py.tmpl (100%)

diff --git a/reframe_config_bot.py b/reframe_config_bot.py.tmpl
similarity index 100%
rename from reframe_config_bot.py
rename to reframe_config_bot.py.tmpl
diff --git a/test_suite.sh b/test_suite.sh
index 99643c42b8..b3884c3b81 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -161,6 +161,7 @@ fi
 
 # Configure ReFrame
 export RFM_CONFIG_FILES=$TOPDIR/reframe_config_bot.py
+export RFM_CONFIG_FILE_TEMPLATE=$TOPDIR/reframe_config_bot.py.tmpl
 export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests
 export RFM_CHECK_SEARCH_RECURSIVE=1
 export RFM_PREFIX=$PWD/reframe_runs
@@ -190,10 +191,10 @@ if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then
 else
     fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu."
 fi
-sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES
-sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES
-sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES
-sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES
+sed "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
+sed "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
+sed "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
+sed "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
 
 # Check we can run reframe
 reframe --version

From 08a7fd8eaa64dce298624e1d4f6054d2938a7fcc Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 18:55:20 +0100
Subject: [PATCH 059/154] Fix typo in variable

---
 bot/check-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index faf6e440fa..ef52206346 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -11,7 +11,7 @@
 #
 job_dir=${PWD}
 job_out="slurm-${SLURM_JOB_ID}.out"
-job_test_result_file="_bot_job${SLURM__JOB_ID}.test"
+job_test_result_file="_bot_job${SLURM_JOB_ID}.test"
 
 # Check that job output file is found
 [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"

From 8b5bde5413553c03f58ce788ef03e78ae87f343b Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 18:56:07 +0100
Subject: [PATCH 060/154] No longer needed

---
 clone_eessi_test_suite.sh | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100755 clone_eessi_test_suite.sh

diff --git a/clone_eessi_test_suite.sh b/clone_eessi_test_suite.sh
deleted file mode 100755
index f7684c8cfd..0000000000
--- a/clone_eessi_test_suite.sh
+++ /dev/null
@@ -1 +0,0 @@
-git clone https://github.com/EESSI/test-suite EESSI-test-suite

From 5d383eb8bc4c729cfa85d1aeae9ea10d82000c5c Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 19:03:29 +0100
Subject: [PATCH 061/154] Added header, rephrased some comments

---
 run_tests.sh | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/run_tests.sh b/run_tests.sh
index 5e9fcb543d..0c51a14807 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -1,13 +1,24 @@
 #!/bin/bash
+#
+# This script gets invoked by the bot/test.sh script to run within the EESSI container
+# Thus, this script defines all of the steps that should run for the tests.
+# Note that, unless we have good reason, we don't run test steps in the prefix environment:
+# users also typically don't run in the prefix environment, and we want to check if the
+# software works well in that specific setup.
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Caspar van Leeuwe (@casparvl)
+#
+# license: GPLv2
+#
+
 base_dir=$(dirname $(realpath $0))
 source ${base_dir}/init/eessi_defaults
 
-# Note: for these tests, we _don't_ run in the compat layer env
-# These tests should mimic what users do, and they are typically not in a prefix environment
-
-# Run eb --sanity-check-only on changed easyconfigs
-# TODO: in the future we may implement this as a light first check.
+# Git clone has to be run in compat layer, to make the git command available
+./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite"
 
 # Run the test suite
-./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite"
 ./test_suite.sh

From cdf79712184e89a31afaaff3880fc7cf0387b2e5 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 19:13:11 +0100
Subject: [PATCH 062/154] Fix headers

---
 run_tests.sh  |  2 +-
 test_suite.sh | 13 +++++++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/run_tests.sh b/run_tests.sh
index 0c51a14807..de7fd8c2e1 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -9,7 +9,7 @@
 # This script is part of the EESSI software layer, see
 # https://github.com/EESSI/software-layer.git
 #
-# author: Caspar van Leeuwe (@casparvl)
+# author: Caspar van Leeuwen (@casparvl)
 #
 # license: GPLv2
 #
diff --git a/test_suite.sh b/test_suite.sh
index b3884c3b81..ffe8955f1e 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -1,9 +1,14 @@
 #!/bin/bash
 #
-# Run sanity check for all requested modules (and built dependencies)?
-# get ec from diff
-# set up EasyBuild
-# run `eb --sanity-only ec`
+# This script creates a ReFrame config file from a template, in which CPU properties get replaced
+# based on where this script is run (typically: a build node). Then, it runs the EESSI test suite.
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Caspar van Leeuwen (@casparvl)
+#
+# license: GPLv2
 
 display_help() {
   echo "usage: $0 [OPTIONS]"

From 79a9ad320aec47ac368ed96f72e0f59a0cfbe9f5 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 19:16:10 +0100
Subject: [PATCH 063/154] Load default ReFrame version

---
 test_suite.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index ffe8955f1e..1f6d8ae44d 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -129,11 +129,11 @@ else
     echo_green ">> MODULEPATH set up: ${MODULEPATH}"
 fi
 
-# TODO: this should not be hardcoded. Ideally, we put some logic in place to discover the newest version
-# of the ReFrame module available in the current environment
-module load ReFrame/4.3.3
+# Load the ReFrame module
+# Currently, we load the default version. Maybe we should somehow make this configurable in the future?
+module load ReFrame
 if [[ $? -eq 0 ]]; then
-    echo_green ">> Loaded ReFrame/4.3.3"
+    echo_green ">> Loaded ReFrame module"
 else
     fatal_error "Failed to load the ReFrame module"
 fi

From 2f280e4304b520de295fdb408218e3125b153853 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 19:21:24 +0100
Subject: [PATCH 064/154] Point to docs for ReFrame config

---
 test_suite.sh | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index 1f6d8ae44d..d2403b91d8 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -130,7 +130,7 @@ else
 fi
 
 # Load the ReFrame module
-# Currently, we load the default version. Maybe we should somehow make this configurable in the future?
+ Currently, we load the default version. Maybe we should somehow make this configurable in the future?
 module load ReFrame
 if [[ $? -eq 0 ]]; then
     echo_green ">> Loaded ReFrame module"
@@ -139,15 +139,16 @@ else
 fi
 
 # Check ReFrame came with the hpctestlib and we can import it
-python3 -c 'import hpctestlib.sciapps.gromacs'
+reframe_import="hpctestlib.sciapps.gromacs"
+python3 -c "import ${reframe_import}"
 if [[ $? -eq 0 ]]; then
-    echo_green "Succesfully found and imported hpctestlib.sciapps.gromas"
+    echo_green "Succesfully found and imported ${reframe_import}"
 else
-    fatal_error "Failed to load hpctestlib"
+    fatal_error "Failed to import ${reframe_import}"
 fi
 
-# Cloning should already be done by clone_eessi_test_suite.sh, which runs in compat layer to have 'git' available
-# git clone https://github.com/EESSI/test-suite EESSI-test-suite
+# Cloning should already be done in run_tests.sh before test_suite.sh is invoked
+# Check if that succeeded
 export TESTSUITEPREFIX=$PWD/EESSI-test-suite
 if [ -d $TESTSUITEPREFIX ]; then
     echo_green "Clone of the test suite $TESTSUITEPREFIX available, OK!"
@@ -157,14 +158,15 @@ fi
 export PYTHONPATH=$TESTSUITEPREFIX:$PYTHONPATH
 
 # Check that we can import from the testsuite
-python3 -c 'import eessi.testsuite'
+testsuite_import="eessi.testsuite"
+python3 -c "import ${testsuite_import}"
 if [[ $? -eq 0 ]]; then
-    echo_green "Succesfully found and imported eessi.testsuite"
+    echo_green "Succesfully found and imported ${testsuite_import}"
 else
-    fatal_error "Failed to import from eessi.testsuite in Python"
+    fatal_error "Failed to import ${testsuite_import}"
 fi
 
-# Configure ReFrame
+# Configure ReFrame, see https://www.eessi.io/docs/test-suite/installation-configuration
 export RFM_CONFIG_FILES=$TOPDIR/reframe_config_bot.py
 export RFM_CONFIG_FILE_TEMPLATE=$TOPDIR/reframe_config_bot.py.tmpl
 export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests

From e5d60b9c6bd2b3a39a32454a9ee6707320693c11 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 19:26:17 +0100
Subject: [PATCH 065/154] Fix silly mistake: I was overwriting the ReFrame
 config file with every sed call

---
 test_suite.sh | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index d2403b91d8..755102e6bc 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -198,17 +198,18 @@ if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then
 else
     fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu."
 fi
-sed "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
-sed "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
-sed "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
-sed "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
+cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES}
+sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES
 
 # Check we can run reframe
 reframe --version
 if [[ $? -eq 0 ]]; then
-    echo_green "Succesfully ran reframe --version"
+    echo_green "Succesfully ran 'reframe --version'"
 else
-    fatal_error "Failed to run ReFrame --version"
+    fatal_error "Failed to run 'reframe --version'"
 fi
 
 # List the tests we want to run
@@ -224,7 +225,8 @@ fi
 # Run all tests
 echo "Running tests: reframe ${REFRAME_ARGS} --run"
 reframe ${REFRAME_ARGS} --run
-if [[ $? -eq 0 ]]; then
+reframe_exit_code=$?
+if [[ ${reframe_exit_code} -eq 0 ]]; then
     echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run."
 else
     fatal_error "ReFrame runtime failed to run with command: reframe ${REFRAME_ARGS} --run."
@@ -233,4 +235,4 @@ fi
 echo ">> Cleaning up ${TMPDIR}..."
 rm -r ${TMPDIR}
 
-exit 0
+exit ${reframe_exit_code}

From b5c60ccfc483602eb6cebb9e1f580c9d99cd964b Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 19:28:32 +0100
Subject: [PATCH 066/154] See if @boegel is correct and if we don't actually
 need ANY software change to have the bot run...

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index 4c73b5887a..82190071ab 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -15,4 +15,3 @@ easyconfigs:
       options:
         from-pr: 19573
   - scikit-learn-1.3.1-gfbf-2023a.eb
-  - patchelf-0.18.0-GCCcore-12.3.0.eb

From 69d6d2e227c6f2ceb0ad39068d6fe4a99d7fd24e Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 14 Feb 2024 14:55:14 +0100
Subject: [PATCH 067/154] Add escape character to star, so that the regex
 pattern prints correctly on GitHub in the report

---
 bot/check-test.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index ef52206346..bc5ba84101 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -188,6 +188,8 @@ success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"
 comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
 
+# Add an escape character to every *, for it to be printed correctly in the comment on GitHub
+GP_failed="${GP_failed//\*/\\*}"
 success_msg="no message matching <code>""${GP_failed}""</code>"
 failure_msg="found message matching <code>""${GP_failed}""</code>"
 comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")

From 2dd048e96cef85b203b76a9ca330c5c72813dbbc Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 15 Feb 2024 15:44:15 +0100
Subject: [PATCH 068/154] Add the list(SCALES.keys()) feature (though
 technically we could limit to singlenode already here...). Also, make sure
 color is not used for printing the ReFrame results, to avoid color-characters
 making it into the slurm output file

---
 reframe_config_bot.py.tmpl | 4 +++-
 test_suite.sh              | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/reframe_config_bot.py.tmpl b/reframe_config_bot.py.tmpl
index ad66a72041..b2f7916e79 100644
--- a/reframe_config_bot.py.tmpl
+++ b/reframe_config_bot.py.tmpl
@@ -19,7 +19,9 @@ site_configuration = {
                     'scheduler': 'local',
                     'launcher': 'mpirun',
                     'environs': ['default'],
-                    'features': [FEATURES[CPU]],
+                    'features': [
+                        FEATURES[CPU]
+                    ] + list(SCALES.keys()),
                     'processor': {
                         'num_cpus': __NUM_CPUS__,
                         'num_sockets': __NUM_SOCKETS__,
diff --git a/test_suite.sh b/test_suite.sh
index 755102e6bc..f8c46ef764 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -213,7 +213,7 @@ else
 fi
 
 # List the tests we want to run
-export REFRAME_ARGS='--tag CI --tag 1_node'
+export REFRAME_ARGS='--tag CI --tag 1_node --nocolor'
 echo "Listing tests: reframe ${REFRAME_ARGS} --list"
 reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then

From 0492b9e6d836ef75126ca5e8f613c7ad7487dc2b Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 15 Feb 2024 15:45:25 +0100
Subject: [PATCH 069/154] Try the original regex patterns, capturing spaces
 more specifically, now that we got rid of color characters

---
 bot/check-test.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index bc5ba84101..4b5f7575e5 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -42,7 +42,7 @@ fi
 # Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails
 FAILED=-1
 if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
-  GP_failed='\[.*FAILED.*\].*Ran .* test case'
+  GP_failed='\[\s*FAILED\s*\].*Ran .* test case'
   grep_reframe_failed=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
   # have to be careful to not add searched for pattern into slurm out file
@@ -65,7 +65,7 @@ fi
 SUCCESS=-1
 # Grep for the success pattern, so we can report the amount of tests run
 if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
-  GP_success='\[.*PASSED.*\].*Ran .* test case'
+  GP_success='\[\s*PASSED\s*\].*Ran .* test case'
   grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
   [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0
   # have to be careful to not add searched for pattern into slurm out file

From ec61759dc7b8a2a39613aff55f6ae74a55dd2620 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Mon, 19 Feb 2024 12:11:54 +0100
Subject: [PATCH 070/154] Remove manual setup of MODULEPATH etc, and source the
 regular EESSI init script to do this

---
 test_suite.sh | 68 +++++++++++----------------------------------------
 1 file changed, 14 insertions(+), 54 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index f8c46ef764..8a98fd59a8 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -74,63 +74,14 @@ fi
 TMPDIR=$(mktemp -d)
 
 echo ">> Setting up environment..."
-
-source $TOPDIR/init/minimal_eessi_env
-
-if [ -d $EESSI_CVMFS_REPO ]; then
-    echo_green "$EESSI_CVMFS_REPO available, OK!"
-else
-    fatal_error "$EESSI_CVMFS_REPO is not available!"
-fi
-
-# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
-export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
-
-echo ">> Determining software subdirectory to use for current build/test host..."
-if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
-  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
-  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
-else
-  echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
-fi
-
-# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE)
-# $EESSI_SILENT - don't print any messages
-# $EESSI_BASIC_ENV - give a basic set of environment variables
-EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables
-
-if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then
-    fatal_error "Failed to determine software subdirectory?!"
-elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then
-    fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!"
-else
-    echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!"
-fi
-
-echo ">> Initializing Lmod..."
-source $EPREFIX/usr/share/Lmod/init/bash
-ml_version_out=$TMPDIR/ml.out
-ml --version &> $ml_version_out
-if [[ $? -eq 0 ]]; then
-    echo_green ">> Found Lmod ${LMOD_VERSION}"
-else
-    fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}"
-fi
-
-echo ">> Setting up \$MODULEPATH..."
-# make sure no modules are loaded
 module --force purge
-# ignore current $MODULEPATH entirely
-module unuse $MODULEPATH
-module use ${EESSI_SOFTWARE_PATH}/modules/all
-if [[ -z ${MODULEPATH} ]]; then
-    fatal_error "Failed to set up \$MODULEPATH?!"
-else
-    echo_green ">> MODULEPATH set up: ${MODULEPATH}"
-fi
+# Make sure defaults are set for EESSI_CVMFS_REPO and EESSI_VERSION, before initializing EESSI
+source $TOPDIR/init/eessi_defaults
+# Initialize EESSI
+source ${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/bash
 
 # Load the ReFrame module
- Currently, we load the default version. Maybe we should somehow make this configurable in the future?
+# Currently, we load the default version. Maybe we should somehow make this configurable in the future?
 module load ReFrame
 if [[ $? -eq 0 ]]; then
     echo_green ">> Loaded ReFrame module"
@@ -138,6 +89,15 @@ else
     fatal_error "Failed to load the ReFrame module"
 fi
 
+# Check that a system python3 is available
+python3_found=$(which python3)
+if [ -z ${python3_found} ]; then
+    fatal_error "No system python3 found"
+else
+    echo_green "System python3 found:"
+    python3 -V
+fi
+
 # Check ReFrame came with the hpctestlib and we can import it
 reframe_import="hpctestlib.sciapps.gromacs"
 python3 -c "import ${reframe_import}"

From 9bae7f590a6bfe34e48fe20bc0be066ee407fb25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 20 Feb 2024 15:26:18 +0100
Subject: [PATCH 071/154] add OpenMPI from PR 19940 (and rebuild: True)

---
 .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml         | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index 75dc86c31a..70fb5bfbe0 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -22,10 +22,6 @@ easyconfigs:
       options:
         from-pr: 19339
   - Qt5-5.15.10-GCCcore-12.3.0.eb
-  - gnuplot-5.4.8-GCCcore-12.3.0.eb: 
-      options:
-        from-pr: 19261
-  - OpenFOAM-10-foss-2023a.eb
   - OSU-Micro-Benchmarks-7.1-1-gompi-2023a.eb
   - LHAPDF-6.5.4-GCC-12.3.0.eb:
       options:
@@ -56,3 +52,11 @@ easyconfigs:
   - BWA-0.7.17-20220923-GCCcore-12.3.0.eb:
       options: 
         from-pr: 19820 
+  - gnuplot-5.4.8-GCCcore-12.3.0.eb: 
+      options:
+        from-pr: 19261
+  - OpenMPI-4.1.5-GCC-12.3.0.eb:
+      options:
+      from-pr: 19940
+      rebuild: True
+  - OpenFOAM-10-foss-2023a.eb

From d3c1f654371126a5187d3a9e07b6def690f8725d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 20 Feb 2024 15:41:24 +0100
Subject: [PATCH 072/154] fix indentation

---
 .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml                  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index 70fb5bfbe0..b971c5adad 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -57,6 +57,6 @@ easyconfigs:
         from-pr: 19261
   - OpenMPI-4.1.5-GCC-12.3.0.eb:
       options:
-      from-pr: 19940
-      rebuild: True
+        from-pr: 19940
+        rebuild: True
   - OpenFOAM-10-foss-2023a.eb

From e666e2c4058a38b251697730a1551ba0ca34601a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 13:42:12 +0100
Subject: [PATCH 073/154] launch with --fakeroot for build jobs triggered by
 bot

---
 eessi_container.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/eessi_container.sh b/eessi_container.sh
index d6e9558202..55ce409e2b 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -585,6 +585,11 @@ if [[ "${ACCESS}" == "ro" ]]; then
 fi
 
 if [[ "${ACCESS}" == "rw" ]]; then
+  if [[ ! -z ${JOB_CFG_FILE} ]]; then
+    # always launch build jobs triggered by the job with --fakeroot,
+    # we drop back to a regular user in the build script
+    ADDITIONAL_CONTAINER_OPTIONS+=("--fakeroot")
+  fi
   mkdir -p ${EESSI_TMPDIR}/overlay-upper
   mkdir -p ${EESSI_TMPDIR}/overlay-work
 

From 34bc1c25bb0cdc926d05bcc0354069658bf73592 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 13:42:36 +0100
Subject: [PATCH 074/154] remove existing dirs for rebuilds

---
 EESSI-install-software.sh | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 69de9d1997..2803880547 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -222,6 +222,23 @@ else
         echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."
     
         if [ -f ${easystack_file} ]; then
+            if [ $(basename $(dirname ${easystack_file} = 'rebuilds' ]; then
+              echo_green "Software rebuild(s) requested, so determining which existing installation have to be removed..."
+              # we need to remove existing installation directories first,
+              # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R])
+              #  * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
+              rebuild_apps=$(${EB} --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
+              for app in ${rebuild_apps}; do
+                app_dir=${EASYBUILD_INSTALLPATH}/software/${app}
+                app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua
+                echo_yellow "Removing ${appdir} and {app_module}..."
+                rm -rf ${appdir}
+                rm -rf ${app_module}
+              done
+            fi
+            # drop back to a regular user
+            su - eessi
+
             echo_green "Feeding easystack file ${easystack_file} to EasyBuild..."
     
             ${EB} --easystack ${TOPDIR}/${easystack_file} --robot

From 46e753208352710344e9c3f6ebb859c9e3975168 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 13:46:55 +0100
Subject: [PATCH 075/154] rebuild all OpenMPI 4.1.x versions

---
 .../20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml     | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml

diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml
new file mode 100644
index 0000000000..7a6d3c63a3
--- /dev/null
+++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml
@@ -0,0 +1,10 @@
+easyconfigs:
+  - OpenMPI-4.1.4-GCC-12.2.0.eb:
+      options:
+        from-pr: 19940
+  - OpenMPI-4.1.5-GCC-12.3.0:
+      options:
+        from-pr: 19940
+  - OpenMPI-4.1.6-GCC-13.2.0:
+      options:
+        from-pr: 19940

From 21959d651c35c0cf70b68c239deed30f55bb95d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 13:58:22 +0100
Subject: [PATCH 076/154] add some comments/links

---
 .../rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml  | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml
index 7a6d3c63a3..042d0a214c 100644
--- a/easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml
+++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml
@@ -1,3 +1,8 @@
+# 2024-03-01
+# Rebuild all OpenMPI 4.1.x versions due to an issue with smcuda:
+# https://github.com/open-mpi/ompi/issues/12270
+# https://github.com/open-mpi/ompi/pull/12344
+# https://github.com/easybuilders/easybuild-easyconfigs/pull/19940
 easyconfigs:
   - OpenMPI-4.1.4-GCC-12.2.0.eb:
       options:

From 7054a587884b2cd56a44bf871d0cd052633a6e1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 13:59:35 +0100
Subject: [PATCH 077/154] fix syntax, add missing parentheses

---
 EESSI-install-software.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 2803880547..2e3641a3ae 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -222,7 +222,7 @@ else
         echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."
     
         if [ -f ${easystack_file} ]; then
-            if [ $(basename $(dirname ${easystack_file} = 'rebuilds' ]; then
+            if [ $(basename $(dirname ${easystack_file})) = 'rebuilds' ]; then
               echo_green "Software rebuild(s) requested, so determining which existing installation have to be removed..."
               # we need to remove existing installation directories first,
               # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R])

From 1fd1609ec960e3b713e940076241f118677a3181 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Fri, 1 Mar 2024 14:00:03 +0100
Subject: [PATCH 078/154] 'which' is not working in the container, command -v
 is

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 8a98fd59a8..e9168184bc 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -90,7 +90,7 @@ else
 fi
 
 # Check that a system python3 is available
-python3_found=$(which python3)
+python3_found=$(command -v python3)
 if [ -z ${python3_found} ]; then
     fatal_error "No system python3 found"
 else

From 7b78861fe6ab49547adbcf22df8c044039fbcc3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 14:01:30 +0100
Subject: [PATCH 079/154] remove spaces from blank lines

---
 EESSI-install-software.sh | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 2e3641a3ae..25069689cc 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -208,19 +208,19 @@ if [ -z ${changed_easystacks} ]; then
     echo "No missing installations, party time!"  # Ensure the bot report success, as there was nothing to be build here
 else
     for easystack_file in ${changed_easystacks}; do
-    
+
         echo -e "Processing easystack file ${easystack_file}...\n\n"
-    
+
         # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
         eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
-    
+
         # load EasyBuild module (will be installed if it's not available yet)
         source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
-    
+
         ${EB} --show-config
-    
+
         echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."
-    
+
         if [ -f ${easystack_file} ]; then
             if [ $(basename $(dirname ${easystack_file})) = 'rebuilds' ]; then
               echo_green "Software rebuild(s) requested, so determining which existing installation have to be removed..."
@@ -240,10 +240,10 @@ else
             su - eessi
 
             echo_green "Feeding easystack file ${easystack_file} to EasyBuild..."
-    
+
             ${EB} --easystack ${TOPDIR}/${easystack_file} --robot
             ec=$?
-    
+
             # copy EasyBuild log file if EasyBuild exited with an error
             if [ ${ec} -ne 0 ]; then
                 eb_last_log=$(unset EB_VERBOSE; eb --last-log)
@@ -253,12 +253,12 @@ else
                 # copy to build logs dir (with context added)
                 copy_build_log "${eb_last_log}" "${build_logs_dir}"
             fi
-    
+
             $TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file}
         else
             fatal_error "Easystack file ${easystack_file} not found!"
         fi
-    
+
     done
 fi
 

From 87e5ff09fbd8720d4cad0bf60647d970cbea0c87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 14:03:49 +0100
Subject: [PATCH 080/154] rename file to be able to find EB version

---
 ...b-4.9.0.yml => 20240301-eb-4.9.0-OpenMPI-4.1.x-fix-smcuda.yml} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename easystacks/software.eessi.io/2023.06/rebuilds/{20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml => 20240301-eb-4.9.0-OpenMPI-4.1.x-fix-smcuda.yml} (100%)

diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240301-eb-4.9.0-OpenMPI-4.1.x-fix-smcuda.yml
similarity index 100%
rename from easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml
rename to easystacks/software.eessi.io/2023.06/rebuilds/20240301-eb-4.9.0-OpenMPI-4.1.x-fix-smcuda.yml

From d07879f4c28a798e84d4e22e30e874ce766c6deb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 14:06:48 +0100
Subject: [PATCH 081/154] clarify comment about fakeroot

---
 eessi_container.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 55ce409e2b..bedb269cfd 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -586,7 +586,8 @@ fi
 
 if [[ "${ACCESS}" == "rw" ]]; then
   if [[ ! -z ${JOB_CFG_FILE} ]]; then
-    # always launch build jobs triggered by the job with --fakeroot,
+    # always launch build jobs triggered by the job with --fakeroot in order to be able to remove existing installations, see:
+    # https://github.com/EESSI/software-layer/issues/312
     # we drop back to a regular user in the build script
     ADDITIONAL_CONTAINER_OPTIONS+=("--fakeroot")
   fi

From 135c3c890f00ca282af5dff53fa8f48c302c7247 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 14:08:27 +0100
Subject: [PATCH 082/154] add --allow-use-as-root-and-accept-consequences

---
 EESSI-install-software.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 25069689cc..85cdcbcbbf 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -227,7 +227,7 @@ else
               # we need to remove existing installation directories first,
               # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R])
               #  * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
-              rebuild_apps=$(${EB} --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
+              rebuild_apps=$(${EB} --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
               for app in ${rebuild_apps}; do
                 app_dir=${EASYBUILD_INSTALLPATH}/software/${app}
                 app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua

From f1831b44d53ecd525717c2c0b1049c0c4da1bca7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 14:14:21 +0100
Subject: [PATCH 083/154] add missing $

---
 EESSI-install-software.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 85cdcbcbbf..4b87a8f0cd 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -231,7 +231,7 @@ else
               for app in ${rebuild_apps}; do
                 app_dir=${EASYBUILD_INSTALLPATH}/software/${app}
                 app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua
-                echo_yellow "Removing ${appdir} and {app_module}..."
+                echo_yellow "Removing ${appdir} and ${app_module}..."
                 rm -rf ${appdir}
                 rm -rf ${app_module}
               done

From 68212a41298d9a90e95214ccbd971b6d9e2a626a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 14:14:51 +0100
Subject: [PATCH 084/154] fix app_dir variable

---
 EESSI-install-software.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 4b87a8f0cd..6b5aeb39bc 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -231,7 +231,7 @@ else
               for app in ${rebuild_apps}; do
                 app_dir=${EASYBUILD_INSTALLPATH}/software/${app}
                 app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua
-                echo_yellow "Removing ${appdir} and ${app_module}..."
+                echo_yellow "Removing ${app_dir} and ${app_module}..."
                 rm -rf ${appdir}
                 rm -rf ${app_module}
               done

From 6bcd0e3a8e35613979ed7e45b4ca546c7a46759f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 14:28:15 +0100
Subject: [PATCH 085/154] split loop, first remove everything that needs to be
 removed, then drop back to regular user and do installations

---
 EESSI-install-software.sh | 43 +++++++++++++++++++++++----------------
 1 file changed, 26 insertions(+), 17 deletions(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 6b5aeb39bc..b7350f9fde 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -207,6 +207,32 @@ changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z
 if [ -z ${changed_easystacks} ]; then
     echo "No missing installations, party time!"  # Ensure the bot report success, as there was nothing to be build here
 else
+    changed_easystacks_rebuilds=$(grep "/rebuilds/" <<< ${changed_easystacks})
+    for easystack_file in ${changed_easystacks_rebuilds}; do
+        # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
+        eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
+
+        # load EasyBuild module (will be installed if it's not available yet)
+        source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
+
+        if [ -f ${easystack_file} ]; then
+            echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..."
+            # we need to remove existing installation directories first,
+            # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R])
+            #  * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
+            rebuild_apps=$(${EB} --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
+            for app in ${rebuild_apps}; do
+                app_dir=${EASYBUILD_INSTALLPATH}/software/${app}
+                app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua
+                echo_yellow "Removing ${app_dir} and ${app_module}..."
+                rm -rf ${appdir}
+                rm -rf ${app_module}
+            done
+    done
+
+    # drop back to a regular user
+    su - eessi
+
     for easystack_file in ${changed_easystacks}; do
 
         echo -e "Processing easystack file ${easystack_file}...\n\n"
@@ -222,23 +248,6 @@ else
         echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."
 
         if [ -f ${easystack_file} ]; then
-            if [ $(basename $(dirname ${easystack_file})) = 'rebuilds' ]; then
-              echo_green "Software rebuild(s) requested, so determining which existing installation have to be removed..."
-              # we need to remove existing installation directories first,
-              # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R])
-              #  * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
-              rebuild_apps=$(${EB} --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
-              for app in ${rebuild_apps}; do
-                app_dir=${EASYBUILD_INSTALLPATH}/software/${app}
-                app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua
-                echo_yellow "Removing ${app_dir} and ${app_module}..."
-                rm -rf ${appdir}
-                rm -rf ${app_module}
-              done
-            fi
-            # drop back to a regular user
-            su - eessi
-
             echo_green "Feeding easystack file ${easystack_file} to EasyBuild..."
 
             ${EB} --easystack ${TOPDIR}/${easystack_file} --robot

From c81c0dec1709cd683c571bd50537999da1bdd8ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 14:31:28 +0100
Subject: [PATCH 086/154] add missing else and fi

---
 EESSI-install-software.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index b7350f9fde..62cc1a3df2 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -228,6 +228,9 @@ else
                 rm -rf ${appdir}
                 rm -rf ${app_module}
             done
+        else
+            fatal_error "Easystack file ${easystack_file} not found!"
+        fi
     done
 
     # drop back to a regular user

From f99458395e44b9b063ad6f785e21a1f16edc16ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 14:40:54 +0100
Subject: [PATCH 087/154] su without hypen

---
 EESSI-install-software.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 62cc1a3df2..01ca19336c 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -234,7 +234,7 @@ else
     done
 
     # drop back to a regular user
-    su - eessi
+    su eessi
 
     for easystack_file in ${changed_easystacks}; do
 

From 21e3735712a7679f385a20fa767d60a261854568 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 15:28:28 +0100
Subject: [PATCH 088/154] rerun the script with a regular user after removing
 installations

---
 EESSI-install-software.sh | 63 ++++++++++++++++++++++-----------------
 1 file changed, 35 insertions(+), 28 deletions(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 01ca19336c..1e517bc243 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -187,6 +187,41 @@ fi
 # assume there's only one diff file that corresponds to the PR patch file
 pr_diff=$(ls [0-9]*.diff | head -1)
 
+# if this script is run as root, use PR patch file to determine if software needs to be removed first
+if [ $UID -eq 0 ]; then
+    changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') | grep "/rebuilds/
+    if [ -z ${changed_easystacks} ]; then
+        echo "No software needs to be removed."
+    else
+        for easystack_file in ${changed_easystacks_rebuilds}; do
+            # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
+            eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
+
+            # load EasyBuild module (will be installed if it's not available yet)
+            source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
+
+            if [ -f ${easystack_file} ]; then
+                echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..."
+                # we need to remove existing installation directories first,
+                # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R])
+                #  * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
+                rebuild_apps=$(${EB} --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
+                for app in ${rebuild_apps}; do
+                    app_dir=${EASYBUILD_INSTALLPATH}/software/${app}
+                    app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua
+                    echo_yellow "Removing ${app_dir} and ${app_module}..."
+                    rm -rf ${appdir}
+                    rm -rf ${app_module}
+                done
+            else
+                fatal_error "Easystack file ${easystack_file} not found!"
+            fi
+        done
+    fi
+    # now rerun the script with a regular user account to do the software installations
+    exec su eessi $( readlink -f "$0" ) -- "$@"
+fi
+
 # install any additional required scripts
 # order is important: these are needed to install a full CUDA SDK in host_injections
 # for now, this just reinstalls all scripts. Note the most elegant, but works
@@ -207,34 +242,6 @@ changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z
 if [ -z ${changed_easystacks} ]; then
     echo "No missing installations, party time!"  # Ensure the bot report success, as there was nothing to be build here
 else
-    changed_easystacks_rebuilds=$(grep "/rebuilds/" <<< ${changed_easystacks})
-    for easystack_file in ${changed_easystacks_rebuilds}; do
-        # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
-        eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
-
-        # load EasyBuild module (will be installed if it's not available yet)
-        source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
-
-        if [ -f ${easystack_file} ]; then
-            echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..."
-            # we need to remove existing installation directories first,
-            # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R])
-            #  * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
-            rebuild_apps=$(${EB} --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
-            for app in ${rebuild_apps}; do
-                app_dir=${EASYBUILD_INSTALLPATH}/software/${app}
-                app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua
-                echo_yellow "Removing ${app_dir} and ${app_module}..."
-                rm -rf ${appdir}
-                rm -rf ${app_module}
-            done
-        else
-            fatal_error "Easystack file ${easystack_file} not found!"
-        fi
-    done
-
-    # drop back to a regular user
-    su eessi
 
     for easystack_file in ${changed_easystacks}; do
 

From 4f436b346bbba403fb8c23c939423ba12b8e1f54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 15:38:24 +0100
Subject: [PATCH 089/154] fix typo and add missing quote

---
 EESSI-install-software.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 1e517bc243..28bfb8534c 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -189,8 +189,8 @@ pr_diff=$(ls [0-9]*.diff | head -1)
 
 # if this script is run as root, use PR patch file to determine if software needs to be removed first
 if [ $UID -eq 0 ]; then
-    changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') | grep "/rebuilds/
-    if [ -z ${changed_easystacks} ]; then
+    changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') | grep "/rebuilds/"
+    if [ -z ${changed_easystacks_rebuilds} ]; then
         echo "No software needs to be removed."
     else
         for easystack_file in ${changed_easystacks_rebuilds}; do

From 0676fdab6d576d0af0b2f107bbd6f92e6cbde146 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 15:44:04 +0100
Subject: [PATCH 090/154] fix syntax

---
 EESSI-install-software.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 28bfb8534c..0f1856891e 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -189,7 +189,7 @@ pr_diff=$(ls [0-9]*.diff | head -1)
 
 # if this script is run as root, use PR patch file to determine if software needs to be removed first
 if [ $UID -eq 0 ]; then
-    changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') | grep "/rebuilds/"
+    changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/")
     if [ -z ${changed_easystacks_rebuilds} ]; then
         echo "No software needs to be removed."
     else

From 95d6aaa983c9a59e42ff28a6feab75f493dc2e7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 15:44:23 +0100
Subject: [PATCH 091/154] fix typo in app_dir variable

---
 EESSI-install-software.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 0f1856891e..7f44403c73 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -210,7 +210,7 @@ if [ $UID -eq 0 ]; then
                     app_dir=${EASYBUILD_INSTALLPATH}/software/${app}
                     app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua
                     echo_yellow "Removing ${app_dir} and ${app_module}..."
-                    rm -rf ${appdir}
+                    rm -rf ${app_dir}
                     rm -rf ${app_module}
                 done
             else

From 314178937a1876eff79300fc0b562031ffc410a8 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Fri, 1 Mar 2024 15:58:01 +0100
Subject: [PATCH 092/154] Workaround for 'PSM3 can't open nic unit' error

---
 test_suite.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test_suite.sh b/test_suite.sh
index e9168184bc..e3bab04aec 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -164,6 +164,9 @@ sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES
 sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES
 sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES
 
+# Workaround for https://github.com/EESSI/software-layer/pull/467#issuecomment-1973341966
+export PSM3_DEVICES='self,shm'  # this is enough, since we only run single node for now
+
 # Check we can run reframe
 reframe --version
 if [[ $? -eq 0 ]]; then

From e52afc192fc900db8476cb4763a5655228ef9d0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 16:17:49 +0100
Subject: [PATCH 093/154] run removal step without Prefix

---
 EESSI-install-software.sh | 3 ++-
 install_software_layer.sh | 4 ++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 7f44403c73..097d0992e5 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -219,7 +219,8 @@ if [ $UID -eq 0 ]; then
         done
     fi
     # now rerun the script with a regular user account to do the software installations
-    exec su eessi $( readlink -f "$0" ) -- "$@"
+    #exec su eessi $( readlink -f "$0" ) -- "$@"
+    exit
 fi
 
 # install any additional required scripts
diff --git a/install_software_layer.sh b/install_software_layer.sh
index 82ca70b73f..85f856e86a 100755
--- a/install_software_layer.sh
+++ b/install_software_layer.sh
@@ -1,4 +1,8 @@
 #!/bin/bash
 base_dir=$(dirname $(realpath $0))
 source ${base_dir}/init/eessi_defaults
+if [ $UID -eq 0 ]; then
+    ./EESSI-install-software.sh "$@"
+    exec runuser -u eessi $( readlink -f "$0" ) -- "$@"
+fi
 ./run_in_compat_layer_env.sh ./EESSI-install-software.sh "$@"

From be4c59280fdf7de49adcc8029970987201f1b551 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 1 Mar 2024 16:36:38 +0100
Subject: [PATCH 094/154] move removal part to its own script, at least for now

---
 EESSI-install-software.sh |  36 --------
 EESSI-remove-software.sh  | 189 ++++++++++++++++++++++++++++++++++++++
 install_software_layer.sh |   2 +-
 3 files changed, 190 insertions(+), 37 deletions(-)
 create mode 100755 EESSI-remove-software.sh

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 097d0992e5..ab91d2c6de 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -187,42 +187,6 @@ fi
 # assume there's only one diff file that corresponds to the PR patch file
 pr_diff=$(ls [0-9]*.diff | head -1)
 
-# if this script is run as root, use PR patch file to determine if software needs to be removed first
-if [ $UID -eq 0 ]; then
-    changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/")
-    if [ -z ${changed_easystacks_rebuilds} ]; then
-        echo "No software needs to be removed."
-    else
-        for easystack_file in ${changed_easystacks_rebuilds}; do
-            # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
-            eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
-
-            # load EasyBuild module (will be installed if it's not available yet)
-            source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
-
-            if [ -f ${easystack_file} ]; then
-                echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..."
-                # we need to remove existing installation directories first,
-                # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R])
-                #  * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
-                rebuild_apps=$(${EB} --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
-                for app in ${rebuild_apps}; do
-                    app_dir=${EASYBUILD_INSTALLPATH}/software/${app}
-                    app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua
-                    echo_yellow "Removing ${app_dir} and ${app_module}..."
-                    rm -rf ${app_dir}
-                    rm -rf ${app_module}
-                done
-            else
-                fatal_error "Easystack file ${easystack_file} not found!"
-            fi
-        done
-    fi
-    # now rerun the script with a regular user account to do the software installations
-    #exec su eessi $( readlink -f "$0" ) -- "$@"
-    exit
-fi
-
 # install any additional required scripts
 # order is important: these are needed to install a full CUDA SDK in host_injections
 # for now, this just reinstalls all scripts. Note the most elegant, but works
diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
new file mode 100755
index 0000000000..89e5be977e
--- /dev/null
+++ b/EESSI-remove-software.sh
@@ -0,0 +1,189 @@
+#!/bin/bash
+#
+# Script to install EESSI software stack (version set through init/eessi_defaults)
+
+# see example parsing of command line arguments at
+#   https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop
+#   https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash
+
+display_help() {
+  echo "usage: $0 [OPTIONS]"
+  echo "  --build-logs-dir       -  location to copy EasyBuild logs to for failed builds"
+  echo "  -g | --generic         -  instructs script to build for generic architecture target"
+  echo "  -h | --help            -  display this usage information"
+  echo "  -x | --http-proxy URL  -  provides URL for the environment variable http_proxy"
+  echo "  -y | --https-proxy URL -  provides URL for the environment variable https_proxy"
+  echo "  --shared-fs-path       -  path to directory on shared filesystem that can be used"
+}
+
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -g|--generic)
+      EASYBUILD_OPTARCH="GENERIC"
+      shift
+      ;;
+    -h|--help)
+      display_help  # Call your function
+      # no shifting needed here, we're done.
+      exit 0
+      ;;
+    -x|--http-proxy)
+      export http_proxy="$2"
+      shift 2
+      ;;
+    -y|--https-proxy)
+      export https_proxy="$2"
+      shift 2
+      ;;
+    --build-logs-dir)
+      export build_logs_dir="${2}"
+      shift 2
+      ;;
+    --shared-fs-path)
+      export shared_fs_path="${2}"
+      shift 2
+      ;;
+    -*|--*)
+      echo "Error: Unknown option: $1" >&2
+      exit 1
+      ;;
+    *)  # No more options
+      POSITIONAL_ARGS+=("$1") # save positional arg
+      shift
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}"
+
+TOPDIR=$(dirname $(realpath $0))
+
+source $TOPDIR/scripts/utils.sh
+
+# honor $TMPDIR if it is already defined, use /tmp otherwise
+if [ -z $TMPDIR ]; then
+    export WORKDIR=/tmp/$USER
+else
+    export WORKDIR=$TMPDIR/$USER
+fi
+
+TMPDIR=$(mktemp -d)
+
+echo ">> Setting up environment..."
+
+source $TOPDIR/init/minimal_eessi_env
+
+if [ -d $EESSI_CVMFS_REPO ]; then
+    echo_green "$EESSI_CVMFS_REPO available, OK!"
+else
+    fatal_error "$EESSI_CVMFS_REPO is not available!"
+fi
+
+# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
+export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
+
+DETECTION_PARAMETERS=''
+GENERIC=0
+EB='eb'
+if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then
+    echo_yellow ">> GENERIC build requested, taking appropriate measures!"
+    DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic"
+    GENERIC=1
+    EB='eb --optarch=GENERIC'
+fi
+
+echo ">> Determining software subdirectory to use for current build host..."
+if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
+  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
+  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
+else
+  echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
+fi
+
+# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE)
+# $EESSI_SILENT - don't print any messages
+# $EESSI_BASIC_ENV - give a basic set of environment variables
+EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables
+
+if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then
+    fatal_error "Failed to determine software subdirectory?!"
+elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then
+    fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!"
+else
+    echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!"
+fi
+
+echo ">> Initializing Lmod..."
+source $EPREFIX/usr/share/Lmod/init/bash
+ml_version_out=$TMPDIR/ml.out
+ml --version &> $ml_version_out
+if [[ $? -eq 0 ]]; then
+    echo_green ">> Found Lmod ${LMOD_VERSION}"
+else
+    fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}"
+fi
+
+echo ">> Configuring EasyBuild..."
+source $TOPDIR/configure_easybuild
+
+if [ ! -z "${shared_fs_path}" ]; then
+    shared_eb_sourcepath=${shared_fs_path}/easybuild/sources
+    echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path"
+    export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH}
+fi
+
+echo ">> Setting up \$MODULEPATH..."
+# make sure no modules are loaded
+module --force purge
+# ignore current $MODULEPATH entirely
+module unuse $MODULEPATH
+module use $EASYBUILD_INSTALLPATH/modules/all
+if [[ -z ${MODULEPATH} ]]; then
+    fatal_error "Failed to set up \$MODULEPATH?!"
+else
+    echo_green ">> MODULEPATH set up: ${MODULEPATH}"
+fi
+
+# assume there's only one diff file that corresponds to the PR patch file
+pr_diff=$(ls [0-9]*.diff | head -1)
+
+# if this script is run as root, use PR patch file to determine if software needs to be removed first
+if [ $UID -eq 0 ]; then
+    changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/")
+    if [ -z ${changed_easystacks_rebuilds} ]; then
+        echo "No software needs to be removed."
+    else
+        for easystack_file in ${changed_easystacks_rebuilds}; do
+            # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
+            eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
+
+            # load EasyBuild module (will be installed if it's not available yet)
+            source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
+
+            if [ -f ${easystack_file} ]; then
+                echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..."
+                # we need to remove existing installation directories first,
+                # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R])
+                #  * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
+                rebuild_apps=$(${EB} --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
+                for app in ${rebuild_apps}; do
+                    app_dir=${EASYBUILD_INSTALLPATH}/software/${app}
+                    app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua
+                    echo_yellow "Removing ${app_dir} and ${app_module}..."
+                    rm -rf ${app_dir}
+                    rm -rf ${app_module}
+                done
+            else
+                fatal_error "Easystack file ${easystack_file} not found!"
+            fi
+        done
+    fi
+    # now rerun the script with a regular user account to do the software installations
+    #exec su eessi $( readlink -f "$0" ) -- "$@"
+    exit
+fi
+
+echo ">> Cleaning up ${TMPDIR}..."
+rm -r ${TMPDIR}
diff --git a/install_software_layer.sh b/install_software_layer.sh
index 85f856e86a..249140a503 100755
--- a/install_software_layer.sh
+++ b/install_software_layer.sh
@@ -2,7 +2,7 @@
 base_dir=$(dirname $(realpath $0))
 source ${base_dir}/init/eessi_defaults
 if [ $UID -eq 0 ]; then
-    ./EESSI-install-software.sh "$@"
+    ./EESSI-remove-software.sh "$@"
     exec runuser -u eessi $( readlink -f "$0" ) -- "$@"
 fi
 ./run_in_compat_layer_env.sh ./EESSI-install-software.sh "$@"

From 340b8332ffb858ba43613f38bd4ac23d260cc31e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 5 Mar 2024 14:55:32 +0100
Subject: [PATCH 095/154] remove su, add error message

---
 EESSI-remove-software.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index 89e5be977e..e55efe823e 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -180,10 +180,10 @@ if [ $UID -eq 0 ]; then
             fi
         done
     fi
-    # now rerun the script with a regular user account to do the software installations
-    #exec su eessi $( readlink -f "$0" ) -- "$@"
-    exit
+else
+    fatal_error "This script can only be run by root!"
 fi
 
-echo ">> Cleaning up ${TMPDIR}..."
-rm -r ${TMPDIR}
+
+#echo ">> Cleaning up ${TMPDIR}..."
+#rm -r ${TMPDIR}

From 88d21d9487272abbe951477be8727165d1c01795 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 5 Mar 2024 15:13:27 +0100
Subject: [PATCH 096/154] first run the EESSI-remove-software.sh script in a
 separate container session

---
 eessi_container.sh | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index bedb269cfd..09cec8964f 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -585,12 +585,12 @@ if [[ "${ACCESS}" == "ro" ]]; then
 fi
 
 if [[ "${ACCESS}" == "rw" ]]; then
-  if [[ ! -z ${JOB_CFG_FILE} ]]; then
+  #if [[ ! -z ${JOB_CFG_FILE} ]]; then
     # always launch build jobs triggered by the job with --fakeroot in order to be able to remove existing installations, see:
     # https://github.com/EESSI/software-layer/issues/312
     # we drop back to a regular user in the build script
-    ADDITIONAL_CONTAINER_OPTIONS+=("--fakeroot")
-  fi
+    #ADDITIONAL_CONTAINER_OPTIONS+=("--fakeroot")
+  #fi
   mkdir -p ${EESSI_TMPDIR}/overlay-upper
   mkdir -p ${EESSI_TMPDIR}/overlay-work
 
@@ -627,6 +627,12 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
+if [[ ! -z ${JOB_CFG_FILE} ]]; then
+    echo "Removing software by launching container with command (next line):"
+    echo "singularity ${RUN_QUIET} exec --fakeroot ${ADDITIONAL_CONTAINER_OPTIONS[@]} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ./EESSI-remove-software.sh"
+    singularity ${RUN_QUIET} exec --fakeroot "${ADDITIONAL_CONTAINER_OPTIONS[@]}" "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ./EESSI-remove-software.sh
+fi
+
 echo "Launching container with command (next line):"
 echo "singularity ${RUN_QUIET} ${MODE} ${ADDITIONAL_CONTAINER_OPTIONS[@]} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 singularity ${RUN_QUIET} ${MODE} "${ADDITIONAL_CONTAINER_OPTIONS[@]}" "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"

From 499328d70e3c6275eec9dcaf717a34afaf75a6e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 5 Mar 2024 15:14:47 +0100
Subject: [PATCH 097/154] undo

---
 install_software_layer.sh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/install_software_layer.sh b/install_software_layer.sh
index 249140a503..82ca70b73f 100755
--- a/install_software_layer.sh
+++ b/install_software_layer.sh
@@ -1,8 +1,4 @@
 #!/bin/bash
 base_dir=$(dirname $(realpath $0))
 source ${base_dir}/init/eessi_defaults
-if [ $UID -eq 0 ]; then
-    ./EESSI-remove-software.sh "$@"
-    exec runuser -u eessi $( readlink -f "$0" ) -- "$@"
-fi
 ./run_in_compat_layer_env.sh ./EESSI-install-software.sh "$@"

From 8100fed0e6a7105a4f1ffc08f6fca89f35145e0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 5 Mar 2024 15:32:27 +0100
Subject: [PATCH 098/154] dont use TMPDIR

---
 EESSI-remove-software.sh | 20 +++-----------------
 1 file changed, 3 insertions(+), 17 deletions(-)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index e55efe823e..91490507c6 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -62,15 +62,6 @@ TOPDIR=$(dirname $(realpath $0))
 
 source $TOPDIR/scripts/utils.sh
 
-# honor $TMPDIR if it is already defined, use /tmp otherwise
-if [ -z $TMPDIR ]; then
-    export WORKDIR=/tmp/$USER
-else
-    export WORKDIR=$TMPDIR/$USER
-fi
-
-TMPDIR=$(mktemp -d)
-
 echo ">> Setting up environment..."
 
 source $TOPDIR/init/minimal_eessi_env
@@ -82,7 +73,7 @@ else
 fi
 
 # avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
-export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
+#export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
 
 DETECTION_PARAMETERS=''
 GENERIC=0
@@ -117,12 +108,11 @@ fi
 
 echo ">> Initializing Lmod..."
 source $EPREFIX/usr/share/Lmod/init/bash
-ml_version_out=$TMPDIR/ml.out
-ml --version &> $ml_version_out
+ml --version
 if [[ $? -eq 0 ]]; then
     echo_green ">> Found Lmod ${LMOD_VERSION}"
 else
-    fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}"
+    fatal_error "Failed to initialize Lmod?!"
 fi
 
 echo ">> Configuring EasyBuild..."
@@ -183,7 +173,3 @@ if [ $UID -eq 0 ]; then
 else
     fatal_error "This script can only be run by root!"
 fi
-
-
-#echo ">> Cleaning up ${TMPDIR}..."
-#rm -r ${TMPDIR}

From c758d74bc70d386ea8c1134d5f9edf500b48db5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 5 Mar 2024 15:42:15 +0100
Subject: [PATCH 099/154] use EUID instead of UID

---
 EESSI-remove-software.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index 91490507c6..4cbc27be98 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -140,7 +140,7 @@ fi
 pr_diff=$(ls [0-9]*.diff | head -1)
 
 # if this script is run as root, use PR patch file to determine if software needs to be removed first
-if [ $UID -eq 0 ]; then
+if [ $EUID -eq 0 ]; then
     changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/")
     if [ -z ${changed_easystacks_rebuilds} ]; then
         echo "No software needs to be removed."

From 78cc503e43cd4731fb5a8c10c4d3499613cb09f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 5 Mar 2024 15:48:15 +0100
Subject: [PATCH 100/154] do set/override TMPDIR

---
 EESSI-remove-software.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index 4cbc27be98..819c08f8ce 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -60,6 +60,8 @@ set -- "${POSITIONAL_ARGS[@]}"
 
 TOPDIR=$(dirname $(realpath $0))
 
+export TMPDIR=$(mktemp -d /tmp/eessi-remove.XXXXXXXX)
+
 source $TOPDIR/scripts/utils.sh
 
 echo ">> Setting up environment..."

From c660a4259c7e391502167203e4a9bc31bbac692e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 8 Mar 2024 10:40:40 +0100
Subject: [PATCH 101/154] dont set fakeroot here

---
 eessi_container.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 09cec8964f..ad6d68640c 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -627,11 +627,11 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
-if [[ ! -z ${JOB_CFG_FILE} ]]; then
-    echo "Removing software by launching container with command (next line):"
-    echo "singularity ${RUN_QUIET} exec --fakeroot ${ADDITIONAL_CONTAINER_OPTIONS[@]} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ./EESSI-remove-software.sh"
-    singularity ${RUN_QUIET} exec --fakeroot "${ADDITIONAL_CONTAINER_OPTIONS[@]}" "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ./EESSI-remove-software.sh
-fi
+#if [[ ! -z ${JOB_CFG_FILE} ]]; then
+#    echo "Removing software by launching container with command (next line):"
+#    echo "singularity ${RUN_QUIET} exec --fakeroot ${ADDITIONAL_CONTAINER_OPTIONS[@]} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ./EESSI-remove-software.sh"
+#    singularity ${RUN_QUIET} exec --fakeroot "${ADDITIONAL_CONTAINER_OPTIONS[@]}" "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ./EESSI-remove-software.sh
+#fi
 
 echo "Launching container with command (next line):"
 echo "singularity ${RUN_QUIET} ${MODE} ${ADDITIONAL_CONTAINER_OPTIONS[@]} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"

From be5aa374e0aea043f97a709afae6fba58c556a59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 8 Mar 2024 10:40:51 +0100
Subject: [PATCH 102/154] run build step with fakeroot

---
 bot/build.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bot/build.sh b/bot/build.sh
index 1622e757e2..ef9b34ab88 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -181,6 +181,8 @@ BUILD_STEP_ARGS+=("--nvidia" "all")
 if [[ ! -z ${SHARED_FS_PATH} ]]; then
     BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections")
 fi
+# add fakeroot option in order to be able to remove software
+BUILD_STEP_ARGS+=("--fakeroot")
 
 # prepare arguments to install_software_layer.sh (specific to build step)
 declare -a INSTALL_SCRIPT_ARGS=()

From 4946b085eb6bff9515aa8e2257d3320f778ab6f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 8 Mar 2024 10:41:21 +0100
Subject: [PATCH 103/154] if run as root, run the removal script

---
 install_software_layer.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/install_software_layer.sh b/install_software_layer.sh
index 82ca70b73f..8137ec78e3 100755
--- a/install_software_layer.sh
+++ b/install_software_layer.sh
@@ -1,4 +1,8 @@
 #!/bin/bash
 base_dir=$(dirname $(realpath $0))
 source ${base_dir}/init/eessi_defaults
+if [ $EUID -eq 0 ]; then
+    ./EESSI-remove-software.sh "$@"
+    exec runuser -u eessi $( readlink -f "$0" ) -- "$@"
+fi
 ./run_in_compat_layer_env.sh ./EESSI-install-software.sh "$@"

From 123c47a84ff20e35c2fbe7dba6a7f371b9a1c694 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 8 Mar 2024 10:51:16 +0100
Subject: [PATCH 104/154] add fakeroot option

---
 eessi_container.sh | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/eessi_container.sh b/eessi_container.sh
index ad6d68640c..af8618f4b8 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -73,6 +73,7 @@ display_help() {
   echo "  -a | --access {ro,rw}  - ro (read-only), rw (read & write) [default: ro]"
   echo "  -c | --container IMG   - image file or URL defining the container to use"
   echo "                           [default: docker://ghcr.io/eessi/build-node:debian11]"
+  echo "  -f | --fakeroot        - run the container with --fakeroot [default: false]"
   echo "  -g | --storage DIR     - directory space on host machine (used for"
   echo "                           temporary data) [default: 1. TMPDIR, 2. /tmp]"
   echo "  -h | --help            - display this usage information [default: false]"
@@ -113,6 +114,7 @@ display_help() {
 ACCESS="ro"
 CONTAINER="docker://ghcr.io/eessi/build-node:debian11"
 #DRY_RUN=0
+FAKEROOT=0
 VERBOSE=0
 STORAGE=
 LIST_REPOS=0
@@ -140,6 +142,10 @@ while [[ $# -gt 0 ]]; do
 #      DRY_RUN=1
 #      shift 1
 #      ;;
+    -f|--fakeroot)
+      FAKEROOT=1
+      shift 1
+      ;;
     -g|--storage)
       STORAGE="$2"
       shift 2
@@ -460,6 +466,11 @@ if [[ ${SETUP_NVIDIA} -eq 1 ]]; then
     fi
 fi
 
+# Configure the fakeroot setting for the container
+if [[ ${FAKEROOT} -eq 1 ]]; then
+  ADDITIONAL_CONTAINER_OPTIONS+=("--fakeroot")
+fi
+
 # set up repository config (always create directory repos_cfg and populate it with info when
 # arg -r|--repository is used)
 mkdir -p ${EESSI_TMPDIR}/repos_cfg

From 93ccf68c0fcb5f48aeba2339f09bdf892d4547d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 8 Mar 2024 11:13:46 +0100
Subject: [PATCH 105/154] add echo

---
 install_software_layer.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/install_software_layer.sh b/install_software_layer.sh
index 8137ec78e3..c5cf40bb5a 100755
--- a/install_software_layer.sh
+++ b/install_software_layer.sh
@@ -1,6 +1,9 @@
 #!/bin/bash
 base_dir=$(dirname $(realpath $0))
 source ${base_dir}/init/eessi_defaults
+
+echo "User ID: $EUID"
+
 if [ $EUID -eq 0 ]; then
     ./EESSI-remove-software.sh "$@"
     exec runuser -u eessi $( readlink -f "$0" ) -- "$@"

From 017803bbf1705045b84a4bf8c8b4e479fffada5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 8 Mar 2024 11:18:57 +0100
Subject: [PATCH 106/154] add echo

---
 EESSI-install-software.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index ab91d2c6de..4d26b30b53 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -87,6 +87,8 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
+echo "Software installation user: $EUID $UID"
+
 set -- "${POSITIONAL_ARGS[@]}"
 
 TOPDIR=$(dirname $(realpath $0))

From c49240680190081b1e0cc6ac8b9e57bffde415c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 8 Mar 2024 11:51:42 +0100
Subject: [PATCH 107/154] add removal step

---
 bot/build.sh | 41 ++++++++++++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 11 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index ef9b34ab88..bc1cd27770 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -168,9 +168,34 @@ COMMON_ARGS+=("--mode" "run")
 # make sure to use the same parent dir for storing tarballs of tmp
 PREVIOUS_TMP_DIR=${PWD}/previous_tmp
 
-# prepare directory to store tarball of tmp for build step
+# prepare directory to store tarball of tmp for removal and build steps
+TARBALL_TMP_REMOVE_STEP_DIR=${PREVIOUS_TMP_DIR}/remove_step
 TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step
-mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR}
+mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} ${TARBALL_TMP_REMOVE_STEP_DIR}
+
+# prepare arguments to install_software_layer.sh (specific to build step)
+declare -a INSTALL_SCRIPT_ARGS=()
+if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
+    INSTALL_SCRIPT_ARGS+=("--generic")
+fi
+[[ ! -z ${BUILD_LOGS_DIR} ]] && INSTALL_SCRIPT_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}")
+[[ ! -z ${SHARED_FS_PATH} ]] && INSTALL_SCRIPT_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}")
+
+# prepare arguments to eessi_container.sh specific to remove step
+declare -a REMOVE_STEP_ARGS=()
+REMOVE_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
+REMOVE_STEP_ARGS+=("--storage" "${STORAGE}")
+# add fakeroot option in order to be able to remove software
+REMOVE_STEP_ARGS+=("--fakeroot")
+
+# create tmp file for output of removal step
+remove_outerr=$(mktemp remove.outerr.XXXX)
+
+echo "Executing command to remove software:"
+echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVE_STEP_ARGS[@]}"
+echo "                     -- ./install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${remove_outerr}"
+./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVE_STEP_ARGS[@]}" \
+                     -- ./install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${remove_outerr}
 
 # prepare arguments to eessi_container.sh specific to build step
 declare -a BUILD_STEP_ARGS=()
@@ -181,16 +206,10 @@ BUILD_STEP_ARGS+=("--nvidia" "all")
 if [[ ! -z ${SHARED_FS_PATH} ]]; then
     BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections")
 fi
-# add fakeroot option in order to be able to remove software
-BUILD_STEP_ARGS+=("--fakeroot")
 
-# prepare arguments to install_software_layer.sh (specific to build step)
-declare -a INSTALL_SCRIPT_ARGS=()
-if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
-    INSTALL_SCRIPT_ARGS+=("--generic")
-fi
-[[ ! -z ${BUILD_LOGS_DIR} ]] && INSTALL_SCRIPT_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}")
-[[ ! -z ${SHARED_FS_PATH} ]] && INSTALL_SCRIPT_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}")
+# determine temporary directory to resume from; this is important in case software was removed for a rebuild
+REMOVE_TMPDIR=$(grep ' as tmp directory ' ${remove_outerr} | cut -d ' ' -f 2)
+BUILD_STEP_ARGS+=("--resume" "${REMOVE_TMPDIR}")
 
 # create tmp file for output of build step
 build_outerr=$(mktemp build.outerr.XXXX)

From 7b57e8130ca9a03cc2fab3a8b74bda0c59d1c6e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 8 Mar 2024 11:52:05 +0100
Subject: [PATCH 108/154] remove removal step

---
 install_software_layer.sh | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/install_software_layer.sh b/install_software_layer.sh
index c5cf40bb5a..82ca70b73f 100755
--- a/install_software_layer.sh
+++ b/install_software_layer.sh
@@ -1,11 +1,4 @@
 #!/bin/bash
 base_dir=$(dirname $(realpath $0))
 source ${base_dir}/init/eessi_defaults
-
-echo "User ID: $EUID"
-
-if [ $EUID -eq 0 ]; then
-    ./EESSI-remove-software.sh "$@"
-    exec runuser -u eessi $( readlink -f "$0" ) -- "$@"
-fi
 ./run_in_compat_layer_env.sh ./EESSI-install-software.sh "$@"

From 485ff46bd4e87d2c9e35d1e42b12d8c884e782f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 8 Mar 2024 11:52:50 +0100
Subject: [PATCH 109/154] call EESSI-remove-software.sh for removal step

---
 bot/build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index bc1cd27770..208209ab28 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -193,9 +193,9 @@ remove_outerr=$(mktemp remove.outerr.XXXX)
 
 echo "Executing command to remove software:"
 echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVE_STEP_ARGS[@]}"
-echo "                     -- ./install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${remove_outerr}"
+echo "                     -- ./EESSI-remove-software.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${remove_outerr}"
 ./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVE_STEP_ARGS[@]}" \
-                     -- ./install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${remove_outerr}
+                     -- ./EESSI-remove-software.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${remove_outerr}
 
 # prepare arguments to eessi_container.sh specific to build step
 declare -a BUILD_STEP_ARGS=()

From 8204a529b4d73aa082bcc0a785555a3f1119fe70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 8 Mar 2024 12:00:49 +0100
Subject: [PATCH 110/154] remove echo command

---
 EESSI-install-software.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 4d26b30b53..ab91d2c6de 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -87,8 +87,6 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-echo "Software installation user: $EUID $UID"
-
 set -- "${POSITIONAL_ARGS[@]}"
 
 TOPDIR=$(dirname $(realpath $0))

From 15eb765405f65966f7c16c2756d7062bbcb4d4a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 8 Mar 2024 12:02:43 +0100
Subject: [PATCH 111/154] remove unused code

---
 eessi_container.sh | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index af8618f4b8..d084fe5fab 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -596,12 +596,6 @@ if [[ "${ACCESS}" == "ro" ]]; then
 fi
 
 if [[ "${ACCESS}" == "rw" ]]; then
-  #if [[ ! -z ${JOB_CFG_FILE} ]]; then
-    # always launch build jobs triggered by the job with --fakeroot in order to be able to remove existing installations, see:
-    # https://github.com/EESSI/software-layer/issues/312
-    # we drop back to a regular user in the build script
-    #ADDITIONAL_CONTAINER_OPTIONS+=("--fakeroot")
-  #fi
   mkdir -p ${EESSI_TMPDIR}/overlay-upper
   mkdir -p ${EESSI_TMPDIR}/overlay-work
 
@@ -638,12 +632,6 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
-#if [[ ! -z ${JOB_CFG_FILE} ]]; then
-#    echo "Removing software by launching container with command (next line):"
-#    echo "singularity ${RUN_QUIET} exec --fakeroot ${ADDITIONAL_CONTAINER_OPTIONS[@]} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ./EESSI-remove-software.sh"
-#    singularity ${RUN_QUIET} exec --fakeroot "${ADDITIONAL_CONTAINER_OPTIONS[@]}" "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ./EESSI-remove-software.sh
-#fi
-
 echo "Launching container with command (next line):"
 echo "singularity ${RUN_QUIET} ${MODE} ${ADDITIONAL_CONTAINER_OPTIONS[@]} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 singularity ${RUN_QUIET} ${MODE} "${ADDITIONAL_CONTAINER_OPTIONS[@]}" "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"

From 1f7501f2e1aad58abb5d85430d8fed95aaa55576 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 8 Mar 2024 13:02:14 +0100
Subject: [PATCH 112/154] rename remove to removal, resume tarball step from
 removal dir

---
 bot/build.sh | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 208209ab28..8f3439b3ad 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -169,9 +169,9 @@ COMMON_ARGS+=("--mode" "run")
 PREVIOUS_TMP_DIR=${PWD}/previous_tmp
 
 # prepare directory to store tarball of tmp for removal and build steps
-TARBALL_TMP_REMOVE_STEP_DIR=${PREVIOUS_TMP_DIR}/remove_step
+TARBALL_TMP_REMOVAL_STEP_DIR=${PREVIOUS_TMP_DIR}/removal_step
 TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step
-mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} ${TARBALL_TMP_REMOVE_STEP_DIR}
+mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} ${TARBALL_TMP_REMOVAL_STEP_DIR}
 
 # prepare arguments to install_software_layer.sh (specific to build step)
 declare -a INSTALL_SCRIPT_ARGS=()
@@ -182,20 +182,20 @@ fi
 [[ ! -z ${SHARED_FS_PATH} ]] && INSTALL_SCRIPT_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}")
 
 # prepare arguments to eessi_container.sh specific to remove step
-declare -a REMOVE_STEP_ARGS=()
-REMOVE_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
-REMOVE_STEP_ARGS+=("--storage" "${STORAGE}")
+declare -a REMOVAL_STEP_ARGS=()
+REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
+REMOVAL_STEP_ARGS+=("--storage" "${STORAGE}")
 # add fakeroot option in order to be able to remove software
-REMOVE_STEP_ARGS+=("--fakeroot")
+REMOVAL_STEP_ARGS+=("--fakeroot")
 
 # create tmp file for output of removal step
-remove_outerr=$(mktemp remove.outerr.XXXX)
+removal_outerr=$(mktemp remove.outerr.XXXX)
 
 echo "Executing command to remove software:"
-echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVE_STEP_ARGS[@]}"
-echo "                     -- ./EESSI-remove-software.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${remove_outerr}"
-./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVE_STEP_ARGS[@]}" \
-                     -- ./EESSI-remove-software.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${remove_outerr}
+echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVAL_STEP_ARGS[@]}"
+echo "                     -- ./EESSI-remove-software.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}"
+./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVAL_STEP_ARGS[@]}" \
+                     -- ./EESSI-remove-software.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr}
 
 # prepare arguments to eessi_container.sh specific to build step
 declare -a BUILD_STEP_ARGS=()
@@ -208,8 +208,8 @@ if [[ ! -z ${SHARED_FS_PATH} ]]; then
 fi
 
 # determine temporary directory to resume from; this is important in case software was removed for a rebuild
-REMOVE_TMPDIR=$(grep ' as tmp directory ' ${remove_outerr} | cut -d ' ' -f 2)
-BUILD_STEP_ARGS+=("--resume" "${REMOVE_TMPDIR}")
+REMOVAL_TMPDIR=$(grep ' as tmp directory ' ${removal_outerr} | cut -d ' ' -f 2)
+BUILD_STEP_ARGS+=("--resume" "${REMOVAL_TMPDIR}")
 
 # create tmp file for output of build step
 build_outerr=$(mktemp build.outerr.XXXX)
@@ -232,8 +232,7 @@ declare -a TARBALL_STEP_ARGS=()
 TARBALL_STEP_ARGS+=("--save" "${TARBALL_TMP_TARBALL_STEP_DIR}")
 
 # determine temporary directory to resume from
-BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)
-TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}")
+TARBALL_STEP_ARGS+=("--resume" "${REMOVAL_TMPDIR}")
 
 timestamp=$(date +%s)
 # to set EESSI_VERSION we need to source init/eessi_defaults now

From 333762b9c1ddab70721826b70a1d432f6d043c49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Mon, 11 Mar 2024 16:41:19 +0100
Subject: [PATCH 113/154] determine whether or not the removal step has to be
 run

---
 bot/build.sh | 60 +++++++++++++++++++++++++++++++---------------------
 1 file changed, 36 insertions(+), 24 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 8f3439b3ad..31d9b576fe 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -168,12 +168,8 @@ COMMON_ARGS+=("--mode" "run")
 # make sure to use the same parent dir for storing tarballs of tmp
 PREVIOUS_TMP_DIR=${PWD}/previous_tmp
 
-# prepare directory to store tarball of tmp for removal and build steps
-TARBALL_TMP_REMOVAL_STEP_DIR=${PREVIOUS_TMP_DIR}/removal_step
-TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step
-mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} ${TARBALL_TMP_REMOVAL_STEP_DIR}
-
 # prepare arguments to install_software_layer.sh (specific to build step)
+declare -a BUILD_STEP_ARGS=()
 declare -a INSTALL_SCRIPT_ARGS=()
 if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
     INSTALL_SCRIPT_ARGS+=("--generic")
@@ -181,24 +177,44 @@ fi
 [[ ! -z ${BUILD_LOGS_DIR} ]] && INSTALL_SCRIPT_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}")
 [[ ! -z ${SHARED_FS_PATH} ]] && INSTALL_SCRIPT_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}")
 
-# prepare arguments to eessi_container.sh specific to remove step
-declare -a REMOVAL_STEP_ARGS=()
-REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
-REMOVAL_STEP_ARGS+=("--storage" "${STORAGE}")
-# add fakeroot option in order to be able to remove software
-REMOVAL_STEP_ARGS+=("--fakeroot")
-
-# create tmp file for output of removal step
-removal_outerr=$(mktemp remove.outerr.XXXX)
+# determine if the removal step has to be run
+# assume there's only one diff file that corresponds to the PR patch file
+pr_diff=$(ls [0-9]*.diff | head -1)
+changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | grep "/rebuilds/")
+if [[ -z ${changed_easystacks_rebuilds} ]]; then
+    echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step."
+else
+    # prepare directory to store tarball of tmp for removal and build steps
+    TARBALL_TMP_REMOVAL_STEP_DIR=${PREVIOUS_TMP_DIR}/removal_step
+    mkdir -p ${TARBALL_TMP_REMOVAL_STEP_DIR}
+
+    # prepare arguments to eessi_container.sh specific to remove step
+    declare -a REMOVAL_STEP_ARGS=()
+    REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
+    REMOVAL_STEP_ARGS+=("--storage" "${STORAGE}")
+    # add fakeroot option in order to be able to remove software
+    REMOVAL_STEP_ARGS+=("--fakeroot")
+
+    # create tmp file for output of removal step
+    removal_outerr=$(mktemp remove.outerr.XXXX)
+
+    echo "Executing command to remove software:"
+    echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVAL_STEP_ARGS[@]}"
+    echo "                     -- ./EESSI-remove-software.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}"
+    ./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVAL_STEP_ARGS[@]}" \
+                         -- ./EESSI-remove-software.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr}
+
+    # make sure that the build step resumes from the same temporary directory
+    # this is important, as otherwise the removed software will still be there
+    REMOVAL_TMPDIR=$(grep ' as tmp directory ' ${removal_outerr} | cut -d ' ' -f 2)
+    BUILD_STEP_ARGS+=("--resume" "${REMOVAL_TMPDIR}")
+fi
 
-echo "Executing command to remove software:"
-echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVAL_STEP_ARGS[@]}"
-echo "                     -- ./EESSI-remove-software.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}"
-./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVAL_STEP_ARGS[@]}" \
-                     -- ./EESSI-remove-software.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr}
+# prepare directory to store tarball of tmp for build step
+TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step
+mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR}
 
 # prepare arguments to eessi_container.sh specific to build step
-declare -a BUILD_STEP_ARGS=()
 BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
 BUILD_STEP_ARGS+=("--storage" "${STORAGE}")
 # add options required to handle NVIDIA support
@@ -207,10 +223,6 @@ if [[ ! -z ${SHARED_FS_PATH} ]]; then
     BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections")
 fi
 
-# determine temporary directory to resume from; this is important in case software was removed for a rebuild
-REMOVAL_TMPDIR=$(grep ' as tmp directory ' ${removal_outerr} | cut -d ' ' -f 2)
-BUILD_STEP_ARGS+=("--resume" "${REMOVAL_TMPDIR}")
-
 # create tmp file for output of build step
 build_outerr=$(mktemp build.outerr.XXXX)
 

From 0b826ae64a31e0b2463c6ea65c9c12e1701922ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 12 Mar 2024 16:53:26 +0100
Subject: [PATCH 114/154] remove proxy arguments

Co-authored-by: Kenneth Hoste <kenneth.hoste@ugent.be>
---
 EESSI-remove-software.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index 819c08f8ce..4786d60fc0 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -11,8 +11,6 @@ display_help() {
   echo "  --build-logs-dir       -  location to copy EasyBuild logs to for failed builds"
   echo "  -g | --generic         -  instructs script to build for generic architecture target"
   echo "  -h | --help            -  display this usage information"
-  echo "  -x | --http-proxy URL  -  provides URL for the environment variable http_proxy"
-  echo "  -y | --https-proxy URL -  provides URL for the environment variable https_proxy"
   echo "  --shared-fs-path       -  path to directory on shared filesystem that can be used"
 }
 

From 0ed12492ed91e0ada5dd142536927a5edc4a817f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 12 Mar 2024 16:58:17 +0100
Subject: [PATCH 115/154] remove code for setting http(s)_proxy

Co-authored-by: Kenneth Hoste <kenneth.hoste@ugent.be>
---
 EESSI-remove-software.sh | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index 4786d60fc0..e995f55a45 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -27,14 +27,6 @@ while [[ $# -gt 0 ]]; do
       # no shifting needed here, we're done.
       exit 0
       ;;
-    -x|--http-proxy)
-      export http_proxy="$2"
-      shift 2
-      ;;
-    -y|--https-proxy)
-      export https_proxy="$2"
-      shift 2
-      ;;
     --build-logs-dir)
       export build_logs_dir="${2}"
       shift 2

From bc4fd890f9ae68b094b164dc85ed217eef420622 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 12 Mar 2024 16:58:42 +0100
Subject: [PATCH 116/154] no need to set build logs dir

Co-authored-by: Kenneth Hoste <kenneth.hoste@ugent.be>
---
 EESSI-remove-software.sh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index e995f55a45..d3823c7036 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -27,10 +27,6 @@ while [[ $# -gt 0 ]]; do
       # no shifting needed here, we're done.
       exit 0
       ;;
-    --build-logs-dir)
-      export build_logs_dir="${2}"
-      shift 2
-      ;;
     --shared-fs-path)
       export shared_fs_path="${2}"
       shift 2

From fd7a9eadd21fd386beaceb0af211dd2f523a6669 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 12 Mar 2024 16:59:52 +0100
Subject: [PATCH 117/154] remove commented code

Co-authored-by: ocaisa <alan.ocais@cecam.org>
---
 EESSI-remove-software.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index d3823c7036..02e91a43d1 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -60,9 +60,6 @@ else
     fatal_error "$EESSI_CVMFS_REPO is not available!"
 fi
 
-# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
-#export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
-
 DETECTION_PARAMETERS=''
 GENERIC=0
 EB='eb'

From 994d573759c1c7707952a9d9a125cc6e1d3df47b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 12 Mar 2024 17:00:27 +0100
Subject: [PATCH 118/154] fix comment about purpose of script

Co-authored-by: ocaisa <alan.ocais@cecam.org>
---
 EESSI-remove-software.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index 02e91a43d1..f111be339e 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Script to install EESSI software stack (version set through init/eessi_defaults)
+# Script to remove part of the EESSI software stack (version set through init/eessi_defaults)
 
 # see example parsing of command line arguments at
 #   https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop

From 3d7a959d0a1e62eb4c29c7b6742b82d5eaff4321 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 12 Mar 2024 17:02:10 +0100
Subject: [PATCH 119/154] remove code and parameter for setting shared fs path

---
 EESSI-remove-software.sh | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index f111be339e..1779a7ed58 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -11,7 +11,6 @@ display_help() {
   echo "  --build-logs-dir       -  location to copy EasyBuild logs to for failed builds"
   echo "  -g | --generic         -  instructs script to build for generic architecture target"
   echo "  -h | --help            -  display this usage information"
-  echo "  --shared-fs-path       -  path to directory on shared filesystem that can be used"
 }
 
 POSITIONAL_ARGS=()
@@ -27,10 +26,6 @@ while [[ $# -gt 0 ]]; do
       # no shifting needed here, we're done.
       exit 0
       ;;
-    --shared-fs-path)
-      export shared_fs_path="${2}"
-      shift 2
-      ;;
     -*|--*)
       echo "Error: Unknown option: $1" >&2
       exit 1
@@ -103,12 +98,6 @@ fi
 echo ">> Configuring EasyBuild..."
 source $TOPDIR/configure_easybuild
 
-if [ ! -z "${shared_fs_path}" ]; then
-    shared_eb_sourcepath=${shared_fs_path}/easybuild/sources
-    echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path"
-    export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH}
-fi
-
 echo ">> Setting up \$MODULEPATH..."
 # make sure no modules are loaded
 module --force purge

From a148920b35a4a70d03ce5f0175b3c730ae279db0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Wed, 13 Mar 2024 22:35:09 +0100
Subject: [PATCH 120/154] remove openmpi hook that provided a temporary
 workaround

---
 create_lmodrc.py | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/create_lmodrc.py b/create_lmodrc.py
index bc69dd4396..0e738a530e 100755
--- a/create_lmodrc.py
+++ b/create_lmodrc.py
@@ -94,28 +94,7 @@
     end
 end
 
-local function openmpi_load_hook(t)
-    -- disable smcuda BTL when loading OpenMPI module for aarch64/neoverse_v1,
-    -- to work around hang/crash due to bug in OpenMPI;
-    -- see https://gitlab.com/eessi/support/-/issues/41
-    local frameStk = require("FrameStk"):singleton()
-    local mt = frameStk:mt()
-    local moduleName = string.match(t.modFullName, "(.-)/")
-    local cpuTarget = os.getenv("EESSI_SOFTWARE_SUBDIR") or ""
-    if (moduleName == "OpenMPI") and (cpuTarget == "aarch64/neoverse_v1") then
-        local msg = "Adding '^smcuda' to $OMPI_MCA_btl to work around bug in OpenMPI"
-        LmodMessage(msg .. " (see https://gitlab.com/eessi/support/-/issues/41)")
-	local ompiMcaBtl = os.getenv("OMPI_MCA_btl")
-	if ompiMcaBtl == nil then
-            setenv("OMPI_MCA_btl", "^smcuda")
-        else
-            setenv("OMPI_MCA_btl", ompiMcaBtl .. ",^smcuda")
-	end
-    end
-end
-
 hook.register("load", cuda_enabled_load_hook)
-hook.register("load", openmpi_load_hook)
 """
 
 def error(msg):

From 5d4542e5a73962da0032427bba7ee7ed19b79ef0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Wed, 13 Mar 2024 22:38:33 +0100
Subject: [PATCH 121/154] remove hook.register for cuda hook

---
 create_lmodrc.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/create_lmodrc.py b/create_lmodrc.py
index 2175f885d2..133743c844 100755
--- a/create_lmodrc.py
+++ b/create_lmodrc.py
@@ -94,8 +94,6 @@
     end
 end
 
-hook.register("load", cuda_enabled_load_hook)
-
 -- Combine both functions into a single one, as we can only register one function as load hook in lmod
 -- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed
 function eessi_load_hook(t)

From 9d62787eb90dcd6a42e5c9e8b9f51f87c4d84d31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 26 Mar 2024 09:19:44 +0100
Subject: [PATCH 122/154] source init/bash instead of minimal eessi env

---
 EESSI-remove-software.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index 1779a7ed58..010ee365af 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -47,7 +47,7 @@ source $TOPDIR/scripts/utils.sh
 
 echo ">> Setting up environment..."
 
-source $TOPDIR/init/minimal_eessi_env
+source $TOPDIR/init/bash
 
 if [ -d $EESSI_CVMFS_REPO ]; then
     echo_green "$EESSI_CVMFS_REPO available, OK!"

From 229ce93ac4c2db86b8fbd29da38f1e78f2d5c167 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 26 Mar 2024 09:20:35 +0100
Subject: [PATCH 123/154] remove GENERIC variable

---
 EESSI-remove-software.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index 010ee365af..6d0c9e3800 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -56,12 +56,10 @@ else
 fi
 
 DETECTION_PARAMETERS=''
-GENERIC=0
 EB='eb'
 if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then
     echo_yellow ">> GENERIC build requested, taking appropriate measures!"
     DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic"
-    GENERIC=1
     EB='eb --optarch=GENERIC'
 fi
 

From 977bcaad3f5ba29d5d38ef95dc5f2acc782c573b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 26 Mar 2024 09:26:41 +0100
Subject: [PATCH 124/154] clean up eb optarch=generic settings

---
 EESSI-remove-software.sh | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index 6d0c9e3800..cdf4ae3b7b 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -18,7 +18,9 @@ POSITIONAL_ARGS=()
 while [[ $# -gt 0 ]]; do
   case $1 in
     -g|--generic)
+      echo_yellow ">> GENERIC build requested, taking appropriate measures!"
       EASYBUILD_OPTARCH="GENERIC"
+      DETECTION_PARAMETERS="--generic"
       shift
       ;;
     -h|--help)
@@ -55,14 +57,6 @@ else
     fatal_error "$EESSI_CVMFS_REPO is not available!"
 fi
 
-DETECTION_PARAMETERS=''
-EB='eb'
-if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then
-    echo_yellow ">> GENERIC build requested, taking appropriate measures!"
-    DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic"
-    EB='eb --optarch=GENERIC'
-fi
-
 echo ">> Determining software subdirectory to use for current build host..."
 if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
   export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
@@ -129,7 +123,7 @@ if [ $EUID -eq 0 ]; then
                 # we need to remove existing installation directories first,
                 # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R])
                 #  * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
-                rebuild_apps=$(${EB} --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
+                rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
                 for app in ${rebuild_apps}; do
                     app_dir=${EASYBUILD_INSTALLPATH}/software/${app}
                     app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua

From f1cfc1b1f4f9f32d3669b46af781b2ab648d9c52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 26 Mar 2024 09:38:48 +0100
Subject: [PATCH 125/154] source init/bash and clean up redundant code

---
 EESSI-remove-software.sh | 30 ++++++++----------------------
 1 file changed, 8 insertions(+), 22 deletions(-)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index cdf4ae3b7b..c1f0bcb3d0 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -47,6 +47,14 @@ export TMPDIR=$(mktemp -d /tmp/eessi-remove.XXXXXXXX)
 
 source $TOPDIR/scripts/utils.sh
 
+echo ">> Determining software subdirectory to use for current build host..."
+if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
+  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
+  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
+else
+  echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
+fi
+
 echo ">> Setting up environment..."
 
 source $TOPDIR/init/bash
@@ -57,19 +65,6 @@ else
     fatal_error "$EESSI_CVMFS_REPO is not available!"
 fi
 
-echo ">> Determining software subdirectory to use for current build host..."
-if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
-  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
-  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
-else
-  echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
-fi
-
-# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE)
-# $EESSI_SILENT - don't print any messages
-# $EESSI_BASIC_ENV - give a basic set of environment variables
-EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables
-
 if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then
     fatal_error "Failed to determine software subdirectory?!"
 elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then
@@ -78,15 +73,6 @@ else
     echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!"
 fi
 
-echo ">> Initializing Lmod..."
-source $EPREFIX/usr/share/Lmod/init/bash
-ml --version
-if [[ $? -eq 0 ]]; then
-    echo_green ">> Found Lmod ${LMOD_VERSION}"
-else
-    fatal_error "Failed to initialize Lmod?!"
-fi
-
 echo ">> Configuring EasyBuild..."
 source $TOPDIR/configure_easybuild
 

From aa4a82dd3c87eccf155715eee9797532f9ed8dd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 26 Mar 2024 09:48:48 +0100
Subject: [PATCH 126/154] set EB variable

---
 EESSI-remove-software.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index c1f0bcb3d0..9993986904 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -74,6 +74,7 @@ else
 fi
 
 echo ">> Configuring EasyBuild..."
+EB="eb"
 source $TOPDIR/configure_easybuild
 
 echo ">> Setting up \$MODULEPATH..."

From 6ca6a70671a019db0e83f54cb74e6049cba2a0f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 26 Mar 2024 09:57:32 +0100
Subject: [PATCH 127/154] remove build-dir argument from display_help

---
 EESSI-remove-software.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index 9993986904..10cea933db 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -8,7 +8,6 @@
 
 display_help() {
   echo "usage: $0 [OPTIONS]"
-  echo "  --build-logs-dir       -  location to copy EasyBuild logs to for failed builds"
   echo "  -g | --generic         -  instructs script to build for generic architecture target"
   echo "  -h | --help            -  display this usage information"
 }

From e89cb05d3bdcdb8df9c16cfdcea24afa81f8abf1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 26 Mar 2024 10:08:12 +0100
Subject: [PATCH 128/154] export the EASYBUILD_OPTARCH variable

Co-authored-by: Kenneth Hoste <kenneth.hoste@ugent.be>
---
 EESSI-remove-software.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index 10cea933db..f465aff304 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -18,7 +18,7 @@ while [[ $# -gt 0 ]]; do
   case $1 in
     -g|--generic)
       echo_yellow ">> GENERIC build requested, taking appropriate measures!"
-      EASYBUILD_OPTARCH="GENERIC"
+      export EASYBUILD_OPTARCH="GENERIC"
       DETECTION_PARAMETERS="--generic"
       shift
       ;;

From 502c02311cae893ef7983b340481c47684716997 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 26 Mar 2024 10:19:46 +0100
Subject: [PATCH 129/154] remove EASYBUILD_OPTARCH

---
 EESSI-remove-software.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index f465aff304..1cd291b16c 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -18,7 +18,6 @@ while [[ $# -gt 0 ]]; do
   case $1 in
     -g|--generic)
       echo_yellow ">> GENERIC build requested, taking appropriate measures!"
-      export EASYBUILD_OPTARCH="GENERIC"
       DETECTION_PARAMETERS="--generic"
       shift
       ;;

From 3206a0ffa0b8759d8db5d67b5c3525d8b25ed674 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 26 Mar 2024 10:24:19 +0100
Subject: [PATCH 130/154] remove echo_yellow call for generic builds

---
 EESSI-remove-software.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh
index 1cd291b16c..446a156cb8 100755
--- a/EESSI-remove-software.sh
+++ b/EESSI-remove-software.sh
@@ -17,7 +17,6 @@ POSITIONAL_ARGS=()
 while [[ $# -gt 0 ]]; do
   case $1 in
     -g|--generic)
-      echo_yellow ">> GENERIC build requested, taking appropriate measures!"
       DETECTION_PARAMETERS="--generic"
       shift
       ;;

From ee1c2cefd1b6461aa675288bef0cd2faa1163d31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 26 Mar 2024 10:27:20 +0100
Subject: [PATCH 131/154] use separate variable for removal script arguments

---
 bot/build.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 31d9b576fe..c76285faf4 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -171,8 +171,10 @@ PREVIOUS_TMP_DIR=${PWD}/previous_tmp
 # prepare arguments to install_software_layer.sh (specific to build step)
 declare -a BUILD_STEP_ARGS=()
 declare -a INSTALL_SCRIPT_ARGS=()
+declare -a REMOVAL_SCRIPT_ARGS=()
 if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
     INSTALL_SCRIPT_ARGS+=("--generic")
+    REMOVAL_SCRIPT_ARGS+=("--generic")
 fi
 [[ ! -z ${BUILD_LOGS_DIR} ]] && INSTALL_SCRIPT_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}")
 [[ ! -z ${SHARED_FS_PATH} ]] && INSTALL_SCRIPT_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}")
@@ -200,9 +202,9 @@ else
 
     echo "Executing command to remove software:"
     echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVAL_STEP_ARGS[@]}"
-    echo "                     -- ./EESSI-remove-software.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}"
+    echo "                     -- ./EESSI-remove-software.sh \"${REMOVAL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}"
     ./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVAL_STEP_ARGS[@]}" \
-                         -- ./EESSI-remove-software.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr}
+                         -- ./EESSI-remove-software.sh "${REMOVAL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr}
 
     # make sure that the build step resumes from the same temporary directory
     # this is important, as otherwise the removed software will still be there

From 330db7a02cf011c2115babf598991217f87bd022 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Mar 2024 10:32:49 +0100
Subject: [PATCH 132/154] also dump status to test result file

---
 bot/check-test.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 4b5f7575e5..f045b9500a 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -200,5 +200,6 @@ comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
 
 # Actually writing the comment description to the result file
 echo "${comment_description}" >> ${job_test_result_file}
+echo "status = ${status}" >> ${job_test_result_file}
 
 exit 0

From f3c62077c40e299bb6a3f157adfd2c1f5a32d067 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Mar 2024 10:33:24 +0100
Subject: [PATCH 133/154] add Caspar as author in `bot/test.sh` script

---
 bot/test.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bot/test.sh b/bot/test.sh
index 89c1a6a8bf..66c2cef129 100755
--- a/bot/test.sh
+++ b/bot/test.sh
@@ -8,6 +8,7 @@
 # https://github.com/EESSI/software-layer.git
 #
 # author: Thomas Roeblitz (@trz42)
+# author: Caspar van Leeuwen (@casparvl)
 #
 # license: GPLv2
 #

From c55ee6b6075786fd0dfd657ba3e2e667a215b9da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 26 Mar 2024 10:33:44 +0100
Subject: [PATCH 134/154] clarify why we need fakeroot, add link to issue

---
 bot/build.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bot/build.sh b/bot/build.sh
index c76285faf4..b007b80172 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -194,7 +194,8 @@ else
     declare -a REMOVAL_STEP_ARGS=()
     REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
     REMOVAL_STEP_ARGS+=("--storage" "${STORAGE}")
-    # add fakeroot option in order to be able to remove software
+    # add fakeroot option in order to be able to remove software, see:
+    # https://github.com/EESSI/software-layer/issues/312
     REMOVAL_STEP_ARGS+=("--fakeroot")
 
     # create tmp file for output of removal step

From 13072df707b96cad40b4d21cbedd6666af756b0d Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Mar 2024 10:34:09 +0100
Subject: [PATCH 135/154] use read-only mode to access software installations
 via container

---
 bot/test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/test.sh b/bot/test.sh
index 66c2cef129..925b91d49a 100755
--- a/bot/test.sh
+++ b/bot/test.sh
@@ -180,7 +180,7 @@ echo "bot/test.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'"
 # prepare arguments to eessi_container.sh common to build and tarball steps
 declare -a COMMON_ARGS=()
 COMMON_ARGS+=("--verbose")
-COMMON_ARGS+=("--access" "rw")
+COMMON_ARGS+=("--access" "ro")
 COMMON_ARGS+=("--mode" "run")
 [[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}")
 [[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}")

From 1fa323ada6d0fddc312ad102cba8619d073f9fb0 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Mar 2024 10:34:53 +0100
Subject: [PATCH 136/154] fix typo: build -> test

---
 bot/test.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bot/test.sh b/bot/test.sh
index 925b91d49a..4984340e6e 100755
--- a/bot/test.sh
+++ b/bot/test.sh
@@ -190,7 +190,7 @@ COMMON_ARGS+=("--mode" "run")
 # make sure to use the same parent dir for storing tarballs of tmp
 PREVIOUS_TMP_DIR=${PWD}/previous_tmp
 
-# prepare directory to store tarball of tmp for build step
+# prepare directory to store tarball of tmp for test step
 TARBALL_TMP_TEST_STEP_DIR=${PREVIOUS_TMP_DIR}/test_step
 mkdir -p ${TARBALL_TMP_TEST_STEP_DIR}
 
@@ -216,7 +216,7 @@ fi
 # create tmp file for output of build step
 test_outerr=$(mktemp test.outerr.XXXX)
 
-echo "Executing command to build software:"
+echo "Executing command to test software:"
 echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TEST_STEP_ARGS[@]}"
 echo "                     -- ./run_tests.sh \"${TEST_SUITE_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${test_outerr}"
 ./eessi_container.sh "${COMMON_ARGS[@]}" "${TEST_STEP_ARGS[@]}" \

From 8d05473eb5eedf0cdaba75f6b4d4a40f38a3182d Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Mar 2024 10:35:30 +0100
Subject: [PATCH 137/154] disable CPU auto-detection in template ReFrame
 configuration script used for running test suite

---
 reframe_config_bot.py.tmpl | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/reframe_config_bot.py.tmpl b/reframe_config_bot.py.tmpl
index b2f7916e79..0cc3e9f530 100644
--- a/reframe_config_bot.py.tmpl
+++ b/reframe_config_bot.py.tmpl
@@ -51,9 +51,8 @@ site_configuration = {
         {
             'purge_environment': True,
             'resolve_module_conflicts': False,  # avoid loading the module before submitting the job
-            # Enable automatic detection of CPU architecture
-            # See https://reframe-hpc.readthedocs.io/en/stable/configure.html#auto-detecting-processor-information
-            'remote_detect': True,
+            # disable automatic detection of CPU architecture (since we're using local scheduler)
+            'remote_detect': False,
         }
     ],
     'logging': common_logging_config(),

From c53f578687e43ae0c7876c3a9d5c48fc2432ad49 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Mar 2024 10:37:09 +0100
Subject: [PATCH 138/154] pass down --generic to script used for running test
 suite, so tests are run with generic software installations when intended

---
 run_tests.sh  | 2 +-
 test_suite.sh | 9 ++++-----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/run_tests.sh b/run_tests.sh
index de7fd8c2e1..1dbb47db9d 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -21,4 +21,4 @@ source ${base_dir}/init/eessi_defaults
 ./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite"
 
 # Run the test suite
-./test_suite.sh
+./test_suite.sh "$@"
diff --git a/test_suite.sh b/test_suite.sh
index e3bab04aec..95eb9daa2a 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -23,7 +23,7 @@ POSITIONAL_ARGS=()
 while [[ $# -gt 0 ]]; do
   case $1 in
     -g|--generic)
-      EASYBUILD_OPTARCH="GENERIC"
+      DETECTION_PARAMETERS="--generic"
       shift
       ;;
     -h|--help)
@@ -75,10 +75,9 @@ TMPDIR=$(mktemp -d)
 
 echo ">> Setting up environment..."
 module --force purge
-# Make sure defaults are set for EESSI_CVMFS_REPO and EESSI_VERSION, before initializing EESSI
-source $TOPDIR/init/eessi_defaults
-# Initialize EESSI
-source ${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/bash
+export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
+
+source $TOPDIR/init/bash
 
 # Load the ReFrame module
 # Currently, we load the default version. Maybe we should somehow make this configurable in the future?

From 57d8b6fd4eceaf951c69cac9f325d5aa5eb113ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 26 Mar 2024 10:54:13 +0100
Subject: [PATCH 139/154] remove openmpi hook for smcuda workaround

---
 create_lmodsitepackage.py | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py
index 9a4a232863..5a7a915494 100755
--- a/create_lmodsitepackage.py
+++ b/create_lmodsitepackage.py
@@ -84,31 +84,10 @@
     end
 end
 
-local function eessi_openmpi_load_hook(t)
-    -- disable smcuda BTL when loading OpenMPI module for aarch64/neoverse_v1,
-    -- to work around hang/crash due to bug in OpenMPI;
-    -- see https://gitlab.com/eessi/support/-/issues/41
-    local frameStk = require("FrameStk"):singleton()
-    local mt = frameStk:mt()
-    local moduleName = string.match(t.modFullName, "(.-)/")
-    local cpuTarget = os.getenv("EESSI_SOFTWARE_SUBDIR") or ""
-    if (moduleName == "OpenMPI") and (cpuTarget == "aarch64/neoverse_v1") then
-        local msg = "Adding '^smcuda' to $OMPI_MCA_btl to work around bug in OpenMPI"
-        LmodMessage(msg .. " (see https://gitlab.com/eessi/support/-/issues/41)")
-	local ompiMcaBtl = os.getenv("OMPI_MCA_btl")
-	if ompiMcaBtl == nil then
-            setenv("OMPI_MCA_btl", "^smcuda")
-        else
-            setenv("OMPI_MCA_btl", ompiMcaBtl .. ",^smcuda")
-	end
-    end
-end
-
 -- Combine both functions into a single one, as we can only register one function as load hook in lmod
 -- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed
 function eessi_load_hook(t)
     eessi_cuda_enabled_load_hook(t)
-    eessi_openmpi_load_hook(t)
 end
 
 

From a0dc2815c4f45162aed56bf2c5e7bfcdc8b0eaf3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 26 Mar 2024 11:03:33 +0100
Subject: [PATCH 140/154] fix comment for creating lmod rc code block

---
 EESSI-install-software.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 31ce30d4fc..b2984af856 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -247,7 +247,7 @@ fi
 
 ### add packages here
 
-echo ">> Creating/updating Lmod cache..."
+echo ">> Creating/updating Lmod RC file..."
 export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod"
 lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua"
 lmodrc_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodrc.py$' > /dev/null; echo $?)

From b2570ed8cca9d7f87d3c506ec33d7b0c7b033671 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 26 Mar 2024 11:03:50 +0100
Subject: [PATCH 141/154] don't update lmod cache here anymore

---
 EESSI-install-software.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index b2984af856..7b7a60686c 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -265,7 +265,5 @@ if [ ! -f "$lmod_sitepackage_file" ] || [ "${sitepackage_changed}" == '0' ]; the
     check_exit_code $? "$lmod_sitepackage_file created" "Failed to create $lmod_sitepackage_file"
 fi
 
-$TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH}
-
 echo ">> Cleaning up ${TMPDIR}..."
 rm -r ${TMPDIR}

From f2844fb210aa865155989811c88d85b7d8b3c1a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 26 Mar 2024 17:09:49 +0100
Subject: [PATCH 142/154] remove OpenMPI, which has been rebuilt in #488

---
 .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml                  | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index b971c5adad..1c3ea8fd78 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -55,8 +55,4 @@ easyconfigs:
   - gnuplot-5.4.8-GCCcore-12.3.0.eb: 
       options:
         from-pr: 19261
-  - OpenMPI-4.1.5-GCC-12.3.0.eb:
-      options:
-        from-pr: 19940
-        rebuild: True
   - OpenFOAM-10-foss-2023a.eb

From 5a43f9ef8d8d66dfa2bd0226f28a8bdf7521892b Mon Sep 17 00:00:00 2001
From: Neves-P <p.m.santos.neves@rug.nl>
Date: Tue, 26 Mar 2024 19:16:00 +0100
Subject: [PATCH 143/154] Don't need gnuplot, install with EasyBuild 4.9.0

---
 .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml                  | 4 ----
 .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml                  | 1 +
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index 1c3ea8fd78..7244219dc3 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -52,7 +52,3 @@ easyconfigs:
   - BWA-0.7.17-20220923-GCCcore-12.3.0.eb:
       options: 
         from-pr: 19820 
-  - gnuplot-5.4.8-GCCcore-12.3.0.eb: 
-      options:
-        from-pr: 19261
-  - OpenFOAM-10-foss-2023a.eb
diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index 157a47a49e..13d7ebdc02 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -29,3 +29,4 @@ easyconfigs:
       # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19554
       options:
         from-pr: 19554
+  - OpenFOAM-10-foss-2023a.eb

From 7c7e1b1ff18c960db5f1233438295d9c60dd6431 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 26 Mar 2024 20:18:03 +0100
Subject: [PATCH 144/154] use older apptainer (1.2.4)

---
 install_apptainer_ubuntu.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/install_apptainer_ubuntu.sh b/install_apptainer_ubuntu.sh
index c35c34cda6..6cf9a6f48e 100755
--- a/install_apptainer_ubuntu.sh
+++ b/install_apptainer_ubuntu.sh
@@ -5,8 +5,11 @@ set -e
 # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181
 sudo apt-get install alien
 alien --version
-apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g')
-curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/${apptainer_rpm}
+#apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g')
+#curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/${apptainer_rpm}
+#/pub/archive/epel/8.8/Everything/x86_64/Packages/a
+apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/archive/epel/8.8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g')
+curl -OL https://dl.fedoraproject.org/pub/archive/epel/8.8/Everything/x86_64/Packages/a/${apptainer_rpm}
 sudo alien -d ${apptainer_rpm}
 sudo apt install ./apptainer*.deb
 apptainer --version

From deb12e9fef3103b8c07504e745b14fe366ed618d Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Mar 2024 20:35:40 +0100
Subject: [PATCH 145/154] clean up downloading RPM for Apptainer < 1.3.0 RPM
 from EPEL 8.8 archive, for now

---
 install_apptainer_ubuntu.sh | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/install_apptainer_ubuntu.sh b/install_apptainer_ubuntu.sh
index 6cf9a6f48e..5eb513db2c 100755
--- a/install_apptainer_ubuntu.sh
+++ b/install_apptainer_ubuntu.sh
@@ -5,11 +5,13 @@ set -e
 # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181
 sudo apt-get install alien
 alien --version
-#apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g')
-#curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/${apptainer_rpm}
-#/pub/archive/epel/8.8/Everything/x86_64/Packages/a
-apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/archive/epel/8.8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g')
-curl -OL https://dl.fedoraproject.org/pub/archive/epel/8.8/Everything/x86_64/Packages/a/${apptainer_rpm}
+# stick to Apptainer < 1.3.0 by downloading from EPEL 8.8 archive,
+# since CI workflow for testing scripts hangs/fails when using Apptainer 1.3.0
+# cfr. https://github.com/EESSI/software-layer/pull/514
+epel_subdir="pub/epel/8"
+epel_subdir="pub/archive/epel/8.8"
+apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/${epel_subdir}/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g')
+curl -OL https://dl.fedoraproject.org/${epel_subdir}/Everything/x86_64/Packages/a/${apptainer_rpm}
 sudo alien -d ${apptainer_rpm}
 sudo apt install ./apptainer*.deb
 apptainer --version

From 547386f9180687627ce506cab0cacc91ed808025 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Wed, 27 Mar 2024 10:08:14 +0100
Subject: [PATCH 146/154] dont fail if grep doesnt find rebuild easystack files

---
 bot/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/build.sh b/bot/build.sh
index b007b80172..2438adbcd9 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -182,7 +182,7 @@ fi
 # determine if the removal step has to be run
 # assume there's only one diff file that corresponds to the PR patch file
 pr_diff=$(ls [0-9]*.diff | head -1)
-changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | grep "/rebuilds/")
+changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | grep "/rebuilds/" || true)
 if [[ -z ${changed_easystacks_rebuilds} ]]; then
     echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step."
 else

From c2fad0032008d0ca73f6603ad7fbae6c2391a6a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Wed, 27 Mar 2024 10:08:32 +0100
Subject: [PATCH 147/154] add quotes around variable in if statement

---
 bot/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/build.sh b/bot/build.sh
index 2438adbcd9..e9f8ecfbb9 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -183,7 +183,7 @@ fi
 # assume there's only one diff file that corresponds to the PR patch file
 pr_diff=$(ls [0-9]*.diff | head -1)
 changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | grep "/rebuilds/" || true)
-if [[ -z ${changed_easystacks_rebuilds} ]]; then
+if [[ -z "${changed_easystacks_rebuilds}" ]]; then
     echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step."
 else
     # prepare directory to store tarball of tmp for removal and build steps

From bce1067e0b9962c591804c87ea150e38123822be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Wed, 27 Mar 2024 10:13:32 +0100
Subject: [PATCH 148/154] use proper tmp dir for removal step

---
 bot/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/build.sh b/bot/build.sh
index e9f8ecfbb9..0ebd55692e 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -192,7 +192,7 @@ else
 
     # prepare arguments to eessi_container.sh specific to remove step
     declare -a REMOVAL_STEP_ARGS=()
-    REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
+    REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_REMOVAL_STEP_DIR}")
     REMOVAL_STEP_ARGS+=("--storage" "${STORAGE}")
     # add fakeroot option in order to be able to remove software, see:
     # https://github.com/EESSI/software-layer/issues/312

From 4a9d9aeedd5fc7dc719bbc9612c6b5e2213a673b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Wed, 27 Mar 2024 10:20:16 +0100
Subject: [PATCH 149/154] let the tarball step resume from the right directory

---
 bot/build.sh | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/bot/build.sh b/bot/build.sh
index 0ebd55692e..0d23f59b66 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -247,7 +247,14 @@ declare -a TARBALL_STEP_ARGS=()
 TARBALL_STEP_ARGS+=("--save" "${TARBALL_TMP_TARBALL_STEP_DIR}")
 
 # determine temporary directory to resume from
-TARBALL_STEP_ARGS+=("--resume" "${REMOVAL_TMPDIR}")
+if [[ -z ${REMOVAL_TMPDIR} ]]; then
+    # no rebuild step was done, so the tarball step should resume from the build directory
+    BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)
+    TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}")
+else
+    # a removal step was done, so resume from its temporary directory (which was also used for the build step)
+    TARBALL_STEP_ARGS+=("--resume" "${REMOVAL_TMPDIR}")
+fi
 
 timestamp=$(date +%s)
 # to set EESSI_VERSION we need to source init/eessi_defaults now

From 2ba8513f770f2f23961a05bc64b98488e4aaaa8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Wed, 27 Mar 2024 11:14:05 +0100
Subject: [PATCH 150/154] group grep and true commands

Co-authored-by: Kenneth Hoste <kenneth.hoste@ugent.be>
---
 bot/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/build.sh b/bot/build.sh
index 0d23f59b66..ff933a447d 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -182,7 +182,7 @@ fi
 # determine if the removal step has to be run
 # assume there's only one diff file that corresponds to the PR patch file
 pr_diff=$(ls [0-9]*.diff | head -1)
-changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | grep "/rebuilds/" || true)
+changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | (grep "/rebuilds/" || true))
 if [[ -z "${changed_easystacks_rebuilds}" ]]; then
     echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step."
 else

From 0e38a4769d7db0e893f9a4eb058ec9333a0e648e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Wed, 27 Mar 2024 11:16:35 +0100
Subject: [PATCH 151/154] add comment about true command

---
 bot/build.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bot/build.sh b/bot/build.sh
index ff933a447d..dcc61c19d4 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -182,6 +182,8 @@ fi
 # determine if the removal step has to be run
 # assume there's only one diff file that corresponds to the PR patch file
 pr_diff=$(ls [0-9]*.diff | head -1)
+# the true at the end of the next command is important: grep will expectedly return 1 if there is no easystack file being added under rebuilds,
+# but due to "set -e" the entire script would otherwise fail
 changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | (grep "/rebuilds/" || true))
 if [[ -z "${changed_easystacks_rebuilds}" ]]; then
     echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step."

From e9bb5b8da5e7634396fca503bdafbe11861c0f42 Mon Sep 17 00:00:00 2001
From: Richard Top <richard.topouchian@uib.no>
Date: Wed, 27 Mar 2024 14:25:47 +0000
Subject: [PATCH 152/154] {2023.06}[GCCcore/13.2.0] Wayland v1.22.0

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index 25cdde27e6..6710c07b49 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -53,4 +53,5 @@ easyconfigs:
   - giflib-5.2.1-GCCcore-13.2.0.eb
   - OpenJPEG-2.5.0-GCCcore-13.2.0.eb
   - libwebp-1.3.2-GCCcore-13.2.0.eb
+  - Wayland-1.22.0-GCCcore-13.2.0.eb
 

From 80609663bf16cf013df9922c2229976c19e821df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Wed, 27 Mar 2024 16:08:52 +0100
Subject: [PATCH 153/154] add quotes around changed_easystacks variable

---
 EESSI-install-software.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 470663e45a..e2d7cf8c51 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -204,7 +204,7 @@ ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12
 
 # use PR patch file to determine in which easystack files stuff was added
 changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') 
-if [ -z ${changed_easystacks} ]; then
+if [ -z "${changed_easystacks}" ]; then
     echo "No missing installations, party time!"  # Ensure the bot report success, as there was nothing to be build here
 else
 

From 70935ea4293d0e7a94b99f573a77fe74b4f68fbe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Wed, 27 Mar 2024 16:13:07 +0100
Subject: [PATCH 154/154] first process rebuilds, then new installations

---
 EESSI-install-software.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index e2d7cf8c51..a905d966f6 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -208,7 +208,11 @@ if [ -z "${changed_easystacks}" ]; then
     echo "No missing installations, party time!"  # Ensure the bot report success, as there was nothing to be build here
 else
 
-    for easystack_file in ${changed_easystacks}; do
+    # first process rebuilds, if any, then easystack files for new installations
+    # "|| true" is used to make sure that the grep command always returns success
+    rebuild_easystacks=$(echo "${changed_easystacks}" | (grep "/rebuilds/" || true))
+    new_easystacks=$(echo "${changed_easystacks}" | (grep -v "/rebuilds/" || true))
+    for easystack_file in ${rebuild_easystacks} ${new_easystacks}; do
 
         echo -e "Processing easystack file ${easystack_file}...\n\n"