From 63a968b088ce3a680bf938687536a70542581703 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sun, 26 Nov 2023 20:48:19 +0100
Subject: [PATCH 01/81] add JSON file + Python script to keep track of software
 licenses

---
 licenses/README.md     |   3 ++
 licenses/licenses.json |  10 +++++
 licenses/spdx.py       | 100 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 113 insertions(+)
 create mode 100644 licenses/README.md
 create mode 100644 licenses/licenses.json
 create mode 100644 licenses/spdx.py

diff --git a/licenses/README.md b/licenses/README.md
new file mode 100644
index 0000000000..36a7615b21
--- /dev/null
+++ b/licenses/README.md
@@ -0,0 +1,3 @@
+see https://spdx.org/licenses
+
+Python function to download SPDX list of licenses is available in `spdx.py`
diff --git a/licenses/licenses.json b/licenses/licenses.json
new file mode 100644
index 0000000000..8831ed368c
--- /dev/null
+++ b/licenses/licenses.json
@@ -0,0 +1,10 @@
+{
+    "EasyBuild": {
+        "spdx": "GPL-2.0-only",
+        "license_url": "https://easybuild.io"
+    },
+    "GCCcore": {
+        "spdx": "GPL-2.0-with-GCC-exception",
+        "license_url": "https://github.com/gcc-mirror/gcc/blob/master/COPYING"
+    }
+}
diff --git a/licenses/spdx.py b/licenses/spdx.py
new file mode 100644
index 0000000000..06c3edb4e6
--- /dev/null
+++ b/licenses/spdx.py
@@ -0,0 +1,100 @@
+import json
+import logging
+import sys
+import urllib.request
+
+SPDX_LICENSE_LIST_URL = 'https://raw.githubusercontent.com/spdx/license-list-data/main/json/licenses.json'
+
+LICENSE_URL = 'license_url'
+SPDX = 'spdx'
+
+spdx_license_list = None
+
+# Configure the logging module
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+
+
+def get_spdx_license_list():
+    """
+    Download JSON file with current list of SPDX licenses, parse it, and return it as a Python dictionary.
+    """
+    global spdx_license_list
+
+    if spdx_license_list is None:
+        with urllib.request.urlopen(SPDX_LICENSE_LIST_URL) as fp:
+            spdx_license_list = json.load(fp)
+        version, release_date = spdx_license_list['licenseListVersion'], spdx_license_list['releaseDate']
+        logging.info(f"Downloaded version {version} of SPDX license list (release date: {release_date})")
+        licenses = spdx_license_list['licenses']
+        logging.info(f"Found info on {len(licenses)} licenses!")
+
+    return spdx_license_list
+
+
+def license_info(spdx_id):
+    """Find license with specified SPDX identifier."""
+ 
+    spdx_license_list = get_spdx_license_list()
+
+    licenses = spdx_license_list['licenses']
+    for lic in licenses:
+        if lic['licenseId'] == spdx_id:
+            return lic
+
+    # if no match is found, return None as result
+    return None
+
+
+def read_licenses(path):
+    """
+    Read software project to license mapping from specified path
+    """
+    with open(path) as fp:
+        licenses = json.loads(fp.read())
+
+    return licenses
+
+
+def check_licenses(licenses):
+    """
+    Check mapping of software licenses: make sure SPDX identifiers are valid.
+    """
+    faulty_licenses = {}
+
+    for software_name in licenses:
+        spdx_lic_id = licenses[software_name][SPDX]
+        lic_info = license_info(spdx_lic_id)
+        if lic_info:
+            lic_url = licenses[software_name][LICENSE_URL]
+            logging.info(f"License for software '{software_name}': {lic_info['name']} (see {lic_url})")
+        else:
+            logging.warning(f"Found faulty SPDX license ID for {software_name}: {spdx_lic_id}")
+            faulty_licenses[software_name] = spdx_lic_id
+
+    if faulty_licenses:
+        logging.warning(f"Found {len(faulty_licenses)} faulty SPDIX license IDs (out of {len(licenses)})!")
+        result = False
+    else:
+        logging.info(f"License check passed for {len(licenses)} licenses!")
+        result = True
+
+    return result
+
+
+def main(args):
+    if len(args) == 1:
+        licenses_path = args[0]
+    else:
+        logging.error("Usage: python spdx.py <path to licenses.json>")
+        sys.exit(1)
+
+    licenses = read_licenses(licenses_path)
+    if check_licenses(licenses):
+        logging.info("All license checks PASSED!")
+    else:
+        logging.error("One or more licence checks failed!")
+        sys.exit(2)
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

From 0421bae6c78040b95c746fb15bfe3c66b7d874b3 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sun, 26 Nov 2023 20:55:38 +0100
Subject: [PATCH 02/81] add CI workflow to check specified software licenses

---
 .github/workflows/test_licenses.yml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 .github/workflows/test_licenses.yml

diff --git a/.github/workflows/test_licenses.yml b/.github/workflows/test_licenses.yml
new file mode 100644
index 0000000000..00a2c90f6b
--- /dev/null
+++ b/.github/workflows/test_licenses.yml
@@ -0,0 +1,20 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Test software licenses
+on: [push, pull_request]
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+        - name: Check out software-layer repository
+          uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+
+        - name: set up Python
+          uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
+          with:
+            python-version: '3.9'
+
+        - name: Check software licenses
+          run: |
+            python licenses/spdx.py licenses/licenses.json

From f94430a60940ecfab5d89b789743b4c6b740b983 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@gcn6.local.snellius.surf.nl>
Date: Fri, 1 Dec 2023 16:43:46 +0100
Subject: [PATCH 03/81] Replicated changes that were initially targetted at
 2023.06-pilot repo

---
 eessi_container.sh                            |  71 +++++-
 .../nvidia/install_cuda_host_injections.sh    | 209 ++++++++++++++++++
 .../nvidia/link_nvidia_host_libraries.sh      | 136 ++++++++++++
 scripts/utils.sh                              |  56 ++++-
 4 files changed, 461 insertions(+), 11 deletions(-)
 create mode 100755 gpu_support/nvidia/install_cuda_host_injections.sh
 create mode 100755 gpu_support/nvidia/link_nvidia_host_libraries.sh

diff --git a/eessi_container.sh b/eessi_container.sh
index bf0294c7bf..81d7be81ad 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -30,8 +30,8 @@
 # -. initial settings & exit codes
 TOPDIR=$(dirname $(realpath $0))
 
-source ${TOPDIR}/scripts/utils.sh
-source ${TOPDIR}/scripts/cfg_files.sh
+source "${TOPDIR}"/scripts/utils.sh
+source "${TOPDIR}"/scripts/cfg_files.sh
 
 # exit codes: bitwise shift codes to allow for combination of exit codes
 # ANY_ERROR_EXITCODE is sourced from ${TOPDIR}/scripts/utils.sh
@@ -46,6 +46,7 @@ SAVE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8))
 HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
 HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
 RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
+NVIDIA_MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 12))
 
 # CernVM-FS settings
 CVMFS_VAR_LIB="var-lib-cvmfs"
@@ -72,12 +73,17 @@ display_help() {
   echo "  -a | --access {ro,rw}  - ro (read-only), rw (read & write) [default: ro]"
   echo "  -c | --container IMG   - image file or URL defining the container to use"
   echo "                           [default: docker://ghcr.io/eessi/build-node:debian11]"
-  echo "  -h | --help            - display this usage information [default: false]"
   echo "  -g | --storage DIR     - directory space on host machine (used for"
   echo "                           temporary data) [default: 1. TMPDIR, 2. /tmp]"
+  echo "  -h | --help            - display this usage information [default: false]"
+  echo "  -i | --host-injections - directory to link to for host_injections "
+  echo "                           [default: /..storage../opt-eessi]"
   echo "  -l | --list-repos      - list available repository identifiers [default: false]"
   echo "  -m | --mode MODE       - with MODE==shell (launch interactive shell) or"
   echo "                           MODE==run (run a script or command) [default: shell]"
+  echo "  -n | --nvidia MODE     - configure the container to work with NVIDIA GPUs,"
+  echo "                           MODE==install for a CUDA installation, MODE==run to"
+  echo "                           attach a GPU, MODE==all for both [default: false]"
   echo "  -r | --repository CFG  - configuration file or identifier defining the"
   echo "                           repository to use [default: EESSI via"
   echo "                           default container, see --container]"
@@ -111,6 +117,8 @@ VERBOSE=0
 STORAGE=
 LIST_REPOS=0
 MODE="shell"
+SETUP_NVIDIA=0
+ADDITIONAL_SINGULARITY_FLAGS=
 REPOSITORY="EESSI"
 RESUME=
 SAVE=
@@ -141,6 +149,10 @@ while [[ $# -gt 0 ]]; do
       display_help
       exit 0
       ;;
+    -i|--host-injections)
+      USER_HOST_INJECTIONS="$2"
+      shift 2
+      ;;
     -l|--list-repos)
       LIST_REPOS=1
       shift 1
@@ -149,6 +161,11 @@ while [[ $# -gt 0 ]]; do
       MODE="$2"
       shift 2
       ;;
+    -n|--nvidia)
+      SETUP_NVIDIA=1
+      NVIDIA_MODE="$2"
+      shift 2
+      ;;
     -r|--repository)
       REPOSITORY="$2"
       shift 2
@@ -224,6 +241,13 @@ if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then
     fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}"
 fi
 
+# Also validate the NVIDIA GPU mode (if present)
+if [[ ${SETUP_NVIDIA} -eq 1 ]]; then
+    if [[ "${NVIDIA_MODE}" != "run" && "${NVIDIA_MODE}" != "install" && "${NVIDIA_MODE}" != "all" ]]; then
+        fatal_error "unknown NVIDIA mode '${NVIDIA_MODE}'" "${NVIDIA_MODE_UNKNOWN_EXITCODE}"
+    fi
+fi
+
 # TODO (arg -r|--repository) check if repository is known
 # REPOSITORY_ERROR_EXITCODE
 if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then
@@ -310,12 +334,25 @@ fi
 #      |-overlay-work
 #      |-home
 #      |-repos_cfg
+#      |-opt-eessi (unless otherwise specificed for host_injections)
 
 # tmp dir for EESSI
 EESSI_TMPDIR=${EESSI_HOST_STORAGE}
 mkdir -p ${EESSI_TMPDIR}
 [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}"
 
+# Set host_injections directory and ensure it is a writable directory (if user provided)
+if [ -z ${USER_HOST_INJECTIONS+x} ]; then
+    # Not set, so use our default
+    HOST_INJECTIONS=${EESSI_TMPDIR}/opt-eessi
+    mkdir -p $HOST_INJECTIONS
+else
+    # Make sure the host_injections directory specified exists and is a folder
+    mkdir -p ${USER_HOST_INJECTIONS} || fatal_error "host_injections directory ${USER_HOST_INJECTIONS} is either not a directory or cannot be created"
+    HOST_INJECTIONS=${USER_HOST_INJECTIONS}
+fi
+[[ ${VERBOSE} -eq 1 ]] && echo "HOST_INJECTIONS=${HOST_INJECTIONS}"
+
 # configure Singularity: if SINGULARITY_CACHEDIR is already defined, use that
 #   a global SINGULARITY_CACHEDIR would ensure that we don't consume
 #   storage space again and again for the container & also speed-up
@@ -394,12 +431,34 @@ fi
 [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}"
 
 # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined)
-BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs"
+BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs,${HOST_INJECTIONS}:/opt/eessi"
 # provide a '/tmp' inside the container
 BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}"
 
 [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
 
+# Configure anything we need for NVIDIA GPUs and CUDA installation
+if [[ ${SETUP_NVIDIA} -eq 1 ]]; then
+    if [[ "${NVIDIA_MODE}" == "run" || "${NVIDIA_MODE}" == "all" ]]; then
+        # Give singularity the appropriate flag
+        ADDITIONAL_SINGULARITY_FLAGS="--nv ${ADDITIONAL_SINGULARITY_FLAGS}"
+        [[ ${VERBOSE} -eq 1 ]] && echo "ADDITIONAL_SINGULARITY_FLAGS=${ADDITIONAL_SINGULARITY_FLAGS}"
+    fi
+    if [[ "${NVIDIA_MODE}" == "install" || "${NVIDIA_MODE}" == "all" ]]; then
+        # Add additional bind mounts to allow CUDA to install within a container
+        # (Experience tells us that these are necessary, but we don't know _why_
+        # as the CUDA installer is a black box. The suspicion is that the CUDA
+        # installer gets confused by the permissions on these directories when
+        # inside a container)
+        EESSI_VAR_LOG=${EESSI_TMPDIR}/var-log
+        EESSI_USR_LOCAL_CUDA=${EESSI_TMPDIR}/usr-local-cuda
+        mkdir -p ${EESSI_VAR_LOG}
+        mkdir -p ${EESSI_USR_LOCAL_CUDA}
+        BIND_PATHS="${BIND_PATHS},${EESSI_VAR_LOG}:/var/log,${EESSI_USR_LOCAL_CUDA}:/usr/local/cuda"
+        [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
+    fi
+fi
+
 # set up repository config (always create directory repos_cfg and populate it with info when
 # arg -r|--repository is used)
 mkdir -p ${EESSI_TMPDIR}/repos_cfg
@@ -562,8 +621,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
 fi
 
 echo "Launching container with command (next line):"
-echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
-singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
+echo "singularity ${RUN_QUIET} ${MODE} ${ADDITIONAL_SINGULARITY_FLAGS} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
+singularity ${RUN_QUIET} ${MODE} ${ADDITIONAL_SINGULARITY_FLAGS} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
 exit_code=$?
 
 # 6. save tmp if requested (arg -s|--save)
diff --git a/gpu_support/nvidia/install_cuda_host_injections.sh b/gpu_support/nvidia/install_cuda_host_injections.sh
new file mode 100755
index 0000000000..f02f0da02e
--- /dev/null
+++ b/gpu_support/nvidia/install_cuda_host_injections.sh
@@ -0,0 +1,209 @@
+#!/usr/bin/env bash
+
+# This script can be used to install CUDA under the `.../host_injections` directory.
+# This provides the parts of the CUDA installation that cannot be redistributed as
+# part of EESSI due to license limitations. While GPU-based software from EESSI will
+# _run_ without these, installation of additional CUDA software requires the CUDA
+# installation(s) under `host_injections` to be present.
+#
+# The `host_injections` directory is a variant symlink that by default points to
+# `/opt/eessi`, unless otherwise defined in the local CVMFS configuration (see
+# https://cvmfs.readthedocs.io/en/stable/cpt-repo.html#variant-symlinks). For the
+# installation to be successful, this directory needs to be writeable by the user
+# executing this script.
+
+# Initialise our bash functions
+TOPDIR=$(dirname $(realpath $BASH_SOURCE))
+source "$TOPDIR"/../../scripts/utils.sh
+
+# Function to display help message
+show_help() {
+    echo "Usage: $0 [OPTIONS]"
+    echo "Options:"
+    echo "  --help                           Display this help message"
+    echo "  --accept-cuda-eula               You _must_ accept the CUDA EULA to install"
+    echo "                                   CUDA, see the EULA at"
+    echo "                                   https://docs.nvidia.com/cuda/eula/index.html"
+    echo "  -c, --cuda-version CUDA_VERSION  Specify a version o CUDA to install (must"
+    echo "                                   have a corresponding easyconfig in the"
+    echo "                                   EasyBuild release)"
+    echo "  -t, --temp-dir /path/to/tmpdir   Specify a location to use for temporary"
+    echo "                                   storage during the CUDA install"
+    echo "                                   (must have >10GB available)"
+}
+
+# Initialize variables
+install_cuda_version=""
+eula_accepted=0
+
+# Parse command-line options
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --help)
+            show_help
+            exit 0
+            ;;
+        -c|--cuda-version)
+            if [ -n "$2" ]; then
+                install_cuda_version="$2"
+                shift 2
+            else
+                echo "Error: Argument required for $1"
+                show_help
+                exit 1
+            fi
+            ;;
+        --accept-cuda-eula)
+            eula_accepted=1
+            shift 1
+            ;;
+        -t|--temp-dir)
+            if [ -n "$2" ]; then
+                CUDA_TEMP_DIR="$2"
+                shift 2
+            else
+                echo "Error: Argument required for $1"
+                show_help
+                exit 1
+            fi
+            ;;
+        *)
+            show_help
+            fatal_error "Error: Unknown option: $1"
+            ;;
+    esac
+done
+
+# Make sure EESSI is initialised
+check_eessi_initialised
+
+# Make sure the CUDA version supplied is a semantic version
+is_semantic_version() {
+    local version=$1
+    local regex='^[0-9]+\.[0-9]+\.[0-9]+$'
+
+    if [[ $version =~ $regex ]]; then
+        return 0  # Return success (0) if it's a semantic version
+    else
+        return 1  # Return failure (1) if it's not a semantic version
+    fi
+}
+if ! is_semantic_version "$install_cuda_version"; then
+  show_help
+  error="\nYou must provide a semantic version for CUDA (e.g., 12.1.1) via the appropriate\n"
+  error="${error}command line option. This script is intended for use with EESSI so the 'correct'\n"
+  error="${error}version to provide is probably one of those available under\n"
+  error="${error}$EESSI_SOFTWARE_PATH/software/CUDA\n"
+  fatal_error "${error}"
+fi
+
+# Make sure they have accepted the CUDA EULA
+if [ "$eula_accepted" -ne 1 ]; then
+  show_help
+  error="\nYou _must_ accept the CUDA EULA via the appropriate command line option.\n"
+  fatal_error "${error}"
+fi
+
+# As an installation location just use $EESSI_SOFTWARE_PATH but replacing `versions` with `host_injections`
+# (CUDA is a binary installation so no need to worry too much about the EasyBuild setup)
+cuda_install_parent=${EESSI_SOFTWARE_PATH/versions/host_injections}
+
+# Only install CUDA if specified version is not found.
+# (existence of easybuild subdir implies a successful install)
+if [ -d "${cuda_install_parent}"/software/CUDA/"${install_cuda_version}"/easybuild ]; then
+  echo_green "CUDA software found! No need to install CUDA again."
+else
+  # We need to be able write to the installation space so let's make sure we can
+  if ! create_directory_structure "${cuda_install_parent}"/software/CUDA ; then
+    fatal_error "No write permissions to directory ${cuda_install_parent}/software/CUDA"
+  fi
+
+  # we need a directory we can use for temporary storage
+  if [[ -z "${CUDA_TEMP_DIR}" ]]; then
+    tmpdir=$(mktemp -d)
+  else
+    tmpdir="${CUDA_TEMP_DIR}"/temp
+    if ! mkdir "$tmpdir" ; then
+      fatal_error "Could not create directory ${tmpdir}"
+    fi
+  fi
+
+  required_space_in_tmpdir=50000
+  # Let's see if we have sources and build locations defined if not, we use the temporary space
+  if [[ -z "${EASYBUILD_BUILDPATH}" ]]; then
+    export EASYBUILD_BUILDPATH=${tmpdir}/build
+    required_space_in_tmpdir=$((required_space_in_tmpdir + 5000000))
+  fi
+  if [[ -z "${EASYBUILD_SOURCEPATH}" ]]; then
+    export EASYBUILD_SOURCEPATH=${tmpdir}/sources
+    required_space_in_tmpdir=$((required_space_in_tmpdir + 5000000))
+  fi
+
+  # The install is pretty fat, you need lots of space for download/unpack/install (~3*5GB),
+  # need to do a space check before we proceed
+  avail_space=$(df --output=avail "${cuda_install_parent}"/ | tail -n 1 | awk '{print $1}')
+  if (( avail_space < 5000000 )); then
+    fatal_error "Need at least 5GB disk space to install CUDA under ${cuda_install_parent}, exiting now..."
+  fi
+  avail_space=$(df --output=avail "${tmpdir}"/ | tail -n 1 | awk '{print $1}')
+  if (( avail_space < required_space_in_tmpdir )); then
+    error="Need at least ${required_space_in_tmpdir} disk space under ${tmpdir}.\n"
+    error="${error}Set the environment variable CUDA_TEMP_DIR to a location with adequate space to pass this check."
+    error="${error}You can alternatively set EASYBUILD_BUILDPATH and/or EASYBUILD_SOURCEPATH "
+    error="${error}to reduce this requirement. Exiting now..."
+    fatal_error "${error}"
+  fi
+
+  if ! command -v "eb" &>/dev/null; then
+    echo_yellow "Attempting to load an EasyBuild module to do actual install"
+    module load EasyBuild
+    # There are some scenarios where this may fail
+    if [ $? -ne 0 ]; then
+      error="'eb' command not found in your environment and\n"
+      error="${error}  module load EasyBuild\n"
+      error="${error}failed for some reason.\n"
+      error="${error}Please re-run this script with the 'eb' command available."
+      fatal_error "${error}"
+    fi
+  fi
+
+  cuda_easyconfig="CUDA-${install_cuda_version}.eb"
+
+  # Check the easyconfig file is available in the release
+  # (eb search always returns 0, so we need a grep to ensure a usable exit code)
+  eb --search ^${cuda_easyconfig}|grep CUDA > /dev/null 2>&1
+  # Check the exit code
+  if [ $? -ne 0 ]; then
+    eb_version=$(eb --version)
+    available_cuda_easyconfigs=$(eb --search ^CUDA-*.eb|grep CUDA)
+
+    error="The easyconfig ${cuda_easyconfig} was not found in EasyBuild version:\n"
+    error="${error}  ${eb_version}\n"
+    error="${error}You either need to give a different version of CUDA to install _or_ \n"
+    error="${error}use a different version of EasyBuild for the installation.\n"
+    error="${error}\nThe versions of available with the current eb command are:\n"
+    error="${error}${available_cuda_easyconfigs}"
+    fatal_error "${error}"
+  fi
+
+  # We need the --rebuild option, as the CUDA module may or may not be on the
+  # `MODULEPATH` yet. Even if it is, we still want to redo this installation
+  # since it will provide the symlinked targets for the parts of the CUDA
+  # installation in the `.../versions/...` prefix
+  # We install the module in our `tmpdir` since we do not need the modulefile,
+  # we only care about providing the targets for the symlinks.
+  extra_args="--rebuild --installpath-modules=${tmpdir}"
+
+  # We don't want hooks used in this install, we need a vanilla CUDA installation
+  touch "$tmpdir"/none.py
+  # shellcheck disable=SC2086  # Intended splitting of extra_args
+  eb --prefix="$tmpdir" ${extra_args} --accept-eula-for=CUDA --hooks="$tmpdir"/none.py --installpath="${cuda_install_parent}"/ "${cuda_easyconfig}"
+  ret=$?
+  if [ $ret -ne 0 ]; then
+    fatal_error  "CUDA installation failed, please check EasyBuild logs..."
+  else
+    echo_green "CUDA installation at ${cuda_install_parent}/software/CUDA/${install_cuda_version} succeeded!"
+  fi
+  # clean up tmpdir
+  rm -rf "${tmpdir}"
+fi
diff --git a/gpu_support/nvidia/link_nvidia_host_libraries.sh b/gpu_support/nvidia/link_nvidia_host_libraries.sh
new file mode 100755
index 0000000000..26760f0b82
--- /dev/null
+++ b/gpu_support/nvidia/link_nvidia_host_libraries.sh
@@ -0,0 +1,136 @@
+#!/bin/bash
+
+# This script links host libraries related to GPU drivers to a location where
+# they can be found by the EESSI linker
+
+# Initialise our bash functions
+TOPDIR=$(dirname $(realpath $BASH_SOURCE))
+source "$TOPDIR"/../../scripts/utils.sh
+
+# We rely on ldconfig to give us the location of the libraries on the host
+command_name="ldconfig"
+# We cannot use a version of ldconfig that's being shipped under CVMFS
+exclude_prefix="/cvmfs"
+
+found_paths=()
+# Always attempt to use /sbin/ldconfig
+if [ -x "/sbin/$command_name" ]; then
+    found_paths+=("/sbin/$command_name")
+fi
+IFS=':' read -ra path_dirs <<< "$PATH"
+for dir in "${path_dirs[@]}"; do
+  if [ "$dir" = "/sbin" ]; then
+    continue  # we've already checked for $command_name in /sbin, don't need to do it twice
+  fi
+  if [[ ! "$dir" =~ ^$exclude_prefix ]]; then
+      if [ -x "$dir/$command_name" ]; then
+          found_paths+=("$dir/$command_name")
+      fi
+  fi
+done
+
+if [ ${#found_paths[@]} -gt 0 ]; then
+    echo "Found $command_name in the following locations:"
+    printf -- "- %s\n" "${found_paths[@]}"
+    echo "Using first version"
+    host_ldconfig=${found_paths[0]}
+else
+    error="$command_name not found in PATH or only found in paths starting with $exclude_prefix."
+    fatal_error $error
+fi
+
+# Make sure EESSI is initialised (doesn't matter what version)
+check_eessi_initialised
+
+# Find the CUDA version of the host CUDA drivers
+# (making sure that this can still work inside prefix environment inside a container)
+export LD_LIBRARY_PATH=/.singularity.d/libs:$LD_LIBRARY_PATH
+nvidia_smi_command="nvidia-smi --query-gpu=driver_version --format=csv,noheader"
+if $nvidia_smi_command > /dev/null; then
+  host_driver_version=$($nvidia_smi_command | tail -n1)
+  # If the first worked, this should work too
+  host_cuda_version=$(nvidia-smi  -q --display=COMPUTE | grep CUDA | awk 'NF>1{print $NF}')
+else
+  error="Failed to successfully execute\n  $nvidia_smi_command\n"
+  fatal_error $error
+fi
+
+# Let's make sure the driver libraries are not already in place
+link_drivers=1
+
+host_injections_nvidia_dir="/cvmfs/pilot.eessi-hpc.org/host_injections/nvidia/${EESSI_CPU_FAMILY}"
+host_injection_driver_dir="${host_injections_nvidia_dir}/host"
+host_injection_driver_version_file="$host_injection_driver_dir/driver_version.txt"
+if [ -e "$host_injection_driver_version_file" ]; then
+  if grep -q "$host_driver_version" "$host_injection_driver_version_file"; then
+    echo_green "The host CUDA driver libraries have already been linked!"
+    link_drivers=0
+  else
+    # There's something there but it is out of date
+    echo_yellow "Cleaning out outdated symlinks"
+    rm $host_injection_driver_dir/*
+    if [ $? -ne 0 ]; then
+      error="Unable to remove files under '$host_injection_driver_dir'."
+      fatal_error $error
+    fi
+  fi
+fi
+
+drivers_linked=0
+if [ "$link_drivers" -eq 1 ]; then
+  if ! create_directory_structure "${host_injection_driver_dir}" ; then
+    fatal_error "No write permissions to directory ${host_injection_driver_dir}"
+  fi
+  cd ${host_injection_driver_dir}
+  # Need a small temporary space to hold a couple of files
+  temp_dir=$(mktemp -d)
+
+  # Gather libraries on the host (_must_ be host ldconfig)
+  $host_ldconfig -p | awk '{print $NF}' > "$temp_dir"/libs.txt
+  # Allow for the fact that we may be in a container so the CUDA libs might be in there
+  ls /.singularity.d/libs/* >> "$temp_dir"/libs.txt 2>/dev/null
+
+  # Leverage singularity to find the full list of libraries we should be linking to
+  echo_yellow "Downloading latest version of nvliblist.conf from Apptainer"
+  curl -o "$temp_dir"/nvliblist.conf https://raw.githubusercontent.com/apptainer/apptainer/main/etc/nvliblist.conf
+
+  # Make symlinks to all the interesting libraries
+  grep '.so$' "$temp_dir"/nvliblist.conf | xargs -i grep {} "$temp_dir"/libs.txt | xargs -i ln -s {}
+
+  # Inject driver and CUDA versions into dir
+  echo $host_driver_version > driver_version.txt
+  echo $host_cuda_version > cuda_version.txt
+  drivers_linked=1
+
+  # Remove the temporary directory when done
+  rm -r "$temp_dir"
+fi
+
+# Make latest symlink for NVIDIA drivers
+cd $host_injections_nvidia_dir
+symlink="latest"
+if [ -L "$symlink" ]; then
+    # Unless the drivers have been installed, leave the symlink alone
+    if [ "$drivers_linked" -eq 1 ]; then
+      ln -sf host latest
+    fi
+else
+    # No link exists yet
+    ln -s host latest
+fi
+
+# Make sure the libraries can be found by the EESSI linker
+host_injection_linker_dir=${EESSI_EPREFIX/versions/host_injections}
+if [ -L "$host_injection_linker_dir/lib" ]; then
+  target_path=$(readlink -f "$host_injection_linker_dir/lib")
+  if [ "$target_path" != "$$host_injections_nvidia_dir/latest" ]; then
+    cd $host_injection_linker_dir
+    ln -sf $host_injections_nvidia_dir/latest lib
+  fi
+else
+  create_directory_structure $host_injection_linker_dir
+  cd $host_injection_linker_dir
+  ln -s $host_injections_nvidia_dir/latest lib
+fi
+
+echo_green "Host NVIDIA gpu drivers linked successfully for EESSI"
diff --git a/scripts/utils.sh b/scripts/utils.sh
index d0da95e87f..b2be3f6221 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -14,7 +14,7 @@ ANY_ERROR_EXITCODE=1
 function fatal_error() {
     echo_red "ERROR: $1" >&2
     if [[ $# -gt 1 ]]; then
-      exit $2
+      exit "$2"
     else
       exit "${ANY_ERROR_EXITCODE}"
     fi
@@ -32,11 +32,57 @@ function check_exit_code {
     fi
 }
 
+function check_eessi_initialised() {
+  if [[ -z "${EESSI_SOFTWARE_PATH}" ]]; then
+    fatal_error "EESSI has not been initialised!"
+  else
+    return 0
+  fi
+}
+
+function check_in_prefix_shell() {
+  # Make sure EPREFIX is defined
+  if [[ -z "${EPREFIX}" ]]; then
+    fatal_error "This script cannot be used without having first defined EPREFIX"
+  fi
+  if [[ ! ${SHELL} = ${EPREFIX}/bin/bash ]]; then
+    fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!"
+  fi
+}
+
+function create_directory_structure() {
+  # Ensure we are given a single path argument
+  if [ $# -ne 1 ]; then
+    echo_red "Function requires a single (relative or absolute) path argument" >&2
+    return $ANY_ERROR_EXITCODE
+  fi
+  dir_structure="$1"
+
+  # Attempt to create the directory structure
+  error_message=$(mkdir -p "$dir_structure" 2>&1)
+  return_code=$?
+  # If it fails be explicit about the error
+  if [ ${return_code} -ne 0 ]; then
+    real_dir=$(realpath -m "$dir_structure")
+    echo_red "Creating ${dir_structure} (real path ${real_dir}) failed with:\n ${error_message}" >&2
+  else
+    # If we're creating it, our use case is that we want to be able to write there
+    # (this is a check in case the directory already existed)
+    if [ ! -w "${dir_structure}" ]; then
+      real_dir=$(realpath -m "$dir_structure")
+      echo_red "You do not have (required) write permissions to ${dir_structure} (real path ${real_dir})!"
+      return_code=$ANY_ERROR_EXITCODE
+    fi
+  fi
+
+  return $return_code
+}
+
 function get_path_for_tool {
     tool_name=$1
     tool_envvar_name=$2
 
-    which_out=$(which ${tool_name} 2>&1)
+    which_out=$(which "${tool_name}" 2>&1)
     exit_code=$?
     if [[ ${exit_code} -eq 0 ]]; then
         echo "INFO: found tool ${tool_name} in PATH (${which_out})" >&2
@@ -68,7 +114,7 @@ function get_host_from_url {
     url=$1
     re="(http|https)://([^/:]+)"
     if [[ $url =~ $re ]]; then
-        echo ${BASH_REMATCH[2]}
+        echo "${BASH_REMATCH[2]}"
         return 0
     else
         echo ""
@@ -80,7 +126,7 @@ function get_port_from_url {
     url=$1
     re="(http|https)://[^:]+:([0-9]+)"
     if [[ $url =~ $re ]]; then
-        echo ${BASH_REMATCH[2]}
+        echo "${BASH_REMATCH[2]}"
         return 0
     else
         echo ""
@@ -90,7 +136,7 @@ function get_port_from_url {
 
 function get_ipv4_address {
     hname=$1
-    hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1)
+    hipv4=$(grep "${hname}" /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1)
     # TODO try other methods if the one above does not work --> tool that verifies
     #      what method can be used?
     echo "${hipv4}"

From c06a7aa392a4d0b719d9ffa86bbafa1e684e54e0 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@gcn6.local.snellius.surf.nl>
Date: Fri, 1 Dec 2023 16:46:19 +0100
Subject: [PATCH 04/81] Update nvidia_dir with new repo name

---
 gpu_support/nvidia/link_nvidia_host_libraries.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gpu_support/nvidia/link_nvidia_host_libraries.sh b/gpu_support/nvidia/link_nvidia_host_libraries.sh
index 26760f0b82..a03ce609a0 100755
--- a/gpu_support/nvidia/link_nvidia_host_libraries.sh
+++ b/gpu_support/nvidia/link_nvidia_host_libraries.sh
@@ -58,7 +58,7 @@ fi
 # Let's make sure the driver libraries are not already in place
 link_drivers=1
 
-host_injections_nvidia_dir="/cvmfs/pilot.eessi-hpc.org/host_injections/nvidia/${EESSI_CPU_FAMILY}"
+host_injections_nvidia_dir="/cvmfs/software.eessi.io/host_injections/nvidia/${EESSI_CPU_FAMILY}"
 host_injection_driver_dir="${host_injections_nvidia_dir}/host"
 host_injection_driver_version_file="$host_injection_driver_dir/driver_version.txt"
 if [ -e "$host_injection_driver_version_file" ]; then

From 24a4ebca030ea2ea55bb06d9b0d362f1f95ec8fd Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Fri, 1 Dec 2023 16:52:40 +0100
Subject: [PATCH 05/81] Update link_nvidia_host_libraries.sh

---
 gpu_support/nvidia/link_nvidia_host_libraries.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gpu_support/nvidia/link_nvidia_host_libraries.sh b/gpu_support/nvidia/link_nvidia_host_libraries.sh
index a03ce609a0..6458be7fae 100755
--- a/gpu_support/nvidia/link_nvidia_host_libraries.sh
+++ b/gpu_support/nvidia/link_nvidia_host_libraries.sh
@@ -58,7 +58,7 @@ fi
 # Let's make sure the driver libraries are not already in place
 link_drivers=1
 
-host_injections_nvidia_dir="/cvmfs/software.eessi.io/host_injections/nvidia/${EESSI_CPU_FAMILY}"
+host_injections_nvidia_dir="${EESSI_CVMFS_REPO}/host_injections/nvidia/${EESSI_CPU_FAMILY}"
 host_injection_driver_dir="${host_injections_nvidia_dir}/host"
 host_injection_driver_version_file="$host_injection_driver_dir/driver_version.txt"
 if [ -e "$host_injection_driver_version_file" ]; then

From 76ae987cafc858b6ed9ae3c9f4f0a4a209b31d25 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 4 Dec 2023 21:16:14 +0100
Subject: [PATCH 06/81] do not use /dev/stdout

---
 init/bash | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/init/bash b/init/bash
index 84fe783bce..0b3bdcb869 100644
--- a/init/bash
+++ b/init/bash
@@ -1,10 +1,17 @@
-# Allow for a silent mode
-if [[ -v EESSI_SILENT ]]; then
-  # EESSI_SILENT set
-  output=/dev/null
-else
-  output=/dev/stdout
-fi
+function show_msg {
+  msg=$1
+  if [[ ! -v EESSI_SILENT ]]; then
+    echo "$msg"
+  fi
+}
+
+## Allow for a silent mode
+#if [[ -v EESSI_SILENT ]]; then
+#  # EESSI_SILENT set
+#  output=/dev/null
+#else
+#  output=/dev/stdout
+#fi
 
 # The following method should be safe, but might break if file is a symlink
 # (could switch to $(dirname "$(readlink -f "$BASH_SOURCE")") in that case)
@@ -20,11 +27,11 @@ if [ $? -eq 0 ]; then
     export PATH=$EPREFIX/usr/bin:$EPREFIX/bin:$PATH
 
     # init Lmod
-    echo "Initializing Lmod..." >> $output
+    show_msg "Initializing Lmod..."
     source $EESSI_EPREFIX/usr/share/Lmod/init/bash
 
     # prepend location of modules for EESSI software stack to $MODULEPATH
-    echo "Prepending $EESSI_MODULEPATH to \$MODULEPATH..." >> $output
+    show_msg "Prepending $EESSI_MODULEPATH to \$MODULEPATH..."
     module use $EESSI_MODULEPATH
 
     #echo >> $output

From 0768ae1bc627ede6bdfe1059d49739437b783b78 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 5 Dec 2023 09:06:43 +0100
Subject: [PATCH 07/81] work around permission denied for writing to
 /dev/stdout

---
 init/bash                        | 21 +++++++--------------
 init/eessi_environment_variables | 32 ++++++++++++++++----------------
 2 files changed, 23 insertions(+), 30 deletions(-)

diff --git a/init/bash b/init/bash
index 0b3bdcb869..3adfb4bc9d 100644
--- a/init/bash
+++ b/init/bash
@@ -1,18 +1,11 @@
 function show_msg {
+  # only echo msg if EESSI_SILENT is unset
   msg=$1
   if [[ ! -v EESSI_SILENT ]]; then
     echo "$msg"
   fi
 }
 
-## Allow for a silent mode
-#if [[ -v EESSI_SILENT ]]; then
-#  # EESSI_SILENT set
-#  output=/dev/null
-#else
-#  output=/dev/stdout
-#fi
-
 # The following method should be safe, but might break if file is a symlink
 # (could switch to $(dirname "$(readlink -f "$BASH_SOURCE")") in that case)
 source $(dirname "$BASH_SOURCE")/eessi_environment_variables
@@ -34,12 +27,12 @@ if [ $? -eq 0 ]; then
     show_msg "Prepending $EESSI_MODULEPATH to \$MODULEPATH..."
     module use $EESSI_MODULEPATH
 
-    #echo >> $output
-    #echo "*** Known problems in the ${EESSI_VERSION} software stack ***" >> $output
-    #echo >> $output
-    #echo "1) ..." >> $output
-    #echo >> $output
-    #echo >> $output
+    #show_msg ""
+    #show_msg "*** Known problems in the ${EESSI_VERSION} software stack ***"
+    #show_msg ""
+    #show_msg "1) ..."
+    #show_msg ""
+    #show_msg ""
 
     echo "Environment set up to use EESSI (${EESSI_VERSION}), have fun!"
 
diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables
index 42f4b6b76a..3584bfaf34 100644
--- a/init/eessi_environment_variables
+++ b/init/eessi_environment_variables
@@ -2,24 +2,24 @@
 # $BASH_SOURCE points to correct path, see also http://mywiki.wooledge.org/BashFAQ/028
 EESSI_INIT_DIR_PATH=$(dirname $(realpath $BASH_SOURCE))
 
-# Allow for a silent mode
-if [[ -v EESSI_SILENT ]]; then
-  # EESSI_SILENT set
-  output=/dev/null
-else
-  output=/dev/stdout
-fi
-
 function error() {
     echo -e "\e[31mERROR: $1\e[0m" >&2
     false
 }
 
+function show_msg {
+  # only echo msg if EESSI_SILENT is unset
+  msg=$1
+  if [[ ! -v EESSI_SILENT ]]; then
+    echo "$msg"
+  fi
+}
+
 # set up minimal environment: $EESSI_PREFIX, $EESSI_VERSION, $EESSI_OS_TYPE, $EESSI_CPU_FAMILY, $EPREFIX
 source $EESSI_INIT_DIR_PATH/minimal_eessi_env
 
 if [ -d $EESSI_PREFIX ]; then
-  echo "Found EESSI repo @ $EESSI_PREFIX!" >> $output
+  show_msg "Found EESSI repo @ $EESSI_PREFIX!"
 
   export EESSI_EPREFIX=$EPREFIX
   if [ -d $EESSI_EPREFIX ]; then
@@ -28,21 +28,21 @@ if [ -d $EESSI_PREFIX ]; then
     if [ "$EESSI_USE_ARCHDETECT" == "1" ]; then
       # if archdetect is enabled, use internal code
       export EESSI_SOFTWARE_SUBDIR=$(${EESSI_INIT_DIR_PATH}/eessi_archdetect.sh cpupath)
-      echo "archdetect says ${EESSI_SOFTWARE_SUBDIR}" >> $output
+      show_msg "archdetect says ${EESSI_SOFTWARE_SUBDIR}"
     elif [ "$EESSI_USE_ARCHSPEC" == "1" ]; then
       # note: eessi_software_subdir_for_host.py will pick up value from $EESSI_SOFTWARE_SUBDIR_OVERRIDE if it's defined!
       export EESSI_EPREFIX_PYTHON=$EESSI_EPREFIX/usr/bin/python3
       export EESSI_SOFTWARE_SUBDIR=$($EESSI_EPREFIX_PYTHON ${EESSI_INIT_DIR_PATH}/eessi_software_subdir_for_host.py $EESSI_PREFIX)
-      echo "archspec says ${EESSI_SOFTWARE_SUBDIR}" >> $output
+      show_msg "archspec says ${EESSI_SOFTWARE_SUBDIR}"
     else
       error "Don't know how to detect host CPU, giving up!"
     fi
     if [ ! -z $EESSI_SOFTWARE_SUBDIR ]; then
 
-        echo "Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory." >> $output
+        show_msg "Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory."
         export EESSI_SOFTWARE_PATH=$EESSI_PREFIX/software/$EESSI_OS_TYPE/$EESSI_SOFTWARE_SUBDIR
         if [ ! -z $EESSI_BASIC_ENV ]; then
-          echo "Only setting up basic environment, so we're done" >> $output
+          show_msg "Only setting up basic environment, so we're done"
         elif [ -d $EESSI_SOFTWARE_PATH ]; then
           # Allow for the use of a custom MNS
           if [ -z ${EESSI_CUSTOM_MODULEPATH+x} ]; then
@@ -55,13 +55,13 @@ if [ -d $EESSI_PREFIX ]; then
             fi
             EESSI_MODULEPATH=$EESSI_SOFTWARE_PATH/$EESSI_MODULE_SUBDIR
           else
-            echo "Using defined environment variable \$EESSI_CUSTOM_MODULEPATH to set EESSI_MODULEPATH." >> $output
+            show_msg "Using defined environment variable \$EESSI_CUSTOM_MODULEPATH to set EESSI_MODULEPATH."
             EESSI_MODULEPATH=$EESSI_CUSTOM_MODULEPATH
           fi
 
           if [ -d $EESSI_MODULEPATH ]; then
             export EESSI_MODULEPATH=$EESSI_MODULEPATH
-            echo "Using ${EESSI_MODULEPATH} as the directory to be added to MODULEPATH." >> $output
+            show_msg "Using ${EESSI_MODULEPATH} as the directory to be added to MODULEPATH."
           else
             error "EESSI module path at $EESSI_MODULEPATH not found!"
             false
@@ -69,7 +69,7 @@ if [ -d $EESSI_PREFIX ]; then
 
           export LMOD_RC="$EESSI_SOFTWARE_PATH/.lmod/lmodrc.lua"
           if [ -f $LMOD_RC ]; then
-            echo "Found Lmod configuration file at $LMOD_RC" >> $output
+            show_msg "Found Lmod configuration file at $LMOD_RC"
           else
             error "Lmod configuration file not found at $LMOD_RC"
           fi

From 3ba0f24de3bd5eec7fcafeea9da28c596aac91bb Mon Sep 17 00:00:00 2001
From: Richard Top <richard.topouchian@uib.no>
Date: Tue, 12 Dec 2023 13:52:38 +0000
Subject: [PATCH 08/81] {2023.06}[foss/2022b] Qt5 v5.15.7

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml
index 39bae9c781..30e89c4485 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml
@@ -1,2 +1,3 @@
 easyconfigs:
   - foss-2022b.eb
+  - Qt5-5.15.7-GCCcore-12.2.0.eb

From aa572b561a95aefdec8d1d44f9842f66f812600d Mon Sep 17 00:00:00 2001
From: Richard Top <richard.topouchian@uib.no>
Date: Tue, 12 Dec 2023 16:28:25 +0000
Subject: [PATCH 09/81] added reference to pr:19339

---
 .../2023.06/eessi-2023.06-eb-4.8.2-2022b.yml                  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml
index 30e89c4485..c618089663 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml
@@ -1,3 +1,5 @@
 easyconfigs:
   - foss-2022b.eb
-  - Qt5-5.15.7-GCCcore-12.2.0.eb
+  - Qt5-5.15.7-GCCcore-12.2.0.eb:
+      options:
+        from-pr: 19339

From cf4fc13080e82caaf18694a95875107a04687cf7 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 15 Dec 2023 21:43:02 +0100
Subject: [PATCH 10/81] {2023.06}[gfbf/2023a] R 4.3.2

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index fab52b4468..a9323d3df6 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -29,3 +29,6 @@ easyconfigs:
   - LoopTools-2.15-GCC-12.3.0.eb:
       options:
         from-pr: 19397
+  - R-4.3.2-gfbf-2023a.eb:
+      options:
+        from-pr: 19185

From 7321ffe247dcf69975108480ea8eca9429a20bad Mon Sep 17 00:00:00 2001
From: TopRichard <121792457+TopRichard@users.noreply.github.com>
Date: Sun, 17 Dec 2023 06:49:10 +0100
Subject: [PATCH 11/81] Update eessi-2023.06-eb-4.8.2-2022b.yml

Trying to build HarfBuzz-5.3.1 before Qt5
---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml
index c618089663..69ffb750a2 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml
@@ -1,5 +1,6 @@
 easyconfigs:
   - foss-2022b.eb
-  - Qt5-5.15.7-GCCcore-12.2.0.eb:
+  - HarfBuzz-5.3.1-GCCcore-12.2.0.eb:
       options:
         from-pr: 19339
+  - Qt5-5.15.7-GCCcore-12.2.0.eb

From c45f5cab3b0a542d4964cc293c7dac8c9b0686cb Mon Sep 17 00:00:00 2001
From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com>
Date: Tue, 19 Dec 2023 11:51:36 +0100
Subject: [PATCH 12/81] Add Boost-1.82.0-GCC-12.3.0.eb

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index a9323d3df6..bae536ff9b 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -32,3 +32,4 @@ easyconfigs:
   - R-4.3.2-gfbf-2023a.eb:
       options:
         from-pr: 19185
+  - Boost-1.82.0-GCC-12.3.0.eb

From ec94fd6db01e2e765834d80b034be05e197b0c9f Mon Sep 17 00:00:00 2001
From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com>
Date: Tue, 19 Dec 2023 12:03:32 +0100
Subject: [PATCH 13/81] Add netCDF-4.9.2-gompi-2023a.eb

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index a9323d3df6..299616b396 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -32,3 +32,4 @@ easyconfigs:
   - R-4.3.2-gfbf-2023a.eb:
       options:
         from-pr: 19185
+  - netCDF-4.9.2-gompi-2023a.eb

From 004351a037c62dbacfbe41cd2d5a6a0678ce76d9 Mon Sep 17 00:00:00 2001
From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com>
Date: Tue, 19 Dec 2023 12:41:07 +0100
Subject: [PATCH 14/81] Add FFmpeg-6.0-GCCcore-12.3.0.eb

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index a9323d3df6..0f538c08d4 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -32,3 +32,4 @@ easyconfigs:
   - R-4.3.2-gfbf-2023a.eb:
       options:
         from-pr: 19185
+  - FFmpeg-6.0-GCCcore-12.3.0.eb

From c87714f0106bc13415d0d97e8749b37216757baa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 19 Dec 2023 15:48:28 +0100
Subject: [PATCH 15/81] explicitly set/override EESSI_CPU_FAMILY based on
 EESSI_SOFTWARE_SUBDIR_OVERRIDE

---
 .github/workflows/test-software.eessi.io.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml
index 8c97601611..e6c7a49ada 100644
--- a/.github/workflows/test-software.eessi.io.yml
+++ b/.github/workflows/test-software.eessi.io.yml
@@ -35,6 +35,9 @@ jobs:
           run: |
               export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
               source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
+              # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash),
+              # to prevent issues with checks in the Easybuild configuration that use this variable
+              export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*}
               module load EasyBuild
               which eb
               eb --version

From 7e71f59f6aed081367a6aee89d0b11a7f338a205 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 19 Dec 2023 15:58:32 +0100
Subject: [PATCH 16/81] add the same check for the other check missing
 installations task

---
 .github/workflows/test-software.eessi.io.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml
index e6c7a49ada..3ac341a177 100644
--- a/.github/workflows/test-software.eessi.io.yml
+++ b/.github/workflows/test-software.eessi.io.yml
@@ -56,6 +56,9 @@ jobs:
           run: |
               export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
               source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
+              # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash),
+              # to prevent issues with checks in the Easybuild configuration that use this variable
+              export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*}
               module load EasyBuild
               which eb
               eb --version

From fcc7ddbab8a57d419736d8a0857943569fdb587c Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@gcn6.local.snellius.surf.nl>
Date: Tue, 19 Dec 2023 18:36:31 +0100
Subject: [PATCH 17/81] Also recreated lmodrc when it was changed in a PR

---
 EESSI-install-software.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index f6087b3cfe..8170897726 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -229,7 +229,8 @@ done
 
 echo ">> Creating/updating Lmod cache..."
 export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua"
-if [ ! -f $LMOD_RC ]; then
+lmodrc_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodrc.py$' > /dev/null; echo $?)
+if [ ! -f $LMOD_RC ] || [ ${lmodrc_changed} == '0' ]; then
     python3 $TOPDIR/create_lmodrc.py ${EASYBUILD_INSTALLPATH}
     check_exit_code $? "$LMOD_RC created" "Failed to create $LMOD_RC"
 fi

From 2b09d1c474e399cd1fa6634e18a5234a2e16ac7d Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@gcn6.local.snellius.surf.nl>
Date: Tue, 19 Dec 2023 18:41:43 +0100
Subject: [PATCH 18/81] Modified lmodrc to add CUDA support. It now checks if
 you load the CUDA module if a full CUDA SDK was also installed in
 host_injections (otherwise you have dead links to the non-redistributable
 parts of the CUDA SDK). Furthermore, for GPU enabled modules, it checks if
 the drivers have been linked in in the host_injections directory. It also
 checks if they are new enough to be used with the CUDA version that was used
 as a dependency for the GPU-enabled module you are trying to load. If any of
 these checks is not true, it prints an error message with advice on how to
 proceed.

---
 create_lmodrc.py | 76 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 76 insertions(+)

diff --git a/create_lmodrc.py b/create_lmodrc.py
index ae65153a20..adf221ecba 100755
--- a/create_lmodrc.py
+++ b/create_lmodrc.py
@@ -17,6 +17,81 @@
 }
 """
 
+GPU_LMOD_RC ="""require("strict")
+local hook = require("Hook")
+local open = io.open
+
+local function read_file(path)
+    local file = open(path, "rb") -- r read mode and b binary mode
+    if not file then return nil end
+    local content = file:read "*a" -- *a or *all reads the whole file
+    file:close()
+    return content
+end
+
+local function cuda_enabled_load_hook(t)
+    local frameStk  = require("FrameStk"):singleton()
+    local mt        = frameStk:mt()
+    local simpleName = string.match(t.modFullName, "(.-)/")
+    -- If we try to load CUDA itself, check if the full CUDA SDK was installed on the host in host_injections. 
+    -- This is required for end users to build additional CUDA software. If the full SDK isn't present, refuse
+    -- to load the CUDA module and print an informative message on how to set up GPU support for EESSI
+    if simpleName == 'CUDA' then
+        -- get the full host_injections path
+        local hostInjections = string.gsub(os.getenv('EESSI_SOFTWARE_PATH') or "", 'versions', 'host_injections')
+        -- build final path where the CUDA software should be installed
+        local cudaEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild"
+        local cudaDirExists = isDir(cudaEasyBuildDir)
+        if not cudaDirExists then
+            local advice = "but while the module file exists, the actual software is not entirely shipped with EESSI "
+            advice = advice .. "due to licencing. In order to be able to use the CUDA module, please follow the "
+            advice = advice .. "instructions available under https://www.eessi.io/docs/gpu/ \\n"
+            LmodError("\\nYou requested to load ", simpleName, " ", advice)
+        end
+    end
+    -- when loading CUDA enabled modules check if the necessary driver libraries are accessible to the EESSI linker,
+    -- otherwise, refuse to load the requested module and print error message
+    local haveGpu = mt:haveProperty(simpleName,"arch","gpu")
+    if haveGpu then
+        local arch = os.getenv("EESSI_CPU_FAMILY") or ""
+        local cudaVersionFile = "/cvmfs/pilot.eessi-hpc.org/host_injections/nvidia/" .. arch .. "/latest/cuda_version.txt"
+        local cudaDriverFile = "/cvmfs/pilot.eessi-hpc.org/host_injections/nvidia/" .. arch .. "/latest/libcuda.so"
+        local cudaDriverExists = isFile(cudaDriverFile)
+        local singularityCudaExists = isFile("/.singularity.d/libs/libcuda.so")
+        if not (cudaDriverExists or singularityCudaExists)  then
+            local advice = "which relies on the CUDA runtime environment and driver libraries. "
+            advice = advice .. "In order to be able to use the module, please follow the instructions "
+            advice = advice .. "available under https://www.eessi.io/docs/gpu/ \\n"
+            LmodError("\\nYou requested to load ", simpleName, " ", advice)
+        else
+            -- CUDA driver exists, now we check its version to see if an update is needed
+            if cudaDriverExists then
+                local cudaVersion = read_file(cudaVersionFile)
+                local cudaVersion_req = os.getenv("EESSICUDAVERSION")
+                -- driver CUDA versions don't give a patch version for CUDA
+                local major, minor = string.match(cudaVersion, "(%d+)%.(%d+)")
+                local major_req, minor_req, patch_req = string.match(cudaVersion_req, "(%d+)%.(%d+)%.(%d+)")
+                local driver_libs_need_update = false
+                if major < major_req then
+                    driver_libs_need_update = true
+                elseif major == major_req then
+                    if minor < minor_req then
+                        driver_libs_need_update = true
+                    end
+                end
+                if driver_libs_need_update == true then
+                    local advice = "but the module you want to load requires CUDA  " .. cudaVersion_req .. ". "
+                    advice = advice .. "Please update your CUDA driver libraries and then follow the instructions "
+                    advice = advice .. "under https://www.eessi.io/docs/gpu/ to let EESSI know about the update.\\n"
+                    LmodError("\\nYour driver CUDA version is ", cudaVersion, " ", advice)
+                end
+            end
+        end
+    end
+end
+
+hook.register("load", cuda_enabled_load_hook)
+"""
 
 def error(msg):
     sys.stderr.write("ERROR: %s\n" % msg)
@@ -36,6 +111,7 @@ def error(msg):
     'dot_lmod': DOT_LMOD,
     'prefix': prefix,
 }
+lmodrc_txt += '\n' + GPU_LMOD_RC
 try:
     os.makedirs(os.path.dirname(lmodrc_path), exist_ok=True)
     with open(lmodrc_path, 'w') as fp:

From 62e70ba9474638b78edeafbb3886e432b7e88e7c Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@gcn6.local.snellius.surf.nl>
Date: Tue, 19 Dec 2023 18:44:50 +0100
Subject: [PATCH 19/81] Adapt created_lmodrc.py for the new domain

---
 create_lmodrc.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/create_lmodrc.py b/create_lmodrc.py
index adf221ecba..9c007c15e6 100755
--- a/create_lmodrc.py
+++ b/create_lmodrc.py
@@ -54,8 +54,8 @@
     local haveGpu = mt:haveProperty(simpleName,"arch","gpu")
     if haveGpu then
         local arch = os.getenv("EESSI_CPU_FAMILY") or ""
-        local cudaVersionFile = "/cvmfs/pilot.eessi-hpc.org/host_injections/nvidia/" .. arch .. "/latest/cuda_version.txt"
-        local cudaDriverFile = "/cvmfs/pilot.eessi-hpc.org/host_injections/nvidia/" .. arch .. "/latest/libcuda.so"
+        local cudaVersionFile = "/cvmfs/software.eessi.io/host_injections/nvidia/" .. arch .. "/latest/cuda_version.txt"
+        local cudaDriverFile = "/cvmfs/software.eessi.io/host_injections/nvidia/" .. arch .. "/latest/libcuda.so"
         local cudaDriverExists = isFile(cudaDriverFile)
         local singularityCudaExists = isFile("/.singularity.d/libs/libcuda.so")
         if not (cudaDriverExists or singularityCudaExists)  then

From 045c099b3c4eee0a742a5859a32ba2142e21cc9c Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@gcn6.local.snellius.surf.nl>
Date: Tue, 19 Dec 2023 18:52:52 +0100
Subject: [PATCH 20/81] Add post_sanitycheck hook for CUDA in order to only
 ship the files we are allowed to redistribute. It will create symlinks to the
 host_injections directory for the rest of the files that we are not allowed
 to redistribute. Additionally, create a hook to inject the GPU lmod property
 when creating module files for modules that have CUDA as a dependency

---
 eb_hooks.py | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 88 insertions(+)

diff --git a/eb_hooks.py b/eb_hooks.py
index 6fe92c7f7b..78580c14b9 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -68,6 +68,9 @@ def parse_hook(ec, *args, **kwargs):
     if ec.name in PARSE_HOOKS:
         PARSE_HOOKS[ec.name](ec, eprefix)
 
+    # inject the GPU property (if required)
+    ec = inject_gpu_property(ec)
+
 
 def post_ready_hook(self, *args, **kwargs):
     """
@@ -247,6 +250,12 @@ def pre_configure_hook(self, *args, **kwargs):
         PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs)
 
 
+def post_sanitycheck_hook(self, *args, **kwargs):
+    """Main post-sanity-check hook: trigger custom functions based on software name."""
+    if self.name in POST_SANITYCHECK_HOOKS:
+        POST_SANITYCHECK_HOOKS[self.name](self, *args, **kwargs)
+
+
 def pre_configure_hook_openblas_optarch_generic(self, *args, **kwargs):
     """
     Pre-configure hook for OpenBLAS: add DYNAMIC_ARCH=1 to build/test/install options when using --optarch=GENERIC
@@ -393,6 +402,81 @@ def pre_single_extension_isoband(ext, *args, **kwargs):
         ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' src/testthat/vendor/catch.h && "
 
 
+def post_sanitycheck_cuda(self, *args, **kwargs):
+    """Delete CUDA files we are not allowed to ship and replace them with a symlink to a possible installation under host_injections."""
+    print_msg("Replacing CUDA stuff we cannot ship with symlinks...")
+    # read CUDA EULA
+    eula_path = os.path.join(self.installdir, "EULA.txt")
+    tmp_buffer = []
+    with open(eula_path) as infile:
+        copy = False
+        for line in infile:
+            if line.strip() == "2.6. Attachment A":
+                copy = True
+                continue
+            elif line.strip() == "2.7. Attachment B":
+                copy = False
+                continue
+            elif copy:
+                tmp_buffer.append(line)
+    # create whitelist without file extensions, they're not really needed and they only complicate things
+    whitelist = ['EULA', 'README']
+    file_extensions = [".so", ".a", ".h", ".bc"]
+    for tmp in tmp_buffer:
+        for word in tmp.split():
+            if any(ext in word for ext in file_extensions):
+                whitelist.append(word.split(".")[0])
+    whitelist = list(set(whitelist))
+    # Do some quick checks for things we should or shouldn't have in the list
+    if "nvcc" in whitelist:
+        raise EasyBuildError("Found 'nvcc' in whitelist: %s" % whitelist)
+    if "libcudart" not in whitelist:
+        raise EasyBuildError("Did not find 'libcudart' in whitelist: %s" % whitelist)
+    # iterate over all files in the CUDA path
+    for root, dirs, files in os.walk(self.installdir):
+        for filename in files:
+            # we only really care about real files, i.e. not symlinks
+            if not os.path.islink(os.path.join(root, filename)):
+                # check if the current file is part of the whitelist
+                basename = filename.split(".")[0]
+                if basename not in whitelist:
+                    # if it is not in the whitelist, delete the file and create a symlink to host_injections
+                    source = os.path.join(root, filename)
+                    target = source.replace("versions", "host_injections")
+                    # Make sure source and target are not the same
+                    if source == target:
+                        raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you are"
+                                             "using this hook for an EESSI installation?")
+                    os.remove(source)
+                    # Using os.symlink requires the existence of the target directory, so we use os.system
+                    system_command="ln -s '%s' '%s'" % (target, source)
+                    if os.system(system_command) != 0:
+                        raise EasyBuildError("Failed to create symbolic link: %s" % system_command)
+
+
+def inject_gpu_property(ec):
+    ec_dict = ec.asdict()
+    # Check if CUDA is in the dependencies, if so add the GPU Lmod tag
+    if ("CUDA" in [dep[0] for dep in iter(ec_dict["dependencies"])]):
+        ec.log.info("[parse hook] Injecting gpu as Lmod arch property and envvar with CUDA version")
+        key = "modluafooter"
+        value = 'add_property("arch","gpu")'
+        cuda_version = 0
+        for dep in iter(ec_dict["dependencies"]):
+            # Make CUDA a build dependency only (rpathing saves us from link errors)
+            if "CUDA" in dep[0]:
+                cuda_version = dep[1]
+                ec_dict["dependencies"].remove(dep)
+                ec_dict["builddependencies"].append(dep) if dep not in ec_dict["builddependencies"] else ec_dict["builddependencies"]
+        value = "\n".join([value, 'setenv("EESSICUDAVERSION","%s")' % cuda_version])
+        if key in ec_dict:
+            if not value in ec_dict[key]:
+                ec[key] = "\n".join([ec_dict[key], value])
+        else:
+            ec[key] = value
+    return ec
+
+
 PARSE_HOOKS = {
     'CGAL': parse_hook_cgal_toolchainopts_precise,
     'fontconfig': parse_hook_fontconfig_add_fonts,
@@ -424,3 +508,7 @@ def pre_single_extension_isoband(ext, *args, **kwargs):
     'isoband': pre_single_extension_isoband,
     'testthat': pre_single_extension_testthat,
 }
+
+POST_SANITYCHECK_HOOKS = {
+    'CUDA': post_sanitycheck_cuda,
+}

From 4a4c6e768d1b469baf4476a32f9721f789ce341a Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@gcn6.local.snellius.surf.nl>
Date: Tue, 19 Dec 2023 18:53:37 +0100
Subject: [PATCH 21/81] Add (the redistributable part of) CUDA to the softare
 stack

---
 .../2023.06/eessi-2023.06-eb-4.8.2-system.yml                 | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml
index f02b9f2802..86d6931820 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml
@@ -5,3 +5,7 @@ easyconfigs:
   - Nextflow-23.10.0.eb:
       options:
         from-pr: 19172
+  - CUDA-12.1.1.eb:
+      options:
+        include-easyblocks-from-pr: 3045
+        accept-eula-for: CUDA

From 0346b22291671d5f03343b11e9a919f13f3ab9c5 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@gcn6.local.snellius.surf.nl>
Date: Tue, 19 Dec 2023 18:55:15 +0100
Subject: [PATCH 22/81] Add CUDA-Samples to the build list

---
 .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml                | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index 567db44e42..0537c448e5 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -35,3 +35,9 @@ easyconfigs:
   - Boost-1.82.0-GCC-12.3.0.eb
   - netCDF-4.9.2-gompi-2023a.eb
   - FFmpeg-6.0-GCCcore-12.3.0.eb
+  - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb
+      # use easyconfig that only install subset of CUDA samples,
+      # to circumvent problem with nvcc linking to glibc of host OS;
+      # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19189
+      options:
+        from-pr: 19189

From 2b1054f7b393adf590201b5a67ddf5622ccd1b3f Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@gcn6.local.snellius.surf.nl>
Date: Tue, 19 Dec 2023 19:32:44 +0100
Subject: [PATCH 23/81] install_scripts.sh should install the scripts required
 to do install the required GPU components in host_injections. The
 EESSI-install-software.sh has been modified to run install_scripts.sh early
 on and then run the actual installed scripts to install a full cuda SDK and
 drivers. This should enable building and using CUDA software anywhere down
 the line in this same environment

---
 EESSI-install-software.sh | 16 ++++++++++
 install_scripts.sh        | 62 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 78 insertions(+)
 create mode 100755 install_scripts.sh

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index f6087b3cfe..4aac3772cf 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -187,6 +187,22 @@ fi
 # assume there's only one diff file that corresponds to the PR patch file
 pr_diff=$(ls [0-9]*.diff | head -1)
 
+# install any additional required scripts
+# order is important: these are needed to install a full CUDA SDK in host_injections
+install_scripts_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^install_scripts.sh$' > /dev/null; echo $?)
+if [ ${install_scripts_changed} == '0' ]; then
+    # for now, this just reinstalls all scripts. Note the most elegant, but works
+    ${TOPDIR}/install_scripts.sh --prefix ${EESSI_CVMFS_REPO}
+fi
+
+# Install full CUDA SDK in host_injections
+# Hardcode this for now, see if it works
+# TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install
+${EESSI_CVMFS_REPO}/gpu_support/nvidia/install_cuda_host_injections.sh 12.1.1
+
+# Install drivers in host_injections
+${EESSI_CVMFS_REPO}/gpu_support/nvidia/link_nvidia_host_libraries.sh
+
 # use PR patch file to determine in which easystack files stuff was added
 for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing'); do
 
diff --git a/install_scripts.sh b/install_scripts.sh
new file mode 100755
index 0000000000..209d953c88
--- /dev/null
+++ b/install_scripts.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+#
+# Script to install scripts from the software-layer repo into the EESSI software stack
+
+display_help() {
+  echo "usage: $0 [OPTIONS]"
+  echo "  -p | --prefix          -  prefix to copy the scripts to"
+  echo "  -h | --help            -  display this usage information"
+}
+
+
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -o|--prefix)
+      INSTALL_PREFIX="$2"
+      shift 2
+      ;;
+    -h|--help)
+      display_help  # Call your function
+      # no shifting needed here, we're done.
+      exit 0
+      ;;
+    -*|--*)
+      echo "Error: Unknown option: $1" >&2
+      exit 1
+      ;;
+    *)  # No more options
+      POSITIONAL_ARGS+=("$1") # save positional arg
+      shift
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}"
+
+TOPDIR=$(dirname $(realpath $0))
+
+# Subdirs for generic scripts
+SCRIPTS_DIR_SOURCE=${TOPDIR}/scripts/  # Source dir
+SCRIPTS_DIR_TARGET=${INSTALL_PREFIX}/scripts/  # Target dir
+
+# Create target dir
+mkdir -p ${SCRIPTS_DIR_TARGET}
+
+# Copy scripts into this prefix
+for file in utils.sh; do
+    cp ${SCRIPTS_DIR_SOURCE}/${file} ${SCRIPTS_DIR_TARGET}/${file}
+done
+# Subdirs for GPU support
+NVIDIA_GPU_SUPPORT_DIR_SOURCE=${TOPDIR}/gpu_support/nvidia/  # Source dir
+NVIDIA_GPU_SUPPORT_DIR_TARGET=${INSTALL_PREFIX}/gpu_support/nvidia/  # Target dir
+
+# Create target dir
+mkdir -p ${NVIDIA_GPU_SUPPORT_DIR_TARGET}
+
+# Copy files from this directory into the prefix
+# To be on the safe side, we dont do recursive copies, but we are explicitely copying each individual file we want to add
+for file in install_cuda_host_injections.sh link_nvidia_host_injections.sh; do
+    cp ${NVIDIA_GPU_SUPPORT_DIR_SOURCE}/${file} ${NVIDIA_GPU_SUPPORT_DIR_TARGET}/${file}
+done

From 5905e727f146480d6472d2f3a1d0ca6ab53105ee Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Wed, 20 Dec 2023 01:07:41 +0100
Subject: [PATCH 24/81] Tweak GPU support implementation

---
 EESSI-install-software.sh                        | 11 ++++-------
 .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml     |  2 +-
 install_scripts.sh                               | 16 ++++++++--------
 3 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 5e9bd5d472..95fb03e9b7 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -189,19 +189,16 @@ pr_diff=$(ls [0-9]*.diff | head -1)
 
 # install any additional required scripts
 # order is important: these are needed to install a full CUDA SDK in host_injections
-install_scripts_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^install_scripts.sh$' > /dev/null; echo $?)
-if [ ${install_scripts_changed} == '0' ]; then
-    # for now, this just reinstalls all scripts. Note the most elegant, but works
-    ${TOPDIR}/install_scripts.sh --prefix ${EESSI_CVMFS_REPO}
-fi
+# for now, this just reinstalls all scripts. Note the most elegant, but works
+${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX}
 
 # Install full CUDA SDK in host_injections
 # Hardcode this for now, see if it works
 # TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install
-${EESSI_CVMFS_REPO}/gpu_support/nvidia/install_cuda_host_injections.sh 12.1.1
+${EESSI_PREFIX}/gpu_support/nvidia/install_cuda_host_injections.sh -c 12.1.1 --accept-cuda-eula
 
 # Install drivers in host_injections
-${EESSI_CVMFS_REPO}/gpu_support/nvidia/link_nvidia_host_libraries.sh
+${EESSI_PREFIX}/gpu_support/nvidia/link_nvidia_host_libraries.sh
 
 # use PR patch file to determine in which easystack files stuff was added
 for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing'); do
diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index 0537c448e5..87ccd69e99 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -35,7 +35,7 @@ easyconfigs:
   - Boost-1.82.0-GCC-12.3.0.eb
   - netCDF-4.9.2-gompi-2023a.eb
   - FFmpeg-6.0-GCCcore-12.3.0.eb
-  - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb
+  - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb:
       # use easyconfig that only install subset of CUDA samples,
       # to circumvent problem with nvcc linking to glibc of host OS;
       # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19189
diff --git a/install_scripts.sh b/install_scripts.sh
index 209d953c88..8fb27826c6 100755
--- a/install_scripts.sh
+++ b/install_scripts.sh
@@ -13,7 +13,7 @@ POSITIONAL_ARGS=()
 
 while [[ $# -gt 0 ]]; do
   case $1 in
-    -o|--prefix)
+    -p|--prefix)
       INSTALL_PREFIX="$2"
       shift 2
       ;;
@@ -38,25 +38,25 @@ set -- "${POSITIONAL_ARGS[@]}"
 TOPDIR=$(dirname $(realpath $0))
 
 # Subdirs for generic scripts
-SCRIPTS_DIR_SOURCE=${TOPDIR}/scripts/  # Source dir
-SCRIPTS_DIR_TARGET=${INSTALL_PREFIX}/scripts/  # Target dir
+SCRIPTS_DIR_SOURCE=${TOPDIR}/scripts  # Source dir
+SCRIPTS_DIR_TARGET=${INSTALL_PREFIX}/scripts  # Target dir
 
 # Create target dir
 mkdir -p ${SCRIPTS_DIR_TARGET}
 
 # Copy scripts into this prefix
 for file in utils.sh; do
-    cp ${SCRIPTS_DIR_SOURCE}/${file} ${SCRIPTS_DIR_TARGET}/${file}
+    cp -u ${SCRIPTS_DIR_SOURCE}/${file} ${SCRIPTS_DIR_TARGET}/${file}
 done
 # Subdirs for GPU support
-NVIDIA_GPU_SUPPORT_DIR_SOURCE=${TOPDIR}/gpu_support/nvidia/  # Source dir
-NVIDIA_GPU_SUPPORT_DIR_TARGET=${INSTALL_PREFIX}/gpu_support/nvidia/  # Target dir
+NVIDIA_GPU_SUPPORT_DIR_SOURCE=${TOPDIR}/gpu_support/nvidia  # Source dir
+NVIDIA_GPU_SUPPORT_DIR_TARGET=${INSTALL_PREFIX}/gpu_support/nvidia  # Target dir
 
 # Create target dir
 mkdir -p ${NVIDIA_GPU_SUPPORT_DIR_TARGET}
 
 # Copy files from this directory into the prefix
 # To be on the safe side, we dont do recursive copies, but we are explicitely copying each individual file we want to add
-for file in install_cuda_host_injections.sh link_nvidia_host_injections.sh; do
-    cp ${NVIDIA_GPU_SUPPORT_DIR_SOURCE}/${file} ${NVIDIA_GPU_SUPPORT_DIR_TARGET}/${file}
+for file in install_cuda_host_injections.sh link_nvidia_host_libraries.sh; do
+    cp -u ${NVIDIA_GPU_SUPPORT_DIR_SOURCE}/${file} ${NVIDIA_GPU_SUPPORT_DIR_TARGET}/${file}
 done

From 73618a00ee06889314d6efc0e362882b39f34161 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Wed, 20 Dec 2023 01:22:21 +0100
Subject: [PATCH 25/81] Add missing quotes on errors

---
 gpu_support/nvidia/link_nvidia_host_libraries.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/gpu_support/nvidia/link_nvidia_host_libraries.sh b/gpu_support/nvidia/link_nvidia_host_libraries.sh
index 6458be7fae..cb7420a0e9 100755
--- a/gpu_support/nvidia/link_nvidia_host_libraries.sh
+++ b/gpu_support/nvidia/link_nvidia_host_libraries.sh
@@ -36,7 +36,7 @@ if [ ${#found_paths[@]} -gt 0 ]; then
     host_ldconfig=${found_paths[0]}
 else
     error="$command_name not found in PATH or only found in paths starting with $exclude_prefix."
-    fatal_error $error
+    fatal_error "$error"
 fi
 
 # Make sure EESSI is initialised (doesn't matter what version)
@@ -52,7 +52,7 @@ if $nvidia_smi_command > /dev/null; then
   host_cuda_version=$(nvidia-smi  -q --display=COMPUTE | grep CUDA | awk 'NF>1{print $NF}')
 else
   error="Failed to successfully execute\n  $nvidia_smi_command\n"
-  fatal_error $error
+  fatal_error "$error"
 fi
 
 # Let's make sure the driver libraries are not already in place
@@ -71,7 +71,7 @@ if [ -e "$host_injection_driver_version_file" ]; then
     rm $host_injection_driver_dir/*
     if [ $? -ne 0 ]; then
       error="Unable to remove files under '$host_injection_driver_dir'."
-      fatal_error $error
+      fatal_error "$error"
     fi
   fi
 fi

From 32925fe17e7576847be87ef3f1786ee374a2356a Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@gcn6.local.snellius.surf.nl>
Date: Wed, 20 Dec 2023 11:12:27 +0100
Subject: [PATCH 26/81] Error messages now refer to the scripts that need to be
 run to install the CUDA SDK or link the CUDA drivers

---
 create_lmodrc.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/create_lmodrc.py b/create_lmodrc.py
index 9c007c15e6..6a72d8dc62 100755
--- a/create_lmodrc.py
+++ b/create_lmodrc.py
@@ -43,9 +43,11 @@
         local cudaEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild"
         local cudaDirExists = isDir(cudaEasyBuildDir)
         if not cudaDirExists then
+            local cvmfsRepo = os.getenv('EESSI_CVMFS_REPO')
             local advice = "but while the module file exists, the actual software is not entirely shipped with EESSI "
-            advice = advice .. "due to licencing. In order to be able to use the CUDA module, please follow the "
-            advice = advice .. "instructions available under https://www.eessi.io/docs/gpu/ \\n"
+            advice = advice .. "due to licencing. Please install a full copy of the CUDA SDK using the script "
+            advice = advice .. cvmfsRepo .. "/gpu_support/nvidia/install_cuda_host_injections.sh.\\n"
+            advice = advice .. "More information, see https://www.eessi.io/docs/gpu/.\\n"
             LmodError("\\nYou requested to load ", simpleName, " ", advice)
         end
     end
@@ -60,8 +62,10 @@
         local singularityCudaExists = isFile("/.singularity.d/libs/libcuda.so")
         if not (cudaDriverExists or singularityCudaExists)  then
             local advice = "which relies on the CUDA runtime environment and driver libraries. "
-            advice = advice .. "In order to be able to use the module, please follow the instructions "
-            advice = advice .. "available under https://www.eessi.io/docs/gpu/ \\n"
+            advice = advice .. "In order to be able to use the module, please run the script "
+            advice = advice .. cvmfsRepo .. "/gpu_support/nvidia/link_nvidia_host_libraries.sh "
+            advice = advice .. "to make sure EESSI can find the drivers from on your host system.\\n"
+            advice = advice .. "More information, see https://www.eessi.io/docs/gpu/.\\n"
             LmodError("\\nYou requested to load ", simpleName, " ", advice)
         else
             -- CUDA driver exists, now we check its version to see if an update is needed
@@ -81,8 +85,10 @@
                 end
                 if driver_libs_need_update == true then
                     local advice = "but the module you want to load requires CUDA  " .. cudaVersion_req .. ". "
-                    advice = advice .. "Please update your CUDA driver libraries and then follow the instructions "
-                    advice = advice .. "under https://www.eessi.io/docs/gpu/ to let EESSI know about the update.\\n"
+                    advice = advice .. "Please update your CUDA driver libraries and rerun the script"
+                    advice = advice .. cvmfsRepo .. "/gpu_support/nvidia/install_cuda_host_injections.sh "
+                    advice = advice .. "to let EESSI know about the update.\\n"
+                    advice = advice .. "More information, see https://www.eessi.io/docs/gpu/.\\n"
                     LmodError("\\nYour driver CUDA version is ", cudaVersion, " ", advice)
                 end
             end

From a33a0cd204a13be3321336caa18c5b3faa5b29f0 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Dec 2023 13:50:46 +0100
Subject: [PATCH 27/81] make install_scripts a bit more verbose

---
 install_scripts.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/install_scripts.sh b/install_scripts.sh
index 8fb27826c6..d53dbb6a9c 100755
--- a/install_scripts.sh
+++ b/install_scripts.sh
@@ -45,8 +45,9 @@ SCRIPTS_DIR_TARGET=${INSTALL_PREFIX}/scripts  # Target dir
 mkdir -p ${SCRIPTS_DIR_TARGET}
 
 # Copy scripts into this prefix
+echo "copying scripts from ${SCRIPTS_DIR_SOURCE} to ${SCRIPTS_DIR_TARGET}"
 for file in utils.sh; do
-    cp -u ${SCRIPTS_DIR_SOURCE}/${file} ${SCRIPTS_DIR_TARGET}/${file}
+    cp -v -u ${SCRIPTS_DIR_SOURCE}/${file} ${SCRIPTS_DIR_TARGET}/${file}
 done
 # Subdirs for GPU support
 NVIDIA_GPU_SUPPORT_DIR_SOURCE=${TOPDIR}/gpu_support/nvidia  # Source dir
@@ -57,6 +58,7 @@ mkdir -p ${NVIDIA_GPU_SUPPORT_DIR_TARGET}
 
 # Copy files from this directory into the prefix
 # To be on the safe side, we dont do recursive copies, but we are explicitely copying each individual file we want to add
+echo "copying scripts from ${NVIDIA_GPU_SUPPORT_DIR_SOURCE} to ${NVIDIA_GPU_SUPPORT_DIR_TARGET}"
 for file in install_cuda_host_injections.sh link_nvidia_host_libraries.sh; do
-    cp -u ${NVIDIA_GPU_SUPPORT_DIR_SOURCE}/${file} ${NVIDIA_GPU_SUPPORT_DIR_TARGET}/${file}
+    cp -v -u ${NVIDIA_GPU_SUPPORT_DIR_SOURCE}/${file} ${NVIDIA_GPU_SUPPORT_DIR_TARGET}/${file}
 done

From c7b380d7b3d7580c935824fb3f3c4c80200b8301 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Dec 2023 14:07:29 +0100
Subject: [PATCH 28/81] use separate easystack file for CUDA + control order in
 which easystack are processed (where needed)

---
 easystacks/software.eessi.io/2023.06/README.md             | 7 +++++++
 ....2-system.yml => eessi-2023.06-eb-4.8.2-001-system.yml} | 4 ----
 .../2023.06/eessi-2023.06-eb-4.8.2-010-CUDA.yml            | 5 +++++
 3 files changed, 12 insertions(+), 4 deletions(-)
 create mode 100644 easystacks/software.eessi.io/2023.06/README.md
 rename easystacks/software.eessi.io/2023.06/{eessi-2023.06-eb-4.8.2-system.yml => eessi-2023.06-eb-4.8.2-001-system.yml} (56%)
 create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-010-CUDA.yml

diff --git a/easystacks/software.eessi.io/2023.06/README.md b/easystacks/software.eessi.io/2023.06/README.md
new file mode 100644
index 0000000000..733ebf9475
--- /dev/null
+++ b/easystacks/software.eessi.io/2023.06/README.md
@@ -0,0 +1,7 @@
+File naming matters, since it determines the order in which easystack files are processed.
+
+Software installed with system toolchain should be installed first,
+this includes EasyBuild itself, see `eessi-2023.06-eb-4.8.2-001-system.yml` .
+
+CUDA installations must be done before CUDA is required as dependency for something
+built with a non-system toolchain, see `eessi-2023.06-eb-4.8.2-010-CUDA.yml` .
diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml
similarity index 56%
rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml
rename to easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml
index 86d6931820..f02b9f2802 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml
@@ -5,7 +5,3 @@ easyconfigs:
   - Nextflow-23.10.0.eb:
       options:
         from-pr: 19172
-  - CUDA-12.1.1.eb:
-      options:
-        include-easyblocks-from-pr: 3045
-        accept-eula-for: CUDA
diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-010-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-010-CUDA.yml
new file mode 100644
index 0000000000..dda274b8db
--- /dev/null
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-010-CUDA.yml
@@ -0,0 +1,5 @@
+easyconfigs:
+  - CUDA-12.1.1.eb:
+      options:
+        include-easyblocks-from-pr: 3045
+        accept-eula-for: CUDA

From f506566c982818bf0ee985c70d7dbe0d24084035 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Dec 2023 14:10:12 +0100
Subject: [PATCH 29/81] copy EasyBuild log file in case CUDA installation
 failed in install_cuda_host_injections.sh

---
 gpu_support/nvidia/install_cuda_host_injections.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/gpu_support/nvidia/install_cuda_host_injections.sh b/gpu_support/nvidia/install_cuda_host_injections.sh
index f02f0da02e..62996fa924 100755
--- a/gpu_support/nvidia/install_cuda_host_injections.sh
+++ b/gpu_support/nvidia/install_cuda_host_injections.sh
@@ -200,7 +200,9 @@ else
   eb --prefix="$tmpdir" ${extra_args} --accept-eula-for=CUDA --hooks="$tmpdir"/none.py --installpath="${cuda_install_parent}"/ "${cuda_easyconfig}"
   ret=$?
   if [ $ret -ne 0 ]; then
-    fatal_error  "CUDA installation failed, please check EasyBuild logs..."
+    eb_last_log=$(unset EB_VERBOSE; eb --last-log)
+    cp -a ${eb_last_log} .
+    fatal_error "CUDA installation failed, please check EasyBuild logs $(basename ${eb_last_log})..."
   else
     echo_green "CUDA installation at ${cuda_install_parent}/software/CUDA/${install_cuda_version} succeeded!"
   fi

From e3ddaccfc04cca311d40ef69eb55028ccf257dcc Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Dec 2023 14:29:00 +0100
Subject: [PATCH 30/81] add additional optional options required for handling
 NVIDIA support to start build container

---
 bot/build.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/bot/build.sh b/bot/build.sh
index 4af217628e..66f93d523e 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -176,6 +176,11 @@ mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR}
 declare -a BUILD_STEP_ARGS=()
 BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
 BUILD_STEP_ARGS+=("--storage" "${STORAGE}")
+# add options required to handle NVIDIA support
+BUILD_STEP_ARGS+=("--nvidia" "all")
+if [[ ! -z ${SHARED_FS_PATH} ]]; then
+    BUILD_STEP_ARGS+=("--host-injections ${SHARED_FS_PATH}/host-injections")
+fi
 
 # prepare arguments to install_software_layer.sh (specific to build step)
 declare -a INSTALL_SCRIPT_ARGS=()

From 16ddf7f8a0d29cfe47c9948933bb2a10b5ee17c0 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Dec 2023 14:41:53 +0100
Subject: [PATCH 31/81] fix typo when passing --host-injections to container
 script

---
 bot/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/build.sh b/bot/build.sh
index 66f93d523e..1622e757e2 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -179,7 +179,7 @@ BUILD_STEP_ARGS+=("--storage" "${STORAGE}")
 # add options required to handle NVIDIA support
 BUILD_STEP_ARGS+=("--nvidia" "all")
 if [[ ! -z ${SHARED_FS_PATH} ]]; then
-    BUILD_STEP_ARGS+=("--host-injections ${SHARED_FS_PATH}/host-injections")
+    BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections")
 fi
 
 # prepare arguments to install_software_layer.sh (specific to build step)

From 35d6084a25ea76b75982e9bbac0154236d860d6a Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Dec 2023 14:50:04 +0100
Subject: [PATCH 32/81] correctly pass --nv to singularity command

---
 eessi_container.sh | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 81d7be81ad..268f94975e 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -118,7 +118,6 @@ STORAGE=
 LIST_REPOS=0
 MODE="shell"
 SETUP_NVIDIA=0
-ADDITIONAL_SINGULARITY_FLAGS=
 REPOSITORY="EESSI"
 RESUME=
 SAVE=
@@ -437,12 +436,14 @@ BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}"
 
 [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
 
+declare -a ADDITIONAL_CONTAINER_OPTIONS=()
+
 # Configure anything we need for NVIDIA GPUs and CUDA installation
 if [[ ${SETUP_NVIDIA} -eq 1 ]]; then
     if [[ "${NVIDIA_MODE}" == "run" || "${NVIDIA_MODE}" == "all" ]]; then
         # Give singularity the appropriate flag
-        ADDITIONAL_SINGULARITY_FLAGS="--nv ${ADDITIONAL_SINGULARITY_FLAGS}"
-        [[ ${VERBOSE} -eq 1 ]] && echo "ADDITIONAL_SINGULARITY_FLAGS=${ADDITIONAL_SINGULARITY_FLAGS}"
+        ADDITIONAL_CONTAINER_OPTIONS+=(--nv)
+        [[ ${VERBOSE} -eq 1 ]] && echo "ADDITIONAL_CONTAINER_OPTIONS=${ADDITIONAL_CONTAINER_OPTIONS[@]}"
     fi
     if [[ "${NVIDIA_MODE}" == "install" || "${NVIDIA_MODE}" == "all" ]]; then
         # Add additional bind mounts to allow CUDA to install within a container
@@ -621,8 +622,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
 fi
 
 echo "Launching container with command (next line):"
-echo "singularity ${RUN_QUIET} ${MODE} ${ADDITIONAL_SINGULARITY_FLAGS} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
-singularity ${RUN_QUIET} ${MODE} ${ADDITIONAL_SINGULARITY_FLAGS} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
+echo "singularity ${RUN_QUIET} ${MODE} ${ADDITIONAL_CONTAINER_OPTIONS[@]} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
+singularity ${RUN_QUIET} ${MODE} "${ADDITIONAL_CONTAINER_OPTIONS[@]}" "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
 exit_code=$?
 
 # 6. save tmp if requested (arg -s|--save)

From fd976675c798b1c79205460eec225469fbf28473 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Dec 2023 14:50:55 +0100
Subject: [PATCH 33/81] use quotes when adding --nv

---
 eessi_container.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 268f94975e..d6e9558202 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -442,7 +442,7 @@ declare -a ADDITIONAL_CONTAINER_OPTIONS=()
 if [[ ${SETUP_NVIDIA} -eq 1 ]]; then
     if [[ "${NVIDIA_MODE}" == "run" || "${NVIDIA_MODE}" == "all" ]]; then
         # Give singularity the appropriate flag
-        ADDITIONAL_CONTAINER_OPTIONS+=(--nv)
+        ADDITIONAL_CONTAINER_OPTIONS+=("--nv")
         [[ ${VERBOSE} -eq 1 ]] && echo "ADDITIONAL_CONTAINER_OPTIONS=${ADDITIONAL_CONTAINER_OPTIONS[@]}"
     fi
     if [[ "${NVIDIA_MODE}" == "install" || "${NVIDIA_MODE}" == "all" ]]; then

From 19171465240f52b3ef0f7eba17e4467031e23a58 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Dec 2023 15:22:20 +0100
Subject: [PATCH 34/81] comment out running of link_nvidia_host_libraries.sh
 script, since it requires working nvidia-smi command

---
 EESSI-install-software.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 95fb03e9b7..4e3f0acee5 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -198,7 +198,9 @@ ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX}
 ${EESSI_PREFIX}/gpu_support/nvidia/install_cuda_host_injections.sh -c 12.1.1 --accept-cuda-eula
 
 # Install drivers in host_injections
-${EESSI_PREFIX}/gpu_support/nvidia/link_nvidia_host_libraries.sh
+# TODO: this is commented out for now, because the script assumes that nvidia-smi is available and works;
+#       if not, an error is produced, and the bot flags the whole build as failed (even when not installing GPU software)
+# ${EESSI_PREFIX}/gpu_support/nvidia/link_nvidia_host_libraries.sh
 
 # use PR patch file to determine in which easystack files stuff was added
 for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing'); do

From f80f0fc0f70026e4be1fa980e13a991102503737 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Dec 2023 20:15:58 +0100
Subject: [PATCH 35/81] clean up post_sanitycheck_cuda hook and
 inject_gpu_property function used in parse_hook

---
 eb_hooks.py | 149 +++++++++++++++++++++++++++++-----------------------
 1 file changed, 83 insertions(+), 66 deletions(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 78580c14b9..27cc873fa1 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -7,7 +7,7 @@
 from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
 from easybuild.tools.build_log import EasyBuildError, print_msg
 from easybuild.tools.config import build_option, update_build_option
-from easybuild.tools.filetools import apply_regex_substitutions, copy_file, which
+from easybuild.tools.filetools import apply_regex_substitutions, copy_file, remove_file, symlink, which
 from easybuild.tools.run import run_cmd
 from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features
 from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC
@@ -250,12 +250,6 @@ def pre_configure_hook(self, *args, **kwargs):
         PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs)
 
 
-def post_sanitycheck_hook(self, *args, **kwargs):
-    """Main post-sanity-check hook: trigger custom functions based on software name."""
-    if self.name in POST_SANITYCHECK_HOOKS:
-        POST_SANITYCHECK_HOOKS[self.name](self, *args, **kwargs)
-
-
 def pre_configure_hook_openblas_optarch_generic(self, *args, **kwargs):
     """
     Pre-configure hook for OpenBLAS: add DYNAMIC_ARCH=1 to build/test/install options when using --optarch=GENERIC
@@ -402,76 +396,99 @@ def pre_single_extension_isoband(ext, *args, **kwargs):
         ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' src/testthat/vendor/catch.h && "
 
 
+def post_sanitycheck_hook(self, *args, **kwargs):
+    """Main post-sanity-check hook: trigger custom functions based on software name."""
+    if self.name in POST_SANITYCHECK_HOOKS:
+        POST_SANITYCHECK_HOOKS[self.name](self, *args, **kwargs)
+
+
 def post_sanitycheck_cuda(self, *args, **kwargs):
-    """Delete CUDA files we are not allowed to ship and replace them with a symlink to a possible installation under host_injections."""
-    print_msg("Replacing CUDA stuff we cannot ship with symlinks...")
-    # read CUDA EULA
-    eula_path = os.path.join(self.installdir, "EULA.txt")
-    tmp_buffer = []
-    with open(eula_path) as infile:
-        copy = False
-        for line in infile:
-            if line.strip() == "2.6. Attachment A":
-                copy = True
-                continue
-            elif line.strip() == "2.7. Attachment B":
-                copy = False
-                continue
-            elif copy:
-                tmp_buffer.append(line)
-    # create whitelist without file extensions, they're not really needed and they only complicate things
-    whitelist = ['EULA', 'README']
-    file_extensions = [".so", ".a", ".h", ".bc"]
-    for tmp in tmp_buffer:
-        for word in tmp.split():
-            if any(ext in word for ext in file_extensions):
-                whitelist.append(word.split(".")[0])
-    whitelist = list(set(whitelist))
-    # Do some quick checks for things we should or shouldn't have in the list
-    if "nvcc" in whitelist:
-        raise EasyBuildError("Found 'nvcc' in whitelist: %s" % whitelist)
-    if "libcudart" not in whitelist:
-        raise EasyBuildError("Did not find 'libcudart' in whitelist: %s" % whitelist)
-    # iterate over all files in the CUDA path
-    for root, dirs, files in os.walk(self.installdir):
-        for filename in files:
-            # we only really care about real files, i.e. not symlinks
-            if not os.path.islink(os.path.join(root, filename)):
-                # check if the current file is part of the whitelist
-                basename = filename.split(".")[0]
-                if basename not in whitelist:
-                    # if it is not in the whitelist, delete the file and create a symlink to host_injections
-                    source = os.path.join(root, filename)
-                    target = source.replace("versions", "host_injections")
-                    # Make sure source and target are not the same
-                    if source == target:
-                        raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you are"
-                                             "using this hook for an EESSI installation?")
-                    os.remove(source)
-                    # Using os.symlink requires the existence of the target directory, so we use os.system
-                    system_command="ln -s '%s' '%s'" % (target, source)
-                    if os.system(system_command) != 0:
-                        raise EasyBuildError("Failed to create symbolic link: %s" % system_command)
+    """
+    Remove files from CUDA installation that we are not allowed to ship,
+    and replace them with a symlink to a corresponding installation under host_injections.
+    """
+    if self.name == 'CUDA':
+        print_msg("Replacing files in CUDA installation that we can not ship with symlinks to host_injections...")
+
+        # read CUDA EULA, construct allowlist based on section 2.6 that specifies list of files that can be shipped
+        eula_path = os.path.join(self.installdir, 'EULA.txt')
+        relevant_eula_lines = []
+        with open(eula_path) as infile:
+            copy = False
+            for line in infile:
+                if line.strip() == "2.6. Attachment A":
+                    copy = True
+                    continue
+                elif line.strip() == "2.7. Attachment B":
+                    copy = False
+                    continue
+                elif copy:
+                    relevant_eula_lines.append(line)
+
+        # create list without file extensions, they're not really needed and they only complicate things
+        allowlist = ['EULA', 'README']
+        file_extensions = ['.so', '.a', '.h', '.bc']
+        for line in relevant_eula_lines:
+            for word in line.split():
+                if any(ext in word for ext in file_extensions):
+                    allowlist.append(os.path.splitext(word)[0])
+        allowlist = sorted(set(allowlist))
+        self.log.info("Allowlist for files in CUDA installation that can be redistributed: " + ', '.join(allowlist))
+
+        # Do some quick sanity checks for things we should or shouldn't have in the list
+        if 'nvcc' in allowlist:
+            raise EasyBuildError("Found 'nvcc' in allowlist: %s" % allowlist)
+        if 'libcudart' not in allowlist:
+            raise EasyBuildError("Did not find 'libcudart' in allowlist: %s" % allowlist)
+
+        # iterate over all files in the CUDA installation directory
+        for dir_path, _, files in os.walk(self.installdir):
+            for filename in files:
+                full_path = os.path.join(dir_path, filename)
+                # we only really care about real files, i.e. not symlinks
+                if not os.path.islink(full_path):
+                    # check if the current file is part of the allowlist
+                    basename = os.path.splitext(filename)[0]
+                    if basename in allowlist:
+                        self.log.debug("%s is found in allowlist, so keeping it: %s", basename, full_path)
+                    else:
+                        self.log.debug("%s is not found in allowlist, so replacing it with symlink: %s",
+                                       basename, full_path)
+                        # if it is not in the allowlist, delete the file and create a symlink to host_injections
+                        host_inj_path = full_path.replace('versions', 'host_injections')
+                        # make sure source and target of symlink are not the same
+                        if full_path == host_inj_path:
+                            raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you "
+                                                 "are using this hook for an EESSI installation?",
+                                                 full_path, host_inj_path)
+                        remove_file(full_path)
+                        symlink(host_inj_path, full_path)
+    else:
+        raise EasyBuildError("CUDA-specific hook triggered for non-CUDA easyconfig?!")
 
 
 def inject_gpu_property(ec):
+    """
+    Add 'gpu' property, via modluafooter easyconfig parameter
+    """
     ec_dict = ec.asdict()
-    # Check if CUDA is in the dependencies, if so add the GPU Lmod tag
-    if ("CUDA" in [dep[0] for dep in iter(ec_dict["dependencies"])]):
-        ec.log.info("[parse hook] Injecting gpu as Lmod arch property and envvar with CUDA version")
-        key = "modluafooter"
+    # Check if CUDA is in the dependencies, if so add the 'gpu' Lmod property
+    if ('CUDA' in [dep[0] for dep in iter(ec_dict['dependencies'])]):
+        ec.log.info("Injecting gpu as Lmod arch property and envvar with CUDA version")
+        key = 'modluafooter'
         value = 'add_property("arch","gpu")'
         cuda_version = 0
-        for dep in iter(ec_dict["dependencies"]):
+        for dep in iter(ec_dict['dependencies']):
             # Make CUDA a build dependency only (rpathing saves us from link errors)
-            if "CUDA" in dep[0]:
+            if 'CUDA' in dep[0]:
                 cuda_version = dep[1]
-                ec_dict["dependencies"].remove(dep)
-                ec_dict["builddependencies"].append(dep) if dep not in ec_dict["builddependencies"] else ec_dict["builddependencies"]
-        value = "\n".join([value, 'setenv("EESSICUDAVERSION","%s")' % cuda_version])
+                ec_dict['dependencies'].remove(dep)
+                if dep not in ec_dict['builddependencies']:
+                    ec_dict['builddependencies'].append(dep)
+        value = '\n'.join([value, 'setenv("EESSICUDAVERSION","%s")' % cuda_version])
         if key in ec_dict:
             if not value in ec_dict[key]:
-                ec[key] = "\n".join([ec_dict[key], value])
+                ec[key] = '\n'.join([ec_dict[key], value])
         else:
             ec[key] = value
     return ec

From 2d378421b697306642467d9019b748a1e67f39bc Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Dec 2023 21:22:42 +0100
Subject: [PATCH 36/81] remove empty line in eessi-2023.06-eb-4.8.2-2023a.yml

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index 91efae766f..87ccd69e99 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -35,7 +35,6 @@ easyconfigs:
   - Boost-1.82.0-GCC-12.3.0.eb
   - netCDF-4.9.2-gompi-2023a.eb
   - FFmpeg-6.0-GCCcore-12.3.0.eb
-
   - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb:
       # use easyconfig that only install subset of CUDA samples,
       # to circumvent problem with nvcc linking to glibc of host OS;

From f007c4061c3104a29b1053e877c61a32a244c30f Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 20 Dec 2023 23:24:15 +0100
Subject: [PATCH 37/81] use easyconfigs PR 19451 for installing CUDA-Samples
 v12.1

---
 .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml              | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index 87ccd69e99..596b9ea21f 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -37,7 +37,9 @@ easyconfigs:
   - FFmpeg-6.0-GCCcore-12.3.0.eb
   - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb:
       # use easyconfig that only install subset of CUDA samples,
-      # to circumvent problem with nvcc linking to glibc of host OS;
-      # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19189
+      # to circumvent problem with nvcc linking to glibc of host OS,
+      # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19189;
+      # and where additional samples are excluded because they fail to build on aarch64,
+      # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19451;
       options:
-        from-pr: 19189
+        from-pr: 19451

From 70fa0f9ac3cd54a582640dc7bdadcd31c210fffc Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Thu, 21 Dec 2023 00:34:38 +0100
Subject: [PATCH 38/81] Ship the scripts, and keep them in a single location

---
 EESSI-install-software.sh                     |  4 ++--
 create_lmodrc.py                              | 22 +++++++++----------
 create_tarball.sh                             |  4 ++++
 install_scripts.sh                            |  4 ++--
 .../nvidia/install_cuda_host_injections.sh    |  2 +-
 .../nvidia/link_nvidia_host_libraries.sh      |  2 +-
 6 files changed, 20 insertions(+), 18 deletions(-)
 rename {gpu_support => scripts/gpu_support}/nvidia/install_cuda_host_injections.sh (99%)
 rename {gpu_support => scripts/gpu_support}/nvidia/link_nvidia_host_libraries.sh (99%)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 4e3f0acee5..b61ca7a579 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -195,12 +195,12 @@ ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX}
 # Install full CUDA SDK in host_injections
 # Hardcode this for now, see if it works
 # TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install
-${EESSI_PREFIX}/gpu_support/nvidia/install_cuda_host_injections.sh -c 12.1.1 --accept-cuda-eula
+${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12.1.1 --accept-cuda-eula
 
 # Install drivers in host_injections
 # TODO: this is commented out for now, because the script assumes that nvidia-smi is available and works;
 #       if not, an error is produced, and the bot flags the whole build as failed (even when not installing GPU software)
-# ${EESSI_PREFIX}/gpu_support/nvidia/link_nvidia_host_libraries.sh
+# ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
 
 # use PR patch file to determine in which easystack files stuff was added
 for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing'); do
diff --git a/create_lmodrc.py b/create_lmodrc.py
index 6a72d8dc62..80635d78cc 100755
--- a/create_lmodrc.py
+++ b/create_lmodrc.py
@@ -36,6 +36,7 @@
     -- If we try to load CUDA itself, check if the full CUDA SDK was installed on the host in host_injections. 
     -- This is required for end users to build additional CUDA software. If the full SDK isn't present, refuse
     -- to load the CUDA module and print an informative message on how to set up GPU support for EESSI
+    local refer_to_docs = "For more information on how to do this, see https://www.eessi.io/docs/gpu/.\\n"
     if simpleName == 'CUDA' then
         -- get the full host_injections path
         local hostInjections = string.gsub(os.getenv('EESSI_SOFTWARE_PATH') or "", 'versions', 'host_injections')
@@ -43,11 +44,10 @@
         local cudaEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild"
         local cudaDirExists = isDir(cudaEasyBuildDir)
         if not cudaDirExists then
-            local cvmfsRepo = os.getenv('EESSI_CVMFS_REPO')
             local advice = "but while the module file exists, the actual software is not entirely shipped with EESSI "
-            advice = advice .. "due to licencing. Please install a full copy of the CUDA SDK using the script "
-            advice = advice .. cvmfsRepo .. "/gpu_support/nvidia/install_cuda_host_injections.sh.\\n"
-            advice = advice .. "More information, see https://www.eessi.io/docs/gpu/.\\n"
+            advice = advice .. "due to licencing. You will need to install a full copy of the CUDA SDK where EESSI "
+            advice = advice .. "can find it.\\n"
+            advice = advice .. refer_to_docs
             LmodError("\\nYou requested to load ", simpleName, " ", advice)
         end
     end
@@ -62,10 +62,9 @@
         local singularityCudaExists = isFile("/.singularity.d/libs/libcuda.so")
         if not (cudaDriverExists or singularityCudaExists)  then
             local advice = "which relies on the CUDA runtime environment and driver libraries. "
-            advice = advice .. "In order to be able to use the module, please run the script "
-            advice = advice .. cvmfsRepo .. "/gpu_support/nvidia/link_nvidia_host_libraries.sh "
-            advice = advice .. "to make sure EESSI can find the drivers from on your host system.\\n"
-            advice = advice .. "More information, see https://www.eessi.io/docs/gpu/.\\n"
+            advice = advice .. "In order to be able to use the module, you will need "
+            advice = advice .. "to make sure EESSI can find the driver libraries on your host system.\\n"
+            advice = advice .. refer_to_docs
             LmodError("\\nYou requested to load ", simpleName, " ", advice)
         else
             -- CUDA driver exists, now we check its version to see if an update is needed
@@ -85,10 +84,9 @@
                 end
                 if driver_libs_need_update == true then
                     local advice = "but the module you want to load requires CUDA  " .. cudaVersion_req .. ". "
-                    advice = advice .. "Please update your CUDA driver libraries and rerun the script"
-                    advice = advice .. cvmfsRepo .. "/gpu_support/nvidia/install_cuda_host_injections.sh "
-                    advice = advice .. "to let EESSI know about the update.\\n"
-                    advice = advice .. "More information, see https://www.eessi.io/docs/gpu/.\\n"
+                    advice = advice .. "Please update your CUDA driver libraries and then "
+                    advice = advice .. "let EESSI know about the update.\\n"
+                    advice = advice .. refer_to_docs
                     LmodError("\\nYour driver CUDA version is ", cudaVersion, " ", advice)
                 end
             end
diff --git a/create_tarball.sh b/create_tarball.sh
index 8510caebf1..09ce94c835 100755
--- a/create_tarball.sh
+++ b/create_tarball.sh
@@ -40,6 +40,10 @@ echo ">> Collecting list of files/directories to include in tarball via ${PWD}..
 files_list=${tmpdir}/files.list.txt
 module_files_list=${tmpdir}/module_files.list.txt
 
+if [ -d ${eessi_version}/scripts ]; then
+    # include scripts we wish to ship along with EESSI,
+    find ${eessi_version}/scripts -type f | grep -v '/\.wh\.' >> ${files_list}
+fi
 if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
     # include Lmod cache and configuration file (lmodrc.lua),
     # skip whiteout files and backup copies of Lmod cache (spiderT.old.*)
diff --git a/install_scripts.sh b/install_scripts.sh
index d53dbb6a9c..224400db1c 100755
--- a/install_scripts.sh
+++ b/install_scripts.sh
@@ -50,8 +50,8 @@ for file in utils.sh; do
     cp -v -u ${SCRIPTS_DIR_SOURCE}/${file} ${SCRIPTS_DIR_TARGET}/${file}
 done
 # Subdirs for GPU support
-NVIDIA_GPU_SUPPORT_DIR_SOURCE=${TOPDIR}/gpu_support/nvidia  # Source dir
-NVIDIA_GPU_SUPPORT_DIR_TARGET=${INSTALL_PREFIX}/gpu_support/nvidia  # Target dir
+NVIDIA_GPU_SUPPORT_DIR_SOURCE=${SCRIPTS_DIR_SOURCE}/gpu_support/nvidia  # Source dir
+NVIDIA_GPU_SUPPORT_DIR_TARGET=${SCRIPTS_DIR_TARGET}/gpu_support/nvidia  # Target dir
 
 # Create target dir
 mkdir -p ${NVIDIA_GPU_SUPPORT_DIR_TARGET}
diff --git a/gpu_support/nvidia/install_cuda_host_injections.sh b/scripts/gpu_support/nvidia/install_cuda_host_injections.sh
similarity index 99%
rename from gpu_support/nvidia/install_cuda_host_injections.sh
rename to scripts/gpu_support/nvidia/install_cuda_host_injections.sh
index 62996fa924..a9310d817a 100755
--- a/gpu_support/nvidia/install_cuda_host_injections.sh
+++ b/scripts/gpu_support/nvidia/install_cuda_host_injections.sh
@@ -14,7 +14,7 @@
 
 # Initialise our bash functions
 TOPDIR=$(dirname $(realpath $BASH_SOURCE))
-source "$TOPDIR"/../../scripts/utils.sh
+source "$TOPDIR"/../../utils.sh
 
 # Function to display help message
 show_help() {
diff --git a/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
similarity index 99%
rename from gpu_support/nvidia/link_nvidia_host_libraries.sh
rename to scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
index cb7420a0e9..e6ff110797 100755
--- a/gpu_support/nvidia/link_nvidia_host_libraries.sh
+++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
@@ -5,7 +5,7 @@
 
 # Initialise our bash functions
 TOPDIR=$(dirname $(realpath $BASH_SOURCE))
-source "$TOPDIR"/../../scripts/utils.sh
+source "$TOPDIR"/../../utils.sh
 
 # We rely on ldconfig to give us the location of the libraries on the host
 command_name="ldconfig"

From db0c14136f7d4e244e42e374f64357eea0f63789 Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Thu, 21 Dec 2023 09:27:28 +0000
Subject: [PATCH 39/81] Update create_lmodrc.py

Co-authored-by: Kenneth Hoste <kenneth.hoste@ugent.be>
---
 create_lmodrc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/create_lmodrc.py b/create_lmodrc.py
index 80635d78cc..0e738a530e 100755
--- a/create_lmodrc.py
+++ b/create_lmodrc.py
@@ -63,7 +63,7 @@
         if not (cudaDriverExists or singularityCudaExists)  then
             local advice = "which relies on the CUDA runtime environment and driver libraries. "
             advice = advice .. "In order to be able to use the module, you will need "
-            advice = advice .. "to make sure EESSI can find the driver libraries on your host system.\\n"
+            advice = advice .. "to make sure EESSI can find the GPU driver libraries on your host system.\\n"
             advice = advice .. refer_to_docs
             LmodError("\\nYou requested to load ", simpleName, " ", advice)
         else

From 293b1075e5619748f043019a5e856e0b52046d0d Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Thu, 21 Dec 2023 09:28:08 +0000
Subject: [PATCH 40/81] Update create_tarball.sh

Co-authored-by: Kenneth Hoste <kenneth.hoste@ugent.be>
---
 create_tarball.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/create_tarball.sh b/create_tarball.sh
index 09ce94c835..4d3ad37311 100755
--- a/create_tarball.sh
+++ b/create_tarball.sh
@@ -40,10 +40,12 @@ echo ">> Collecting list of files/directories to include in tarball via ${PWD}..
 files_list=${tmpdir}/files.list.txt
 module_files_list=${tmpdir}/module_files.list.txt
 
+# include scripts that were copied by install_scripts.sh
 if [ -d ${eessi_version}/scripts ]; then
     # include scripts we wish to ship along with EESSI,
     find ${eessi_version}/scripts -type f | grep -v '/\.wh\.' >> ${files_list}
 fi
+
 if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
     # include Lmod cache and configuration file (lmodrc.lua),
     # skip whiteout files and backup copies of Lmod cache (spiderT.old.*)

From 73476b2b20c551ab7e6f0947f544f5f44d3f2537 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Thu, 21 Dec 2023 12:16:38 +0100
Subject: [PATCH 41/81] Only copy scripts if the contents differ

---
 install_scripts.sh                            |  21 +-
 .../nvidia/install_cuda_host_injections.sh    | 211 ++++++++++++++++++
 .../nvidia/link_nvidia_host_libraries.sh      | 136 +++++++++++
 temp/scripts/utils.sh                         | 144 ++++++++++++
 4 files changed, 510 insertions(+), 2 deletions(-)
 create mode 100755 temp/scripts/gpu_support/nvidia/install_cuda_host_injections.sh
 create mode 100755 temp/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
 create mode 100644 temp/scripts/utils.sh

diff --git a/install_scripts.sh b/install_scripts.sh
index 224400db1c..588248e8d2 100755
--- a/install_scripts.sh
+++ b/install_scripts.sh
@@ -8,6 +8,23 @@ display_help() {
   echo "  -h | --help            -  display this usage information"
 }
 
+compare_and_copy() {
+    if [ "$#" -ne 2 ]; then
+        echo "Usage of function: compare_and_copy <source_file> <destination_file>"
+        return 1
+    fi
+
+    source_file="$1"
+    destination_file="$2"
+
+    if [ ! -f "$destination_file" ] || ! diff -q "$source_file" "$destination_file" ; then
+        cp "$source_file" "$destination_file"
+        echo "File $1 copied to $2."
+    else
+        echo "Files $1 and $2 are identical. No copy needed."
+    fi
+}
+
 
 POSITIONAL_ARGS=()
 
@@ -47,7 +64,7 @@ mkdir -p ${SCRIPTS_DIR_TARGET}
 # Copy scripts into this prefix
 echo "copying scripts from ${SCRIPTS_DIR_SOURCE} to ${SCRIPTS_DIR_TARGET}"
 for file in utils.sh; do
-    cp -v -u ${SCRIPTS_DIR_SOURCE}/${file} ${SCRIPTS_DIR_TARGET}/${file}
+    compare_and_copy ${SCRIPTS_DIR_SOURCE}/${file} ${SCRIPTS_DIR_TARGET}/${file}
 done
 # Subdirs for GPU support
 NVIDIA_GPU_SUPPORT_DIR_SOURCE=${SCRIPTS_DIR_SOURCE}/gpu_support/nvidia  # Source dir
@@ -60,5 +77,5 @@ mkdir -p ${NVIDIA_GPU_SUPPORT_DIR_TARGET}
 # To be on the safe side, we dont do recursive copies, but we are explicitely copying each individual file we want to add
 echo "copying scripts from ${NVIDIA_GPU_SUPPORT_DIR_SOURCE} to ${NVIDIA_GPU_SUPPORT_DIR_TARGET}"
 for file in install_cuda_host_injections.sh link_nvidia_host_libraries.sh; do
-    cp -v -u ${NVIDIA_GPU_SUPPORT_DIR_SOURCE}/${file} ${NVIDIA_GPU_SUPPORT_DIR_TARGET}/${file}
+    compare_and_copy ${NVIDIA_GPU_SUPPORT_DIR_SOURCE}/${file} ${NVIDIA_GPU_SUPPORT_DIR_TARGET}/${file}
 done
diff --git a/temp/scripts/gpu_support/nvidia/install_cuda_host_injections.sh b/temp/scripts/gpu_support/nvidia/install_cuda_host_injections.sh
new file mode 100755
index 0000000000..a9310d817a
--- /dev/null
+++ b/temp/scripts/gpu_support/nvidia/install_cuda_host_injections.sh
@@ -0,0 +1,211 @@
+#!/usr/bin/env bash
+
+# This script can be used to install CUDA under the `.../host_injections` directory.
+# This provides the parts of the CUDA installation that cannot be redistributed as
+# part of EESSI due to license limitations. While GPU-based software from EESSI will
+# _run_ without these, installation of additional CUDA software requires the CUDA
+# installation(s) under `host_injections` to be present.
+#
+# The `host_injections` directory is a variant symlink that by default points to
+# `/opt/eessi`, unless otherwise defined in the local CVMFS configuration (see
+# https://cvmfs.readthedocs.io/en/stable/cpt-repo.html#variant-symlinks). For the
+# installation to be successful, this directory needs to be writeable by the user
+# executing this script.
+
+# Initialise our bash functions
+TOPDIR=$(dirname $(realpath $BASH_SOURCE))
+source "$TOPDIR"/../../utils.sh
+
+# Function to display help message
+show_help() {
+    echo "Usage: $0 [OPTIONS]"
+    echo "Options:"
+    echo "  --help                           Display this help message"
+    echo "  --accept-cuda-eula               You _must_ accept the CUDA EULA to install"
+    echo "                                   CUDA, see the EULA at"
+    echo "                                   https://docs.nvidia.com/cuda/eula/index.html"
+    echo "  -c, --cuda-version CUDA_VERSION  Specify a version o CUDA to install (must"
+    echo "                                   have a corresponding easyconfig in the"
+    echo "                                   EasyBuild release)"
+    echo "  -t, --temp-dir /path/to/tmpdir   Specify a location to use for temporary"
+    echo "                                   storage during the CUDA install"
+    echo "                                   (must have >10GB available)"
+}
+
+# Initialize variables
+install_cuda_version=""
+eula_accepted=0
+
+# Parse command-line options
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --help)
+            show_help
+            exit 0
+            ;;
+        -c|--cuda-version)
+            if [ -n "$2" ]; then
+                install_cuda_version="$2"
+                shift 2
+            else
+                echo "Error: Argument required for $1"
+                show_help
+                exit 1
+            fi
+            ;;
+        --accept-cuda-eula)
+            eula_accepted=1
+            shift 1
+            ;;
+        -t|--temp-dir)
+            if [ -n "$2" ]; then
+                CUDA_TEMP_DIR="$2"
+                shift 2
+            else
+                echo "Error: Argument required for $1"
+                show_help
+                exit 1
+            fi
+            ;;
+        *)
+            show_help
+            fatal_error "Error: Unknown option: $1"
+            ;;
+    esac
+done
+
+# Make sure EESSI is initialised
+check_eessi_initialised
+
+# Make sure the CUDA version supplied is a semantic version
+is_semantic_version() {
+    local version=$1
+    local regex='^[0-9]+\.[0-9]+\.[0-9]+$'
+
+    if [[ $version =~ $regex ]]; then
+        return 0  # Return success (0) if it's a semantic version
+    else
+        return 1  # Return failure (1) if it's not a semantic version
+    fi
+}
+if ! is_semantic_version "$install_cuda_version"; then
+  show_help
+  error="\nYou must provide a semantic version for CUDA (e.g., 12.1.1) via the appropriate\n"
+  error="${error}command line option. This script is intended for use with EESSI so the 'correct'\n"
+  error="${error}version to provide is probably one of those available under\n"
+  error="${error}$EESSI_SOFTWARE_PATH/software/CUDA\n"
+  fatal_error "${error}"
+fi
+
+# Make sure they have accepted the CUDA EULA
+if [ "$eula_accepted" -ne 1 ]; then
+  show_help
+  error="\nYou _must_ accept the CUDA EULA via the appropriate command line option.\n"
+  fatal_error "${error}"
+fi
+
+# As an installation location just use $EESSI_SOFTWARE_PATH but replacing `versions` with `host_injections`
+# (CUDA is a binary installation so no need to worry too much about the EasyBuild setup)
+cuda_install_parent=${EESSI_SOFTWARE_PATH/versions/host_injections}
+
+# Only install CUDA if specified version is not found.
+# (existence of easybuild subdir implies a successful install)
+if [ -d "${cuda_install_parent}"/software/CUDA/"${install_cuda_version}"/easybuild ]; then
+  echo_green "CUDA software found! No need to install CUDA again."
+else
+  # We need to be able write to the installation space so let's make sure we can
+  if ! create_directory_structure "${cuda_install_parent}"/software/CUDA ; then
+    fatal_error "No write permissions to directory ${cuda_install_parent}/software/CUDA"
+  fi
+
+  # we need a directory we can use for temporary storage
+  if [[ -z "${CUDA_TEMP_DIR}" ]]; then
+    tmpdir=$(mktemp -d)
+  else
+    tmpdir="${CUDA_TEMP_DIR}"/temp
+    if ! mkdir "$tmpdir" ; then
+      fatal_error "Could not create directory ${tmpdir}"
+    fi
+  fi
+
+  required_space_in_tmpdir=50000
+  # Let's see if we have sources and build locations defined if not, we use the temporary space
+  if [[ -z "${EASYBUILD_BUILDPATH}" ]]; then
+    export EASYBUILD_BUILDPATH=${tmpdir}/build
+    required_space_in_tmpdir=$((required_space_in_tmpdir + 5000000))
+  fi
+  if [[ -z "${EASYBUILD_SOURCEPATH}" ]]; then
+    export EASYBUILD_SOURCEPATH=${tmpdir}/sources
+    required_space_in_tmpdir=$((required_space_in_tmpdir + 5000000))
+  fi
+
+  # The install is pretty fat, you need lots of space for download/unpack/install (~3*5GB),
+  # need to do a space check before we proceed
+  avail_space=$(df --output=avail "${cuda_install_parent}"/ | tail -n 1 | awk '{print $1}')
+  if (( avail_space < 5000000 )); then
+    fatal_error "Need at least 5GB disk space to install CUDA under ${cuda_install_parent}, exiting now..."
+  fi
+  avail_space=$(df --output=avail "${tmpdir}"/ | tail -n 1 | awk '{print $1}')
+  if (( avail_space < required_space_in_tmpdir )); then
+    error="Need at least ${required_space_in_tmpdir} disk space under ${tmpdir}.\n"
+    error="${error}Set the environment variable CUDA_TEMP_DIR to a location with adequate space to pass this check."
+    error="${error}You can alternatively set EASYBUILD_BUILDPATH and/or EASYBUILD_SOURCEPATH "
+    error="${error}to reduce this requirement. Exiting now..."
+    fatal_error "${error}"
+  fi
+
+  if ! command -v "eb" &>/dev/null; then
+    echo_yellow "Attempting to load an EasyBuild module to do actual install"
+    module load EasyBuild
+    # There are some scenarios where this may fail
+    if [ $? -ne 0 ]; then
+      error="'eb' command not found in your environment and\n"
+      error="${error}  module load EasyBuild\n"
+      error="${error}failed for some reason.\n"
+      error="${error}Please re-run this script with the 'eb' command available."
+      fatal_error "${error}"
+    fi
+  fi
+
+  cuda_easyconfig="CUDA-${install_cuda_version}.eb"
+
+  # Check the easyconfig file is available in the release
+  # (eb search always returns 0, so we need a grep to ensure a usable exit code)
+  eb --search ^${cuda_easyconfig}|grep CUDA > /dev/null 2>&1
+  # Check the exit code
+  if [ $? -ne 0 ]; then
+    eb_version=$(eb --version)
+    available_cuda_easyconfigs=$(eb --search ^CUDA-*.eb|grep CUDA)
+
+    error="The easyconfig ${cuda_easyconfig} was not found in EasyBuild version:\n"
+    error="${error}  ${eb_version}\n"
+    error="${error}You either need to give a different version of CUDA to install _or_ \n"
+    error="${error}use a different version of EasyBuild for the installation.\n"
+    error="${error}\nThe versions of available with the current eb command are:\n"
+    error="${error}${available_cuda_easyconfigs}"
+    fatal_error "${error}"
+  fi
+
+  # We need the --rebuild option, as the CUDA module may or may not be on the
+  # `MODULEPATH` yet. Even if it is, we still want to redo this installation
+  # since it will provide the symlinked targets for the parts of the CUDA
+  # installation in the `.../versions/...` prefix
+  # We install the module in our `tmpdir` since we do not need the modulefile,
+  # we only care about providing the targets for the symlinks.
+  extra_args="--rebuild --installpath-modules=${tmpdir}"
+
+  # We don't want hooks used in this install, we need a vanilla CUDA installation
+  touch "$tmpdir"/none.py
+  # shellcheck disable=SC2086  # Intended splitting of extra_args
+  eb --prefix="$tmpdir" ${extra_args} --accept-eula-for=CUDA --hooks="$tmpdir"/none.py --installpath="${cuda_install_parent}"/ "${cuda_easyconfig}"
+  ret=$?
+  if [ $ret -ne 0 ]; then
+    eb_last_log=$(unset EB_VERBOSE; eb --last-log)
+    cp -a ${eb_last_log} .
+    fatal_error "CUDA installation failed, please check EasyBuild logs $(basename ${eb_last_log})..."
+  else
+    echo_green "CUDA installation at ${cuda_install_parent}/software/CUDA/${install_cuda_version} succeeded!"
+  fi
+  # clean up tmpdir
+  rm -rf "${tmpdir}"
+fi
diff --git a/temp/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/temp/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
new file mode 100755
index 0000000000..e6ff110797
--- /dev/null
+++ b/temp/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
@@ -0,0 +1,136 @@
+#!/bin/bash
+
+# This script links host libraries related to GPU drivers to a location where
+# they can be found by the EESSI linker
+
+# Initialise our bash functions
+TOPDIR=$(dirname $(realpath $BASH_SOURCE))
+source "$TOPDIR"/../../utils.sh
+
+# We rely on ldconfig to give us the location of the libraries on the host
+command_name="ldconfig"
+# We cannot use a version of ldconfig that's being shipped under CVMFS
+exclude_prefix="/cvmfs"
+
+found_paths=()
+# Always attempt to use /sbin/ldconfig
+if [ -x "/sbin/$command_name" ]; then
+    found_paths+=("/sbin/$command_name")
+fi
+IFS=':' read -ra path_dirs <<< "$PATH"
+for dir in "${path_dirs[@]}"; do
+  if [ "$dir" = "/sbin" ]; then
+    continue  # we've already checked for $command_name in /sbin, don't need to do it twice
+  fi
+  if [[ ! "$dir" =~ ^$exclude_prefix ]]; then
+      if [ -x "$dir/$command_name" ]; then
+          found_paths+=("$dir/$command_name")
+      fi
+  fi
+done
+
+if [ ${#found_paths[@]} -gt 0 ]; then
+    echo "Found $command_name in the following locations:"
+    printf -- "- %s\n" "${found_paths[@]}"
+    echo "Using first version"
+    host_ldconfig=${found_paths[0]}
+else
+    error="$command_name not found in PATH or only found in paths starting with $exclude_prefix."
+    fatal_error "$error"
+fi
+
+# Make sure EESSI is initialised (doesn't matter what version)
+check_eessi_initialised
+
+# Find the CUDA version of the host CUDA drivers
+# (making sure that this can still work inside prefix environment inside a container)
+export LD_LIBRARY_PATH=/.singularity.d/libs:$LD_LIBRARY_PATH
+nvidia_smi_command="nvidia-smi --query-gpu=driver_version --format=csv,noheader"
+if $nvidia_smi_command > /dev/null; then
+  host_driver_version=$($nvidia_smi_command | tail -n1)
+  # If the first worked, this should work too
+  host_cuda_version=$(nvidia-smi  -q --display=COMPUTE | grep CUDA | awk 'NF>1{print $NF}')
+else
+  error="Failed to successfully execute\n  $nvidia_smi_command\n"
+  fatal_error "$error"
+fi
+
+# Let's make sure the driver libraries are not already in place
+link_drivers=1
+
+host_injections_nvidia_dir="${EESSI_CVMFS_REPO}/host_injections/nvidia/${EESSI_CPU_FAMILY}"
+host_injection_driver_dir="${host_injections_nvidia_dir}/host"
+host_injection_driver_version_file="$host_injection_driver_dir/driver_version.txt"
+if [ -e "$host_injection_driver_version_file" ]; then
+  if grep -q "$host_driver_version" "$host_injection_driver_version_file"; then
+    echo_green "The host CUDA driver libraries have already been linked!"
+    link_drivers=0
+  else
+    # There's something there but it is out of date
+    echo_yellow "Cleaning out outdated symlinks"
+    rm $host_injection_driver_dir/*
+    if [ $? -ne 0 ]; then
+      error="Unable to remove files under '$host_injection_driver_dir'."
+      fatal_error "$error"
+    fi
+  fi
+fi
+
+drivers_linked=0
+if [ "$link_drivers" -eq 1 ]; then
+  if ! create_directory_structure "${host_injection_driver_dir}" ; then
+    fatal_error "No write permissions to directory ${host_injection_driver_dir}"
+  fi
+  cd ${host_injection_driver_dir}
+  # Need a small temporary space to hold a couple of files
+  temp_dir=$(mktemp -d)
+
+  # Gather libraries on the host (_must_ be host ldconfig)
+  $host_ldconfig -p | awk '{print $NF}' > "$temp_dir"/libs.txt
+  # Allow for the fact that we may be in a container so the CUDA libs might be in there
+  ls /.singularity.d/libs/* >> "$temp_dir"/libs.txt 2>/dev/null
+
+  # Leverage singularity to find the full list of libraries we should be linking to
+  echo_yellow "Downloading latest version of nvliblist.conf from Apptainer"
+  curl -o "$temp_dir"/nvliblist.conf https://raw.githubusercontent.com/apptainer/apptainer/main/etc/nvliblist.conf
+
+  # Make symlinks to all the interesting libraries
+  grep '.so$' "$temp_dir"/nvliblist.conf | xargs -i grep {} "$temp_dir"/libs.txt | xargs -i ln -s {}
+
+  # Inject driver and CUDA versions into dir
+  echo $host_driver_version > driver_version.txt
+  echo $host_cuda_version > cuda_version.txt
+  drivers_linked=1
+
+  # Remove the temporary directory when done
+  rm -r "$temp_dir"
+fi
+
+# Make latest symlink for NVIDIA drivers
+cd $host_injections_nvidia_dir
+symlink="latest"
+if [ -L "$symlink" ]; then
+    # Unless the drivers have been installed, leave the symlink alone
+    if [ "$drivers_linked" -eq 1 ]; then
+      ln -sf host latest
+    fi
+else
+    # No link exists yet
+    ln -s host latest
+fi
+
+# Make sure the libraries can be found by the EESSI linker
+host_injection_linker_dir=${EESSI_EPREFIX/versions/host_injections}
+if [ -L "$host_injection_linker_dir/lib" ]; then
+  target_path=$(readlink -f "$host_injection_linker_dir/lib")
+  if [ "$target_path" != "$$host_injections_nvidia_dir/latest" ]; then
+    cd $host_injection_linker_dir
+    ln -sf $host_injections_nvidia_dir/latest lib
+  fi
+else
+  create_directory_structure $host_injection_linker_dir
+  cd $host_injection_linker_dir
+  ln -s $host_injections_nvidia_dir/latest lib
+fi
+
+echo_green "Host NVIDIA gpu drivers linked successfully for EESSI"
diff --git a/temp/scripts/utils.sh b/temp/scripts/utils.sh
new file mode 100644
index 0000000000..b2be3f6221
--- /dev/null
+++ b/temp/scripts/utils.sh
@@ -0,0 +1,144 @@
+function echo_green() {
+    echo -e "\e[32m$1\e[0m"
+}
+
+function echo_red() {
+    echo -e "\e[31m$1\e[0m"
+}
+
+function echo_yellow() {
+    echo -e "\e[33m$1\e[0m"
+}
+
+ANY_ERROR_EXITCODE=1
+function fatal_error() {
+    echo_red "ERROR: $1" >&2
+    if [[ $# -gt 1 ]]; then
+      exit "$2"
+    else
+      exit "${ANY_ERROR_EXITCODE}"
+    fi
+}
+
+function check_exit_code {
+    ec=$1
+    ok_msg=$2
+    fail_msg=$3
+
+    if [[ $ec -eq 0 ]]; then
+        echo_green "${ok_msg}"
+    else
+        fatal_error "${fail_msg}"
+    fi
+}
+
+function check_eessi_initialised() {
+  if [[ -z "${EESSI_SOFTWARE_PATH}" ]]; then
+    fatal_error "EESSI has not been initialised!"
+  else
+    return 0
+  fi
+}
+
+function check_in_prefix_shell() {
+  # Make sure EPREFIX is defined
+  if [[ -z "${EPREFIX}" ]]; then
+    fatal_error "This script cannot be used without having first defined EPREFIX"
+  fi
+  if [[ ! ${SHELL} = ${EPREFIX}/bin/bash ]]; then
+    fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!"
+  fi
+}
+
+function create_directory_structure() {
+  # Ensure we are given a single path argument
+  if [ $# -ne 1 ]; then
+    echo_red "Function requires a single (relative or absolute) path argument" >&2
+    return $ANY_ERROR_EXITCODE
+  fi
+  dir_structure="$1"
+
+  # Attempt to create the directory structure
+  error_message=$(mkdir -p "$dir_structure" 2>&1)
+  return_code=$?
+  # If it fails be explicit about the error
+  if [ ${return_code} -ne 0 ]; then
+    real_dir=$(realpath -m "$dir_structure")
+    echo_red "Creating ${dir_structure} (real path ${real_dir}) failed with:\n ${error_message}" >&2
+  else
+    # If we're creating it, our use case is that we want to be able to write there
+    # (this is a check in case the directory already existed)
+    if [ ! -w "${dir_structure}" ]; then
+      real_dir=$(realpath -m "$dir_structure")
+      echo_red "You do not have (required) write permissions to ${dir_structure} (real path ${real_dir})!"
+      return_code=$ANY_ERROR_EXITCODE
+    fi
+  fi
+
+  return $return_code
+}
+
+function get_path_for_tool {
+    tool_name=$1
+    tool_envvar_name=$2
+
+    which_out=$(which "${tool_name}" 2>&1)
+    exit_code=$?
+    if [[ ${exit_code} -eq 0 ]]; then
+        echo "INFO: found tool ${tool_name} in PATH (${which_out})" >&2
+        echo "${which_out}"
+        return 0
+    fi
+    if [[ -z "${tool_envvar_name}" ]]; then
+        msg="no env var holding the full path to tool '${tool_name}' provided"
+        echo "${msg}" >&2
+        return 1
+    else
+        tool_envvar_value=${!tool_envvar_name}
+        if [[ -x "${tool_envvar_value}" ]]; then
+            msg="INFO: found tool ${tool_envvar_value} via env var ${tool_envvar_name}"
+            echo "${msg}" >&2
+            echo "${tool_envvar_value}"
+            return 0
+        else
+            msg="ERROR: tool '${tool_name}' not in PATH\n"
+            msg+="ERROR: tool '${tool_envvar_value}' via '${tool_envvar_name}' not in PATH"
+            echo "${msg}" >&2
+            echo ""
+            return 2
+        fi
+    fi
+}
+
+function get_host_from_url {
+    url=$1
+    re="(http|https)://([^/:]+)"
+    if [[ $url =~ $re ]]; then
+        echo "${BASH_REMATCH[2]}"
+        return 0
+    else
+        echo ""
+        return 1
+    fi
+}
+
+function get_port_from_url {
+    url=$1
+    re="(http|https)://[^:]+:([0-9]+)"
+    if [[ $url =~ $re ]]; then
+        echo "${BASH_REMATCH[2]}"
+        return 0
+    else
+        echo ""
+        return 1
+    fi
+}
+
+function get_ipv4_address {
+    hname=$1
+    hipv4=$(grep "${hname}" /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1)
+    # TODO try other methods if the one above does not work --> tool that verifies
+    #      what method can be used?
+    echo "${hipv4}"
+    return 0
+}

From a333a741bb75a68f6d29cb718472af70e3b5c912 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Thu, 21 Dec 2023 12:19:19 +0100
Subject: [PATCH 42/81] Remove temporary test directory

---
 .../nvidia/install_cuda_host_injections.sh    | 211 ------------------
 .../nvidia/link_nvidia_host_libraries.sh      | 136 -----------
 temp/scripts/utils.sh                         | 144 ------------
 3 files changed, 491 deletions(-)
 delete mode 100755 temp/scripts/gpu_support/nvidia/install_cuda_host_injections.sh
 delete mode 100755 temp/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
 delete mode 100644 temp/scripts/utils.sh

diff --git a/temp/scripts/gpu_support/nvidia/install_cuda_host_injections.sh b/temp/scripts/gpu_support/nvidia/install_cuda_host_injections.sh
deleted file mode 100755
index a9310d817a..0000000000
--- a/temp/scripts/gpu_support/nvidia/install_cuda_host_injections.sh
+++ /dev/null
@@ -1,211 +0,0 @@
-#!/usr/bin/env bash
-
-# This script can be used to install CUDA under the `.../host_injections` directory.
-# This provides the parts of the CUDA installation that cannot be redistributed as
-# part of EESSI due to license limitations. While GPU-based software from EESSI will
-# _run_ without these, installation of additional CUDA software requires the CUDA
-# installation(s) under `host_injections` to be present.
-#
-# The `host_injections` directory is a variant symlink that by default points to
-# `/opt/eessi`, unless otherwise defined in the local CVMFS configuration (see
-# https://cvmfs.readthedocs.io/en/stable/cpt-repo.html#variant-symlinks). For the
-# installation to be successful, this directory needs to be writeable by the user
-# executing this script.
-
-# Initialise our bash functions
-TOPDIR=$(dirname $(realpath $BASH_SOURCE))
-source "$TOPDIR"/../../utils.sh
-
-# Function to display help message
-show_help() {
-    echo "Usage: $0 [OPTIONS]"
-    echo "Options:"
-    echo "  --help                           Display this help message"
-    echo "  --accept-cuda-eula               You _must_ accept the CUDA EULA to install"
-    echo "                                   CUDA, see the EULA at"
-    echo "                                   https://docs.nvidia.com/cuda/eula/index.html"
-    echo "  -c, --cuda-version CUDA_VERSION  Specify a version o CUDA to install (must"
-    echo "                                   have a corresponding easyconfig in the"
-    echo "                                   EasyBuild release)"
-    echo "  -t, --temp-dir /path/to/tmpdir   Specify a location to use for temporary"
-    echo "                                   storage during the CUDA install"
-    echo "                                   (must have >10GB available)"
-}
-
-# Initialize variables
-install_cuda_version=""
-eula_accepted=0
-
-# Parse command-line options
-while [[ $# -gt 0 ]]; do
-    case "$1" in
-        --help)
-            show_help
-            exit 0
-            ;;
-        -c|--cuda-version)
-            if [ -n "$2" ]; then
-                install_cuda_version="$2"
-                shift 2
-            else
-                echo "Error: Argument required for $1"
-                show_help
-                exit 1
-            fi
-            ;;
-        --accept-cuda-eula)
-            eula_accepted=1
-            shift 1
-            ;;
-        -t|--temp-dir)
-            if [ -n "$2" ]; then
-                CUDA_TEMP_DIR="$2"
-                shift 2
-            else
-                echo "Error: Argument required for $1"
-                show_help
-                exit 1
-            fi
-            ;;
-        *)
-            show_help
-            fatal_error "Error: Unknown option: $1"
-            ;;
-    esac
-done
-
-# Make sure EESSI is initialised
-check_eessi_initialised
-
-# Make sure the CUDA version supplied is a semantic version
-is_semantic_version() {
-    local version=$1
-    local regex='^[0-9]+\.[0-9]+\.[0-9]+$'
-
-    if [[ $version =~ $regex ]]; then
-        return 0  # Return success (0) if it's a semantic version
-    else
-        return 1  # Return failure (1) if it's not a semantic version
-    fi
-}
-if ! is_semantic_version "$install_cuda_version"; then
-  show_help
-  error="\nYou must provide a semantic version for CUDA (e.g., 12.1.1) via the appropriate\n"
-  error="${error}command line option. This script is intended for use with EESSI so the 'correct'\n"
-  error="${error}version to provide is probably one of those available under\n"
-  error="${error}$EESSI_SOFTWARE_PATH/software/CUDA\n"
-  fatal_error "${error}"
-fi
-
-# Make sure they have accepted the CUDA EULA
-if [ "$eula_accepted" -ne 1 ]; then
-  show_help
-  error="\nYou _must_ accept the CUDA EULA via the appropriate command line option.\n"
-  fatal_error "${error}"
-fi
-
-# As an installation location just use $EESSI_SOFTWARE_PATH but replacing `versions` with `host_injections`
-# (CUDA is a binary installation so no need to worry too much about the EasyBuild setup)
-cuda_install_parent=${EESSI_SOFTWARE_PATH/versions/host_injections}
-
-# Only install CUDA if specified version is not found.
-# (existence of easybuild subdir implies a successful install)
-if [ -d "${cuda_install_parent}"/software/CUDA/"${install_cuda_version}"/easybuild ]; then
-  echo_green "CUDA software found! No need to install CUDA again."
-else
-  # We need to be able write to the installation space so let's make sure we can
-  if ! create_directory_structure "${cuda_install_parent}"/software/CUDA ; then
-    fatal_error "No write permissions to directory ${cuda_install_parent}/software/CUDA"
-  fi
-
-  # we need a directory we can use for temporary storage
-  if [[ -z "${CUDA_TEMP_DIR}" ]]; then
-    tmpdir=$(mktemp -d)
-  else
-    tmpdir="${CUDA_TEMP_DIR}"/temp
-    if ! mkdir "$tmpdir" ; then
-      fatal_error "Could not create directory ${tmpdir}"
-    fi
-  fi
-
-  required_space_in_tmpdir=50000
-  # Let's see if we have sources and build locations defined if not, we use the temporary space
-  if [[ -z "${EASYBUILD_BUILDPATH}" ]]; then
-    export EASYBUILD_BUILDPATH=${tmpdir}/build
-    required_space_in_tmpdir=$((required_space_in_tmpdir + 5000000))
-  fi
-  if [[ -z "${EASYBUILD_SOURCEPATH}" ]]; then
-    export EASYBUILD_SOURCEPATH=${tmpdir}/sources
-    required_space_in_tmpdir=$((required_space_in_tmpdir + 5000000))
-  fi
-
-  # The install is pretty fat, you need lots of space for download/unpack/install (~3*5GB),
-  # need to do a space check before we proceed
-  avail_space=$(df --output=avail "${cuda_install_parent}"/ | tail -n 1 | awk '{print $1}')
-  if (( avail_space < 5000000 )); then
-    fatal_error "Need at least 5GB disk space to install CUDA under ${cuda_install_parent}, exiting now..."
-  fi
-  avail_space=$(df --output=avail "${tmpdir}"/ | tail -n 1 | awk '{print $1}')
-  if (( avail_space < required_space_in_tmpdir )); then
-    error="Need at least ${required_space_in_tmpdir} disk space under ${tmpdir}.\n"
-    error="${error}Set the environment variable CUDA_TEMP_DIR to a location with adequate space to pass this check."
-    error="${error}You can alternatively set EASYBUILD_BUILDPATH and/or EASYBUILD_SOURCEPATH "
-    error="${error}to reduce this requirement. Exiting now..."
-    fatal_error "${error}"
-  fi
-
-  if ! command -v "eb" &>/dev/null; then
-    echo_yellow "Attempting to load an EasyBuild module to do actual install"
-    module load EasyBuild
-    # There are some scenarios where this may fail
-    if [ $? -ne 0 ]; then
-      error="'eb' command not found in your environment and\n"
-      error="${error}  module load EasyBuild\n"
-      error="${error}failed for some reason.\n"
-      error="${error}Please re-run this script with the 'eb' command available."
-      fatal_error "${error}"
-    fi
-  fi
-
-  cuda_easyconfig="CUDA-${install_cuda_version}.eb"
-
-  # Check the easyconfig file is available in the release
-  # (eb search always returns 0, so we need a grep to ensure a usable exit code)
-  eb --search ^${cuda_easyconfig}|grep CUDA > /dev/null 2>&1
-  # Check the exit code
-  if [ $? -ne 0 ]; then
-    eb_version=$(eb --version)
-    available_cuda_easyconfigs=$(eb --search ^CUDA-*.eb|grep CUDA)
-
-    error="The easyconfig ${cuda_easyconfig} was not found in EasyBuild version:\n"
-    error="${error}  ${eb_version}\n"
-    error="${error}You either need to give a different version of CUDA to install _or_ \n"
-    error="${error}use a different version of EasyBuild for the installation.\n"
-    error="${error}\nThe versions of available with the current eb command are:\n"
-    error="${error}${available_cuda_easyconfigs}"
-    fatal_error "${error}"
-  fi
-
-  # We need the --rebuild option, as the CUDA module may or may not be on the
-  # `MODULEPATH` yet. Even if it is, we still want to redo this installation
-  # since it will provide the symlinked targets for the parts of the CUDA
-  # installation in the `.../versions/...` prefix
-  # We install the module in our `tmpdir` since we do not need the modulefile,
-  # we only care about providing the targets for the symlinks.
-  extra_args="--rebuild --installpath-modules=${tmpdir}"
-
-  # We don't want hooks used in this install, we need a vanilla CUDA installation
-  touch "$tmpdir"/none.py
-  # shellcheck disable=SC2086  # Intended splitting of extra_args
-  eb --prefix="$tmpdir" ${extra_args} --accept-eula-for=CUDA --hooks="$tmpdir"/none.py --installpath="${cuda_install_parent}"/ "${cuda_easyconfig}"
-  ret=$?
-  if [ $ret -ne 0 ]; then
-    eb_last_log=$(unset EB_VERBOSE; eb --last-log)
-    cp -a ${eb_last_log} .
-    fatal_error "CUDA installation failed, please check EasyBuild logs $(basename ${eb_last_log})..."
-  else
-    echo_green "CUDA installation at ${cuda_install_parent}/software/CUDA/${install_cuda_version} succeeded!"
-  fi
-  # clean up tmpdir
-  rm -rf "${tmpdir}"
-fi
diff --git a/temp/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/temp/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
deleted file mode 100755
index e6ff110797..0000000000
--- a/temp/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
+++ /dev/null
@@ -1,136 +0,0 @@
-#!/bin/bash
-
-# This script links host libraries related to GPU drivers to a location where
-# they can be found by the EESSI linker
-
-# Initialise our bash functions
-TOPDIR=$(dirname $(realpath $BASH_SOURCE))
-source "$TOPDIR"/../../utils.sh
-
-# We rely on ldconfig to give us the location of the libraries on the host
-command_name="ldconfig"
-# We cannot use a version of ldconfig that's being shipped under CVMFS
-exclude_prefix="/cvmfs"
-
-found_paths=()
-# Always attempt to use /sbin/ldconfig
-if [ -x "/sbin/$command_name" ]; then
-    found_paths+=("/sbin/$command_name")
-fi
-IFS=':' read -ra path_dirs <<< "$PATH"
-for dir in "${path_dirs[@]}"; do
-  if [ "$dir" = "/sbin" ]; then
-    continue  # we've already checked for $command_name in /sbin, don't need to do it twice
-  fi
-  if [[ ! "$dir" =~ ^$exclude_prefix ]]; then
-      if [ -x "$dir/$command_name" ]; then
-          found_paths+=("$dir/$command_name")
-      fi
-  fi
-done
-
-if [ ${#found_paths[@]} -gt 0 ]; then
-    echo "Found $command_name in the following locations:"
-    printf -- "- %s\n" "${found_paths[@]}"
-    echo "Using first version"
-    host_ldconfig=${found_paths[0]}
-else
-    error="$command_name not found in PATH or only found in paths starting with $exclude_prefix."
-    fatal_error "$error"
-fi
-
-# Make sure EESSI is initialised (doesn't matter what version)
-check_eessi_initialised
-
-# Find the CUDA version of the host CUDA drivers
-# (making sure that this can still work inside prefix environment inside a container)
-export LD_LIBRARY_PATH=/.singularity.d/libs:$LD_LIBRARY_PATH
-nvidia_smi_command="nvidia-smi --query-gpu=driver_version --format=csv,noheader"
-if $nvidia_smi_command > /dev/null; then
-  host_driver_version=$($nvidia_smi_command | tail -n1)
-  # If the first worked, this should work too
-  host_cuda_version=$(nvidia-smi  -q --display=COMPUTE | grep CUDA | awk 'NF>1{print $NF}')
-else
-  error="Failed to successfully execute\n  $nvidia_smi_command\n"
-  fatal_error "$error"
-fi
-
-# Let's make sure the driver libraries are not already in place
-link_drivers=1
-
-host_injections_nvidia_dir="${EESSI_CVMFS_REPO}/host_injections/nvidia/${EESSI_CPU_FAMILY}"
-host_injection_driver_dir="${host_injections_nvidia_dir}/host"
-host_injection_driver_version_file="$host_injection_driver_dir/driver_version.txt"
-if [ -e "$host_injection_driver_version_file" ]; then
-  if grep -q "$host_driver_version" "$host_injection_driver_version_file"; then
-    echo_green "The host CUDA driver libraries have already been linked!"
-    link_drivers=0
-  else
-    # There's something there but it is out of date
-    echo_yellow "Cleaning out outdated symlinks"
-    rm $host_injection_driver_dir/*
-    if [ $? -ne 0 ]; then
-      error="Unable to remove files under '$host_injection_driver_dir'."
-      fatal_error "$error"
-    fi
-  fi
-fi
-
-drivers_linked=0
-if [ "$link_drivers" -eq 1 ]; then
-  if ! create_directory_structure "${host_injection_driver_dir}" ; then
-    fatal_error "No write permissions to directory ${host_injection_driver_dir}"
-  fi
-  cd ${host_injection_driver_dir}
-  # Need a small temporary space to hold a couple of files
-  temp_dir=$(mktemp -d)
-
-  # Gather libraries on the host (_must_ be host ldconfig)
-  $host_ldconfig -p | awk '{print $NF}' > "$temp_dir"/libs.txt
-  # Allow for the fact that we may be in a container so the CUDA libs might be in there
-  ls /.singularity.d/libs/* >> "$temp_dir"/libs.txt 2>/dev/null
-
-  # Leverage singularity to find the full list of libraries we should be linking to
-  echo_yellow "Downloading latest version of nvliblist.conf from Apptainer"
-  curl -o "$temp_dir"/nvliblist.conf https://raw.githubusercontent.com/apptainer/apptainer/main/etc/nvliblist.conf
-
-  # Make symlinks to all the interesting libraries
-  grep '.so$' "$temp_dir"/nvliblist.conf | xargs -i grep {} "$temp_dir"/libs.txt | xargs -i ln -s {}
-
-  # Inject driver and CUDA versions into dir
-  echo $host_driver_version > driver_version.txt
-  echo $host_cuda_version > cuda_version.txt
-  drivers_linked=1
-
-  # Remove the temporary directory when done
-  rm -r "$temp_dir"
-fi
-
-# Make latest symlink for NVIDIA drivers
-cd $host_injections_nvidia_dir
-symlink="latest"
-if [ -L "$symlink" ]; then
-    # Unless the drivers have been installed, leave the symlink alone
-    if [ "$drivers_linked" -eq 1 ]; then
-      ln -sf host latest
-    fi
-else
-    # No link exists yet
-    ln -s host latest
-fi
-
-# Make sure the libraries can be found by the EESSI linker
-host_injection_linker_dir=${EESSI_EPREFIX/versions/host_injections}
-if [ -L "$host_injection_linker_dir/lib" ]; then
-  target_path=$(readlink -f "$host_injection_linker_dir/lib")
-  if [ "$target_path" != "$$host_injections_nvidia_dir/latest" ]; then
-    cd $host_injection_linker_dir
-    ln -sf $host_injections_nvidia_dir/latest lib
-  fi
-else
-  create_directory_structure $host_injection_linker_dir
-  cd $host_injection_linker_dir
-  ln -s $host_injections_nvidia_dir/latest lib
-fi
-
-echo_green "Host NVIDIA gpu drivers linked successfully for EESSI"
diff --git a/temp/scripts/utils.sh b/temp/scripts/utils.sh
deleted file mode 100644
index b2be3f6221..0000000000
--- a/temp/scripts/utils.sh
+++ /dev/null
@@ -1,144 +0,0 @@
-function echo_green() {
-    echo -e "\e[32m$1\e[0m"
-}
-
-function echo_red() {
-    echo -e "\e[31m$1\e[0m"
-}
-
-function echo_yellow() {
-    echo -e "\e[33m$1\e[0m"
-}
-
-ANY_ERROR_EXITCODE=1
-function fatal_error() {
-    echo_red "ERROR: $1" >&2
-    if [[ $# -gt 1 ]]; then
-      exit "$2"
-    else
-      exit "${ANY_ERROR_EXITCODE}"
-    fi
-}
-
-function check_exit_code {
-    ec=$1
-    ok_msg=$2
-    fail_msg=$3
-
-    if [[ $ec -eq 0 ]]; then
-        echo_green "${ok_msg}"
-    else
-        fatal_error "${fail_msg}"
-    fi
-}
-
-function check_eessi_initialised() {
-  if [[ -z "${EESSI_SOFTWARE_PATH}" ]]; then
-    fatal_error "EESSI has not been initialised!"
-  else
-    return 0
-  fi
-}
-
-function check_in_prefix_shell() {
-  # Make sure EPREFIX is defined
-  if [[ -z "${EPREFIX}" ]]; then
-    fatal_error "This script cannot be used without having first defined EPREFIX"
-  fi
-  if [[ ! ${SHELL} = ${EPREFIX}/bin/bash ]]; then
-    fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!"
-  fi
-}
-
-function create_directory_structure() {
-  # Ensure we are given a single path argument
-  if [ $# -ne 1 ]; then
-    echo_red "Function requires a single (relative or absolute) path argument" >&2
-    return $ANY_ERROR_EXITCODE
-  fi
-  dir_structure="$1"
-
-  # Attempt to create the directory structure
-  error_message=$(mkdir -p "$dir_structure" 2>&1)
-  return_code=$?
-  # If it fails be explicit about the error
-  if [ ${return_code} -ne 0 ]; then
-    real_dir=$(realpath -m "$dir_structure")
-    echo_red "Creating ${dir_structure} (real path ${real_dir}) failed with:\n ${error_message}" >&2
-  else
-    # If we're creating it, our use case is that we want to be able to write there
-    # (this is a check in case the directory already existed)
-    if [ ! -w "${dir_structure}" ]; then
-      real_dir=$(realpath -m "$dir_structure")
-      echo_red "You do not have (required) write permissions to ${dir_structure} (real path ${real_dir})!"
-      return_code=$ANY_ERROR_EXITCODE
-    fi
-  fi
-
-  return $return_code
-}
-
-function get_path_for_tool {
-    tool_name=$1
-    tool_envvar_name=$2
-
-    which_out=$(which "${tool_name}" 2>&1)
-    exit_code=$?
-    if [[ ${exit_code} -eq 0 ]]; then
-        echo "INFO: found tool ${tool_name} in PATH (${which_out})" >&2
-        echo "${which_out}"
-        return 0
-    fi
-    if [[ -z "${tool_envvar_name}" ]]; then
-        msg="no env var holding the full path to tool '${tool_name}' provided"
-        echo "${msg}" >&2
-        return 1
-    else
-        tool_envvar_value=${!tool_envvar_name}
-        if [[ -x "${tool_envvar_value}" ]]; then
-            msg="INFO: found tool ${tool_envvar_value} via env var ${tool_envvar_name}"
-            echo "${msg}" >&2
-            echo "${tool_envvar_value}"
-            return 0
-        else
-            msg="ERROR: tool '${tool_name}' not in PATH\n"
-            msg+="ERROR: tool '${tool_envvar_value}' via '${tool_envvar_name}' not in PATH"
-            echo "${msg}" >&2
-            echo ""
-            return 2
-        fi
-    fi
-}
-
-function get_host_from_url {
-    url=$1
-    re="(http|https)://([^/:]+)"
-    if [[ $url =~ $re ]]; then
-        echo "${BASH_REMATCH[2]}"
-        return 0
-    else
-        echo ""
-        return 1
-    fi
-}
-
-function get_port_from_url {
-    url=$1
-    re="(http|https)://[^:]+:([0-9]+)"
-    if [[ $url =~ $re ]]; then
-        echo "${BASH_REMATCH[2]}"
-        return 0
-    else
-        echo ""
-        return 1
-    fi
-}
-
-function get_ipv4_address {
-    hname=$1
-    hipv4=$(grep "${hname}" /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1)
-    # TODO try other methods if the one above does not work --> tool that verifies
-    #      what method can be used?
-    echo "${hipv4}"
-    return 0
-}

From 43c73c07643cf756ff8411b9068bf4cb2a64eec4 Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Thu, 21 Dec 2023 13:06:53 +0100
Subject: [PATCH 43/81] Get rid of copy/paste unfriendly '.'

---
 install_scripts.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/install_scripts.sh b/install_scripts.sh
index 588248e8d2..6e6cd825ac 100755
--- a/install_scripts.sh
+++ b/install_scripts.sh
@@ -19,7 +19,7 @@ compare_and_copy() {
 
     if [ ! -f "$destination_file" ] || ! diff -q "$source_file" "$destination_file" ; then
         cp "$source_file" "$destination_file"
-        echo "File $1 copied to $2."
+        echo "File $1 copied to $2"
     else
         echo "Files $1 and $2 are identical. No copy needed."
     fi

From 3ec3df8f4a91c6684ed83c6b021c231ca74ba7c0 Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Thu, 21 Dec 2023 14:11:09 +0100
Subject: [PATCH 44/81] Update create_tarball.sh

---
 create_tarball.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/create_tarball.sh b/create_tarball.sh
index 4d3ad37311..a172428af1 100755
--- a/create_tarball.sh
+++ b/create_tarball.sh
@@ -43,13 +43,13 @@ module_files_list=${tmpdir}/module_files.list.txt
 # include scripts that were copied by install_scripts.sh
 if [ -d ${eessi_version}/scripts ]; then
     # include scripts we wish to ship along with EESSI,
-    find ${eessi_version}/scripts -type f | grep -v '/\.wh\.' >> ${files_list}
+    find ${eessi_version}/scripts -type f | grep -v '/\.wh\.' > ${files_list}
 fi
 
 if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
     # include Lmod cache and configuration file (lmodrc.lua),
     # skip whiteout files and backup copies of Lmod cache (spiderT.old.*)
-    find ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' > ${files_list}
+    find ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' >> ${files_list}
 fi
 if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules ]; then
     # module files

From 42e3404a0015d7b4049c5cd3e5a01ee764261d44 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 21 Dec 2023 14:13:23 +0100
Subject: [PATCH 45/81] always append to list of files to include in tarball,
 to avoid overwriting it

---
 create_tarball.sh | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/create_tarball.sh b/create_tarball.sh
index a172428af1..faaa9fda6f 100755
--- a/create_tarball.sh
+++ b/create_tarball.sh
@@ -40,10 +40,10 @@ echo ">> Collecting list of files/directories to include in tarball via ${PWD}..
 files_list=${tmpdir}/files.list.txt
 module_files_list=${tmpdir}/module_files.list.txt
 
-# include scripts that were copied by install_scripts.sh
-if [ -d ${eessi_version}/scripts ]; then
-    # include scripts we wish to ship along with EESSI,
-    find ${eessi_version}/scripts -type f | grep -v '/\.wh\.' > ${files_list}
+# include Lmod cache and configuration file (lmodrc.lua),
+if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
+    # skip whiteout files and backup copies of Lmod cache (spiderT.old.*)
+    find ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' >> ${files_list}
 fi
 
 if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
@@ -51,6 +51,12 @@ if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
     # skip whiteout files and backup copies of Lmod cache (spiderT.old.*)
     find ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' >> ${files_list}
 fi
+
+# include scripts that were copied by install_scripts.sh, which we want to ship in EESSI repository
+if [ -d ${eessi_version}/scripts ]; then
+    find ${eessi_version}/scripts -type f | grep -v '/\.wh\.' >> ${files_list}
+fi
+
 if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules ]; then
     # module files
     find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list}
@@ -61,6 +67,7 @@ if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules ]; then
         | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \
         >> ${module_files_list}
 fi
+
 if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then
     # installation directories but only those for which module files were created
     # Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua'

From 60741ae979757b72c87551a9d0e15afefda4a4a6 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 21 Dec 2023 20:48:36 +0100
Subject: [PATCH 46/81] make link_nvidia_host_libraries.sh script a bit more
 robust, in case target of host_injections directory is a non-existing
 directory

---
 .../nvidia/link_nvidia_host_libraries.sh         | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
index e6ff110797..e8d7f0d0a7 100755
--- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
+++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
@@ -48,8 +48,10 @@ export LD_LIBRARY_PATH=/.singularity.d/libs:$LD_LIBRARY_PATH
 nvidia_smi_command="nvidia-smi --query-gpu=driver_version --format=csv,noheader"
 if $nvidia_smi_command > /dev/null; then
   host_driver_version=$($nvidia_smi_command | tail -n1)
+  echo_green "Found NVIDIA GPU driver version ${host_driver_version}"
   # If the first worked, this should work too
   host_cuda_version=$(nvidia-smi  -q --display=COMPUTE | grep CUDA | awk 'NF>1{print $NF}')
+  echo_green "Found host CUDA version ${host_cuda_version}"
 else
   error="Failed to successfully execute\n  $nvidia_smi_command\n"
   fatal_error "$error"
@@ -58,12 +60,18 @@ fi
 # Let's make sure the driver libraries are not already in place
 link_drivers=1
 
+# first make sure that target of host_injections variant symlink is an existing directory
+host_injections_target=$(realpath -m ${EESSI_CVMFS_REPO}/host_injections)
+if [ ! -d ${host_injections_target} ]; then
+    create_directory_structure ${host_injections_target}
+fi
+
 host_injections_nvidia_dir="${EESSI_CVMFS_REPO}/host_injections/nvidia/${EESSI_CPU_FAMILY}"
 host_injection_driver_dir="${host_injections_nvidia_dir}/host"
 host_injection_driver_version_file="$host_injection_driver_dir/driver_version.txt"
 if [ -e "$host_injection_driver_version_file" ]; then
   if grep -q "$host_driver_version" "$host_injection_driver_version_file"; then
-    echo_green "The host CUDA driver libraries have already been linked!"
+    echo_green "The host GPU driver libraries (v${host_driver_version}) have already been linked! (based on ${host_injection_driver_version_file})"
     link_drivers=0
   else
     # There's something there but it is out of date
@@ -91,8 +99,8 @@ if [ "$link_drivers" -eq 1 ]; then
   ls /.singularity.d/libs/* >> "$temp_dir"/libs.txt 2>/dev/null
 
   # Leverage singularity to find the full list of libraries we should be linking to
-  echo_yellow "Downloading latest version of nvliblist.conf from Apptainer"
-  curl -o "$temp_dir"/nvliblist.conf https://raw.githubusercontent.com/apptainer/apptainer/main/etc/nvliblist.conf
+  echo_yellow "Downloading latest version of nvliblist.conf from Apptainer to ${temp_dir}/nvliblist.conf"
+  curl --silent --output "$temp_dir"/nvliblist.conf https://raw.githubusercontent.com/apptainer/apptainer/main/etc/nvliblist.conf
 
   # Make symlinks to all the interesting libraries
   grep '.so$' "$temp_dir"/nvliblist.conf | xargs -i grep {} "$temp_dir"/libs.txt | xargs -i ln -s {}
@@ -133,4 +141,4 @@ else
   ln -s $host_injections_nvidia_dir/latest lib
 fi
 
-echo_green "Host NVIDIA gpu drivers linked successfully for EESSI"
+echo_green "Host NVIDIA GPU drivers linked successfully for EESSI"

From 5c248d1dcb844ec11a76f9f368ba523b7edf3e87 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@gcn6.local.snellius.surf.nl>
Date: Fri, 22 Dec 2023 11:39:31 +0100
Subject: [PATCH 47/81] Ensure that bot reports success if no EasyStacks were
 changed in a PR. Relevant for PRs that e.g. only update things in .../scripts

---
 EESSI-install-software.sh | 75 +++++++++++++++++++++------------------
 1 file changed, 40 insertions(+), 35 deletions(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index b61ca7a579..edbcf7040b 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -203,42 +203,47 @@ ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12
 # ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
 
 # use PR patch file to determine in which easystack files stuff was added
-for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing'); do
-
-    echo -e "Processing easystack file ${easystack_file}...\n\n"
-
-    # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
-    eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
-
-    # load EasyBuild module (will be installed if it's not available yet)
-    source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
-
-    ${EB} --show-config
-
-    echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."
-
-    if [ -f ${easystack_file} ]; then
-        echo_green "Feeding easystack file ${easystack_file} to EasyBuild..."
-
-        ${EB} --easystack ${TOPDIR}/${easystack_file} --robot
-        ec=$?
-
-        # copy EasyBuild log file if EasyBuild exited with an error
-        if [ ${ec} -ne 0 ]; then
-            eb_last_log=$(unset EB_VERBOSE; eb --last-log)
-            # copy to current working directory
-            cp -a ${eb_last_log} .
-            echo "Last EasyBuild log file copied from ${eb_last_log} to ${PWD}"
-            # copy to build logs dir (with context added)
-            copy_build_log "${eb_last_log}" "${build_logs_dir}"
+changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') 
+if [ -z ${changed_easystacks} ]; then
+    echo "No missing installations"  # Ensure the bot report success, as there was nothing to be build here
+else
+    for easystack_file in ${changed_easystacks}; do
+    
+        echo -e "Processing easystack file ${easystack_file}...\n\n"
+    
+        # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
+        eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
+    
+        # load EasyBuild module (will be installed if it's not available yet)
+        source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
+    
+        ${EB} --show-config
+    
+        echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."
+    
+        if [ -f ${easystack_file} ]; then
+            echo_green "Feeding easystack file ${easystack_file} to EasyBuild..."
+    
+            ${EB} --easystack ${TOPDIR}/${easystack_file} --robot
+            ec=$?
+    
+            # copy EasyBuild log file if EasyBuild exited with an error
+            if [ ${ec} -ne 0 ]; then
+                eb_last_log=$(unset EB_VERBOSE; eb --last-log)
+                # copy to current working directory
+                cp -a ${eb_last_log} .
+                echo "Last EasyBuild log file copied from ${eb_last_log} to ${PWD}"
+                # copy to build logs dir (with context added)
+                copy_build_log "${eb_last_log}" "${build_logs_dir}"
+            fi
+    
+            $TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file}
+        else
+            fatal_error "Easystack file ${easystack_file} not found!"
         fi
-
-        $TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file}
-    else
-        fatal_error "Easystack file ${easystack_file} not found!"
-    fi
-
-done
+    
+    done
+fi
 
 ### add packages here
 

From ac53cf0ea0160d47e302393ef60d2829586708af Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@gcn6.local.snellius.surf.nl>
Date: Fri, 22 Dec 2023 14:54:46 +0100
Subject: [PATCH 48/81] Make the pedantic deploy step of the bot happy...

---
 EESSI-install-software.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index edbcf7040b..69de9d1997 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -205,7 +205,7 @@ ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12
 # use PR patch file to determine in which easystack files stuff was added
 changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') 
 if [ -z ${changed_easystacks} ]; then
-    echo "No missing installations"  # Ensure the bot report success, as there was nothing to be build here
+    echo "No missing installations, party time!"  # Ensure the bot report success, as there was nothing to be build here
 else
     for easystack_file in ${changed_easystacks}; do
     

From 203cbd104d8189fbc007c88fa505efba3d02826e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Sat, 23 Dec 2023 14:49:44 +0100
Subject: [PATCH 49/81] add ALL-0.9.2-foss-2023a.eb

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index 567db44e42..7d879044d8 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -35,3 +35,6 @@ easyconfigs:
   - Boost-1.82.0-GCC-12.3.0.eb
   - netCDF-4.9.2-gompi-2023a.eb
   - FFmpeg-6.0-GCCcore-12.3.0.eb
+  - ALL-0.9.2-foss-2023a.eb:
+      options:
+        from-pr: 19455

From b1daa1253af0a77804f5bee803886322209db641 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 2 Jan 2024 09:42:46 +0100
Subject: [PATCH 50/81] {2023.06}[system] EasyBuild v4.9.0

---
 .../2023.06/eessi-2023.06-eb-4.9.0-001-system.yml             | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-001-system.yml

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-001-system.yml
new file mode 100644
index 0000000000..a7bce002c7
--- /dev/null
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-001-system.yml
@@ -0,0 +1,4 @@
+easyconfigs:
+  - EasyBuild-4.9.0.eb:
+      options:
+        from-pr: 19464

From d299c7459681facd6316d7a0c7889e217c7b72f6 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 2 Jan 2024 14:26:45 +0100
Subject: [PATCH 51/81] don't include files in .lmod directory twice in tarball

---
 create_tarball.sh | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/create_tarball.sh b/create_tarball.sh
index faaa9fda6f..a619df9439 100755
--- a/create_tarball.sh
+++ b/create_tarball.sh
@@ -40,12 +40,6 @@ echo ">> Collecting list of files/directories to include in tarball via ${PWD}..
 files_list=${tmpdir}/files.list.txt
 module_files_list=${tmpdir}/module_files.list.txt
 
-# include Lmod cache and configuration file (lmodrc.lua),
-if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
-    # skip whiteout files and backup copies of Lmod cache (spiderT.old.*)
-    find ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' >> ${files_list}
-fi
-
 if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
     # include Lmod cache and configuration file (lmodrc.lua),
     # skip whiteout files and backup copies of Lmod cache (spiderT.old.*)

From 2d0d7e852fe958ff55e1e5a2c1f6e4fb4c3a5f70 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 2 Jan 2024 20:00:13 +0100
Subject: [PATCH 52/81] {2023.06} foss/2023b

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
new file mode 100644
index 0000000000..88ab50b8e7
--- /dev/null
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -0,0 +1,3 @@
+easyconfigs:
+  - GCC-13.2.0.eb
+  - foss-2023b.eb

From 5c0061510b7ffdcfe733307555db8f9e1bc5ed09 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 3 Jan 2024 11:47:09 +0100
Subject: [PATCH 53/81] update pre-configure hook for OpenBLAS to use
 -mtune=generic rather than -mcpu=generic when building with DYNAMIC_ARCH=1 on
 aarch64

---
 eb_hooks.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/eb_hooks.py b/eb_hooks.py
index 27cc873fa1..7a31e87937 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -3,6 +3,7 @@
 import os
 import re
 
+import easybuild.tools.environment as env
 from easybuild.easyblocks.generic.configuremake import obtain_config_guess
 from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
 from easybuild.tools.build_log import EasyBuildError, print_msg
@@ -258,6 +259,13 @@ def pre_configure_hook_openblas_optarch_generic(self, *args, **kwargs):
         if build_option('optarch') == OPTARCH_GENERIC:
             for step in ('build', 'test', 'install'):
                 self.cfg.update(f'{step}opts', "DYNAMIC_ARCH=1")
+
+            # use -mtune=generic rather than -mcpu=generic in $CFLAGS on aarch64,
+            # because -mcpu=generic implies a particular -march=armv* which clashes with those used by OpenBLAS
+            # when building with DYNAMIC_ARCH=1
+            if get_cpu_architecture() == AARCH64:
+                cflags = os.getenv('CFLAGS').replace('-mcpu=generic', '-mtune=generic')
+                env.setvar('CFLAGS', cflags)
     else:
         raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!")
 

From 4936618476b3b30572ca8fece14c890d89f4e34e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 5 Jan 2024 15:12:16 +0100
Subject: [PATCH 54/81] add SciPy-bundle-2023.11-gfbf-2023b.eb

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index 88ab50b8e7..b184b88701 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -1,3 +1,4 @@
 easyconfigs:
   - GCC-13.2.0.eb
   - foss-2023b.eb
+  - SciPy-bundle-2023.11-gfbf-2023b.eb

From 142a65b881183c6c889f3d9cdaef95c0543c4853 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 5 Jan 2024 19:22:50 +0100
Subject: [PATCH 55/81] also apply SciPy-bundle hook to version 2023.11

---
 eb_hooks.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 7a31e87937..73eac9286d 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -367,15 +367,15 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs):
         FAILED optimize/tests/test_linprog.py::TestLinprogIPSparse::test_bug_6139 - A...
         FAILED optimize/tests/test_linprog.py::TestLinprogIPSparsePresolve::test_bug_6139
         = 2 failed, 30554 passed, 2064 skipped, 10992 deselected, 76 xfailed, 7 xpassed, 40 warnings in 380.27s (0:06:20) =
-    In versions 2023.07, 2 failing tests in scipy 1.11.1:
+    In versions 2023.07 and 2023.11, 2 failing tests in scipy 1.11.1 and 1.11.4:
         FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris
         FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris_float32
         = 2 failed, 54409 passed, 3016 skipped, 223 xfailed, 13 xpassed, 10917 warnings in 892.04s (0:14:52) =
     In previous versions we were not as strict yet on the numpy/SciPy tests
     """
     cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
-    if self.name == 'SciPy-bundle' and self.version in ['2021.10', '2023.07'] and cpu_target == CPU_TARGET_NEOVERSE_V1:
-        self.cfg['testopts'] = "|| echo ignoring failing tests" 
+    if self.name == 'SciPy-bundle' and self.version in ['2021.10', '2023.07', '2023.11'] and cpu_target == CPU_TARGET_NEOVERSE_V1:
+        self.cfg['testopts'] = "|| echo ignoring failing tests"
 
 
 def pre_single_extension_hook(ext, *args, **kwargs):

From 78466f569ca028d0119c293cd2f1d2409626619a Mon Sep 17 00:00:00 2001
From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com>
Date: Wed, 10 Jan 2024 16:39:40 +0100
Subject: [PATCH 56/81] Add OpenFOAM-11-foss-2023a

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index aab910d1cc..803b6138e7 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -46,3 +46,6 @@ easyconfigs:
   - ALL-0.9.2-foss-2023a.eb:
       options:
         from-pr: 19455
+  - OpenFOAM-11-foss-2023a.eb:
+      options:
+        from-pr: 19545

From f6ee972f2a8d1c38fd23fc93fe45415793a85513 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 10 Jan 2024 17:43:10 +0100
Subject: [PATCH 57/81] {2023.06}[2022b] SciPy-bundle v2023.02

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml  | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml
new file mode 100644
index 0000000000..fdacd95c55
--- /dev/null
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml
@@ -0,0 +1,2 @@
+easyconfigs:
+  - SciPy-bundle-2023.02-gfbf-2022b.eb

From 838ceffc09b6007384fc41697c2cbee335c8a234 Mon Sep 17 00:00:00 2001
From: Neves-P <p.m.santos.neves@rug.nl>
Date: Fri, 12 Jan 2024 12:07:53 +0100
Subject: [PATCH 58/81] Use EasyBuildv4.9.0 for OpenFOAMv11 install

---
 .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml                  | 3 ---
 .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml                  | 4 ++++
 2 files changed, 4 insertions(+), 3 deletions(-)
 create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index 803b6138e7..aab910d1cc 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -46,6 +46,3 @@ easyconfigs:
   - ALL-0.9.2-foss-2023a.eb:
       options:
         from-pr: 19455
-  - OpenFOAM-11-foss-2023a.eb:
-      options:
-        from-pr: 19545
diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
new file mode 100644
index 0000000000..e33a8af48c
--- /dev/null
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -0,0 +1,4 @@
+easyconfigs:
+  - OpenFOAM-11-foss-2023a.eb:
+      options:
+        from-pr: 19545

From b55efbf4d6d066dd292240e3863c8028c6336098 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 10 Jan 2024 18:41:52 +0100
Subject: [PATCH 59/81] add pre_single_extension_scipy hook to replace
 -mcpu=native with -march=armv8.4-a when building scipy 1.10.1 on
 aarch64/neoverse_v1

---
 eb_hooks.py | 38 ++++++++++++++++++++++++++++----------
 1 file changed, 28 insertions(+), 10 deletions(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 73eac9286d..883d79c6e8 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -384,16 +384,6 @@ def pre_single_extension_hook(ext, *args, **kwargs):
         PRE_SINGLE_EXTENSION_HOOKS[ext.name](ext, *args, **kwargs)
 
 
-def pre_single_extension_testthat(ext, *args, **kwargs):
-    """
-    Pre-extension hook for testthat R package, to fix build on top of recent glibc.
-    """
-    if ext.name == 'testthat' and LooseVersion(ext.version) < LooseVersion('3.1.0'):
-        # use constant value instead of SIGSTKSZ for stack size,
-        # cfr. https://github.com/r-lib/testthat/issues/1373 + https://github.com/r-lib/testthat/pull/1403
-        ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' inst/include/testthat/vendor/catch.h && "
-
-
 def pre_single_extension_isoband(ext, *args, **kwargs):
     """
     Pre-extension hook for isoband R package, to fix build on top of recent glibc.
@@ -404,6 +394,33 @@ def pre_single_extension_isoband(ext, *args, **kwargs):
         ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' src/testthat/vendor/catch.h && "
 
 
+def pre_single_extension_scipy(ext, *args, **kwargs):
+    """
+    Pre-extension hook for scipy, to change -march=native to -march=armv8.4-a for scipy 1.10.x when buidling for
+    aarch64/neoverse_v1 CPU target.
+    """
+    cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
+    if ext.name == 'scipy' and ext.version == '1.10.1' and cpu_target == CPU_TARGET_NEOVERSE_V1:
+        cflags = os.getenv('CFLAGS')
+        if '-mcpu=native' in cflags:
+            cflags = cflags.replace('-mcpu=native', '-march=armv8.4-a')
+            ext.cfg.update('configopts', ' '.join([
+                "-Dc_args='%s'" % cflags,
+                "-Dcpp_args='%s'" % cflags,
+                "-Dfortran_args='%s'" % cflags,
+            ]))
+
+
+def pre_single_extension_testthat(ext, *args, **kwargs):
+    """
+    Pre-extension hook for testthat R package, to fix build on top of recent glibc.
+    """
+    if ext.name == 'testthat' and LooseVersion(ext.version) < LooseVersion('3.1.0'):
+        # use constant value instead of SIGSTKSZ for stack size,
+        # cfr. https://github.com/r-lib/testthat/issues/1373 + https://github.com/r-lib/testthat/pull/1403
+        ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' inst/include/testthat/vendor/catch.h && "
+
+
 def post_sanitycheck_hook(self, *args, **kwargs):
     """Main post-sanity-check hook: trigger custom functions based on software name."""
     if self.name in POST_SANITYCHECK_HOOKS:
@@ -531,6 +548,7 @@ def inject_gpu_property(ec):
 
 PRE_SINGLE_EXTENSION_HOOKS = {
     'isoband': pre_single_extension_isoband,
+    'scipy': pre_single_extension_scipy,
     'testthat': pre_single_extension_testthat,
 }
 

From 684abebfe2a55cd66aef8ceb31169ea23e12ccc9 Mon Sep 17 00:00:00 2001
From: Xin An <xina@int5.local.snellius.surf.nl>
Date: Fri, 12 Jan 2024 15:37:13 +0100
Subject: [PATCH 60/81] {2023.06}[GCC/12.3.0] ReFrame v4.3.3

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index aab910d1cc..feab9d2c19 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -46,3 +46,4 @@ easyconfigs:
   - ALL-0.9.2-foss-2023a.eb:
       options:
         from-pr: 19455
+  - ReFrame-4.3.3.eb

From da6dc24615a7892dbea809bae173db588f2140a5 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 12 Jan 2024 18:34:08 +0100
Subject: [PATCH 61/81] build numpy with -march=armv8.4-a instead of
 -mcpu=native (instead of scipy)

---
 eb_hooks.py | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 883d79c6e8..9d84a9e9f8 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -394,21 +394,19 @@ def pre_single_extension_isoband(ext, *args, **kwargs):
         ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' src/testthat/vendor/catch.h && "
 
 
-def pre_single_extension_scipy(ext, *args, **kwargs):
+def pre_single_extension_numpy(ext, *args, **kwargs):
     """
-    Pre-extension hook for scipy, to change -march=native to -march=armv8.4-a for scipy 1.10.x when buidling for
+    Pre-extension hook for numpy, to change -march=native to -march=armv8.4-a for scipy 1.10.x when buidling for
     aarch64/neoverse_v1 CPU target.
     """
     cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
-    if ext.name == 'scipy' and ext.version == '1.10.1' and cpu_target == CPU_TARGET_NEOVERSE_V1:
-        cflags = os.getenv('CFLAGS')
-        if '-mcpu=native' in cflags:
-            cflags = cflags.replace('-mcpu=native', '-march=armv8.4-a')
-            ext.cfg.update('configopts', ' '.join([
-                "-Dc_args='%s'" % cflags,
-                "-Dcpp_args='%s'" % cflags,
-                "-Dfortran_args='%s'" % cflags,
-            ]))
+    if ext.name == 'numpy' and ext.version == '1.24.2' and cpu_target == CPU_TARGET_NEOVERSE_V1:
+        # unsure which of these actually matter for numpy, so changing all of them
+        for envvar in ('CFLAGS', 'CXXFLAGS', 'F90FLAGS', 'FFLAGS'):
+            value = os.getenv(envvar)
+            if '-mcpu=native' in value:
+                value = value.replace('-mcpu=native', '-march=armv8.4-a')
+                env.setvar(envvar, value)
 
 
 def pre_single_extension_testthat(ext, *args, **kwargs):
@@ -548,7 +546,7 @@ def inject_gpu_property(ec):
 
 PRE_SINGLE_EXTENSION_HOOKS = {
     'isoband': pre_single_extension_isoband,
-    'scipy': pre_single_extension_scipy,
+    'numpy': pre_single_extension_numpy,
     'testthat': pre_single_extension_testthat,
 }
 

From 9943e357127f3ecea98517b1c78c09929710626d Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 13 Jan 2024 18:23:00 +0100
Subject: [PATCH 62/81] update optarch build option to make sure that
 -march=armv8.4-a is used instead of -mcpu=native for numpy, reset it after
 installing numpy, and allow 2 failing scipy tests in SciPy-bundle v2023.02

---
 eb_hooks.py | 39 +++++++++++++++++++++++++++++----------
 1 file changed, 29 insertions(+), 10 deletions(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 9d84a9e9f8..5e6ffef823 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -367,23 +367,30 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs):
         FAILED optimize/tests/test_linprog.py::TestLinprogIPSparse::test_bug_6139 - A...
         FAILED optimize/tests/test_linprog.py::TestLinprogIPSparsePresolve::test_bug_6139
         = 2 failed, 30554 passed, 2064 skipped, 10992 deselected, 76 xfailed, 7 xpassed, 40 warnings in 380.27s (0:06:20) =
-    In versions 2023.07 and 2023.11, 2 failing tests in scipy 1.11.1 and 1.11.4:
+    In versions 2023.02, 2023.07, and 2023.11, 2 failing tests in scipy (versions 1.10.1, 1.11.1, 1.11.4):
         FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris
         FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris_float32
         = 2 failed, 54409 passed, 3016 skipped, 223 xfailed, 13 xpassed, 10917 warnings in 892.04s (0:14:52) =
     In previous versions we were not as strict yet on the numpy/SciPy tests
     """
     cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
-    if self.name == 'SciPy-bundle' and self.version in ['2021.10', '2023.07', '2023.11'] and cpu_target == CPU_TARGET_NEOVERSE_V1:
+    scipy_bundle_versions = ('2021.10', '2023.02', '2023.07', '2023.11')
+    if self.name == 'SciPy-bundle' and self.version in scipy_bundle_versions and cpu_target == CPU_TARGET_NEOVERSE_V1:
         self.cfg['testopts'] = "|| echo ignoring failing tests"
 
 
 def pre_single_extension_hook(ext, *args, **kwargs):
-    """Main pre-configure hook: trigger custom functions based on software name."""
+    """Main pre-extension: trigger custom functions based on software name."""
     if ext.name in PRE_SINGLE_EXTENSION_HOOKS:
         PRE_SINGLE_EXTENSION_HOOKS[ext.name](ext, *args, **kwargs)
 
 
+def post_single_extension_hook(ext, *args, **kwargs):
+    """Main post-extension hook: trigger custom functions based on software name."""
+    if ext.name in POST_SINGLE_EXTENSION_HOOKS:
+        POST_SINGLE_EXTENSION_HOOKS[ext.name](ext, *args, **kwargs)
+
+
 def pre_single_extension_isoband(ext, *args, **kwargs):
     """
     Pre-extension hook for isoband R package, to fix build on top of recent glibc.
@@ -396,17 +403,25 @@ def pre_single_extension_isoband(ext, *args, **kwargs):
 
 def pre_single_extension_numpy(ext, *args, **kwargs):
     """
-    Pre-extension hook for numpy, to change -march=native to -march=armv8.4-a for scipy 1.10.x when buidling for
-    aarch64/neoverse_v1 CPU target.
+    Pre-extension hook for numpy, to change -march=native to -march=armv8.4-a for scipy 1.10.x
+    when building for aarch64/neoverse_v1 CPU target.
     """
     cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
     if ext.name == 'numpy' and ext.version == '1.24.2' and cpu_target == CPU_TARGET_NEOVERSE_V1:
+        # note: this hook is called before build environment is set up (by calling toolchain.prepare()),
+        # so environment variables like $CFLAGS are not defined yet
         # unsure which of these actually matter for numpy, so changing all of them
-        for envvar in ('CFLAGS', 'CXXFLAGS', 'F90FLAGS', 'FFLAGS'):
-            value = os.getenv(envvar)
-            if '-mcpu=native' in value:
-                value = value.replace('-mcpu=native', '-march=armv8.4-a')
-                env.setvar(envvar, value)
+        ext.orig_optarch = build_option('optarch')
+        update_build_option('optarch', 'march=armv8.4-a')
+
+
+def post_single_extension_numpy(ext, *args, **kwargs):
+    """
+    Post-extension hook for numpy, to reset 'optarch' build option.
+    """
+    cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
+    if ext.name == 'numpy' and ext.version == '1.24.2' and cpu_target == CPU_TARGET_NEOVERSE_V1:
+        update_build_option('optarch', ext.orig_optarch)
 
 
 def pre_single_extension_testthat(ext, *args, **kwargs):
@@ -550,6 +565,10 @@ def inject_gpu_property(ec):
     'testthat': pre_single_extension_testthat,
 }
 
+POST_SINGLE_EXTENSION_HOOKS = {
+    'numpy': post_single_extension_numpy,
+}
+
 POST_SANITYCHECK_HOOKS = {
     'CUDA': post_sanitycheck_cuda,
 }

From 258292c99e5aff3c24dc52652f7ad3ddc55af48b Mon Sep 17 00:00:00 2001
From: Xin An <xina@int5.local.snellius.surf.nl>
Date: Mon, 15 Jan 2024 11:10:23 +0100
Subject: [PATCH 63/81] {2023.06} Add ReFrame v4.3.3 to system easystack

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml   | 1 -
 .../2023.06/eessi-2023.06-eb-4.9.0-001-system.yml                | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
index feab9d2c19..aab910d1cc 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
@@ -46,4 +46,3 @@ easyconfigs:
   - ALL-0.9.2-foss-2023a.eb:
       options:
         from-pr: 19455
-  - ReFrame-4.3.3.eb
diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-001-system.yml
index a7bce002c7..25c13e49c9 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-001-system.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-001-system.yml
@@ -2,3 +2,4 @@ easyconfigs:
   - EasyBuild-4.9.0.eb:
       options:
         from-pr: 19464
+  - ReFrame-4.3.3.eb

From 847eb715bb33825de47c2a90b99d7806c2b1e1d9 Mon Sep 17 00:00:00 2001
From: Xin An <xina@int5.local.snellius.surf.nl>
Date: Mon, 15 Jan 2024 11:15:22 +0100
Subject: [PATCH 64/81] {2023.06} Add ReFrame v4.3.3 to system easystack eb
 v4.8.2

---
 .../2023.06/eessi-2023.06-eb-4.8.2-001-system.yml                | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml
index f02b9f2802..d85440ec5b 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml
@@ -5,3 +5,4 @@ easyconfigs:
   - Nextflow-23.10.0.eb:
       options:
         from-pr: 19172
+  - ReFrame-4.3.3.eb

From b16dc0a1cc01e054f27940d10a9a038399fc1478 Mon Sep 17 00:00:00 2001
From: Richard Top <richard.topouchian@uib.no>
Date: Mon, 15 Jan 2024 12:47:09 +0000
Subject: [PATCH 65/81] {2023.06}[gompi/2023b] netCDF v4.9.2

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index b184b88701..9a3b8efa6d 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -2,3 +2,6 @@ easyconfigs:
   - GCC-13.2.0.eb
   - foss-2023b.eb
   - SciPy-bundle-2023.11-gfbf-2023b.eb
+  - netCDF-4.9.2-gompi-2023b.eb:
+      options:
+        from-pr: 19534

From 937efa6d8520e5af5f8292a21cbd0d3a9c5a8344 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 15 Jan 2024 18:06:00 +0100
Subject: [PATCH 66/81] fix docstring in pre_single_extension_numpy hook

---
 eb_hooks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 5e6ffef823..7c52e24ff1 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -403,7 +403,7 @@ def pre_single_extension_isoband(ext, *args, **kwargs):
 
 def pre_single_extension_numpy(ext, *args, **kwargs):
     """
-    Pre-extension hook for numpy, to change -march=native to -march=armv8.4-a for scipy 1.10.x
+    Pre-extension hook for numpy, to change -march=native to -march=armv8.4-a for numpy 1.24.2
     when building for aarch64/neoverse_v1 CPU target.
     """
     cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')

From 15c228b555590b0da28d174ce79d1a8cea44c64b Mon Sep 17 00:00:00 2001
From: Richard Top <richard.topouchian@uib.no>
Date: Tue, 16 Jan 2024 08:53:33 +0000
Subject: [PATCH 67/81] added a hook for neoverse_v1 failing tests

---
 eb_hooks.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/eb_hooks.py b/eb_hooks.py
index 7c52e24ff1..41d196b097 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -378,6 +378,18 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs):
     if self.name == 'SciPy-bundle' and self.version in scipy_bundle_versions and cpu_target == CPU_TARGET_NEOVERSE_V1:
         self.cfg['testopts'] = "|| echo ignoring failing tests"
 
+def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs):
+    """
+    Pre-test hook for SciPy-bundle: skip failing tests for selected netCDF versions on neoverse_v1
+    cfr. https://github.com/EESSI/software-layer/issues/425
+    The following tests are problematic:
+        163 - nc_test4_run_par_test (Timeout)
+        190 - h5_test_run_par_tests (Timeout)
+    A few other tests are skipped in the easyconfig and patches for similar issues, see above issue for details.
+    """
+    cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
+    if self.name == 'netCDF' and self.version == '4.9.2' and cpu_target == CPU_TARGET_NEOVERSE_V1:
+        self.cfg['testopts'] = "|| echo ignoring failing tests" 
 
 def pre_single_extension_hook(ext, *args, **kwargs):
     """Main pre-extension: trigger custom functions based on software name."""
@@ -557,6 +569,7 @@ def inject_gpu_property(ec):
     'ESPResSo': pre_test_hook_ignore_failing_tests_ESPResSo,
     'FFTW.MPI': pre_test_hook_ignore_failing_tests_FFTWMPI,
     'SciPy-bundle': pre_test_hook_ignore_failing_tests_SciPybundle,
+    'netCDF': pre_test_hook_ignore_failing_tests_netCDF,
 }
 
 PRE_SINGLE_EXTENSION_HOOKS = {

From 90e25455b6e1525165d42425ee6035825d040aec Mon Sep 17 00:00:00 2001
From: Richard Top <richard.topouchian@uib.no>
Date: Tue, 16 Jan 2024 12:26:37 +0000
Subject: [PATCH 68/81] added eessi-2023.06-known-issues.yml

---
 eessi-2023.06-known-issues.yml | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 eessi-2023.06-known-issues.yml

diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml
new file mode 100644
index 0000000000..cb2d945af2
--- /dev/null
+++ b/eessi-2023.06-known-issues.yml
@@ -0,0 +1,4 @@
+- aarch64/neoverse_v1:
+  - netCDF-4.9.2-GCC-12.3.0/GCC-13.2.0:
+      - issue: https://github.com/EESSI/software-layer/issues/425
+      - info: "netCDF intermittent test failures"

From 67e6a258860325c3242e22939c0df9f91c07334e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 16 Jan 2024 14:02:14 +0100
Subject: [PATCH 69/81] add at-spi2-core-2.49.91-GCCcore-12.3.0.eb

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index e33a8af48c..1586c16827 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -2,3 +2,4 @@ easyconfigs:
   - OpenFOAM-11-foss-2023a.eb:
       options:
         from-pr: 19545
+  - at-spi2-core-2.49.91-GCCcore-12.3.0.eb

From b49ce5d7929c64687acb8e114ae4938f232647db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 16 Jan 2024 14:12:40 +0100
Subject: [PATCH 70/81] add pre-configure hook for at-spi2-core

---
 eb_hooks.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/eb_hooks.py b/eb_hooks.py
index 7c52e24ff1..ff9072c793 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -335,6 +335,19 @@ def pre_configure_hook_LAMMPS_aarch64(self, *args, **kwargs):
         raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!")
 
 
+def pre_configure_hook_atspi2core_filter_ld_library_path(self, *args, **kwargs):
+    """
+    pre-configure hook for at-spi2-core:
+    - instruct GObject-Introspection's g-ir-scanner tool to not set $LD_LIBRARY_PATH
+      when EasyBuild is configured to filter it
+    """
+    if self.name == 'at-spi2-core':
+        if build_option('filter_env_vars') and 'LD_LIBRARY_PATH' in build_option('filter_env_vars'):
+            self.cfg.update('preconfigopts', 'sed -i "s/gir_extra_args = \[/gir_extra_args = \[\\n  \'--lib-dirs-envvar=FILTER_LD_LIBRARY_PATH\',/g" %(start_dir)s/atspi/meson.build && ')
+    else:
+        raise EasyBuildError("at-spi2-core-specific hook triggered for non-at-spi2-core easyconfig?!")
+
+
 def pre_test_hook(self,*args, **kwargs):
     """Main pre-test hook: trigger custom functions based on software name."""
     if self.name in PRE_TEST_HOOKS:
@@ -551,6 +564,7 @@ def inject_gpu_property(ec):
     'OpenBLAS': pre_configure_hook_openblas_optarch_generic,
     'WRF': pre_configure_hook_wrf_aarch64,
     'LAMMPS': pre_configure_hook_LAMMPS_aarch64,
+    'at-spi2-core': pre_configure_hook_atspi2core_filter_ld_library_path,
 }
 
 PRE_TEST_HOOKS = {

From 23aa7ce5e0e59f2280cbc5feda7e3f213788b9b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 16 Jan 2024 14:18:42 +0100
Subject: [PATCH 71/81] split long line

---
 eb_hooks.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index ff9072c793..c63121afad 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -343,7 +343,8 @@ def pre_configure_hook_atspi2core_filter_ld_library_path(self, *args, **kwargs):
     """
     if self.name == 'at-spi2-core':
         if build_option('filter_env_vars') and 'LD_LIBRARY_PATH' in build_option('filter_env_vars'):
-            self.cfg.update('preconfigopts', 'sed -i "s/gir_extra_args = \[/gir_extra_args = \[\\n  \'--lib-dirs-envvar=FILTER_LD_LIBRARY_PATH\',/g" %(start_dir)s/atspi/meson.build && ')
+            sed_cmd = 'sed -i "s/gir_extra_args = \[/gir_extra_args = \[\\n  \'--lib-dirs-envvar=FILTER_LD_LIBRARY_PATH\',/g" %(start_dir)s/atspi/meson.build && ')
+            self.cfg.update('preconfigopts', sed_cmd)
     else:
         raise EasyBuildError("at-spi2-core-specific hook triggered for non-at-spi2-core easyconfig?!")
 

From 75bf2dec2d333df9bf0bee84116f0ea9bdce5d9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 16 Jan 2024 14:58:29 +0100
Subject: [PATCH 72/81] fix syntax, remove )

---
 eb_hooks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index c63121afad..a9169d60b8 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -343,7 +343,7 @@ def pre_configure_hook_atspi2core_filter_ld_library_path(self, *args, **kwargs):
     """
     if self.name == 'at-spi2-core':
         if build_option('filter_env_vars') and 'LD_LIBRARY_PATH' in build_option('filter_env_vars'):
-            sed_cmd = 'sed -i "s/gir_extra_args = \[/gir_extra_args = \[\\n  \'--lib-dirs-envvar=FILTER_LD_LIBRARY_PATH\',/g" %(start_dir)s/atspi/meson.build && ')
+            sed_cmd = 'sed -i "s/gir_extra_args = \[/gir_extra_args = \[\\n  \'--lib-dirs-envvar=FILTER_LD_LIBRARY_PATH\',/g" %(start_dir)s/atspi/meson.build && '
             self.cfg.update('preconfigopts', sed_cmd)
     else:
         raise EasyBuildError("at-spi2-core-specific hook triggered for non-at-spi2-core easyconfig?!")

From 9a1b4c82583d9022fb9a50bff9ce7d2e080cc290 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 16 Jan 2024 15:53:57 +0100
Subject: [PATCH 73/81] add link to github issue

---
 eb_hooks.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index a9169d60b8..b3cbc4b28e 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -339,7 +339,8 @@ def pre_configure_hook_atspi2core_filter_ld_library_path(self, *args, **kwargs):
     """
     pre-configure hook for at-spi2-core:
     - instruct GObject-Introspection's g-ir-scanner tool to not set $LD_LIBRARY_PATH
-      when EasyBuild is configured to filter it
+      when EasyBuild is configured to filter it, see:
+      https://github.com/EESSI/software-layer/issues/196
     """
     if self.name == 'at-spi2-core':
         if build_option('filter_env_vars') and 'LD_LIBRARY_PATH' in build_option('filter_env_vars'):

From 9944ab6546d856acdf0d92231dac56af8aab82b8 Mon Sep 17 00:00:00 2001
From: TopRichard <121792457+TopRichard@users.noreply.github.com>
Date: Tue, 16 Jan 2024 17:17:29 +0100
Subject: [PATCH 74/81] Update eb_hooks.py

Co-authored-by: ocaisa <alan.ocais@cecam.org>
---
 eb_hooks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 41d196b097..3b18f330ed 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -380,7 +380,7 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs):
 
 def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs):
     """
-    Pre-test hook for SciPy-bundle: skip failing tests for selected netCDF versions on neoverse_v1
+    Pre-test hook for netCDF: skip failing tests for selected netCDF versions on neoverse_v1
     cfr. https://github.com/EESSI/software-layer/issues/425
     The following tests are problematic:
         163 - nc_test4_run_par_test (Timeout)

From 5577d243ccb43df34256036a617b0ab1b53665e0 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 16 Jan 2024 20:14:26 +0100
Subject: [PATCH 75/81] {2023.06}[foss/2023a] ESPResSo v4.2.1

---
 .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml                 | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index e33a8af48c..0569affbca 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -1,4 +1,9 @@
 easyconfigs:
   - OpenFOAM-11-foss-2023a.eb:
+      # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19545
       options:
         from-pr: 19545
+  - ESPResSo-4.2.1-foss-2023a.eb:
+      # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19592    
+      options:
+        from-pr: 19592

From 07a99b43b7b7577e881095a73b74300a00793567 Mon Sep 17 00:00:00 2001
From: Xin An <xina@int5.local.snellius.surf.nl>
Date: Wed, 17 Jan 2024 11:01:27 +0100
Subject: [PATCH 76/81] remove reframe v433 from eb v4.8.2

---
 .../2023.06/eessi-2023.06-eb-4.8.2-001-system.yml                | 1 -
 1 file changed, 1 deletion(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml
index d85440ec5b..f02b9f2802 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml
@@ -5,4 +5,3 @@ easyconfigs:
   - Nextflow-23.10.0.eb:
       options:
         from-pr: 19172
-  - ReFrame-4.3.3.eb

From 34f4c453044ea6acd324b462f47de38db33c3b61 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 18 Jan 2024 12:03:25 +0100
Subject: [PATCH 77/81] fix known issues YAML file for netCDF

---
 eessi-2023.06-known-issues.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml
index cb2d945af2..0ee661be73 100644
--- a/eessi-2023.06-known-issues.yml
+++ b/eessi-2023.06-known-issues.yml
@@ -1,4 +1,7 @@
 - aarch64/neoverse_v1:
-  - netCDF-4.9.2-GCC-12.3.0/GCC-13.2.0:
+  - netCDF-4.9.2-gompi-2023a.eb:
+      - issue: https://github.com/EESSI/software-layer/issues/425
+      - info: "netCDF intermittent test failures"
+  - netCDF-4.9.2-gompi-2023b.eb:
       - issue: https://github.com/EESSI/software-layer/issues/425
       - info: "netCDF intermittent test failures"

From abfc7ae9f5b8f52c9cb5a348bcf263d3d0bf650a Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 19 Jan 2024 09:19:59 +0100
Subject: [PATCH 78/81] complete overview of known issues (so far) for
 software.eessi.io version 2023.06

---
 eessi-2023.06-known-issues.yml | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml
index 0ee661be73..8d8cfe4d07 100644
--- a/eessi-2023.06-known-issues.yml
+++ b/eessi-2023.06-known-issues.yml
@@ -1,7 +1,28 @@
 - aarch64/neoverse_v1:
+  - ESPResSo-4.2.1-foss-2023a:
+      - issue: https://github.com/EESSI/software-layer/issues/363
+      - info: "ESPResSo tests failing due to timeouts"
+  - FFTW.MPI-3.3.10-gompi-2023a:
+      - issue: https://github.com/EESSI/software-layer/issues/325
+      - info: "Flaky FFTW tests, random failures"
+  - FFTW.MPI-3.3.10-gompi-2023b:
+      - issue: https://github.com/EESSI/software-layer/issues/325
+      - info: "Flaky FFTW tests, random failures"
   - netCDF-4.9.2-gompi-2023a.eb:
       - issue: https://github.com/EESSI/software-layer/issues/425
       - info: "netCDF intermittent test failures"
   - netCDF-4.9.2-gompi-2023b.eb:
       - issue: https://github.com/EESSI/software-layer/issues/425
       - info: "netCDF intermittent test failures"
+  - OpenBLAS-0.3.21-GCC-12.2.0:
+    - issue: https://github.com/EESSI/software-layer/issues/314
+    - info: "Increased number of numerical errors in OpenBLAS test suite (344 vs max. 150 on x86_64/*)"
+  - SciPy-bundle-2023.11-gfbf-2023b:
+    - issue: https://github.com/EESSI/software-layer/issues/318
+    - info: "numpy built with -march=armv8.4-a instead of -mcpu=native (no SVE) + 2 failing tests (vs 50005 passed) in scipy test suite"
+  - SciPy-bundle-2023.11-gfbf-2023b:
+    - issue: https://github.com/EESSI/software-layer/issues/318
+    - info: "2 failing tests (vs 54409 passed) in scipy test suite"
+  - SciPy-bundle-2023.11-gfbf-2023b:
+    - issue: https://github.com/EESSI/software-layer/issues/318
+    - info: "2 failing tests (vs 54876 passed) in scipy test suite"

From efbc538248634d06dc8e09374af87188dd373f0e Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 19 Jan 2024 09:20:32 +0100
Subject: [PATCH 79/81] limit hook to increase max failing numerical tests on
 neoverse_v1 to OpenBLAS < 0.3.23

---
 eb_hooks.py | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 14742678bb..2bc2fdebd5 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -185,20 +185,21 @@ def parse_hook_fontconfig_add_fonts(ec, eprefix):
 
 
 def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix):
-    """Relax number of failing numerical LAPACK tests for aarch64/neoverse_v1 CPU target."""
+    """Relax number of failing numerical LAPACK tests for aarch64/neoverse_v1 CPU target for OpenBLAS < 0.3.23"""
     cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
     if ec.name == 'OpenBLAS':
-        # relax maximum number of failed numerical LAPACK tests for aarch64/neoverse_v1 CPU target
-        # since the default setting of 150 that works well on other aarch64 targets and x86_64 is a bit too strict
-        # See https://github.com/EESSI/software-layer/issues/314
-        cfg_option = 'max_failing_lapack_tests_num_errors'
-        if cpu_target == CPU_TARGET_NEOVERSE_V1:
-            orig_value = ec[cfg_option]
-            ec[cfg_option] = 400
-            print_msg("Maximum number of failing LAPACK tests with numerical errors for %s relaxed to %s (was %s)",
-                      ec.name, ec[cfg_option], orig_value)
-        else:
-            print_msg("Not changing option %s for %s on non-AARCH64", cfg_option, ec.name)
+        if LooseVersion(ec.version) < LooseVersion('0.3.23'):
+            # relax maximum number of failed numerical LAPACK tests for aarch64/neoverse_v1 CPU target
+            # since the default setting of 150 that works well on other aarch64 targets and x86_64 is a bit too strict
+            # See https://github.com/EESSI/software-layer/issues/314
+            cfg_option = 'max_failing_lapack_tests_num_errors'
+            if cpu_target == CPU_TARGET_NEOVERSE_V1:
+                orig_value = ec[cfg_option]
+                ec[cfg_option] = 400
+                print_msg("Maximum number of failing LAPACK tests with numerical errors for %s relaxed to %s (was %s)",
+                          ec.name, ec[cfg_option], orig_value)
+            else:
+                print_msg("Not changing option %s for %s on non-AARCH64", cfg_option, ec.name)
     else:
         raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!")
 

From 855251262cb228c8adf1ea37ad22af6e71ddb8c7 Mon Sep 17 00:00:00 2001
From: Richard Top <richard.topouchian@uib.no>
Date: Fri, 19 Jan 2024 11:07:59 +0000
Subject: [PATCH 80/81] {2023.06}[gfbf/2023b] matplotlib v3.8.2

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index 9a3b8efa6d..4dd31dbd5d 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -5,3 +5,6 @@ easyconfigs:
   - netCDF-4.9.2-gompi-2023b.eb:
       options:
         from-pr: 19534
+  - matplotlib-3.8.2-gfbf-2023b.eb:
+      options:
+        from-pr: 19552

From 6705bfc57c35bf4ee4a1c3e8c08356c57c0b91c5 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 19 Jan 2024 16:31:12 +0100
Subject: [PATCH 81/81] fix entries in known issues for SciPy-bundle

---
 eessi-2023.06-known-issues.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml
index 8d8cfe4d07..475ee2c1d7 100644
--- a/eessi-2023.06-known-issues.yml
+++ b/eessi-2023.06-known-issues.yml
@@ -17,10 +17,10 @@
   - OpenBLAS-0.3.21-GCC-12.2.0:
     - issue: https://github.com/EESSI/software-layer/issues/314
     - info: "Increased number of numerical errors in OpenBLAS test suite (344 vs max. 150 on x86_64/*)"
-  - SciPy-bundle-2023.11-gfbf-2023b:
+  - SciPy-bundle-2023.02-gfbf-2022b:
     - issue: https://github.com/EESSI/software-layer/issues/318
     - info: "numpy built with -march=armv8.4-a instead of -mcpu=native (no SVE) + 2 failing tests (vs 50005 passed) in scipy test suite"
-  - SciPy-bundle-2023.11-gfbf-2023b:
+  - SciPy-bundle-2023.07-gfbf-2023a:
     - issue: https://github.com/EESSI/software-layer/issues/318
     - info: "2 failing tests (vs 54409 passed) in scipy test suite"
   - SciPy-bundle-2023.11-gfbf-2023b: