From 9766267f475b1f266923b6c03b545d10caabd42a Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 16 Dec 2021 19:26:11 +0100
Subject: [PATCH 01/31] also install SciPy-bundle-2021.05-foss-2021a.eb into
 2021.12 pilot

---
 EESSI-pilot-install-software.sh | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index 10c57c6d13..4cbf0d3b8e 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -326,6 +326,24 @@ fail_msg="Installation of IPython failed, that's unexpected..."
 $EB IPython-7.15.0-foss-2020a-Python-3.8.2.eb -r
 check_exit_code $? "${ok_msg}" "${fail_msg}"
 
+echo ">> Installing EasyBuild 4.5.1..."
+ok_msg="EasyBuild v4.5.1 installed"
+fail_msg="EasyBuild v4.5.1 failed to install"
+$EB --from-pr 14545
+check_exit_code $? "${ok_msg}" "${fail_msg}"
+
+LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1
+check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5.1"
+
+echo ">> Installing SciPy-bundle with foss/2021a..."
+ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age"
+fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..."
+$EB SciPy-bundle-2021.05-foss-2021a.eb -M
+$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248
+$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot
+$EB SciPy-bundle-2021.05-foss-2021a.eb -r --buildpath /dev/shm/$USER/easybuild_build
+check_exit_code $? "${ok_msg}" "${fail_msg}"
+
 echo ">> Creating/updating Lmod cache..."
 export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua"
 if [ ! -f $LMOD_RC ]; then

From 596ab7adc3ac5d7fe045b4be6fc3b4b6aad4f840 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 16 Dec 2021 19:41:37 +0100
Subject: [PATCH 02/31] update hooks to add --disable-psm3 when building
 libfabric on x86_64 without AVX support

---
 eb_hooks.py | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 126d89ca46..7eb7627643 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -4,7 +4,8 @@
 
 from easybuild.tools.build_log import EasyBuildError, print_msg
 from easybuild.tools.config import build_option, update_build_option
-from easybuild.tools.systemtools import POWER, get_cpu_architecture
+from easybuild.tools.systemtools import POWER, X86_64, get_cpu_architecture, get_cpu_features
+from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC
 
 EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs'
 
@@ -52,6 +53,13 @@ def parse_hook(ec, *args, **kwargs):
         PARSE_HOOKS[ec.name](ec, eprefix)
 
 
+def pre_configure_hook(self, *args, **kwargs):
+    """Main pre-configure hook: trigger custom functions based on software name."""
+
+    if self.name in PRE_CONFIGURE_HOOKS:
+        PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs)
+
+
 def pre_prepare_hook(self, *args, **kwargs):
     """Main pre-prepare hook: trigger custom functions."""
 
@@ -114,12 +122,25 @@ def fontconfig_add_fonts(ec, eprefix):
         raise EasyBuildError("fontconfig-specific hook triggered for non-fontconfig easyconfig?!")
 
 
+def libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs):
+    """Add --disable-psm3 to libfabric configure options when building with --optarch=GENERIC on x86_64."""
+    if self.name == 'libfabric':
+        if get_cpu_architecture() == X86_64:
+            generic = build_option('optarch') == OPTARCH_GENERIC
+            no_avx = 'avx' not in get_cpu_features()
+            if generic or no_avx:
+                self.cfg.update('configopts', '--disable-psm3')
+                print_msg("Using custom configure options for %s: %s", self.name, self.cfg['configopts'])
+    else:
+        raise EasyBuildError("libfabric-specific hook triggered for non-libfabric easyconfig?!")
+
+
 def ucx_eprefix(ec, eprefix):
     """Make UCX aware of compatibility layer via additional configuration options."""
     if ec.name == 'UCX':
         ec.update('configopts', '--with-sysroot=%s' % eprefix)
         ec.update('configopts', '--with-rdmacm=%s' % os.path.join(eprefix, 'usr'))
-        print_msg("Using custom configure option for %s: %s", ec.name, ec['configopts'])
+        print_msg("Using custom configure options for %s: %s", ec.name, ec['configopts'])
     else:
         raise EasyBuildError("UCX-specific hook triggered for non-UCX easyconfig?!")
 
@@ -129,3 +150,7 @@ def ucx_eprefix(ec, eprefix):
     'fontconfig': fontconfig_add_fonts,
     'UCX': ucx_eprefix,
 }
+
+PRE_CONFIGURE_HOOKS = {
+    'libfabric': libfabric_disable_psm3_x86_64_generic,
+}

From 7adc0764864f45b85b1e0feeb27c90cd86fc736e Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 6 Jan 2022 17:07:41 +0100
Subject: [PATCH 03/31] reorder software alphabetically in easystack file for
 2021.12

---
 eessi-2021.12.yml | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/eessi-2021.12.yml b/eessi-2021.12.yml
index 0d060f55ac..cc1519e30b 100644
--- a/eessi-2021.12.yml
+++ b/eessi-2021.12.yml
@@ -1,10 +1,8 @@
 software:
-  R-bundle-Bioconductor:
+  code-server:
     toolchains:
-      foss-2020a:
-        versions:
-          '3.11':
-            versionsuffix: -R-4.0.0
+      SYSTEM:
+        versions: '3.7.3'
   GROMACS:
     toolchains:
       foss-2020a:
@@ -31,19 +29,21 @@ software:
      toolchains:
        foss-2020a:
          versions: ['6.6']
-  TensorFlow:
+  R-bundle-Bioconductor:
     toolchains:
       foss-2020a:
         versions:
-          '2.3.1':
-            versionsuffix: -Python-3.8.2
+          '3.11':
+            versionsuffix: -R-4.0.0
   RStudio-Server:
     toolchains:
       foss-2020a:
         versions:
           '1.3.1093':
             versionsuffix: -Java-11-R-4.0.0
-  code-server:
+  TensorFlow:
     toolchains:
-      SYSTEM:
-        versions: '3.7.3'
+      foss-2020a:
+        versions:
+          '2.3.1':
+            versionsuffix: -Python-3.8.2

From 1e3d482ed47a4defd819d7826f29678dfcf4f8d3 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 6 Jan 2022 17:12:22 +0100
Subject: [PATCH 04/31] add SciPy-bundle with foss/2021a to easystack file for
 2021.12 EESSI pilot

---
 eessi-2021.12.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/eessi-2021.12.yml b/eessi-2021.12.yml
index cc1519e30b..d2e1ccb1ba 100644
--- a/eessi-2021.12.yml
+++ b/eessi-2021.12.yml
@@ -41,6 +41,10 @@ software:
         versions:
           '1.3.1093':
             versionsuffix: -Java-11-R-4.0.0
+  SciPy-bundle:
+    toolchains:
+      foss-2021a:
+        versions: ['2021.05']
   TensorFlow:
     toolchains:
       foss-2020a:

From e5d67b2c16650c40551a6513ed64d25053e14613 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 7 Jan 2022 15:59:49 +0100
Subject: [PATCH 05/31] use enhanced Perl easyblock to fix problem with
 excessive long shebang line in Perl scripts

---
 EESSI-pilot-install-software.sh | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index 4cbf0d3b8e..be699069d6 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -339,7 +339,14 @@ echo ">> Installing SciPy-bundle with foss/2021a..."
 ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age"
 fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..."
 $EB SciPy-bundle-2021.05-foss-2021a.eb -M
+# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640
+# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...)
+$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640
+# use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set
+# from https://github.com/easybuilders/easybuild-easyblocks/pull/2248
 $EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248
+# use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584
+# that includes patch to fix bootstrap problem when using alternate sysroot
 $EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot
 $EB SciPy-bundle-2021.05-foss-2021a.eb -r --buildpath /dev/shm/$USER/easybuild_build
 check_exit_code $? "${ok_msg}" "${fail_msg}"

From b80be7a8f723323fb7a3662b0a4ef856f2bb573d Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 7 Nov 2022 19:52:34 +0100
Subject: [PATCH 06/31] use merged PRs to fix installation of GCCcore/10.3.0
 with recent kernel headers + workaround for compiler bug affecting OpenBLAS
 on POWER

---
 EESSI-pilot-install-software.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index 193f5600a1..7fdcbde6b6 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -344,7 +344,9 @@ check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5.
 echo ">> Installing SciPy-bundle with foss/2021a..."
 ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age"
 fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..."
-$EB SciPy-bundle-2021.05-foss-2021a.eb -M
+# use GCCcore easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14454
+# which includes patch to fix installation with recent Linux kernel headers
+$EB --from-pr 14454 GCCcore-10.3.0.eb --robot
 # use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640
 # to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...)
 $EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640
@@ -354,6 +356,10 @@ $EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248
 # use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584
 # that includes patch to fix bootstrap problem when using alternate sysroot
 $EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot
+# use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885
+# which includes a patch to fix installation on POWER
+$EB --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot
+
 $EB SciPy-bundle-2021.05-foss-2021a.eb -r --buildpath /dev/shm/$USER/easybuild_build
 check_exit_code $? "${ok_msg}" "${fail_msg}"
 

From 65812efb82578b5cc584e0bd3ab9b16f4e00681f Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 7 Nov 2022 20:06:52 +0100
Subject: [PATCH 07/31] use updated EasyBuildMeta easyblock to install
 EasyBuild 4.5.1

---
 EESSI-pilot-install-software.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index f23d60a328..983ff9d1bd 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -359,7 +359,7 @@ check_exit_code $? "${ok_msg}" "${fail_msg}"
 echo ">> Installing EasyBuild 4.5.1..."
 ok_msg="EasyBuild v4.5.1 installed"
 fail_msg="EasyBuild v4.5.1 failed to install"
-$EB --from-pr 14545
+$EB --from-pr 14545 --include-easyblocks-from-pr 2805
 check_exit_code $? "${ok_msg}" "${fail_msg}"
 
 LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1

From 83b283c66720942371f88ce987b932f94d26c8c5 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 8 Nov 2022 19:34:17 +0100
Subject: [PATCH 08/31] fix alphabetical sorting in eessi-2021.12.yml

---
 eessi-2021.12.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/eessi-2021.12.yml b/eessi-2021.12.yml
index 102349551b..977c0f9804 100644
--- a/eessi-2021.12.yml
+++ b/eessi-2021.12.yml
@@ -17,6 +17,10 @@ software:
         versions:
           '0.21.3':
             versionsuffix: -TensorFlow-2.3.1-Python-3.8.2
+  Nextflow:
+    toolchains:
+      SYSTEM:
+        versions: '22.10.1'
   OpenFOAM:
     toolchains:
       foss-2020a:
@@ -57,7 +61,3 @@ software:
          versions: 
            '3.9.1.1': 
              versionsuffix: -dmpar
-  Nextflow:
-    toolchains:
-      SYSTEM:
-        versions: '22.10.1'

From 79a580475c2d0f5d9677d06ee7e61c2f9a2cb1ef Mon Sep 17 00:00:00 2001
From: hugo meiland <hugo.meiland@microsoft.com>
Date: Wed, 9 Nov 2022 12:15:29 +0100
Subject: [PATCH 09/31] take out nameref of update_arch_specs function to
 add/keep support for bash4 (centos7)

---
 init/eessi_archdetect.sh | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh
index af63e36f3a..c901bdfa48 100755
--- a/init/eessi_archdetect.sh
+++ b/init/eessi_archdetect.sh
@@ -27,17 +27,13 @@ log () {
 # Supported CPU specifications
 update_arch_specs(){
     # Add contents of given spec file into an array
-    # 1: name of array with CPU arch specs
-    # 2: spec file with the additional specs
+    # 1: spec file with the additional specs
 
-    [ -z "$1" ] && echo "[ERROR] update_arch_specs: missing array in argument list" >&2 && exit 1
-    local -n arch_specs=$1
-
-    [ ! -f "$2" ] && echo "[ERROR] update_arch_specs: spec file not found: $2" >&2 && exit 1
-    local spec_file="$2"
+    [ ! -f "$1" ] && echo "[ERROR] update_arch_specs: spec file not found: $1" >&2 && exit 1
+    local spec_file="$1"
     while read spec_line; do
        # format spec line as an array and append it to array with all CPU arch specs
-       arch_specs+=("(${spec_line})")
+       cpu_arch_spec+=("(${spec_line})")
     # remove comments from spec file
     done < <(sed -E 's/(^|[\s\t])#.*$//g;/^\s*$/d' "$spec_file")
 }
@@ -87,7 +83,7 @@ cpupath(){
     esac
     # spec files are located in a subfolder with this script
     local base_dir=$(dirname $(realpath $0))
-    update_arch_specs cpu_arch_spec "$base_dir/arch_specs/${spec_file}"
+    update_arch_specs "$base_dir/arch_specs/${spec_file}"
   
     # Identify the host CPU vendor
     local cpu_vendor_tag="vendor[ _]id"

From 15b40770ee99aa75272dff9a335fb7e41f3598ee Mon Sep 17 00:00:00 2001
From: hugo meiland <hugo.meiland@microsoft.com>
Date: Wed, 9 Nov 2022 12:44:43 +0100
Subject: [PATCH 10/31] adding test for override variable

---
 init/eessi_archdetect.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh
index c901bdfa48..d2b6dacf04 100755
--- a/init/eessi_archdetect.sh
+++ b/init/eessi_archdetect.sh
@@ -66,6 +66,10 @@ check_allinfirst(){
 }
 
 cpupath(){
+    # If EESSI_SOFTWARE_SUBDIR_OVERRIDE is set, use it
+    log "DEBUG" "cpupath: Override variable set as '$EESI_SOFTWARE_SUBDIR_OVERRIDE' "
+    [ $EESI_SOFTWARE_SUBDIR_OVERRIDE ] && echo ${EESI_SOFTWARE_SUBDIR_OVERRIDE} && exit
+
     # Identify the best matching CPU architecture from a list of supported specifications for the host CPU
     # Return the path to the installation files in EESSI of the best matching architecture
     local cpu_arch_spec=()

From deb34a761a545609693966dc37a323ab75553fa6 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 9 Nov 2022 23:24:40 +0100
Subject: [PATCH 11/31] add OSU-Micro-Benchmarks/5.7.1-gompi-2021a to EESSI
 pilot 2021.12

---
 EESSI-pilot-install-software.sh | 6 ++++++
 eessi-2021.12.yml               | 2 ++
 2 files changed, 8 insertions(+)

diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index dc50f7fae2..63b5812b99 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -362,6 +362,12 @@ fail_msg="Installation of Nextflow failed, that's unexpected..."
 $EB -r --from-pr 16531 Nextflow-22.10.1.eb
 check_exit_code $? "${ok_msg}" "${fail_msg}"
 
+echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..."
+ok_msg="OSU-Micro-Benchmarks installed, yihaa!"
+fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..."
+$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb -r
+check_exit_code $? "${ok_msg}" "${fail_msg}"
+
 ### add packages here
 
 echo ">> Creating/updating Lmod cache..."
diff --git a/eessi-2021.12.yml b/eessi-2021.12.yml
index 58101777ca..04a9854b9c 100644
--- a/eessi-2021.12.yml
+++ b/eessi-2021.12.yml
@@ -27,6 +27,8 @@ software:
      toolchains:
        gompi-2020a:
          versions: ['5.6.3']
+       gompi-2021a:
+         versions: ['5.7.1']
   QuantumESPRESSO:
      toolchains:
        foss-2020a:

From 85e2ed64554bb027d6201c5a782eb00fe36c9804 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 12 Nov 2022 17:29:44 +0100
Subject: [PATCH 12/31] enhance EasyBuild hooks to fix installation of MetaBAT
 in EESSI

---
 eb_hooks.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/eb_hooks.py b/eb_hooks.py
index 653094266d..850b3da3b4 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -1,6 +1,7 @@
 # Hooks to customize how EasyBuild installs software in EESSI
 # see https://docs.easybuild.io/en/latest/Hooks.html
 import os
+import re
 
 from easybuild.tools.build_log import EasyBuildError, print_msg
 from easybuild.tools.config import build_option, update_build_option
@@ -130,6 +131,23 @@ def pre_configure_hook(self, *args, **kwargs):
         PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs)
 
 
+def metabat_preconfigure(self, *args, **kwargs):
+    """
+    Pre-configure hook for MetaBAT:
+    - take into account that zlib is a filtered dependency,
+      and that there's no libz.a in the EESSI compat layer
+    """
+    if self.name == 'MetaBAT':
+        print("from metadata_preconfigure hook: %s" % self.cfg['configopts'])
+        configopts = self.cfg['configopts']
+        regex = re.compile(r"\$EBROOTZLIB/lib/libz.a")
+        print(regex.search(configopts))
+        self.cfg['configopts'] = regex.sub('$EPREFIX/usr/lib64/libz.so', configopts)
+        print(self.cfg['configopts'])
+    else:
+        raise EasyBuildError("MetaBAT-specific hook triggered for non-MetaBAT easyconfig?!")
+
+
 def wrf_preconfigure(self, *args, **kwargs):
     """
     Pre-configure hook for WRF:
@@ -144,6 +162,7 @@ def wrf_preconfigure(self, *args, **kwargs):
     else:
         raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!")
 
+
 PARSE_HOOKS = {
     'CGAL': cgal_toolchainopts_precise,
     'fontconfig': fontconfig_add_fonts,
@@ -151,5 +170,6 @@ def wrf_preconfigure(self, *args, **kwargs):
 }
 
 PRE_CONFIGURE_HOOKS = {
+    'MetaBAT': metabat_preconfigure,
     'WRF': wrf_preconfigure,
 }

From 0f83623924bc03a427d5c22fbd1f2ec6e0267d4a Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Fri, 2 Dec 2022 11:05:16 +0100
Subject: [PATCH 13/31] Update actions to avoid deprecation errors

---
 .github/workflows/tests.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 34bc06979e..a4dbd3cd4c 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -3,16 +3,16 @@ name: Tests
 on: [push, pull_request]
 jobs:
   build:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         python: [3.6, 3.7, 3.8, 3.9, '3.10']
       fail-fast: false
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v4
       with:
         python-version: ${{matrix.python}}
         architecture: x64

From a7c8f161f8fbe1db2718a7a54b5be46bfd719ca8 Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Fri, 2 Dec 2022 11:07:39 +0100
Subject: [PATCH 14/31] Update tests.yml

---
 .github/workflows/tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index a4dbd3cd4c..ae04e115c7 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -3,7 +3,7 @@ name: Tests
 on: [push, pull_request]
 jobs:
   build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         python: [3.6, 3.7, 3.8, 3.9, '3.10']

From 9b553faae00510c7ada8de12bb76cc36c9ef9741 Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Fri, 2 Dec 2022 12:06:36 +0100
Subject: [PATCH 15/31] Update tests.yml

---
 .github/workflows/tests.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index ae04e115c7..c2aaeafbe2 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -1,6 +1,8 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
 name: Tests
 on: [push, pull_request]
+permissions:
+  contents: read # to fetch code (actions/checkout)
 jobs:
   build:
     runs-on: ubuntu-20.04

From d6eeda6cc5568b38306b0ef68b3de84fe10ab4fb Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Fri, 2 Dec 2022 12:08:31 +0100
Subject: [PATCH 16/31] Update tests_archdetect.yml

---
 .github/workflows/tests_archdetect.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml
index 4fffce0dde..5732574ea2 100644
--- a/.github/workflows/tests_archdetect.yml
+++ b/.github/workflows/tests_archdetect.yml
@@ -1,6 +1,8 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
 name: Tests for eessi_archdetect.sh
 on: [push, pull_request]
+permissions:
+  contents: read # to fetch code (actions/checkout)
 jobs:
   build:
     runs-on: ubuntu-20.04

From a6072f82917ccd65c9621c5e5e2d541d0fe7ce42 Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Fri, 2 Dec 2022 12:08:47 +0100
Subject: [PATCH 17/31] Update tests_init.yml

---
 .github/workflows/tests_init.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/tests_init.yml b/.github/workflows/tests_init.yml
index 027cc1a540..6df8b0a039 100644
--- a/.github/workflows/tests_init.yml
+++ b/.github/workflows/tests_init.yml
@@ -1,6 +1,8 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
 name: Tests for init scripts
 on: [push, pull_request]
+permissions:
+  contents: read # to fetch code (actions/checkout)
 jobs:
   build:
     runs-on: ubuntu-18.04

From d10b2bfdf468834b6ffb3af6098f98b9f1a98468 Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Fri, 2 Dec 2022 12:09:17 +0100
Subject: [PATCH 18/31] Update tests_scripts.yml

---
 .github/workflows/tests_scripts.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml
index ca8c2b930a..18fcd7b255 100644
--- a/.github/workflows/tests_scripts.yml
+++ b/.github/workflows/tests_scripts.yml
@@ -18,6 +18,8 @@ on:
       - run_in_compat_layer_env.sh
       - utils.sh
       - update_lmod_cache.sh
+permissions:
+  contents: read # to fetch code (actions/checkout)
 jobs:
   build:
     runs-on: ubuntu-20.04

From 94147e35703648f82da5ed08a9d7540b504e021e Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Thu, 8 Dec 2022 16:22:33 +0100
Subject: [PATCH 19/31] Add CI security action

---
 .github/workflows/scorecards.yml       | 75 ++++++++++++++++++++++++++
 .github/workflows/tests.yml            |  5 +-
 .github/workflows/tests_archdetect.yml |  3 +-
 .github/workflows/tests_init.yml       |  7 +--
 .github/workflows/tests_scripts.yml    |  3 +-
 5 files changed, 86 insertions(+), 7 deletions(-)
 create mode 100644 .github/workflows/scorecards.yml

diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
new file mode 100644
index 0000000000..ec018bd049
--- /dev/null
+++ b/.github/workflows/scorecards.yml
@@ -0,0 +1,75 @@
+# This workflow uses actions that are not certified by GitHub. They are provided
+# by a third-party and are governed by separate terms of service, privacy
+# policy, and support documentation.
+
+name: Scorecards supply-chain security
+on:
+  # For Branch-Protection check. Only the default branch is supported. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
+  branch_protection_rule:
+  # To guarantee Maintained check is occasionally updated. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
+  schedule:
+    - cron: '25 15 * * 3'
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches:    
+      - main
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecards analysis
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+      # Uncomment the permissions below if installing in a private repository.
+      # contents: read
+      # actions: read
+
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # v2.0.6
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          # (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
+          # - you want to enable the Branch-Protection check on a *public* repository, or
+          # - you are installing Scorecards on a *private* repository
+          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
+          # repo_token: ${{ secrets.SCORECARD_TOKEN }}
+
+          # Public repositories:
+          #   - Publish results to OpenSSF REST API for easy access by consumers
+          #   - Allows the repository to include the Scorecard badge.
+          #   - See https://github.com/ossf/scorecard-action#publishing-results.
+          # For private repositories:
+          #   - `publish_results` will always be set to `false`, regardless
+          #     of the value entered here.
+          publish_results: true
+
+      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
+      # format to the repository Actions tab.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard.
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@807578363a7869ca324a79039e6db9c843e0e100 # v2.1.27
+        with:
+          sarif_file: results.sarif
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index c2aaeafbe2..cc00685a40 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -11,10 +11,11 @@ jobs:
         python: [3.6, 3.7, 3.8, 3.9, '3.10']
       fail-fast: false
     steps:
-    - uses: actions/checkout@v3
+    - name: checkout
+      uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
     - name: set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
       with:
         python-version: ${{matrix.python}}
         architecture: x64
diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml
index 5732574ea2..618f6eb142 100644
--- a/.github/workflows/tests_archdetect.yml
+++ b/.github/workflows/tests_archdetect.yml
@@ -19,7 +19,8 @@ jobs:
           - aarch64/arm/neoverse-v1/AWS-awslinux-graviton3
       fail-fast: false
     steps:
-    - uses: actions/checkout@v2
+    - name: checkout
+      uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
     - name: test eessi_archdetect.sh
       run: |
diff --git a/.github/workflows/tests_init.yml b/.github/workflows/tests_init.yml
index 6df8b0a039..417b7851f1 100644
--- a/.github/workflows/tests_init.yml
+++ b/.github/workflows/tests_init.yml
@@ -5,16 +5,17 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   build:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         python: [3.6, 3.7, 3.8, 3.9, '3.10']
       fail-fast: false
     steps:
-    - uses: actions/checkout@v2
+    - name: checkout
+      uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
     - name: set up Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
       with:
         python-version: ${{matrix.python}}
         architecture: x64
diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml
index 18fcd7b255..1a2b793750 100644
--- a/.github/workflows/tests_scripts.yml
+++ b/.github/workflows/tests_scripts.yml
@@ -24,7 +24,8 @@ jobs:
   build:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v2
+    - name: checkout
+      uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
     # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181
     - name: install Singularity

From d48fbdf78ed0a60da70660f23a5a8643b3ea7bfe Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 14 Dec 2022 11:04:03 +0100
Subject: [PATCH 20/31] remove debug print statements in eb_hooks.py

---
 eb_hooks.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 850b3da3b4..1877e27b44 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -138,12 +138,9 @@ def metabat_preconfigure(self, *args, **kwargs):
       and that there's no libz.a in the EESSI compat layer
     """
     if self.name == 'MetaBAT':
-        print("from metadata_preconfigure hook: %s" % self.cfg['configopts'])
         configopts = self.cfg['configopts']
         regex = re.compile(r"\$EBROOTZLIB/lib/libz.a")
-        print(regex.search(configopts))
         self.cfg['configopts'] = regex.sub('$EPREFIX/usr/lib64/libz.so', configopts)
-        print(self.cfg['configopts'])
     else:
         raise EasyBuildError("MetaBAT-specific hook triggered for non-MetaBAT easyconfig?!")
 

From edaae9db896a57d66babb2e4a7c516b6b1db0578 Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Wed, 14 Dec 2022 15:44:51 +0100
Subject: [PATCH 21/31] Whoops

---
 eb_hooks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 391cf8813b..df7742f999 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -163,7 +163,7 @@ def metabat_preconfigure(self, *args, **kwargs):
         regex = re.compile(r"\$EBROOTZLIB/lib/libz.a")
         self.cfg['configopts'] = regex.sub('$EPREFIX/usr/lib64/libz.so', configopts)
     else:
-        raise EasyBuildError("MetaBAT-specific hook triggered for non-MetaBAT easyconfig?!"
+        raise EasyBuildError("MetaBAT-specific hook triggered for non-MetaBAT easyconfig?!")
 
 
 def wrf_preconfigure(self, *args, **kwargs):

From cec7938957a6ae7184368031684dfdce5eae9d41 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 15 Dec 2022 15:11:48 +0100
Subject: [PATCH 22/31] use correct OpenBLAS easyblock to install
 OpenBLAS/0.3.15-GCC-10.3.0 for */generic + ignore failing FlexiBLAS tests for
 ppc64le/generic

---
 EESSI-pilot-install-software.sh | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index 2bc6876965..10311b7a15 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -224,10 +224,11 @@ ok_msg="Done with OpenBLAS!"
 fail_msg="Installation of OpenBLAS failed!"
 if [[ $GENERIC -eq 1 ]]; then
     echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS."
-    $EB --include-easyblocks-from-pr 1946 OpenBLAS-0.3.9-GCC-9.3.0.eb --robot
+    openblas_include_easyblocks_from_pr="--include-easyblocks-from-pr 1946"
 else
-    $EB OpenBLAS-0.3.9-GCC-9.3.0.eb --robot
+    openblas_include_easyblocks_from_pr=''
 fi
+$EB $openblas_include_easyblocks_from_pr OpenBLAS-0.3.9-GCC-9.3.0.eb --robot
 check_exit_code $? "${ok_msg}" "${fail_msg}"
 
 echo ">> Installing OpenMPI..."
@@ -389,9 +390,19 @@ $EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248
 $EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot
 # use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885
 # which includes a patch to fix installation on POWER
-$EB --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot
+$EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot
+# ignore failing FlexiBLAS tests when building on POWER;
+# some tests are failing due to a segmentation fault due to "invalid memory reference",
+# see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476;
+# using -fstack-protector-strong -fstack-clash-protection should fix that,
+# but it doesn't for some reason when building for ppc64le/generic...
+if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then
+    $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --ignore-test-failure
+else
+    $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb
+fi
 
-$EB SciPy-bundle-2021.05-foss-2021a.eb -r --buildpath /dev/shm/$USER/easybuild_build
+$EB SciPy-bundle-2021.05-foss-2021a.eb --robot
 check_exit_code $? "${ok_msg}" "${fail_msg}"
 
 ### add packages here

From d3ceed500c86852200e475ed74f2d95a3fddefcd Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 13 Jan 2023 21:45:01 +0100
Subject: [PATCH 23/31] flesh out script for checking on missing installations
 + test it in CI

---
 .github/workflows/test_eessi.yml | 41 ++++++++++++++++++++++++++++++++
 EESSI-pilot-install-software.sh  | 10 ++------
 check_missing_installations.sh   | 21 ++++++++++++++++
 3 files changed, 64 insertions(+), 8 deletions(-)
 create mode 100644 .github/workflows/test_eessi.yml
 create mode 100755 check_missing_installations.sh

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
new file mode 100644
index 0000000000..2b57eccc90
--- /dev/null
+++ b/.github/workflows/test_eessi.yml
@@ -0,0 +1,41 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Tests relying on having EESSI pilot repo mounted
+on: [push, pull_request, workflow_dispatch]
+jobs:
+  eessi_pilot_repo:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        EESSI_VERSION:
+        - 2021.12
+        EESSI_SOFTWARE_SUBDIR:
+        - aarch64/generic
+        - aarch64/graviton2
+        - aarch64/graviton3
+        - x86_64/amd/zen2
+        - x86_64/amd/zen3
+        - x86_64/intel/haswell
+        - x86_64/intel/skylake_avx512
+        - x86_64/generic
+    steps:
+        - name: Check out software-layer repository
+          uses: actions/checkout@v2
+
+        - name: Mount EESSI CernVM-FS pilot repository
+          uses: cvmfs-contrib/github-action-cvmfs@main
+          with:
+              cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
+              cvmfs_http_proxy: DIRECT
+              cvmfs_repositories: pilot.eessi-hpc.org
+
+        - name: Test check_missing_installations.sh script
+          run: |
+              source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
+              module load EasyBuild
+              eb --version
+              export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
+              export EESSI_OS_TYPE=linux
+              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+              env | grep ^EESSI | sort
+              ./check_missing_installations.sh
diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index d03c2bdce7..57c078f511 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -118,7 +118,7 @@ else
 fi
 
 echo ">> Configuring EasyBuild..."
-source configure_easybuild
+source $TOPDIR/configure_easybuild
 
 echo ">> Setting up \$MODULEPATH..."
 # make sure no modules are loaded
@@ -421,13 +421,7 @@ fi
 
 $TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH}
 
-echo ">> Checking for missing installations..."
-ok_msg="No missing installations, party time!"
-fail_msg="On no, some installations are still missing, how did that happen?!"
-eb_missing_out=$TMPDIR/eb_missing.out
-$EB --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing --robot $EASYBUILD_PREFIX/ebfiles_repo | tee ${eb_missing_out}
-grep "No missing modules" ${eb_missing_out} > /dev/null
-check_exit_code $? "${ok_msg}" "${fail_msg}"
+$TOPDIR/check_missing_installations.sh
 
 echo ">> Cleaning up ${TMPDIR}..."
 rm -r ${TMPDIR}
diff --git a/check_missing_installations.sh b/check_missing_installations.sh
new file mode 100755
index 0000000000..9675397164
--- /dev/null
+++ b/check_missing_installations.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Script to check for missing installations in EESSI pilot software stack (version 2021.12)
+
+TOPDIR=$(dirname $(realpath $0))
+
+TMPDIR=$(mktemp -d)
+
+source $TOPDIR/utils.sh
+
+source $TOPDIR/configure_easybuild
+
+echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..."
+ok_msg="No missing installations, party time!"
+fail_msg="On no, some installations are still missing, how did that happen?!"
+eb_missing_out=$TMPDIR/eb_missing.out
+# we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used
+# PR #16531: Nextflow-22.10.1.eb
+${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out}
+grep "No missing modules" ${eb_missing_out} > /dev/null
+check_exit_code $? "${ok_msg}" "${fail_msg}"

From cc15fd577a811529941ef80ae89797c685d95686 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 13 Jan 2023 22:17:53 +0100
Subject: [PATCH 24/31] fix issues in CI workflow raised by OSSF scorecard

---
 .github/workflows/test_eessi.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index 2b57eccc90..92b1f71cad 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -1,6 +1,8 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
 name: Tests relying on having EESSI pilot repo mounted
 on: [push, pull_request, workflow_dispatch]
+permissions:
+  contents: read # to fetch code (actions/checkout)
 jobs:
   eessi_pilot_repo:
     runs-on: ubuntu-20.04
@@ -20,10 +22,10 @@ jobs:
         - x86_64/generic
     steps:
         - name: Check out software-layer repository
-          uses: actions/checkout@v2
+          uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
         - name: Mount EESSI CernVM-FS pilot repository
-          uses: cvmfs-contrib/github-action-cvmfs@main
+          uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1
           with:
               cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
               cvmfs_http_proxy: DIRECT

From a7909eee1a768c4f7a3723386eb561a71716f35d Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 14 Jan 2023 08:20:50 +0100
Subject: [PATCH 25/31] add license header + reuse $TMPDIR if set + make sure
 that $EESSI_PILOT_VERSION is set in check_missing_installations.sh

---
 check_missing_installations.sh | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index 9675397164..cfddb0ee14 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -1,10 +1,21 @@
 #!/bin/bash
 #
 # Script to check for missing installations in EESSI pilot software stack (version 2021.12)
+#
+# author: Kenneth Hoste (@boegel)
+#
+# license: GPLv2
+#
 
 TOPDIR=$(dirname $(realpath $0))
 
-TMPDIR=$(mktemp -d)
+if [ -z ${EESSI_PILOT_VERSION} ]; then
+    echo "ERROR: \${EESSI_PILOT_VERSION} must be set to run $0!" >&2
+    exit 1
+fi
+
+# reuse existing $TMPDIR if set, create a new tmpdir if $TMPDIR is unset
+TMPDIR=${TMPDIR:-$(mktemp -d)}
 
 source $TOPDIR/utils.sh
 

From 5c940744415fa9688bbd4a9e64c2ae146469f348 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 14 Jan 2023 14:40:43 +0100
Subject: [PATCH 26/31] rename $TMPDIR to $LOCAL_TMPDIR in
 check_missing_installations.sh, since $TMPDIR is a common environment
 variable (picked up by mktemp, etc.)

---
 check_missing_installations.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index cfddb0ee14..30f9cc6ff7 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -14,8 +14,7 @@ if [ -z ${EESSI_PILOT_VERSION} ]; then
     exit 1
 fi
 
-# reuse existing $TMPDIR if set, create a new tmpdir if $TMPDIR is unset
-TMPDIR=${TMPDIR:-$(mktemp -d)}
+LOCAL_TMPDIR=$(mktemp -d)
 
 source $TOPDIR/utils.sh
 
@@ -24,7 +23,7 @@ source $TOPDIR/configure_easybuild
 echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..."
 ok_msg="No missing installations, party time!"
 fail_msg="On no, some installations are still missing, how did that happen?!"
-eb_missing_out=$TMPDIR/eb_missing.out
+eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
 # we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used
 # PR #16531: Nextflow-22.10.1.eb
 ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out}

From 61d8e8fd8b61ee3e14cdb268ee0956f0d16a1ce1 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 14 Jan 2023 21:01:03 +0100
Subject: [PATCH 27/31] use newer build container (Debian 11 w/ fuse-overlayfs
 1.9)

---
 build_container.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_container.sh b/build_container.sh
index d7e7256b4e..f8e9c6bf59 100755
--- a/build_container.sh
+++ b/build_container.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian10"
+BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian11"
 
 if [ $# -lt 2 ]; then
     echo "Usage: $0 <shell|run> <path for temporary directories>" >&2

From f66397ef6aa7ca60ea20ce4b73aed8ba29dd2898 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 14 Jan 2023 21:16:43 +0100
Subject: [PATCH 28/31] install Apptainer in CI workflow used to test
 build_container.sh script

---
 .github/workflows/tests_scripts.yml | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml
index 1a2b793750..9c4975c381 100644
--- a/.github/workflows/tests_scripts.yml
+++ b/.github/workflows/tests_scripts.yml
@@ -28,14 +28,16 @@ jobs:
       uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
     # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181
-    - name: install Singularity
+    - name: install Apptainer
       run: |
         sudo apt-get install alien
         alien --version
-        singularity_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/ | grep singularity | sed 's/.*\(singularity[0-9._a-z-]*.rpm\).*/\1/g')
-        curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/$singularity_rpm
-        sudo alien -d $singularity_rpm
-        sudo apt install ./singularity*.deb
+        apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g')
+        curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/$apptainer_rpm
+        sudo alien -d $apptainer_rpm
+        sudo apt install ./apptainer*.deb
+        apptainer --version
+        # also check whether 'singularity' command is still provided by Apptainer installation
         singularity --version
 
     - name: test install_software_layer.sh script

From 8fbae4a6acc5c4c97286ab885a73f8156a675abf Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 14 Jan 2023 22:08:38 +0100
Subject: [PATCH 29/31] don't blindly overwrite $EESSI_SOFTWARE_SUBDIR_OVERRIDE
 in EESSI-pilot-install-software.sh

---
 EESSI-pilot-install-software.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index 57c078f511..2c7622b45a 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -92,7 +92,12 @@ if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then
 fi
 
 echo ">> Determining software subdirectory to use for current build host..."
-export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
+if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
+  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
+  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
+else
+  echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
+fi
 
 # Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE)
 # $EESSI_SILENT - don't print any messages

From 9778c10f3f392f3a8ac9625ff1eeb9327aa2f3e3 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 14 Jan 2023 22:38:50 +0100
Subject: [PATCH 30/31] pass down $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build
 container and Prefix environment

---
 build_container.sh         |  7 +++++++
 run_in_compat_layer_env.sh | 10 ++++++++--
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/build_container.sh b/build_container.sh
index f8e9c6bf59..bddd3dfffc 100755
--- a/build_container.sh
+++ b/build_container.sh
@@ -43,6 +43,13 @@ fi
 export EESSI_PILOT_READONLY="container:cvmfs2 pilot.eessi-hpc.org /cvmfs_ro/pilot.eessi-hpc.org"
 export EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs -o lowerdir=/cvmfs_ro/pilot.eessi-hpc.org -o upperdir=$EESSI_TMPDIR/overlay-upper -o workdir=$EESSI_TMPDIR/overlay-work /cvmfs/pilot.eessi-hpc.org"
 
+# pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set)
+if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
+    export SINGULARITYENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+    # also specify via $APPTAINERENV_* (future proof, cfr. https://apptainer.org/docs/user/latest/singularity_compatibility.html#singularity-environment-variable-compatibility)
+    export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+fi
+
 if [ "$SHELL_OR_RUN" == "shell" ]; then
     # start shell in Singularity container, with EESSI repository mounted with writable overlay
     echo "Starting Singularity build container..."
diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh
index 118aac65d5..561a311588 100755
--- a/run_in_compat_layer_env.sh
+++ b/run_in_compat_layer_env.sh
@@ -8,5 +8,11 @@ if [ ! -d ${EESSI_COMPAT_LAYER_DIR} ]; then
     echo "ERROR: ${EESSI_COMPAT_LAYER_DIR} does not exist!" >&2
     exit 1
 fi
-echo "Running '$@' in EESSI ${EESSI_PILOT_VERSION} compatibility layer environment..."
-${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "$@"
+
+INPUT=$(echo "$@")
+if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
+    INPUT="export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}; ${INPUT}"
+fi
+
+echo "Running '${INPUT}' in EESSI ${EESSI_PILOT_VERSION} compatibility layer environment..."
+${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "${INPUT}"

From 7eba3bf6c742ef7972bd271875f8bb4bd5973593 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sun, 15 Jan 2023 07:52:33 +0100
Subject: [PATCH 31/31] always exclude '.wh.*' files when creating tarball in
 overlay

---
 create_tarball.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/create_tarball.sh b/create_tarball.sh
index 619d6ea2e1..f2643ccfc5 100755
--- a/create_tarball.sh
+++ b/create_tarball.sh
@@ -43,13 +43,13 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
 fi
 if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then
     # module files
-    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f >> ${files_list}
+    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list}
     # module symlinks
-    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l >> ${files_list}
+    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list}
 fi
 if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then
     # installation directories
-    ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* >> ${files_list}
+    ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* | grep -v '/\.wh\.' >> ${files_list}
 fi
 
 topdir=${cvmfs_repo}/versions/