syn local branch with target

.github/workflows/test_eessi.yml

@@ -4,14 +4,14 @@ on: [push, pull_request, workflow_dispatch]
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
-  eessi_pilot_repo:
+  pilot:
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
         EESSI_VERSION:
         - 2023.06
-        EESSI_SOFTWARE_SUBDIR_OVERRIDE:
+        EESSI_SOFTWARE_SUBDIR:
         - aarch64/generic
         - x86_64/amd/zen2
         - x86_64/intel/broadwell
@@ -25,6 +25,8 @@ jobs:
         - eessi-2023.06-eb-4.7.2-2022b.yml
         - eessi-2023.06-eb-4.7.2-system.yml
         - eessi-2023.06-eb-4.8.0-system.yml
+        - eessi-2023.06-eb-4.8.1-2022a.yml
+        - eessi-2023.06-eb-4.8.1-system.yml
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
@@ -36,38 +38,55 @@ jobs:
               cvmfs_http_proxy: DIRECT
               cvmfs_repositories: pilot.nessi.no
 
-        - name: Test check_missing_installations.sh script
+        - name: Test check_missing_installations.sh script with EESSI_SOFTWARE_SUBDIR_OVERRIDE
+          if: '!cancelled()'
+          run: |
+              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+              source /cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}}/init/bash
+              module load EasyBuild
+              eb --version
+              export EESSI_PREFIX=/cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}}
+              export EESSI_OS_TYPE=linux
+              env | grep ^EESSI | sort
+              echo "just run check_missing_installations.sh (should use ${{matrix.EASYSTACK_FILE}})"
+              ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}}
+
+        - name: Test check_missing_installations.sh script without EESSI_SOFTWARE_SUBDIR_OVERRIDE
+          if: '!cancelled()'
           run: |
-              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
               source /cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}}/init/bash
               module load EasyBuild
-              which eb
               eb --version
               export EESSI_PREFIX=/cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
+              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
-              echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
+              echo "just run check_missing_installations.sh (should use ${{matrix.EASYSTACK_FILE}})"
               ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}}
 
         - name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
+          if: '!cancelled()'
           run: |
-              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
               source /cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}}/init/bash
               module load EasyBuild
               which eb
               eb --version
               export EESSI_PREFIX=/cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
+              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
-              echo "modify easystack file by adding a missing package (GCC/8.3.0)"
-              echo "  - GCC-8.3.0:" >> ${{matrix.EASYSTACK_FILE}}
-              tail -n 5 ${{matrix.EASYSTACK_FILE}}
+              # create dummy easystack file with a single entry (something that is not installed in EESSI)
+              easystack_file="test.yml"
+              echo "easyconfigs:" > ${easystack_file}
+              echo "  - GCC-8.3.0:" >> ${easystack_file}
+              echo "created easystack file '${easystack_file}' with a missing installation (GCC/8.3.0):"
+              cat ${easystack_file}
               # note, check_missing_installations.sh exits 1 if a package was
               # missing, which is intepreted as false (exit code based, not
               # boolean logic), hence when the script exits 0 if no package was
               # missing it is interpreted as true, thus the test did not capture
               # the missing package
-              if ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}}; then
+              if ./check_missing_installations.sh ${easystack_file}; then
                 echo "did NOT capture missing package; test FAILED"
                 exit 1
               else

EESSI-pilot-install-software.sh

@@ -172,8 +172,6 @@ if [ ! -z "${shared_fs_path}" ]; then
     export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH}
 fi
 
-${EB} --show-config
-
 echo ">> Setting up \$MODULEPATH..."
 # make sure no modules are loaded
 module --force purge
@@ -200,6 +198,8 @@ for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[
     # load EasyBuild module (will be installed if it's not available yet)
     source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
 
+    ${EB} --show-config
+
     echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."
 
     if [ -f ${easystack_file} ]; then

check_missing_installations.sh

@@ -22,6 +22,9 @@ source $TOPDIR/scripts/utils.sh
 
 source $TOPDIR/configure_easybuild
 
+echo ">> Active EasyBuild configuration when checking for missing installations:"
+${EB:-eb} --show-config
+
 echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..."
 eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
 ${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out}

create_tarball.sh

@@ -52,7 +52,7 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then
     find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list}
     # module files and symlinks
     find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \
-        | grep -v '/\.wh\.' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \
+        | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \
         >> ${module_files_list}
 fi
 if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then

eb_hooks.py

@@ -18,6 +18,9 @@
     from distutils.version import LooseVersion
 
 
+CPU_TARGET_NEOVERSE_V1 = 'aarch64/neoverse_v1'
+
+
 EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs'
 
 
@@ -248,6 +251,23 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs):
     else:
         raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!")
 
+
+def pre_test_hook(self,*args, **kwargs):
+    """Main pre-test hook: trigger custom functions based on software name."""
+    if self.name in PRE_TEST_HOOKS:
+        PRE_TEST_HOOKS[self.name](self, *args, **kwargs)
+
+
+def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs):
+    """
+    Pre-test hook for SciPy-bundle: skip failing tests for SciPy-bundle 2021.10 (currently the only version that is failing).
+    In previous versions we were not as strict yet on the numpy/SciPy tests
+    """
+    cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
+    if self.name == 'SciPy-bundle' and self.version == '2021.10' and cpu_target == CPU_TARGET_NEOVERSE_V1:
+        self.cfg['testopts'] = "|| echo ignoring failing tests" 
+
+
 PARSE_HOOKS = {
     'CGAL': parse_hook_cgal_toolchainopts_precise,
     'fontconfig': parse_hook_fontconfig_add_fonts,
@@ -265,3 +285,7 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs):
     'OpenBLAS': pre_configure_hook_openblas_optarch_generic,
     'WRF': pre_configure_hook_wrf_aarch64,
 }
+
+PRE_TEST_HOOKS = {
+    'SciPy-bundle': pre_test_hook_ignore_failing_tests_SciPybundle,
+}

eessi-2023.06-eb-4.7.2-2021a.yml

@@ -21,3 +21,7 @@ easyconfigs:
         options:
           from-pr: 18446
   - WRF-4.3-foss-2021a-dmpar.eb
+  - Yambo-5.1.1-foss-2021a.eb:
+        options:
+          from-pr: 18905
+

eessi-2023.06-eb-4.8.1-2022a.yml

@@ -1,2 +1,35 @@
 easyconfigs:
+  - BAMM-2.5.0-foss-2022a.eb
+  - BamTools-2.5.2-GCC-11.3.0.eb
+  - BCFtools-1.15.1-GCC-11.3.0.eb
+  - Bowtie2-2.4.5-GCC-11.3.0.eb
+  - FastTree-2.1.11-GCCcore-11.3.0.eb
+  - SAMtools-1.16.1-GCC-11.3.0.eb
+  - gzip-1.12-GCCcore-11.3.0.eb
+  - lz4-1.9.3-GCCcore-11.3.0.eb
+  - zstd-1.5.2-GCCcore-11.3.0.eb
+  - Boost.Python-1.79.0-GCC-11.3.0.eb
+  - netCDF-4.9.0-gompi-2022a.eb:
+      # use updated CMakeMake easyblock to avoid that -DCMAKE_SKIP_RPATH=ON is used, which breaks the netCDF test step
+      # see https://github.com/easybuilders/easybuild-easyblocks/pull/3012
+      options:
+        include-easyblocks-from-pr: 3012
+  - NCO-5.1.0-foss-2022a.eb
+  - AdapterRemoval-2.3.3-GCC-11.3.0.eb
+  - BEDTools-2.30.0-GCC-11.3.0.eb
+  - Pysam-0.20.0-GCC-11.3.0.eb
+  - TransDecoder-5.5.0-GCC-11.3.0.eb
+  - VCFtools-0.1.16-GCC-11.3.0.eb
+  - XML-LibXML-2.0207-GCCcore-11.3.0.eb
+  - CMake-3.24.3-GCCcore-11.3.0.eb
+  - elfutils-0.187-GCCcore-11.3.0.eb
+  - Ninja-1.10.2-GCCcore-11.3.0.eb
+  - Z3-4.10.2-GCCcore-11.3.0.eb
+  - SciPy-bundle-2022.05-foss-2022a.eb
+  - Xvfb-21.1.3-GCCcore-11.3.0.eb:
+      # enable exec permissions for xvfb-run after copying;
+      # need to also enable user write permissions on xvfb-run to ensure that copying with preserved permissions works
+      options:
+        from-pr: 18834
+  - tbb-2021.5.0-GCCcore-11.3.0.eb
   - GROMACS-2023.1-foss-2022a.eb

eessi-2023.06-eb-4.8.1-system.yml

@@ -0,0 +1,3 @@
+easyconfigs:
+  # wraps around Java/11.0.20
+  - Java-11.eb