Merge branch '2023.06' of github.com:EESSI/software-layer into eessi-…

…2023.06-WRF-dmpar/4.4.1-foss/2022b

.github/workflows/test_eessi.yml

@@ -20,16 +20,6 @@ jobs:
         - x86_64/intel/haswell
         - x86_64/intel/skylake_avx512
         - x86_64/generic
-        EASYSTACK_FILE:
-        - eessi-2023.06-eb-4.7.2-2021a.yml
-        - eessi-2023.06-eb-4.7.2-2021b.yml
-        - eessi-2023.06-eb-4.7.2-2022a.yml
-        - eessi-2023.06-eb-4.7.2-2022b.yml
-        - eessi-2023.06-eb-4.7.2-system.yml
-        - eessi-2023.06-eb-4.8.0-2021a.yml
-        - eessi-2023.06-eb-4.8.0-system.yml
-        - eessi-2023.06-eb-4.8.0-2021b.yml
-        - eessi-2023.06-eb-4.8.1-system.yml
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
@@ -51,7 +41,12 @@ jobs:
               export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
               echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
-              ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}}
+              for easystack_file in $(ls eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do
+                  echo "check missing installations for ${easystack_file}..."
+                  ./check_missing_installations.sh ${easystack_file}
+                  ec=$?
+                  if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
+              done
 
         - name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
           run: |
@@ -62,15 +57,18 @@ jobs:
               export EESSI_OS_TYPE=linux
               export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
-              echo "modify easystack file by adding a missing package (GCC/8.3.0)"
-              echo "  - GCC-8.3.0:" >> ${{matrix.EASYSTACK_FILE}}
-              tail -n 5 ${{matrix.EASYSTACK_FILE}}
+              # create dummy easystack file with a single entry (something that is not installed in EESSI)
+              easystack_file="test.yml"
+              echo "easyconfigs:" > ${easystack_file}
+              echo "  - GCC-8.3.0:" >> ${easystack_file}
+              echo "created easystack file '${easystack_file}' with a missing installation (GCC/8.3.0):"
+              cat ${easystack_file}
               # note, check_missing_installations.sh exits 1 if a package was
               # missing, which is intepreted as false (exit code based, not
               # boolean logic), hence when the script exits 0 if no package was
               # missing it is interpreted as true, thus the test did not capture
               # the missing package
-              if ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}}; then
+              if ./check_missing_installations.sh ${easystack_file}; then
                 echo "did NOT capture missing package; test FAILED"
                 exit 1
               else

EESSI-pilot-install-software.sh

@@ -8,10 +8,12 @@
 
 display_help() {
   echo "usage: $0 [OPTIONS]"
+  echo "  --build-logs-dir       -  location to copy EasyBuild logs to for failed builds"
   echo "  -g | --generic         -  instructs script to build for generic architecture target"
   echo "  -h | --help            -  display this usage information"
   echo "  -x | --http-proxy URL  -  provides URL for the environment variable http_proxy"
   echo "  -y | --https-proxy URL -  provides URL for the environment variable https_proxy"
+  echo "  --shared-fs-path       -  path to directory on shared filesystem that can be used"
 }
 
 function copy_build_log() {
@@ -39,7 +41,7 @@ function copy_build_log() {
         echo "- working directory: ${PWD}" >> ${build_log_path}
         echo "- Slurm job ID: ${SLURM_OUT}" >> ${build_log_path}
         echo "- EasyBuild version: ${eb_version}" >> ${build_log_path}
-        echo "- easystack file: ${es}" >> ${build_log_path}
+        echo "- easystack file: ${easystack_file}" >> ${build_log_path}
 
         echo "EasyBuild log file ${build_log} copied to ${build_log_path} (with context appended)"
     fi
@@ -70,6 +72,10 @@ while [[ $# -gt 0 ]]; do
       export build_logs_dir="${2}"
       shift 2
       ;;
+    --shared-fs-path)
+      export shared_fs_path="${2}"
+      shift 2
+      ;;
     -*|--*)
       echo "Error: Unknown option: $1" >&2
       exit 1
@@ -160,6 +166,14 @@ fi
 echo ">> Configuring EasyBuild..."
 source $TOPDIR/configure_easybuild
 
+if [ ! -z "${shared_fs_path}" ]; then
+    shared_eb_sourcepath=${shared_fs_path}/easybuild/sources
+    echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path"
+    export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH}
+fi
+
+${EB} --show-config
+
 echo ">> Setting up \$MODULEPATH..."
 # make sure no modules are loaded
 module --force purge
@@ -172,36 +186,42 @@ else
     echo_green ">> MODULEPATH set up: ${MODULEPATH}"
 fi
 
-for eb_version in '4.7.2' '4.8.0' '4.8.1'; do
+# assume there's only one diff file that corresponds to the PR patch file
+pr_diff=$(ls [0-9]*.diff | head -1)
+
+# use PR patch file to determine in which easystack files stuff was added
+for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^eessi.*yml$'); do
+
+    echo -e "Processing easystack file ${easystack_file}...\n\n"
+
+    # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
+    eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
 
     # load EasyBuild module (will be installed if it's not available yet)
     source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
 
     echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."
 
-    for es in $(ls eessi-${EESSI_PILOT_VERSION}-eb-${eb_version}-*.yml); do
-
-        if [ -f ${es} ]; then
-            echo_green "Feeding easystack file ${es} to EasyBuild..."
-
-            ${EB} --easystack ${TOPDIR}/${es} --robot
-            ec=$?
+    if [ -f ${easystack_file} ]; then
+        echo_green "Feeding easystack file ${easystack_file} to EasyBuild..."
 
-            # copy EasyBuild log file if EasyBuild exited with an error
-            if [ ${ec} -ne 0 ]; then
-                eb_last_log=$(unset EB_VERBOSE; eb --last-log)
-                # copy to current working directory
-                cp -a ${eb_last_log} .
-                echo "Last EasyBuild log file copied from ${eb_last_log} to ${PWD}"
-                # copy to build logs dir (with context added)
-                copy_build_log "${eb_last_log}" "${build_logs_dir}"
-            fi
+        ${EB} --easystack ${TOPDIR}/${easystack_file} --robot
+        ec=$?
 
-            $TOPDIR/check_missing_installations.sh ${TOPDIR}/${es}
-        else
-            fatal_error "Easystack file ${es} not found!"
+        # copy EasyBuild log file if EasyBuild exited with an error
+        if [ ${ec} -ne 0 ]; then
+            eb_last_log=$(unset EB_VERBOSE; eb --last-log)
+            # copy to current working directory
+            cp -a ${eb_last_log} .
+            echo "Last EasyBuild log file copied from ${eb_last_log} to ${PWD}"
+            # copy to build logs dir (with context added)
+            copy_build_log "${eb_last_log}" "${build_logs_dir}"
         fi
-    done
+
+        $TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file}
+    else
+        fatal_error "Easystack file ${easystack_file} not found!"
+    fi
 
 done