Merge pull request #795 from ocaisa/streamline_install_check_ci

Streamline the checks for missing installations

.github/workflows/scripts/only_latest_easystacks.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+#
+# This script figures out the latest version of EasyBuild being used for the installation of easystack
+# files.
+#
+# This file is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Alan O'Cais (CECAM)
+#
+# license: GPLv2
+#
+
+EESSI_VERSION=${EESSI_VERSION:-"2023.06"}
+
+directory="easystacks/software.eessi.io/${EESSI_VERSION}"
+# List of example filenames
+files=($(find "$directory" -name "*.yml" | grep -e '-eb-'))
+[ -n "$DEBUG" ] && echo "${files[@]}"
+
+versions=()
+# Loop over each filename
+for filename in "${files[@]}"; do
+    # Extract the semantic version using grep
+    version=$(echo "$filename" | grep -oP '(?<=eb-)\d+\.\d+\.\d+?(?=-)')
+
+    # Output the result
+    [ -n "$DEBUG" ] && echo "Filename: $filename"
+    [ -n "$DEBUG" ] && echo "Extracted version: $version"
+    [ -n "$DEBUG" ] && echo
+    versions+=("$version")
+done
+highest_version=$(printf "%s\n" "${versions[@]}" | sort -V | tail -n 1)
+
+[ -n "$DEBUG" ] && echo "Highest version: $highest_version"
+[ -n "$DEBUG" ] && echo
+[ -n "$DEBUG" ] && echo "Matching files:"
+all_latest_easystacks=($(find $directory -type f -name "*eb-$highest_version*.yml"))
+
+accel_latest_easystacks=()
+cpu_latest_easystacks=()
+
+# Loop through the array and split based on partial matching of string
+accel="/accel/"
+for item in "${all_latest_easystacks[@]}"; do
+  if [[ "$item" == *"$accel"* ]]; then
+    accel_latest_easystacks+=("$item")
+  else
+    cpu_latest_easystacks+=("$item")
+  fi
+done
+
+# Output the results
+if [ -n "$ACCEL_EASYSTACKS" ]; then
+    echo "${accel_latest_easystacks[@]}"
+else
+    echo "${cpu_latest_easystacks[@]}"
+fi

.github/workflows/test-software.eessi.io.yml

@@ -57,15 +57,20 @@ jobs:
               env | grep ^EESSI | sort
 
               # first check the CPU-only builds for this CPU target
-              echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)"
-              for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do
+              echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml with latest EasyBuild release)"
+              for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do
                   if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ]; then
                       if grep -q 2022b <<<"${easystack_file}"; then
-                          # skip the check of installed software on zen4 for foss/2022b builds
-                          continue
-                      elif grep -q CUDA <<<"${easystack_file}"; then
-                          # skip the check of install CUDA software in the CPU path for zen4
-                          continue
+                        # skip the check of installed software on zen4 for foss/2022b builds
+                        continue
+                      fi
+                      if [[ $easystack_file == *"rebuilds"* ]]; then
+                        # Also handle rebuilds, make a temporary EasyStack file where we clean out all 2022b stuff and use that
+                        new_easystack=$(mktemp pruned_easystackXXX --suffix=.yml)
+                        # first clean out the options then clean out the .eb name
+                        sed '/2022b\|12\.2\.0/,/\.eb/{/\.eb/!d}' "${easystack_file}" | sed '/2022b\|12\.2\.0/d' > $new_easystack
+                        diff --unified=0 "$easystack_file" "$new_easystack" || :
+                        easystack_file="$new_easystack"
                       fi
                   fi
                   echo "check missing installations for ${easystack_file}..."
@@ -82,7 +87,7 @@ jobs:
                   for accel in ${accelerators}; do
                       module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all
                       echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}"
-                      for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/accel/$(dirname ${accel})/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do
+                      for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do
                           echo "check missing installations for ${easystack_file}..."
                           ./check_missing_installations.sh ${easystack_file}
                           ec=$?

check_missing_installations.sh

@@ -25,17 +25,6 @@ easystack=$1
 
 LOCAL_TMPDIR=$(mktemp -d)
 
-# Clone the develop branch of EasyBuild and use that to search for easyconfigs
-
-if [[ -z ${EASYBUILD_ROBOT_PATHS} ]]; then
-    git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs
-    export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs
-fi
-
-# All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr
-tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack})
-grep -v from-pr ${easystack} > ${tmp_easystack}
-
 source $TOPDIR/scripts/utils.sh
 
 source $TOPDIR/configure_easybuild
@@ -45,34 +34,11 @@ ${EB:-eb} --show-config
 
 echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..."
 eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
-${EB:-eb} --easystack ${tmp_easystack} --missing 2>&1 | tee ${eb_missing_out}
+${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out}
 exit_code=${PIPESTATUS[0]}
 
 ok_msg="Command 'eb --missing ...' succeeded, analysing output..."
 fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'"
-if [ "$exit_code" -ne 0 ] && [ ! -z "$pr_exceptions" ]; then
-    # We might have failed due to unmerged PRs. Try to make exceptions for --from-pr added in this PR
-    # to software-layer, and see if then it passes. If so, we can report a more specific fail_msg
-    # Note that if no --from-pr's were used in this PR, $pr_exceptions will be empty and we might as
-    # well skip this check - unmerged PRs can not be the reason for the non-zero exit code in that scenario
-
-    # Let's use awk so we can allow for exceptions if we are given a PR diff file
-    awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack"
-    awk_command=${awk_command/\\/}  # Strip out the backslash we needed for !
-    eval ${awk_command/EXCEPTIONS/$pr_exceptions} > ${tmp_easystack}
-
-    msg=">> Checking for missing installations in ${EASYBUILD_INSTALLPATH},"
-    msg="${msg} allowing for --from-pr's that were added in this PR..."
-    echo ${msg}
-    eb_missing_out=$LOCAL_TMPDIR/eb_missing_with_from_pr.out
-    ${EB:-eb} --easystack ${tmp_easystack} --missing 2>&1 | tee ${eb_missing_out}
-    exit_code_with_from_pr=${PIPESTATUS[0]}
-
-    # If now we succeeded, the reason must be that we originally stripped the --from-pr's
-    if [ "$exit_code_with_from_pr" -eq 0 ]; then
-        fail_msg="$fail_msg (are you sure all PRs referenced have been merged in EasyBuild?)"
-    fi
-fi
 
 check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}"