Merge branch 'feature-archdetect-rpi' of github.com:hmeiland/software…

…-layer into feature-archdetect-rpi
EESSI · Jan 18, 2023 · 24b84d4 · 24b84d4
2 parents ef1ae0b + b4fd3b0
commit 24b84d4
Show file tree

Hide file tree

Showing 14 changed files with 324 additions and 56 deletions.
diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
@@ -0,0 +1,75 @@
+# This workflow uses actions that are not certified by GitHub. They are provided
+# by a third-party and are governed by separate terms of service, privacy
+# policy, and support documentation.
+
+name: Scorecards supply-chain security
+on:
+  # For Branch-Protection check. Only the default branch is supported. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
+  branch_protection_rule:
+  # To guarantee Maintained check is occasionally updated. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
+  schedule:
+    - cron: '25 15 * * 3'
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches:    
+      - main
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecards analysis
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+      # Uncomment the permissions below if installing in a private repository.
+      # contents: read
+      # actions: read
+
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # v2.0.6
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          # (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
+          # - you want to enable the Branch-Protection check on a *public* repository, or
+          # - you are installing Scorecards on a *private* repository
+          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
+          # repo_token: ${{ secrets.SCORECARD_TOKEN }}
+
+          # Public repositories:
+          #   - Publish results to OpenSSF REST API for easy access by consumers
+          #   - Allows the repository to include the Scorecard badge.
+          #   - See https://github.com/ossf/scorecard-action#publishing-results.
+          # For private repositories:
+          #   - `publish_results` will always be set to `false`, regardless
+          #     of the value entered here.
+          publish_results: true
+
+      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
+      # format to the repository Actions tab.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard.
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@807578363a7869ca324a79039e6db9c843e0e100 # v2.1.27
+        with:
+          sarif_file: results.sarif
diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
@@ -0,0 +1,43 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Tests relying on having EESSI pilot repo mounted
+on: [push, pull_request, workflow_dispatch]
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  eessi_pilot_repo:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        EESSI_VERSION:
+        - 2021.12
+        EESSI_SOFTWARE_SUBDIR:
+        - aarch64/generic
+        - aarch64/graviton2
+        - aarch64/graviton3
+        - x86_64/amd/zen2
+        - x86_64/amd/zen3
+        - x86_64/intel/haswell
+        - x86_64/intel/skylake_avx512
+        - x86_64/generic
+    steps:
+        - name: Check out software-layer repository
+          uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+
+        - name: Mount EESSI CernVM-FS pilot repository
+          uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1
+          with:
+              cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
+              cvmfs_http_proxy: DIRECT
+              cvmfs_repositories: pilot.eessi-hpc.org
+
+        - name: Test check_missing_installations.sh script
+          run: |
+              source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
+              module load EasyBuild
+              eb --version
+              export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
+              export EESSI_OS_TYPE=linux
+              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+              env | grep ^EESSI | sort
+              ./check_missing_installations.sh
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -1,18 +1,21 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
 name: Tests
 on: [push, pull_request]
+permissions:
+  contents: read # to fetch code (actions/checkout)
 jobs:
   build:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         python: [3.6, 3.7, 3.8, 3.9, '3.10']
       fail-fast: false
     steps:
-    - uses: actions/checkout@v2
+    - name: checkout
+      uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
     - name: set up Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
       with:
         python-version: ${{matrix.python}}
         architecture: x64

diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml
@@ -1,6 +1,8 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
 name: Tests for eessi_archdetect.sh
 on: [push, pull_request]
+permissions:
+  contents: read # to fetch code (actions/checkout)
 jobs:
   build:
     runs-on: ubuntu-20.04
@@ -18,7 +20,8 @@ jobs:
           - aarch64/arm/cortex-a72/debian-rpi4
       fail-fast: false
     steps:
-    - uses: actions/checkout@v2
+    - name: checkout
+      uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
     - name: test eessi_archdetect.sh
       run: |

diff --git a/.github/workflows/tests_init.yml b/.github/workflows/tests_init.yml
@@ -1,18 +1,21 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
 name: Tests for init scripts
 on: [push, pull_request]
+permissions:
+  contents: read # to fetch code (actions/checkout)
 jobs:
   build:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         python: [3.6, 3.7, 3.8, 3.9, '3.10']
       fail-fast: false
     steps:
-    - uses: actions/checkout@v2
+    - name: checkout
+      uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
     - name: set up Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
       with:
         python-version: ${{matrix.python}}
         architecture: x64

diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml
@@ -18,21 +18,26 @@ on:
       - run_in_compat_layer_env.sh
       - utils.sh
       - update_lmod_cache.sh
+permissions:
+  contents: read # to fetch code (actions/checkout)
 jobs:
   build:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v2
+    - name: checkout
+      uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
     # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181
-    - name: install Singularity
+    - name: install Apptainer
       run: |
         sudo apt-get install alien
         alien --version
-        singularity_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/ | grep singularity | sed 's/.*\(singularity[0-9._a-z-]*.rpm\).*/\1/g')
-        curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/$singularity_rpm
-        sudo alien -d $singularity_rpm
-        sudo apt install ./singularity*.deb
+        apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g')
+        curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/$apptainer_rpm
+        sudo alien -d $apptainer_rpm
+        sudo apt install ./apptainer*.deb
+        apptainer --version
+        # also check whether 'singularity' command is still provided by Apptainer installation
         singularity --version
 
     - name: test install_software_layer.sh script

diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
@@ -92,7 +92,12 @@ if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then
 fi
 
 echo ">> Determining software subdirectory to use for current build host..."
-export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
+if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
+  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
+  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
+else
+  echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
+fi
 
 # Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE)
 # $EESSI_SILENT - don't print any messages
@@ -118,7 +123,7 @@ else
 fi
 
 echo ">> Configuring EasyBuild..."
-source configure_easybuild
+source $TOPDIR/configure_easybuild
 
 echo ">> Setting up \$MODULEPATH..."
 # make sure no modules are loaded
@@ -224,10 +229,11 @@ ok_msg="Done with OpenBLAS!"
 fail_msg="Installation of OpenBLAS failed!"
 if [[ $GENERIC -eq 1 ]]; then
     echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS."
-    $EB --include-easyblocks-from-pr 1946 OpenBLAS-0.3.9-GCC-9.3.0.eb --robot
+    openblas_include_easyblocks_from_pr="--include-easyblocks-from-pr 1946"
 else
-    $EB OpenBLAS-0.3.9-GCC-9.3.0.eb --robot
+    openblas_include_easyblocks_from_pr=''
 fi
+$EB $openblas_include_easyblocks_from_pr OpenBLAS-0.3.9-GCC-9.3.0.eb --robot
 check_exit_code $? "${ok_msg}" "${fail_msg}"
 
 echo ">> Installing OpenMPI..."
@@ -362,6 +368,53 @@ fail_msg="Installation of Nextflow failed, that's unexpected..."
 $EB -r --from-pr 16531 Nextflow-22.10.1.eb
 check_exit_code $? "${ok_msg}" "${fail_msg}"
 
+echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..."
+ok_msg="OSU-Micro-Benchmarks installed, yihaa!"
+fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..."
+$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb -r
+check_exit_code $? "${ok_msg}" "${fail_msg}"
+
+echo ">> Installing EasyBuild 4.5.1..."
+ok_msg="EasyBuild v4.5.1 installed"
+fail_msg="EasyBuild v4.5.1 failed to install"
+$EB --from-pr 14545 --include-easyblocks-from-pr 2805
+check_exit_code $? "${ok_msg}" "${fail_msg}"
+
+LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1
+check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5.1"
+
+echo ">> Installing SciPy-bundle with foss/2021a..."
+ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age"
+fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..."
+# use GCCcore easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14454
+# which includes patch to fix installation with recent Linux kernel headers
+$EB --from-pr 14454 GCCcore-10.3.0.eb --robot
+# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640
+# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...)
+$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640
+# use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set
+# from https://github.com/easybuilders/easybuild-easyblocks/pull/2248
+$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248
+# use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584
+# that includes patch to fix bootstrap problem when using alternate sysroot
+$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot
+# use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885
+# which includes a patch to fix installation on POWER
+$EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot
+# ignore failing FlexiBLAS tests when building on POWER;
+# some tests are failing due to a segmentation fault due to "invalid memory reference",
+# see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476;
+# using -fstack-protector-strong -fstack-clash-protection should fix that,
+# but it doesn't for some reason when building for ppc64le/generic...
+if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then
+    $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --ignore-test-failure
+else
+    $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb
+fi
+
+$EB SciPy-bundle-2021.05-foss-2021a.eb --robot
+check_exit_code $? "${ok_msg}" "${fail_msg}"
+
 ### add packages here
 
 echo ">> Creating/updating Lmod cache..."
@@ -373,13 +426,7 @@ fi
 
 $TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH}
 
-echo ">> Checking for missing installations..."
-ok_msg="No missing installations, party time!"
-fail_msg="On no, some installations are still missing, how did that happen?!"
-eb_missing_out=$TMPDIR/eb_missing.out
-$EB --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing --robot $EASYBUILD_PREFIX/ebfiles_repo | tee ${eb_missing_out}
-grep "No missing modules" ${eb_missing_out} > /dev/null
-check_exit_code $? "${ok_msg}" "${fail_msg}"
+$TOPDIR/check_missing_installations.sh
 
 echo ">> Cleaning up ${TMPDIR}..."
 rm -r ${TMPDIR}
diff --git a/build_container.sh b/build_container.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian10"
+BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian11"
 
 if [ $# -lt 2 ]; then
     echo "Usage: $0 <shell|run> <path for temporary directories>" >&2
@@ -43,6 +43,13 @@ fi
 export EESSI_PILOT_READONLY="container:cvmfs2 pilot.eessi-hpc.org /cvmfs_ro/pilot.eessi-hpc.org"
 export EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs -o lowerdir=/cvmfs_ro/pilot.eessi-hpc.org -o upperdir=$EESSI_TMPDIR/overlay-upper -o workdir=$EESSI_TMPDIR/overlay-work /cvmfs/pilot.eessi-hpc.org"
 
+# pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set)
+if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
+    export SINGULARITYENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+    # also specify via $APPTAINERENV_* (future proof, cfr. https://apptainer.org/docs/user/latest/singularity_compatibility.html#singularity-environment-variable-compatibility)
+    export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+fi
+
 if [ "$SHELL_OR_RUN" == "shell" ]; then
     # start shell in Singularity container, with EESSI repository mounted with writable overlay
     echo "Starting Singularity build container..."

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# Script to check for missing installations in EESSI pilot software stack (version 2021.12)
+#
+# author: Kenneth Hoste (@boegel)
+#
+# license: GPLv2
+#
+
+TOPDIR=$(dirname $(realpath $0))
+
+if [ -z ${EESSI_PILOT_VERSION} ]; then
+    echo "ERROR: \${EESSI_PILOT_VERSION} must be set to run $0!" >&2
+    exit 1
+fi
+
+LOCAL_TMPDIR=$(mktemp -d)
+
+source $TOPDIR/utils.sh
+
+source $TOPDIR/configure_easybuild
+
+echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..."
+ok_msg="No missing installations, party time!"
+fail_msg="On no, some installations are still missing, how did that happen?!"
+eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
+# we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used
+# PR #16531: Nextflow-22.10.1.eb
+${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out}
+grep "No missing modules" ${eb_missing_out} > /dev/null
+check_exit_code $? "${ok_msg}" "${fail_msg}"
diff --git a/create_tarball.sh b/create_tarball.sh
@@ -43,13 +43,13 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
 fi
 if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then
     # module files
-    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f >> ${files_list}
+    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list}
     # module symlinks
-    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l >> ${files_list}
+    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list}
 fi
 if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then
     # installation directories
-    ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* >> ${files_list}
+    ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* | grep -v '/\.wh\.' >> ${files_list}
 fi
 
 topdir=${cvmfs_repo}/versions/