diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml new file mode 100644 index 0000000000..ec018bd049 --- /dev/null +++ b/.github/workflows/scorecards.yml @@ -0,0 +1,75 @@ +# This workflow uses actions that are not certified by GitHub. They are provided +# by a third-party and are governed by separate terms of service, privacy +# policy, and support documentation. + +name: Scorecards supply-chain security +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + schedule: + - cron: '25 15 * * 3' + push: + branches: [ "main" ] + pull_request: + branches: + - main + +# Declare default permissions as read only. +permissions: read-all + +jobs: + analysis: + name: Scorecards analysis + runs-on: ubuntu-latest + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + # Needed to publish results and get a badge (see publish_results below). + id-token: write + # Uncomment the permissions below if installing in a private repository. + # contents: read + # actions: read + + steps: + - name: "Checkout code" + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # v2.0.6 + with: + results_file: results.sarif + results_format: sarif + # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: + # - you want to enable the Branch-Protection check on a *public* repository, or + # - you are installing Scorecards on a *private* repository + # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. + # repo_token: ${{ secrets.SCORECARD_TOKEN }} + + # Public repositories: + # - Publish results to OpenSSF REST API for easy access by consumers + # - Allows the repository to include the Scorecard badge. + # - See https://github.com/ossf/scorecard-action#publishing-results. + # For private repositories: + # - `publish_results` will always be set to `false`, regardless + # of the value entered here. + publish_results: true + + # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF + # format to the repository Actions tab. + - name: "Upload artifact" + uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + # Upload the results to GitHub's code scanning dashboard. + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@807578363a7869ca324a79039e6db9c843e0e100 # v2.1.27 + with: + sarif_file: results.sarif diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml new file mode 100644 index 0000000000..92b1f71cad --- /dev/null +++ b/.github/workflows/test_eessi.yml @@ -0,0 +1,43 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Tests relying on having EESSI pilot repo mounted +on: [push, pull_request, workflow_dispatch] +permissions: + contents: read # to fetch code (actions/checkout) +jobs: + eessi_pilot_repo: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + EESSI_VERSION: + - 2021.12 + EESSI_SOFTWARE_SUBDIR: + - aarch64/generic + - aarch64/graviton2 + - aarch64/graviton3 + - x86_64/amd/zen2 + - x86_64/amd/zen3 + - x86_64/intel/haswell + - x86_64/intel/skylake_avx512 + - x86_64/generic + steps: + - name: Check out software-layer repository + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: Mount EESSI CernVM-FS pilot repository + uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1 + with: + cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb + cvmfs_http_proxy: DIRECT + cvmfs_repositories: pilot.eessi-hpc.org + + - name: Test check_missing_installations.sh script + run: | + source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash + module load EasyBuild + eb --version + export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} + export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} + env | grep ^EESSI | sort + ./check_missing_installations.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 34bc06979e..cc00685a40 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,18 +1,21 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions name: Tests on: [push, pull_request] +permissions: + contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 strategy: matrix: python: [3.6, 3.7, 3.8, 3.9, '3.10'] fail-fast: false steps: - - uses: actions/checkout@v2 + - name: checkout + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - name: set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0 with: python-version: ${{matrix.python}} architecture: x64 diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 227071fcd3..45c835a165 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -1,6 +1,8 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions name: Tests for eessi_archdetect.sh on: [push, pull_request] +permissions: + contents: read # to fetch code (actions/checkout) jobs: build: runs-on: ubuntu-20.04 @@ -18,7 +20,8 @@ jobs: - aarch64/arm/cortex-a72/debian-rpi4 fail-fast: false steps: - - uses: actions/checkout@v2 + - name: checkout + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - name: test eessi_archdetect.sh run: | diff --git a/.github/workflows/tests_init.yml b/.github/workflows/tests_init.yml index 027cc1a540..417b7851f1 100644 --- a/.github/workflows/tests_init.yml +++ b/.github/workflows/tests_init.yml @@ -1,18 +1,21 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions name: Tests for init scripts on: [push, pull_request] +permissions: + contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 strategy: matrix: python: [3.6, 3.7, 3.8, 3.9, '3.10'] fail-fast: false steps: - - uses: actions/checkout@v2 + - name: checkout + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - name: set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0 with: python-version: ${{matrix.python}} architecture: x64 diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index ca8c2b930a..9c4975c381 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -18,21 +18,26 @@ on: - run_in_compat_layer_env.sh - utils.sh - update_lmod_cache.sh +permissions: + contents: read # to fetch code (actions/checkout) jobs: build: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - name: checkout + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181 - - name: install Singularity + - name: install Apptainer run: | sudo apt-get install alien alien --version - singularity_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/ | grep singularity | sed 's/.*\(singularity[0-9._a-z-]*.rpm\).*/\1/g') - curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/$singularity_rpm - sudo alien -d $singularity_rpm - sudo apt install ./singularity*.deb + apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g') + curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/$apptainer_rpm + sudo alien -d $apptainer_rpm + sudo apt install ./apptainer*.deb + apptainer --version + # also check whether 'singularity' command is still provided by Apptainer installation singularity --version - name: test install_software_layer.sh script diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index dc50f7fae2..2c7622b45a 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -92,7 +92,12 @@ if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then fi echo ">> Determining software subdirectory to use for current build host..." -export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) +if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) + echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" +else + echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" +fi # Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) # $EESSI_SILENT - don't print any messages @@ -118,7 +123,7 @@ else fi echo ">> Configuring EasyBuild..." -source configure_easybuild +source $TOPDIR/configure_easybuild echo ">> Setting up \$MODULEPATH..." # make sure no modules are loaded @@ -224,10 +229,11 @@ ok_msg="Done with OpenBLAS!" fail_msg="Installation of OpenBLAS failed!" if [[ $GENERIC -eq 1 ]]; then echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." - $EB --include-easyblocks-from-pr 1946 OpenBLAS-0.3.9-GCC-9.3.0.eb --robot + openblas_include_easyblocks_from_pr="--include-easyblocks-from-pr 1946" else - $EB OpenBLAS-0.3.9-GCC-9.3.0.eb --robot + openblas_include_easyblocks_from_pr='' fi +$EB $openblas_include_easyblocks_from_pr OpenBLAS-0.3.9-GCC-9.3.0.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" echo ">> Installing OpenMPI..." @@ -362,6 +368,53 @@ fail_msg="Installation of Nextflow failed, that's unexpected..." $EB -r --from-pr 16531 Nextflow-22.10.1.eb check_exit_code $? "${ok_msg}" "${fail_msg}" +echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..." +ok_msg="OSU-Micro-Benchmarks installed, yihaa!" +fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." +$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb -r +check_exit_code $? "${ok_msg}" "${fail_msg}" + +echo ">> Installing EasyBuild 4.5.1..." +ok_msg="EasyBuild v4.5.1 installed" +fail_msg="EasyBuild v4.5.1 failed to install" +$EB --from-pr 14545 --include-easyblocks-from-pr 2805 +check_exit_code $? "${ok_msg}" "${fail_msg}" + +LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1 +check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5.1" + +echo ">> Installing SciPy-bundle with foss/2021a..." +ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age" +fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..." +# use GCCcore easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14454 +# which includes patch to fix installation with recent Linux kernel headers +$EB --from-pr 14454 GCCcore-10.3.0.eb --robot +# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 +# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) +$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640 +# use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set +# from https://github.com/easybuilders/easybuild-easyblocks/pull/2248 +$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 +# use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584 +# that includes patch to fix bootstrap problem when using alternate sysroot +$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot +# use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885 +# which includes a patch to fix installation on POWER +$EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot +# ignore failing FlexiBLAS tests when building on POWER; +# some tests are failing due to a segmentation fault due to "invalid memory reference", +# see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476; +# using -fstack-protector-strong -fstack-clash-protection should fix that, +# but it doesn't for some reason when building for ppc64le/generic... +if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then + $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --ignore-test-failure +else + $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb +fi + +$EB SciPy-bundle-2021.05-foss-2021a.eb --robot +check_exit_code $? "${ok_msg}" "${fail_msg}" + ### add packages here echo ">> Creating/updating Lmod cache..." @@ -373,13 +426,7 @@ fi $TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH} -echo ">> Checking for missing installations..." -ok_msg="No missing installations, party time!" -fail_msg="On no, some installations are still missing, how did that happen?!" -eb_missing_out=$TMPDIR/eb_missing.out -$EB --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing --robot $EASYBUILD_PREFIX/ebfiles_repo | tee ${eb_missing_out} -grep "No missing modules" ${eb_missing_out} > /dev/null -check_exit_code $? "${ok_msg}" "${fail_msg}" +$TOPDIR/check_missing_installations.sh echo ">> Cleaning up ${TMPDIR}..." rm -r ${TMPDIR} diff --git a/build_container.sh b/build_container.sh index d7e7256b4e..bddd3dfffc 100755 --- a/build_container.sh +++ b/build_container.sh @@ -1,6 +1,6 @@ #!/bin/bash -BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian10" +BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian11" if [ $# -lt 2 ]; then echo "Usage: $0 " >&2 @@ -43,6 +43,13 @@ fi export EESSI_PILOT_READONLY="container:cvmfs2 pilot.eessi-hpc.org /cvmfs_ro/pilot.eessi-hpc.org" export EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs -o lowerdir=/cvmfs_ro/pilot.eessi-hpc.org -o upperdir=$EESSI_TMPDIR/overlay-upper -o workdir=$EESSI_TMPDIR/overlay-work /cvmfs/pilot.eessi-hpc.org" +# pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set) +if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then + export SINGULARITYENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + # also specify via $APPTAINERENV_* (future proof, cfr. https://apptainer.org/docs/user/latest/singularity_compatibility.html#singularity-environment-variable-compatibility) + export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} +fi + if [ "$SHELL_OR_RUN" == "shell" ]; then # start shell in Singularity container, with EESSI repository mounted with writable overlay echo "Starting Singularity build container..." diff --git a/check_missing_installations.sh b/check_missing_installations.sh new file mode 100755 index 0000000000..30f9cc6ff7 --- /dev/null +++ b/check_missing_installations.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Script to check for missing installations in EESSI pilot software stack (version 2021.12) +# +# author: Kenneth Hoste (@boegel) +# +# license: GPLv2 +# + +TOPDIR=$(dirname $(realpath $0)) + +if [ -z ${EESSI_PILOT_VERSION} ]; then + echo "ERROR: \${EESSI_PILOT_VERSION} must be set to run $0!" >&2 + exit 1 +fi + +LOCAL_TMPDIR=$(mktemp -d) + +source $TOPDIR/utils.sh + +source $TOPDIR/configure_easybuild + +echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." +ok_msg="No missing installations, party time!" +fail_msg="On no, some installations are still missing, how did that happen?!" +eb_missing_out=$LOCAL_TMPDIR/eb_missing.out +# we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used +# PR #16531: Nextflow-22.10.1.eb +${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out} +grep "No missing modules" ${eb_missing_out} > /dev/null +check_exit_code $? "${ok_msg}" "${fail_msg}" diff --git a/create_tarball.sh b/create_tarball.sh index 619d6ea2e1..f2643ccfc5 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -43,13 +43,13 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then fi if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then # module files - find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f >> ${files_list} + find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} # module symlinks - find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l >> ${files_list} + find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} fi if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then # installation directories - ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* >> ${files_list} + ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* | grep -v '/\.wh\.' >> ${files_list} fi topdir=${cvmfs_repo}/versions/ diff --git a/eb_hooks.py b/eb_hooks.py index 653094266d..df7742f999 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -1,10 +1,12 @@ # Hooks to customize how EasyBuild installs software in EESSI # see https://docs.easybuild.io/en/latest/Hooks.html import os +import re from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option, update_build_option -from easybuild.tools.systemtools import AARCH64, POWER, get_cpu_architecture +from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features +from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs' @@ -52,6 +54,13 @@ def parse_hook(ec, *args, **kwargs): PARSE_HOOKS[ec.name](ec, eprefix) +def pre_configure_hook(self, *args, **kwargs): + """Main pre-configure hook: trigger custom functions based on software name.""" + + if self.name in PRE_CONFIGURE_HOOKS: + PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs) + + def pre_prepare_hook(self, *args, **kwargs): """Main pre-prepare hook: trigger custom functions.""" @@ -119,7 +128,7 @@ def ucx_eprefix(ec, eprefix): if ec.name == 'UCX': ec.update('configopts', '--with-sysroot=%s' % eprefix) ec.update('configopts', '--with-rdmacm=%s' % os.path.join(eprefix, 'usr')) - print_msg("Using custom configure option for %s: %s", ec.name, ec['configopts']) + print_msg("Using custom configure options for %s: %s", ec.name, ec['configopts']) else: raise EasyBuildError("UCX-specific hook triggered for non-UCX easyconfig?!") @@ -130,6 +139,33 @@ def pre_configure_hook(self, *args, **kwargs): PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs) +def libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs): + """Add --disable-psm3 to libfabric configure options when building with --optarch=GENERIC on x86_64.""" + if self.name == 'libfabric': + if get_cpu_architecture() == X86_64: + generic = build_option('optarch') == OPTARCH_GENERIC + no_avx = 'avx' not in get_cpu_features() + if generic or no_avx: + self.cfg.update('configopts', '--disable-psm3') + print_msg("Using custom configure options for %s: %s", self.name, self.cfg['configopts']) + else: + raise EasyBuildError("libfabric-specific hook triggered for non-libfabric easyconfig?!") + + +def metabat_preconfigure(self, *args, **kwargs): + """ + Pre-configure hook for MetaBAT: + - take into account that zlib is a filtered dependency, + and that there's no libz.a in the EESSI compat layer + """ + if self.name == 'MetaBAT': + configopts = self.cfg['configopts'] + regex = re.compile(r"\$EBROOTZLIB/lib/libz.a") + self.cfg['configopts'] = regex.sub('$EPREFIX/usr/lib64/libz.so', configopts) + else: + raise EasyBuildError("MetaBAT-specific hook triggered for non-MetaBAT easyconfig?!") + + def wrf_preconfigure(self, *args, **kwargs): """ Pre-configure hook for WRF: @@ -144,6 +180,7 @@ def wrf_preconfigure(self, *args, **kwargs): else: raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") + PARSE_HOOKS = { 'CGAL': cgal_toolchainopts_precise, 'fontconfig': fontconfig_add_fonts, @@ -151,5 +188,7 @@ def wrf_preconfigure(self, *args, **kwargs): } PRE_CONFIGURE_HOOKS = { + 'libfabric': libfabric_disable_psm3_x86_64_generic, + 'MetaBAT': metabat_preconfigure, 'WRF': wrf_preconfigure, } diff --git a/eessi-2021.12.yml b/eessi-2021.12.yml index 58101777ca..8674c5979d 100644 --- a/eessi-2021.12.yml +++ b/eessi-2021.12.yml @@ -1,10 +1,8 @@ software: - R-bundle-Bioconductor: + code-server: toolchains: - foss-2020a: - versions: - '3.11': - versionsuffix: -R-4.0.0 + SYSTEM: + versions: '3.7.3' GROMACS: toolchains: foss-2020a: @@ -19,6 +17,10 @@ software: versions: '0.21.3': versionsuffix: -TensorFlow-2.3.1-Python-3.8.2 + Nextflow: + toolchains: + SYSTEM: + versions: '22.10.1' OpenFOAM: toolchains: foss-2020a: @@ -27,33 +29,37 @@ software: toolchains: gompi-2020a: versions: ['5.6.3'] + gompi-2021a: + versions: ['5.7.1'] QuantumESPRESSO: toolchains: foss-2020a: versions: ['6.6'] - TensorFlow: + R-bundle-Bioconductor: toolchains: foss-2020a: versions: - '2.3.1': - versionsuffix: -Python-3.8.2 + '3.11': + versionsuffix: -R-4.0.0 RStudio-Server: toolchains: foss-2020a: versions: '1.3.1093': versionsuffix: -Java-11-R-4.0.0 - code-server: + SciPy-bundle: toolchains: - SYSTEM: - versions: '3.7.3' + foss-2021a: + versions: ['2021.05'] + TensorFlow: + toolchains: + foss-2020a: + versions: + '2.3.1': + versionsuffix: -Python-3.8.2 WRF: toolchains: foss-2020a: versions: '3.9.1.1': versionsuffix: -dmpar - Nextflow: - toolchains: - SYSTEM: - versions: '22.10.1' diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index 68d3e600b4..5458028d15 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -27,17 +27,13 @@ log () { # Supported CPU specifications update_arch_specs(){ # Add contents of given spec file into an array - # 1: name of array with CPU arch specs - # 2: spec file with the additional specs + # 1: spec file with the additional specs - [ -z "$1" ] && echo "[ERROR] update_arch_specs: missing array in argument list" >&2 && exit 1 - local -n arch_specs=$1 - - [ ! -f "$2" ] && echo "[ERROR] update_arch_specs: spec file not found: $2" >&2 && exit 1 - local spec_file="$2" + [ ! -f "$1" ] && echo "[ERROR] update_arch_specs: spec file not found: $1" >&2 && exit 1 + local spec_file="$1" while read spec_line; do # format spec line as an array and append it to array with all CPU arch specs - arch_specs+=("(${spec_line})") + cpu_arch_spec+=("(${spec_line})") # remove comments from spec file done < <(sed -E 's/(^|[\s\t])#.*$//g;/^\s*$/d' "$spec_file") } @@ -70,6 +66,10 @@ check_allinfirst(){ } cpupath(){ + # If EESSI_SOFTWARE_SUBDIR_OVERRIDE is set, use it + log "DEBUG" "cpupath: Override variable set as '$EESI_SOFTWARE_SUBDIR_OVERRIDE' " + [ $EESI_SOFTWARE_SUBDIR_OVERRIDE ] && echo ${EESI_SOFTWARE_SUBDIR_OVERRIDE} && exit + # Identify the best matching CPU architecture from a list of supported specifications for the host CPU # Return the path to the installation files in EESSI of the best matching architecture local cpu_arch_spec=() @@ -101,7 +101,7 @@ cpupath(){ # spec files are located in a subfolder with this script local base_dir=$(dirname $(realpath $0)) - update_arch_specs cpu_arch_spec "$base_dir/arch_specs/${spec_file}" + update_arch_specs "$base_dir/arch_specs/${spec_file}" # Identify the host CPU flags or features local cpu_flag_tag='flags' diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index 118aac65d5..561a311588 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -8,5 +8,11 @@ if [ ! -d ${EESSI_COMPAT_LAYER_DIR} ]; then echo "ERROR: ${EESSI_COMPAT_LAYER_DIR} does not exist!" >&2 exit 1 fi -echo "Running '$@' in EESSI ${EESSI_PILOT_VERSION} compatibility layer environment..." -${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "$@" + +INPUT=$(echo "$@") +if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then + INPUT="export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}; ${INPUT}" +fi + +echo "Running '${INPUT}' in EESSI ${EESSI_PILOT_VERSION} compatibility layer environment..." +${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "${INPUT}"