diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 590aa4225b..33122e6ff4 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -45,7 +45,7 @@ jobs: elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then outfile=out_listrepos.txt ./eessi_container.sh --verbose --list-repos | tee ${outfile} - grep "EESSI-pilot" ${outfile} + grep "EESSI" ${outfile} # test use of --list-repos with custom repos.cfg elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then @@ -57,7 +57,7 @@ jobs: echo "[EESSI/20HT.TP]" >> cfg/repos.cfg echo "repo_version = 20HT.TP" >> cfg/repos.cfg ./eessi_container.sh --verbose --list-repos | tee ${outfile} - grep "EESSI-pilot" ${outfile} + grep "EESSI" ${outfile} export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh --verbose --list-repos | tee ${outfile2} diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 618f6eb142..1e8b830e14 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -13,24 +13,59 @@ jobs: - x86_64/intel/skylake_avx512/archspec-linux-6132 - x86_64/amd/zen2/Azure-CentOS7-7V12 - x86_64/amd/zen3/Azure-CentOS7-7V73X - - ppc64le/power9le/unknown-power9le - - aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra - - aarch64/arm/neoverse-n1/AWS-awslinux-graviton2 - - aarch64/arm/neoverse-v1/AWS-awslinux-graviton3 + - aarch64/neoverse_n1/Azure-Ubuntu20-Altra + - aarch64/neoverse_n1/AWS-awslinux-graviton2 + - aarch64/neoverse_v1/AWS-awslinux-graviton3 + # commented out since these targets are currently not supported in software.eessi.io repo + # (and some tests assume that the corresponding subdirectory in software layer is there) + # - ppc64le/power9le/unknown-power9le fail-fast: false steps: - name: checkout uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + - name: Mount EESSI CernVM-FS pilot repository + uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1 + with: + cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb + cvmfs_http_proxy: DIRECT + cvmfs_repositories: software.eessi.io + - name: test eessi_archdetect.sh run: | export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}} export EESSI_MACHINE_TYPE=${EESSI_MACHINE_TYPE%%/*} export EESSI_PROC_CPUINFO=./tests/archdetect/${{matrix.proc_cpuinfo}}.cpuinfo + # check that printing of best match works correctly CPU_ARCH=$(./init/eessi_archdetect.sh cpupath) if [[ $CPU_ARCH == "$( cat ./tests/archdetect/${{matrix.proc_cpuinfo}}.output )" ]]; then - echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCH" >&2 + echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCH" else echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCH" >&2 exit 1 fi + # check that $EESSI_SOFTWARE_SUBDIR_OVERRIDE is honored + export EESSI_SOFTWARE_SUBDIR_OVERRIDE='dummy/cpu' + CPU_ARCH=$(./init/eessi_archdetect.sh cpupath) + if [[ $CPU_ARCH == "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then + echo "Test for picking up on \$EESSI_SOFTWARE_SUBDIR_OVERRIDE PASSED" + else + echo "Test for picking up on \$EESSI_SOFTWARE_SUBDIR_OVERRIDE FAILED" >&2 + exit 1 + fi + unset EESSI_SOFTWARE_SUBDIR_OVERRIDE + # check that printing of all matches works correctly (-a option for cpupath action) + CPU_ARCHES=$(./init/eessi_archdetect.sh -a cpupath) + if [[ $CPU_ARCHES == "$( cat ./tests/archdetect/${{matrix.proc_cpuinfo}}.all.output )" ]]; then + echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCHES" + else + echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCHES" >&2 + exit 1 + fi + # Check all those architectures actually exist (if this EESSI version has been populated already) + if [ -d ${EESSI_PREFIX}/software/linux ]; then + for dir in $(echo "$CPU_ARCHES" | tr ':' '\n'); do + # Search all EESSI versions as we may drop support at some point + ls -d ${EESSI_PREFIX}/software/linux/${dir} + done + fi diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index a8bbf9d762..d67532ec27 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -5,7 +5,7 @@ on: paths: - build_container.sh - create_directory_tarballs.sh - - EESSI-pilot-install-software.sh + - EESSI-install-software.sh - install_software_layer.sh - load_easybuild_module.sh - run_in_compat_layer_env.sh @@ -16,7 +16,7 @@ on: paths: - build_container.sh - create_directory_tarballs.sh - - EESSI-pilot-install-software.sh + - EESSI-install-software.sh - install_software_layer.sh - load_easybuild_module.sh - run_in_compat_layer_env.sh @@ -40,7 +40,7 @@ jobs: # bind current directory into container as /software-layer export SINGULARITY_BIND="${PWD}:/software-layer" - # can't test with EasyBuild versions older than v4.5.2 when using EESSI pilot 2023.06, + # can't test with EasyBuild versions older than v4.5.2 when using EESSI 2023.06, # since Python in compat layer is Python 3.11.x; # testing with a single EasyBuild version takes a while in GitHub Actions, so stick to a single sensible version for EB_VERSION in '4.6.0'; do @@ -81,20 +81,18 @@ jobs: - name: test install_software_layer.sh script run: | - # scripts need to be copied to /tmp, - # since install_software_layer.sh must be accessible from within build container - cp -a * /tmp/ - cd /tmp + # bind current directory into container as /software-layer + export SINGULARITY_BIND="${PWD}:/software-layer" # force using x86_64/generic, to avoid triggering an installation from scratch - sed -i "s@./EESSI-pilot-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-pilot-install-software.sh\"@g" install_software_layer.sh - ./build_container.sh run /tmp/$USER/EESSI /tmp/install_software_layer.sh + sed -i "s@./EESSI-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-install-software.sh\"@g" install_software_layer.sh + ./eessi_container.sh --mode run --verbose /software-layer/install_software_layer.sh - name: test create_directory_tarballs.sh script run: | + # bind current directory into container as /software-layer + export SINGULARITY_BIND="${PWD}:/software-layer" # scripts need to be copied to /tmp, # since create_directory_tarballs.sh must be accessible from within build container - cp -a * /tmp/ - cd /tmp - ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2023.06 + ./eessi_container.sh --mode run --verbose /software-layer/create_directory_tarballs.sh 2023.06 # check if tarballs have been produced ls -l *.tar.gz diff --git a/eessi-2021.06.yml b/eessi-2021.06.yml deleted file mode 100644 index 3587827746..0000000000 --- a/eessi-2021.06.yml +++ /dev/null @@ -1,53 +0,0 @@ -software: - R-bundle-Bioconductor: - toolchains: - foss-2020a: - versions: - '3.11': - versionsuffix: -R-4.0.0 - GROMACS: - toolchains: - foss-2020a: - versions: - '2020.1': - versionsuffix: -Python-3.8.2 - '2020.4': - versionsuffix: -Python-3.8.2 - Horovod: - toolchains: - foss-2020a: - versions: - '0.21.3': - versionsuffix: -TensorFlow-2.3.1-Python-3.8.2 - OpenFOAM: - toolchains: - foss-2020a: - versions: ['8', 'v2006'] - OSU-Micro-Benchmarks: - toolchains: - gompi-2020a: - versions: ['5.6.3'] - QuantumESPRESSO: - toolchains: - foss-2020a: - versions: ['6.6'] - TensorFlow: - toolchains: - foss-2020a: - versions: - '2.3.1': - versionsuffix: -Python-3.8.2 - RStudio-Server: - toolchains: - foss-2020a: - versions: - '1.3.1093': - versionsuffix: -Java-11-R-4.0.0 - ReFrame: - toolchains: - SYSTEM: - versions: '3.6.2' - code-server: - toolchains: - SYSTEM: - versions: '3.7.3' diff --git a/eessi-2021.12.yml b/eessi-2021.12.yml deleted file mode 100644 index 210bbb2845..0000000000 --- a/eessi-2021.12.yml +++ /dev/null @@ -1,69 +0,0 @@ -software: - code-server: - toolchains: - SYSTEM: - versions: '3.7.3' - GROMACS: - toolchains: - foss-2020a: - versions: - '2020.1': - versionsuffix: -Python-3.8.2 - '2020.4': - versionsuffix: -Python-3.8.2 - Horovod: - toolchains: - foss-2020a: - versions: - '0.21.3': - versionsuffix: -TensorFlow-2.3.1-Python-3.8.2 - Nextflow: - toolchains: - SYSTEM: - versions: '22.10.1' - OpenFOAM: - toolchains: - foss-2020a: - versions: ['8', 'v2006'] - OSU-Micro-Benchmarks: - toolchains: - gompi-2020a: - versions: ['5.6.3'] - gompi-2021a: - versions: ['5.7.1'] - QuantumESPRESSO: - toolchains: - foss-2020a: - versions: ['6.6'] - R: - toolchains: - foss-2021a: - versions: '4.1.0' - R-bundle-Bioconductor: - toolchains: - foss-2020a: - versions: - '3.11': - versionsuffix: -R-4.0.0 - RStudio-Server: - toolchains: - foss-2020a: - versions: - '1.3.1093': - versionsuffix: -Java-11-R-4.0.0 - SciPy-bundle: - toolchains: - foss-2021a: - versions: ['2021.05'] - TensorFlow: - toolchains: - foss-2020a: - versions: - '2.3.1': - versionsuffix: -Python-3.8.2 - WRF: - toolchains: - foss-2020a: - versions: - '3.9.1.1': - versionsuffix: -dmpar diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml deleted file mode 100644 index 42a42f2236..0000000000 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ /dev/null @@ -1,26 +0,0 @@ -easyconfigs: - - GCC-10.3.0 - - CMake-3.20.1-GCCcore-10.3.0.eb: - options: - include-easyblocks-from-pr: 2248 - - foss-2021a - - libpng-1.6.37-GCCcore-10.3.0.eb: - - libjpeg-turbo-2.0.6-GCCcore-10.3.0.eb - - git-2.32.0-GCCcore-10.3.0-nodocs.eb - - QuantumESPRESSO-6.7-foss-2021a.eb - - GROMACS-2021.3-foss-2021a.eb - - Qt5-5.15.2-GCCcore-10.3.0.eb: - # add missing patch files for Qt5 5.15.2 to fix build problems with glibc 2.34, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18087 - options: - from-pr: 18087 - - RapidJSON-1.1.0-GCCcore-10.3.0.eb: - # strip out hardcoded -march=native, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18725 - options: - from-pr: 18725 - - Arrow-6.0.0-foss-2021a.eb: - # fix installation of pyarrow Python bindings - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18348 - options: - from-pr: 18348 diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml deleted file mode 100644 index 43a0fbd4bd..0000000000 --- a/eessi-2023.06-eb-4.7.2-2021b.yml +++ /dev/null @@ -1,17 +0,0 @@ -easyconfigs: - - GCC-11.2.0 - - CMake-3.21.1-GCCcore-11.2.0.eb: - options: - include-easyblocks-from-pr: 2248 - - CMake-3.22.1-GCCcore-11.2.0.eb: - options: - include-easyblocks-from-pr: 2248 - - zlib-1.2.11-GCCcore-11.2.0 - - OpenMPI-4.1.1-GCC-11.2.0 - - gompi-2021b - - FlexiBLAS-3.0.4-GCC-11.2.0.eb - - foss-2021b.eb - - git-2.33.1-GCCcore-11.2.0-nodocs.eb - - QuantumESPRESSO-6.8-foss-2021b.eb - - SciPy-bundle-2021.10-foss-2021b - - GROMACS-2021.5-foss-2021b.eb diff --git a/eessi-2023.06-eb-4.7.2-2022a.yml b/eessi-2023.06-eb-4.7.2-2022a.yml deleted file mode 100644 index d40ddff261..0000000000 --- a/eessi-2023.06-eb-4.7.2-2022a.yml +++ /dev/null @@ -1,3 +0,0 @@ -easyconfigs: - - GCC-11.3.0 - - OpenMPI-4.1.4-GCC-11.3.0.eb diff --git a/eessi-2023.06-eb-4.7.2-2022b.yml b/eessi-2023.06-eb-4.7.2-2022b.yml deleted file mode 100644 index b97be34dcf..0000000000 --- a/eessi-2023.06-eb-4.7.2-2022b.yml +++ /dev/null @@ -1,13 +0,0 @@ -easyconfigs: - - GCC-12.2.0 - - OpenMPI-4.1.4-GCC-12.2.0 - - CMake-3.24.3-GCCcore-12.2.0.eb: - options: - include-easyblocks-from-pr: 2248 - - foss-2022b - - netCDF-4.9.0-gompi-2022b.eb: - # use updated CMakeMake easyblock to avoid that -DCMAKE_SKIP_RPATH=ON is used, which breaks the netCDF test step - # see https://github.com/easybuilders/easybuild-easyblocks/pull/3012 - options: - include-easyblocks-from-pr: 3012 - - WRF-4.4.1-foss-2022b-dmpar.eb diff --git a/eessi-2023.06-eb-4.7.2-system.yml b/eessi-2023.06-eb-4.7.2-system.yml deleted file mode 100644 index 220fd82e87..0000000000 --- a/eessi-2023.06-eb-4.7.2-system.yml +++ /dev/null @@ -1,7 +0,0 @@ -easyconfigs: - - ReFrame-4.2.0.eb: - options: - from-pr: 18320 - - EasyBuild-4.8.0.eb: - options: - from-pr: 18282 diff --git a/eessi-2023.06-eb-4.8.0-2021a.yml b/eessi-2023.06-eb-4.8.0-2021a.yml deleted file mode 100644 index 1adf0572d3..0000000000 --- a/eessi-2023.06-eb-4.8.0-2021a.yml +++ /dev/null @@ -1,2 +0,0 @@ -easyconfigs: - - GDAL-3.3.0-foss-2021a.eb diff --git a/eessi-2023.06-eb-4.8.0-2021b.yml b/eessi-2023.06-eb-4.8.0-2021b.yml deleted file mode 100644 index a2562a4ed1..0000000000 --- a/eessi-2023.06-eb-4.8.0-2021b.yml +++ /dev/null @@ -1,12 +0,0 @@ -easyconfigs: - - MPFR-4.1.0-GCCcore-11.2.0.eb: - # use patch for MPFR 4.1.0 to fix failing tsprintf test with glibc >= 2.37, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18746 - options: - from-pr: 18746 - - gnuplot-5.4.2-GCCcore-11.2.0.eb: - # make sure that Lua dependency is correctly picked up, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19261 - options: - from-pr: 19261 - - OpenFOAM-v2112-foss-2021b.eb diff --git a/eessi-2023.06-eb-4.8.0-system.yml b/eessi-2023.06-eb-4.8.0-system.yml deleted file mode 100644 index 2928f110b0..0000000000 --- a/eessi-2023.06-eb-4.8.0-system.yml +++ /dev/null @@ -1,10 +0,0 @@ -easyconfigs: - - Java-11.eb: - # patch Java binaries/libraries when using alternate sysroot to ensure correct glibc & co are picked up - # see https://github.com/easybuilders/easybuild-easyblocks/pull/2557 - # + https://github.com/easybuilders/easybuild-easyblocks/pull/2995 - options: - include-easyblocks-from-pr: 2995 - - EasyBuild-4.8.1.eb: - options: - from-pr: 18761 diff --git a/eessi-2023.06-eb-4.8.1-2021a.yml b/eessi-2023.06-eb-4.8.1-2021a.yml deleted file mode 100644 index 999eb09925..0000000000 --- a/eessi-2023.06-eb-4.8.1-2021a.yml +++ /dev/null @@ -1,7 +0,0 @@ -easyconfigs: - - Xvfb-1.20.11-GCCcore-10.3.0.eb: - # fix permission issues for xvfb-run script, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18834 - options: - from-pr: 18834 - - R-4.1.0-foss-2021a.eb diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml deleted file mode 100644 index 6ee44926ba..0000000000 --- a/eessi-2023.06-eb-4.8.1-2021b.yml +++ /dev/null @@ -1,15 +0,0 @@ -easyconfigs: - - Xvfb-1.20.13-GCCcore-11.2.0.eb: - # enable exec permissions for xvfb-run after copying; - # need to also enable user write permissions on xvfb-run to ensure that copying with preserved permissions works - options: - from-pr: 18834 - - R-4.2.0-foss-2021b.eb - - PLUMED-2.7.3-foss-2021b.eb: - # the --enable-asmjit is not supported on Aarch64 - options: - from-pr: 19110 - - LAMMPS-23Jun2022-foss-2021b-kokkos.eb: - # TBB and ScaFaCos are optional dependencies when building on Intel arch - options: - from-pr: 19246 diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml deleted file mode 100644 index 8317a40b7a..0000000000 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ /dev/null @@ -1,12 +0,0 @@ -easyconfigs: - - OpenBLAS-0.3.20-GCC-11.3.0: - # use easyconfig that includes patch to fix detection of Neoverse V1 CPUs, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18870 - options: - from-pr: 18870 - - foss-2022a - - SciPy-bundle-2022.05-foss-2022a - - BAMM-2.5.0-foss-2022a.eb - - ESPResSo-4.2.1-foss-2022a: - options: - from-pr: 18963 diff --git a/eessi-2023.06-eb-4.8.1-2023a.yml b/eessi-2023.06-eb-4.8.1-2023a.yml deleted file mode 100644 index c766815934..0000000000 --- a/eessi-2023.06-eb-4.8.1-2023a.yml +++ /dev/null @@ -1,11 +0,0 @@ -easyconfigs: - - GCC-12.3.0 - - OpenBLAS-0.3.23-GCC-12.3.0: - # add patch to disable flaky DDRGES3 LAPACK test, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18887; - # also pulls in patch to avoid hang when running OpenBLAS test suite, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18790 - options: - from-pr: 18887 - - foss-2023a - - X11-20230603-GCCcore-12.3.0 diff --git a/eessi-2023.06-eb-4.8.1-system.yml b/eessi-2023.06-eb-4.8.1-system.yml deleted file mode 100644 index b0731d2534..0000000000 --- a/eessi-2023.06-eb-4.8.1-system.yml +++ /dev/null @@ -1,9 +0,0 @@ -easyconfigs: - # rebuilt with EasyBuild v4.8.1 (now wraps around Java/11.0.20) - - Java-11.eb - - ReFrame-4.3.3.eb: - options: - from-pr: 18851 - - EasyBuild-4.8.2.eb: - options: - from-pr: 19105 diff --git a/eessi-2023.06-eb-4.8.2-2021b.yml b/eessi-2023.06-eb-4.8.2-2021b.yml deleted file mode 100644 index 00c02adf91..0000000000 --- a/eessi-2023.06-eb-4.8.2-2021b.yml +++ /dev/null @@ -1,6 +0,0 @@ -easyconfigs: - - Pillow-8.3.2-GCCcore-11.2.0.eb: - # avoid that hardcoded paths like /usr/include are used in build commands - options: - from-pr: 19226 - - matplotlib-3.4.3-foss-2021b.eb diff --git a/eessi-2023.06-eb-4.8.2-2022a.yml b/eessi-2023.06-eb-4.8.2-2022a.yml deleted file mode 100644 index ccd3fcd65d..0000000000 --- a/eessi-2023.06-eb-4.8.2-2022a.yml +++ /dev/null @@ -1,21 +0,0 @@ -easyconfigs: - - casacore-3.5.0-foss-2022a: - options: - from-pr: 19119 - - AOFlagger-3.4.0-foss-2022a: - options: - from-pr: 19119 - - EveryBeam-0.5.2-foss-2022a: - options: - from-pr: 19119 - - DP3-6.0-foss-2022a: - options: - from-pr: 19119 - - WSClean-3.4-foss-2022a: - options: - from-pr: 19119 - - Pillow-9.1.1-GCCcore-11.3.0.eb: - # avoid that hardcoded paths like /usr/include are used in build commands - options: - from-pr: 19226 - - matplotlib-3.5.2-foss-2022a.eb diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml deleted file mode 100644 index ade1c069df..0000000000 --- a/eessi-2023.06-known-issues.yml +++ /dev/null @@ -1,30 +0,0 @@ -- x86_64/*: - - ESPResSo-4.2.1-foss-2022a: - - issue: https://github.com/EESSI/software-layer/issues/363 - - info: "Failing tests in ESPResSo v4.2.1 due to timeouts" -- aarch64/*: - - ESPResSo-4.2.1-foss-2022a: - - issue: https://github.com/EESSI/software-layer/issues/363 - - info: "Failing tests in ESPResSo v4.2.1 due to timeouts" -- aarch64/neoverse_v1: - - FFTW-3.3.10-GCC-11.3.0: - - issue: https://github.com/EESSI/software-layer/issues/325 - - info: "Flaky FFTW tests, random failures" - - FFTW-3.3.10-gompi-2021b: - - issue: https://github.com/EESSI/software-layer/issues/325 - - info: "Flaky FFTW tests, random failures" - - OpenBLAS-0.3.18-GCC-11.2.0: - - issue: https://github.com/EESSI/software-layer/issues/314 - - info: "Increased number of numerical errors in OpenBLAS test suite (238 vs max. 150 on x86_64/*)" - - OpenBLAS-0.3.20-GCC-11.3.0: - - issue: https://github.com/EESSI/software-layer/issues/314 - - info: "Increased number of numerical errors in OpenBLAS test suite (238 vs max. 150 on x86_64/*)" - - OpenBLAS-0.3.21-GCC-12.2.0: - - issue: https://github.com/EESSI/software-layer/issues/314 - - info: "Increased number of numerical errors in OpenBLAS test suite (344 vs max. 150 on x86_64/*)" - - SciPy-bundle-2021.05-foss-2021a: - - issue: https://github.com/EESSI/software-layer/issues/318 - - info: "2 failing tests (vs 30554 passed) in scipy test suite" - - SciPy-bundle-2021.10-foss-2021b: - - issue: https://github.com/EESSI/software-layer/issues/318 - - info: "20 failing tests (vs 14429 passed) in numpy test suite + 55 failing tests (vs 32438 passed) in scipy test suite" diff --git a/eessi_container.sh b/eessi_container.sh index 838e05923c..bf0294c7bf 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -518,9 +518,9 @@ declare -a EESSI_FUSE_MOUNTS=() EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch") if [[ "${ACCESS}" == "ro" ]]; then - export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}" + export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}" - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}") + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") export EESSI_FUSE_MOUNTS fi @@ -529,18 +529,18 @@ if [[ "${ACCESS}" == "rw" ]]; then mkdir -p ${EESSI_TMPDIR}/overlay-work # set environment variables for fuse mounts in Singularity container - export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" + export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}") + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") - EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work" - EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" - export EESSI_PILOT_WRITABLE_OVERLAY + EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" + EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" + EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper" + EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work" + EESSI_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" + export EESSI_WRITABLE_OVERLAY - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_WRITABLE_OVERLAY}") + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_WRITABLE_OVERLAY}") export EESSI_FUSE_MOUNTS fi diff --git a/init/arch_specs/eessi_arch_arm.spec b/init/arch_specs/eessi_arch_arm.spec index 92f32a76d8..b5c9275043 100755 --- a/init/arch_specs/eessi_arch_arm.spec +++ b/init/arch_specs/eessi_arch_arm.spec @@ -1,6 +1,6 @@ # ARM CPU architecture specifications # Software path in EESSI | Vendor ID | List of defining CPU features -"aarch64/arm/neoverse-n1" "ARM" "asimd" # Ampere Altra -"aarch64/arm/neoverse-n1" "" "asimd" # AWS Graviton2 -"aarch64/arm/neoverse-v1" "ARM" "asimd svei8mm" -"aarch64/arm/neoverse-v1" "" "asimd svei8mm" # AWS Graviton3 +"aarch64/neoverse_n1" "ARM" "asimd" # Ampere Altra +"aarch64/neoverse_n1" "" "asimd" # AWS Graviton2 +"aarch64/neoverse_v1" "ARM" "asimd svei8mm" +"aarch64/neoverse_v1" "" "asimd svei8mm" # AWS Graviton3 diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index d2b6dacf04..58b79b0f2a 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash -VERSION="1.0.0" +VERSION="1.1.0" -# Logging -LOG_LEVEL="INFO" +# default log level: only emit warnings or errors +LOG_LEVEL="WARN" +# Default result type is a best match +CPUPATH_RESULT="best" timestamp () { date "+%Y-%m-%d %H:%M:%S" @@ -67,8 +69,8 @@ check_allinfirst(){ cpupath(){ # If EESSI_SOFTWARE_SUBDIR_OVERRIDE is set, use it - log "DEBUG" "cpupath: Override variable set as '$EESI_SOFTWARE_SUBDIR_OVERRIDE' " - [ $EESI_SOFTWARE_SUBDIR_OVERRIDE ] && echo ${EESI_SOFTWARE_SUBDIR_OVERRIDE} && exit + log "DEBUG" "cpupath: Override variable set as '$EESSI_SOFTWARE_SUBDIR_OVERRIDE' " + [ $EESSI_SOFTWARE_SUBDIR_OVERRIDE ] && echo ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} && exit # Identify the best matching CPU architecture from a list of supported specifications for the host CPU # Return the path to the installation files in EESSI of the best matching architecture @@ -105,7 +107,8 @@ cpupath(){ log "DEBUG" "cpupath: CPU flags of host system: '$cpu_flags'" # Default to generic CPU - local best_arch_match="generic" + local best_arch_match="$machine_type/generic" + local all_arch_matches=$best_arch_match # Iterate over the supported CPU specifications to find the best match for host CPU # Order of the specifications matters, the last one to match will be selected @@ -114,22 +117,29 @@ cpupath(){ if [ "${cpu_vendor}x" == "${arch_spec[1]}x" ]; then # each flag in this CPU specification must be found in the list of flags of the host check_allinfirst "${cpu_flags[*]}" ${arch_spec[2]} && best_arch_match=${arch_spec[0]} && \ - log "DEBUG" "cpupath: host CPU best match updated to $best_arch_match" + all_arch_matches="$best_arch_match:$all_arch_matches" && \ + log "DEBUG" "cpupath: host CPU best match updated to $best_arch_match" fi done - log "INFO" "cpupath: best match for host CPU: $best_arch_match" - echo "$best_arch_match" + if [ "allx" == "${CPUPATH_RESULT}x" ]; then + log "INFO" "cpupath: all matches for host CPU: $all_arch_matches" + echo "$all_arch_matches" + else + log "INFO" "cpupath: best match for host CPU: $best_arch_match" + echo "$best_arch_match" + fi } # Parse command line arguments -USAGE="Usage: eessi_archdetect.sh [-h][-d] " +USAGE="Usage: eessi_archdetect.sh [-h][-d][-a] " -while getopts 'hdv' OPTION; do +while getopts 'hdva' OPTION; do case "$OPTION" in h) echo "$USAGE"; exit 0;; d) LOG_LEVEL="DEBUG";; v) echo "eessi_archdetect.sh v$VERSION"; exit 0;; + a) CPUPATH_RESULT="all";; ?) echo "$USAGE"; exit 1;; esac done @@ -139,5 +149,5 @@ ARGUMENT=${1:-none} case "$ARGUMENT" in "cpupath") cpupath; exit;; - *) echo "$USAGE"; log "ERROR" "Missing argument";; + *) echo "$USAGE"; log "ERROR" "Missing argument (possible actions: 'cpupath')";; esac diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.output b/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.output deleted file mode 100644 index b4dc5e9f1b..0000000000 --- a/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.output +++ /dev/null @@ -1 +0,0 @@ -aarch64/arm/neoverse-n1 diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.output b/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.output deleted file mode 100644 index b4dc5e9f1b..0000000000 --- a/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.output +++ /dev/null @@ -1 +0,0 @@ -aarch64/arm/neoverse-n1 diff --git a/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.output b/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.output deleted file mode 100644 index 20db96d01f..0000000000 --- a/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.output +++ /dev/null @@ -1 +0,0 @@ -aarch64/arm/neoverse-v1 diff --git a/tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.all.output b/tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.all.output new file mode 100644 index 0000000000..340aaa5d02 --- /dev/null +++ b/tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.all.output @@ -0,0 +1 @@ +aarch64/neoverse_n1:aarch64/generic diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.cpuinfo b/tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.cpuinfo similarity index 100% rename from tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.cpuinfo rename to tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.cpuinfo diff --git a/tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.output b/tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.output new file mode 100644 index 0000000000..a9bd49c75c --- /dev/null +++ b/tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.output @@ -0,0 +1 @@ +aarch64/neoverse_n1 diff --git a/tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.all.output b/tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.all.output new file mode 100644 index 0000000000..340aaa5d02 --- /dev/null +++ b/tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.all.output @@ -0,0 +1 @@ +aarch64/neoverse_n1:aarch64/generic diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.cpuinfo b/tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.cpuinfo similarity index 100% rename from tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.cpuinfo rename to tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.cpuinfo diff --git a/tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.output b/tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.output new file mode 100644 index 0000000000..a9bd49c75c --- /dev/null +++ b/tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.output @@ -0,0 +1 @@ +aarch64/neoverse_n1 diff --git a/tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.all.output b/tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.all.output new file mode 100644 index 0000000000..920d5f9996 --- /dev/null +++ b/tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.all.output @@ -0,0 +1 @@ +aarch64/neoverse_v1:aarch64/neoverse_n1:aarch64/generic diff --git a/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.cpuinfo b/tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.cpuinfo similarity index 100% rename from tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.cpuinfo rename to tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.cpuinfo diff --git a/tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.output b/tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.output new file mode 100644 index 0000000000..a8e072a9c6 --- /dev/null +++ b/tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.output @@ -0,0 +1 @@ +aarch64/neoverse_v1 diff --git a/tests/archdetect/ppc64le/power9le/unknown-power9le.all.output b/tests/archdetect/ppc64le/power9le/unknown-power9le.all.output new file mode 100644 index 0000000000..7ecf79d0a7 --- /dev/null +++ b/tests/archdetect/ppc64le/power9le/unknown-power9le.all.output @@ -0,0 +1 @@ +ppc64le/power9le:ppc64le/generic \ No newline at end of file diff --git a/tests/archdetect/x86_64/amd/zen2/Azure-CentOS7-7V12.all.output b/tests/archdetect/x86_64/amd/zen2/Azure-CentOS7-7V12.all.output new file mode 100644 index 0000000000..180de26f0e --- /dev/null +++ b/tests/archdetect/x86_64/amd/zen2/Azure-CentOS7-7V12.all.output @@ -0,0 +1 @@ +x86_64/amd/zen2:x86_64/generic \ No newline at end of file diff --git a/tests/archdetect/x86_64/amd/zen3/Azure-CentOS7-7V73X.all.output b/tests/archdetect/x86_64/amd/zen3/Azure-CentOS7-7V73X.all.output new file mode 100644 index 0000000000..798a0aa565 --- /dev/null +++ b/tests/archdetect/x86_64/amd/zen3/Azure-CentOS7-7V73X.all.output @@ -0,0 +1 @@ +x86_64/amd/zen3:x86_64/amd/zen2:x86_64/generic \ No newline at end of file diff --git a/tests/archdetect/x86_64/intel/haswell/archspec-linux-E5-2680-v3.all.output b/tests/archdetect/x86_64/intel/haswell/archspec-linux-E5-2680-v3.all.output new file mode 100644 index 0000000000..a047dd42cc --- /dev/null +++ b/tests/archdetect/x86_64/intel/haswell/archspec-linux-E5-2680-v3.all.output @@ -0,0 +1 @@ +x86_64/intel/haswell:x86_64/generic \ No newline at end of file diff --git a/tests/archdetect/x86_64/intel/skylake_avx512/archspec-linux-6132.all.output b/tests/archdetect/x86_64/intel/skylake_avx512/archspec-linux-6132.all.output new file mode 100644 index 0000000000..c9fa524ea6 --- /dev/null +++ b/tests/archdetect/x86_64/intel/skylake_avx512/archspec-linux-6132.all.output @@ -0,0 +1 @@ +x86_64/intel/skylake_avx512:x86_64/intel/haswell:x86_64/generic \ No newline at end of file