From 94fa4c4bd89fcb0d168dd79c9a369ba1175bd9e5 Mon Sep 17 00:00:00 2001 From: casparvl Date: Tue, 18 Jul 2023 21:24:36 +0000 Subject: [PATCH 01/41] Added foss-2022a --- eessi-2023.06-eb-4.7.2-2022a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2022a.yml b/eessi-2023.06-eb-4.7.2-2022a.yml index b8f0cc17bc..0721a93257 100644 --- a/eessi-2023.06-eb-4.7.2-2022a.yml +++ b/eessi-2023.06-eb-4.7.2-2022a.yml @@ -1,2 +1,3 @@ easyconfigs: - GCC-11.3.0 + - foss-2022a From cea6c3b63714e34edca7a39e6fd7ab45ad11e617 Mon Sep 17 00:00:00 2001 From: casparvl Date: Tue, 18 Jul 2023 23:37:33 +0000 Subject: [PATCH 02/41] Added Cmake explicitely, since we need to pass an easyblocks-from-pr --- eessi-2023.06-eb-4.7.2-2022a.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-2022a.yml b/eessi-2023.06-eb-4.7.2-2022a.yml index 6c6e0de3f1..f4a3da0143 100644 --- a/eessi-2023.06-eb-4.7.2-2022a.yml +++ b/eessi-2023.06-eb-4.7.2-2022a.yml @@ -1,4 +1,10 @@ easyconfigs: - GCC-11.3.0 - OpenMPI-4.1.4-GCC-11.3.0.eb - - foss-2022a \ No newline at end of file + - CMake-3.23.1-GCCcore-11.3.0.eb: + options: + include-easyblocks-from-pr: 2248 + - CMake-3.24.3-GCCcore-11.3.0.eb: + options: + include-easyblocks-from-pr: 2248 + - foss-2022a From 978da670c998499a4378850beb67463474c1904c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 5 Sep 2023 18:16:10 +0200 Subject: [PATCH 03/41] As agreed, ignore test failures for FFTW.MPI test suite --- eb_hooks.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 488e53d929..9560d84eff 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -274,6 +274,14 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): self.cfg['testopts'] = "|| echo ignoring failing tests" +def pre_test_hook_ignore_failing_tests_FFTWMPI(self, *args, **kwargs): + """ + Pre-test hook for FFTW.MPI: skip failing tests for FFTWMPI 3.3.10 + """ + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if self.name == 'FFTW.MPI' and self.version == '3.3.10' and cpu_target == 'aarch64/neoverse_v1': + self.cfg['testopts'] = "|| echo ignoring failing tests" + PARSE_HOOKS = { 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, @@ -295,4 +303,5 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): PRE_TEST_HOOKS = { 'SciPy-bundle': pre_test_hook_ignore_failing_tests_SciPybundle, + 'FFTW.MPI': pre_test_hook_ignore_failing_tests_FFTWMPI, } From 0a1ff49d22fa612aa34b4e6e98dce5efe5a89b36 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Wed, 6 Sep 2023 14:06:04 +0200 Subject: [PATCH 04/41] Update eb_hooks.py Co-authored-by: Kenneth Hoste --- eb_hooks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 9560d84eff..83bdca4cb5 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -276,7 +276,8 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): def pre_test_hook_ignore_failing_tests_FFTWMPI(self, *args, **kwargs): """ - Pre-test hook for FFTW.MPI: skip failing tests for FFTWMPI 3.3.10 + Pre-test hook for FFTW.MPI: skip failing tests for FFTW.MPI 3.3.10 on neoverse_v1 + cfr. https://github.com/EESSI/software-layer/issues/325 """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if self.name == 'FFTW.MPI' and self.version == '3.3.10' and cpu_target == 'aarch64/neoverse_v1': From 401a35faba7b43ea5a67d752bede3f5b2067709a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 7 Sep 2023 13:34:25 +0200 Subject: [PATCH 05/41] {2023.06}[foss/2021a] R v4.1.0 --- eessi-2023.06-eb-4.7.2-2021a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index 42a42f2236..8824c6414f 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -24,3 +24,4 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18348 options: from-pr: 18348 + - R-4.1.0-foss-2021a.eb From 3a99d54bad7a1a7b7d6e8154d4b08c81d4fa0ebe Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Sep 2023 18:20:22 +0200 Subject: [PATCH 06/41] {2023.03} foss/2023a --- .github/workflows/test_eessi.yml | 1 + EESSI-pilot-install-software.sh | 2 +- eessi-2023.06-eb-4.8.1-2023a.yml | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 eessi-2023.06-eb-4.8.1-2023a.yml diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index da1379043a..8c1aede4c9 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -29,6 +29,7 @@ jobs: - eessi-2023.06-eb-4.8.0-2021a.yml - eessi-2023.06-eb-4.8.0-system.yml - eessi-2023.06-eb-4.8.0-2021b.yml + - eessi-2023.06-eb-4.8.1-2023a.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 29fbb7aa62..4605b77873 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -172,7 +172,7 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -for eb_version in '4.7.2' '4.8.0'; do +for eb_version in '4.7.2' '4.8.0' '4.8.1'; do # load EasyBuild module (will be installed if it's not available yet) source ${TOPDIR}/load_easybuild_module.sh ${eb_version} diff --git a/eessi-2023.06-eb-4.8.1-2023a.yml b/eessi-2023.06-eb-4.8.1-2023a.yml new file mode 100644 index 0000000000..20aafb11fe --- /dev/null +++ b/eessi-2023.06-eb-4.8.1-2023a.yml @@ -0,0 +1,3 @@ +easyconfigs: + - GCC-12.3.0 + - foss-2023a From 76d7e9f6a02b3df29c59f89cf07c4a8de5df74c2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 15 Sep 2023 08:58:12 +0200 Subject: [PATCH 07/41] install OpenBLAS for foss/2023a via easyconfig PR that adds patch to fix hang in OpenBLAS test suite --- eessi-2023.06-eb-4.8.1-2023a.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eessi-2023.06-eb-4.8.1-2023a.yml b/eessi-2023.06-eb-4.8.1-2023a.yml index 20aafb11fe..8c2313b8a4 100644 --- a/eessi-2023.06-eb-4.8.1-2023a.yml +++ b/eessi-2023.06-eb-4.8.1-2023a.yml @@ -1,3 +1,8 @@ easyconfigs: - GCC-12.3.0 + - OpenBLAS-0.3.23-GCC-12.3.0: + # add patch to avoid hang when running OpenBLAS test suite, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18790 + options: + from-pr: 18790 - foss-2023a From 1a28493b4fb379198e2c113edaac07a8a869d079 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 18 Sep 2023 15:59:45 +0200 Subject: [PATCH 08/41] {2023.06}[foss/2021b] R 4.2.0 --- eessi-2023.06-eb-4.8.1-2021b.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 eessi-2023.06-eb-4.8.1-2021b.yml diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml new file mode 100644 index 0000000000..a0ef0fa303 --- /dev/null +++ b/eessi-2023.06-eb-4.8.1-2021b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - R-4.2.0-foss-2021b.eb From 57be9d9ce95a12c3a98a689d6dfcf32e324264eb Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 18 Sep 2023 16:34:38 +0200 Subject: [PATCH 09/41] update .github/workflows/test_eessi.yml --- .github/workflows/test_eessi.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 2db4809b21..1a67ad29ea 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -30,6 +30,7 @@ jobs: - eessi-2023.06-eb-4.8.0-system.yml - eessi-2023.06-eb-4.8.0-2021b.yml - eessi-2023.06-eb-4.8.1-system.yml + - eessi-2023.06-eb-4.8.1-2021b.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 From 65305199a81a5be603f37c39465e9250d045d183 Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 19 Sep 2023 15:23:50 +0200 Subject: [PATCH 10/41] add fix permission issues when copying xvfb-run script in Xvfb easyconfigs --- eessi-2023.06-eb-4.8.1-2021b.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml index a0ef0fa303..c2c50ba27b 100644 --- a/eessi-2023.06-eb-4.8.1-2021b.yml +++ b/eessi-2023.06-eb-4.8.1-2021b.yml @@ -1,2 +1,7 @@ easyconfigs: + - Xvfb-1.20.13-GCCcore-11.2.0.eb: + # enable exec permissions for xvfb-run after copying; + # need to also enable user write permissions on xvfb-run to ensure that copying with preserved permissions works + options: + from-pr: 18834 - R-4.2.0-foss-2021b.eb From 2bc2d2fcb023141ebfc1f71879158c21eda66860 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 19 Sep 2023 15:34:21 +0200 Subject: [PATCH 11/41] use PR to fix xvfb-run permission issue for Xvfb 1.20.11 --- eessi-2023.06-eb-4.7.2-2021a.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index 8824c6414f..aaa23207d1 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -24,4 +24,9 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18348 options: from-pr: 18348 + - Xvfb-1.20.11-GCCcore-10.3.0.eb: + # fix permission issues for xvfb-run script, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18834 + options: + from-pr: 18834 - R-4.1.0-foss-2021a.eb From b8b8c2b6e31cfaff3dbe7160148822cbb90c2878 Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 19 Sep 2023 15:59:05 +0200 Subject: [PATCH 12/41] resolve parsing error of easystack --- eessi-2023.06-eb-4.8.1-2021b.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml index c2c50ba27b..62f529563a 100644 --- a/eessi-2023.06-eb-4.8.1-2021b.yml +++ b/eessi-2023.06-eb-4.8.1-2021b.yml @@ -2,6 +2,6 @@ easyconfigs: - Xvfb-1.20.13-GCCcore-11.2.0.eb: # enable exec permissions for xvfb-run after copying; # need to also enable user write permissions on xvfb-run to ensure that copying with preserved permissions works - options: - from-pr: 18834 + options: + from-pr: 18834 - R-4.2.0-foss-2021b.eb From f3a1f35b5acf46d241ed3f3262e86ae6ed054771 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Sep 2023 17:34:32 +0200 Subject: [PATCH 13/41] fix failing build for testthat R extension on top of recent glibc via pre_single_extension hook that replaces SIGSTKSZ with constant value --- eb_hooks.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 488e53d929..95092fbdc2 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -274,6 +274,22 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): self.cfg['testopts'] = "|| echo ignoring failing tests" +def pre_single_extension_hook(ext, *args, **kwargs): + """Main pre-configure hook: trigger custom functions based on software name.""" + if ext.name in PRE_SINGLE_EXTENSION_HOOKS: + PRE_SINGLE_EXTENSION_HOOKS[ext.name](ext, *args, **kwargs) + + +def pre_single_extension_testthat(ext, *args, **kwargs): + """ + Pre-extension hook for testthat, to fix build on top of recent glibc. + """ + if ext.name == 'testthat' and LooseVersion(ext.version) < LooseVersion('3.1.0'): + # use constant value instead of SIGSTKSZ for stack size, + # cfr. https://github.com/r-lib/testthat/issues/1373 + https://github.com/r-lib/testthat/pull/1403 + ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' inst/include/testthat/vendor/catch.h && " + + PARSE_HOOKS = { 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, @@ -296,3 +312,7 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): PRE_TEST_HOOKS = { 'SciPy-bundle': pre_test_hook_ignore_failing_tests_SciPybundle, } + +PRE_SINGLE_EXTENSION_HOOKS = { + 'testthat': pre_single_extension_testthat, +} From 74f74ae32465fd2410cde9c448cc6ccf86d01db2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Sep 2023 18:36:27 +0200 Subject: [PATCH 14/41] use easybuild/source subdirectory in shared filesystem path as EasyBuild source path, if specified via shared_fs_path in provided configuration --- EESSI-pilot-install-software.sh | 14 ++++++++++++++ bot/build.sh | 13 +++++++++++++ 2 files changed, 27 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 4605b77873..ed5319a6db 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -8,10 +8,12 @@ display_help() { echo "usage: $0 [OPTIONS]" + echo " --build-logs-dir - location to copy EasyBuild logs to for failed builds" echo " -g | --generic - instructs script to build for generic architecture target" echo " -h | --help - display this usage information" echo " -x | --http-proxy URL - provides URL for the environment variable http_proxy" echo " -y | --https-proxy URL - provides URL for the environment variable https_proxy" + echo " --shared-fs-path - path to directory on shared filesystem that can be used" } function copy_build_log() { @@ -70,6 +72,10 @@ while [[ $# -gt 0 ]]; do export build_logs_dir="${2}" shift 2 ;; + --shared-fs-path) + export shared_fs_path="${2}" + shift 2 + ;; -*|--*) echo "Error: Unknown option: $1" >&2 exit 1 @@ -160,6 +166,14 @@ fi echo ">> Configuring EasyBuild..." source $TOPDIR/configure_easybuild +if [ ! -z "${shared_fs_path}" ]; then + shared_eb_sourcepath=${shared_fs_path}/easybuild/sources + echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path" + export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH} +fi + +${EB} --show-config + echo ">> Setting up \$MODULEPATH..." # make sure no modules are loaded module --force purge diff --git a/bot/build.sh b/bot/build.sh index bd7b2fa8d8..1e7aac49c1 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -64,6 +64,18 @@ else export SINGULARITY_BIND="${SINGULARITY_BIND},${BUILD_LOGS_DIR}" fi +# check if path to directory on shared filesystem is specified, +# and use it as location for source tarballs used by EasyBuild if so +SHARED_FS_PATH=$(cfg_get_value "site_config" "shared_fs_path") +echo "bot/build.sh: SHARED_FS_PATH='${SHARED_FS_PATH}'" +# if $SHARED_FS_PATH is set, add it to $SINGULARITY_BIND so the path is available in the build container +mkdir -p ${SHARED_FS_PATH} +if [[ -z ${SINGULARITY_BIND} ]]; then + export SINGULARITY_BIND="${SHARED_FS_PATH}" +else + export SINGULARITY_BIND="${SINGULARITY_BIND},${SHARED_FS_PATH}" +fi + SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then @@ -167,6 +179,7 @@ if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then INSTALL_SCRIPT_ARGS+=("--generic") fi [[ ! -z ${BUILD_LOGS_DIR} ]] && INSTALL_SCRIPT_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}") +[[ ! -z ${SHARED_FS_PATH} ]] && INSTALL_SCRIPT_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}") # create tmp file for output of build step build_outerr=$(mktemp build.outerr.XXXX) From 24146ee35424ee5f9e3c621611a1df13118d1315 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Sep 2023 20:18:36 +0200 Subject: [PATCH 15/41] {2023.06}[system] ReFrame 4.3.3 --- eessi-2023.06-eb-4.8.1-system.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eessi-2023.06-eb-4.8.1-system.yml b/eessi-2023.06-eb-4.8.1-system.yml index 8966fbfb7a..1e9664e10f 100644 --- a/eessi-2023.06-eb-4.8.1-system.yml +++ b/eessi-2023.06-eb-4.8.1-system.yml @@ -1,3 +1,6 @@ easyconfigs: # rebuilt with EasyBuild v4.8.1 (now wraps around Java/11.0.20) - Java-11.eb + - ReFrame-4.3.3.eb: + options: + from-pr: 18851 From 8da226c5eec4aa204e3d522cc322f228b85f4aa5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 22 Sep 2023 10:12:32 +0200 Subject: [PATCH 16/41] also fix SIGSTKSZ for isoband R package via pre-extension hook --- eb_hooks.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 95092fbdc2..ce84ba655e 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -282,7 +282,7 @@ def pre_single_extension_hook(ext, *args, **kwargs): def pre_single_extension_testthat(ext, *args, **kwargs): """ - Pre-extension hook for testthat, to fix build on top of recent glibc. + Pre-extension hook for testthat R package, to fix build on top of recent glibc. """ if ext.name == 'testthat' and LooseVersion(ext.version) < LooseVersion('3.1.0'): # use constant value instead of SIGSTKSZ for stack size, @@ -290,6 +290,16 @@ def pre_single_extension_testthat(ext, *args, **kwargs): ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' inst/include/testthat/vendor/catch.h && " +def pre_single_extension_isoband(ext, *args, **kwargs): + """ + Pre-extension hook for isoband R package, to fix build on top of recent glibc. + """ + if ext.name == 'isoband' and LooseVersion(ext.version) < LooseVersion('0.2.5'): + # use constant value instead of SIGSTKSZ for stack size in vendored testthat included in isoband sources, + # cfr. https://github.com/r-lib/isoband/commit/6984e6ce8d977f06e0b5ff73f5d88e5c9a44c027 + ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' src/testthat/vendor/catch.h && " + + PARSE_HOOKS = { 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, @@ -314,5 +324,6 @@ def pre_single_extension_testthat(ext, *args, **kwargs): } PRE_SINGLE_EXTENSION_HOOKS = { + 'isoband': pre_single_extension_isoband, 'testthat': pre_single_extension_testthat, } From 39ecb81b31568f82863c9c66709aaf7f8eaf13a8 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 25 Sep 2023 12:48:26 +0200 Subject: [PATCH 17/41] add yml for known issues --- eessi-2023.06-missing.yml | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 eessi-2023.06-missing.yml diff --git a/eessi-2023.06-missing.yml b/eessi-2023.06-missing.yml new file mode 100644 index 0000000000..cd7967043d --- /dev/null +++ b/eessi-2023.06-missing.yml @@ -0,0 +1,44 @@ + - aarch64/generic: + - foss/2022b: + - OpenBlas: + - issue: 314 + - foss/2021a: + - OpenBlas: + - issue: 314 + - foss/2021b: + - OpenBlas: + - issue: 314 + - foss/2022a: + - OpenBlas: + - issue: 314 + - aarch64/neoverse_n1: + - foss/2022b: + - OpenBlas: + - issue: 314 + - foss/2021a: + - OpenBlas: + - issue: 314 + - foss/2021b: + - OpenBlas: + - issue: 314 + - foss/2022a: + - OpenBlas: + - issue: 314 + - aarch64/neoverse_v1: + - foss/2022a: + - OpenBlas: + - issue: 314 + - foss/2022b: + - OpenBlas: + - issue: 314 + - foss/2021a: + - OpenBlas: + - issue: 314 + - foss/2021b: + - OpenBlas: + - issue: 314 + - SciPy-bundle: + - issue: 318 + - foss/2022a: + - OpenBlas: + - issue: 314 From 9d96b667e458b6b0bd12808281379527aa570121 Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 26 Sep 2023 10:30:49 +0200 Subject: [PATCH 18/41] update known issue to make more compact --- eessi-2023.06-known_issues.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 eessi-2023.06-known_issues.yml diff --git a/eessi-2023.06-known_issues.yml b/eessi-2023.06-known_issues.yml new file mode 100644 index 0000000000..b4a48684a5 --- /dev/null +++ b/eessi-2023.06-known_issues.yml @@ -0,0 +1,16 @@ +- aarch64/generic: + OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 + +- aarch64/generic: + - OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 + - OpenBLAS/0.3.18-GCC-11.2.0: https://github.com/EESSI/software-layer/issues/314 + - OpenBLAS/0.3.15-GCC-10.3.0: https://github.com/EESSI/software-layer/issues/314 +- aarch64/neoverse_n1: + - OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 + - OpenBLAS/0.3.18-GCC-11.2.0: https://github.com/EESSI/software-layer/issues/314 + - OpenBLAS/0.3.15-GCC-10.3.0: https://github.com/EESSI/software-layer/issues/314 +- aarch64/neoverse_v1: + - OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 + - OpenBLAS/0.3.18-GCC-11.2.0: https://github.com/EESSI/software-layer/issues/314 + - OpenBLAS/0.3.15-GCC-10.3.0: https://github.com/EESSI/software-layer/issues/314 +- SciPy-bundle/2021.10-foss-2021b: https://github.com/EESSI/software-layer/issues/318 From ab7759fbd61e6c708ef8bcd696041312ae458599 Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 26 Sep 2023 10:34:14 +0200 Subject: [PATCH 19/41] remove missing yml --- eessi-2023.06-missing.yml | 44 --------------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 eessi-2023.06-missing.yml diff --git a/eessi-2023.06-missing.yml b/eessi-2023.06-missing.yml deleted file mode 100644 index cd7967043d..0000000000 --- a/eessi-2023.06-missing.yml +++ /dev/null @@ -1,44 +0,0 @@ - - aarch64/generic: - - foss/2022b: - - OpenBlas: - - issue: 314 - - foss/2021a: - - OpenBlas: - - issue: 314 - - foss/2021b: - - OpenBlas: - - issue: 314 - - foss/2022a: - - OpenBlas: - - issue: 314 - - aarch64/neoverse_n1: - - foss/2022b: - - OpenBlas: - - issue: 314 - - foss/2021a: - - OpenBlas: - - issue: 314 - - foss/2021b: - - OpenBlas: - - issue: 314 - - foss/2022a: - - OpenBlas: - - issue: 314 - - aarch64/neoverse_v1: - - foss/2022a: - - OpenBlas: - - issue: 314 - - foss/2022b: - - OpenBlas: - - issue: 314 - - foss/2021a: - - OpenBlas: - - issue: 314 - - foss/2021b: - - OpenBlas: - - issue: 314 - - SciPy-bundle: - - issue: 318 - - foss/2022a: - - OpenBlas: - - issue: 314 From e8ddf5ed20de44548e462e09cc3c02e2c672cb36 Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 26 Sep 2023 10:35:31 +0200 Subject: [PATCH 20/41] update known issue yml --- eessi-2023.06-known_issues.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/eessi-2023.06-known_issues.yml b/eessi-2023.06-known_issues.yml index b4a48684a5..523ca0d712 100644 --- a/eessi-2023.06-known_issues.yml +++ b/eessi-2023.06-known_issues.yml @@ -1,6 +1,3 @@ -- aarch64/generic: - OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 - - aarch64/generic: - OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 - OpenBLAS/0.3.18-GCC-11.2.0: https://github.com/EESSI/software-layer/issues/314 From 63352534ae3a1e1cd437e41ecf371436f2cc569d Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Tue, 26 Sep 2023 10:49:37 +0200 Subject: [PATCH 21/41] Update eessi-2023.06-known_issues.yml --- eessi-2023.06-known_issues.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-known_issues.yml b/eessi-2023.06-known_issues.yml index 523ca0d712..de01f1ff89 100644 --- a/eessi-2023.06-known_issues.yml +++ b/eessi-2023.06-known_issues.yml @@ -10,4 +10,4 @@ - OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 - OpenBLAS/0.3.18-GCC-11.2.0: https://github.com/EESSI/software-layer/issues/314 - OpenBLAS/0.3.15-GCC-10.3.0: https://github.com/EESSI/software-layer/issues/314 -- SciPy-bundle/2021.10-foss-2021b: https://github.com/EESSI/software-layer/issues/318 + - SciPy-bundle/2021.10-foss-2021b: https://github.com/EESSI/software-layer/issues/318 From acf6b94cd301619311a4245c7c38b524baf7e52f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Sep 2023 20:27:30 +0200 Subject: [PATCH 22/41] install OpenBLAS for foss/2022a via easyconfig PR that adds patch to fix detection of Neoverse V1 CPUs + use EasyBuild v4.8.1 for installing foss/2022a --- .github/workflows/test_eessi.yml | 1 + eessi-2023.06-eb-4.7.2-2022a.yml | 7 ------- eessi-2023.06-eb-4.8.1-2022a.yml | 7 +++++++ 3 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 eessi-2023.06-eb-4.8.1-2022a.yml diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 1a67ad29ea..abade73d27 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -31,6 +31,7 @@ jobs: - eessi-2023.06-eb-4.8.0-2021b.yml - eessi-2023.06-eb-4.8.1-system.yml - eessi-2023.06-eb-4.8.1-2021b.yml + - eessi-2023.06-eb-4.8.1-2022a.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 diff --git a/eessi-2023.06-eb-4.7.2-2022a.yml b/eessi-2023.06-eb-4.7.2-2022a.yml index f4a3da0143..d40ddff261 100644 --- a/eessi-2023.06-eb-4.7.2-2022a.yml +++ b/eessi-2023.06-eb-4.7.2-2022a.yml @@ -1,10 +1,3 @@ easyconfigs: - GCC-11.3.0 - OpenMPI-4.1.4-GCC-11.3.0.eb - - CMake-3.23.1-GCCcore-11.3.0.eb: - options: - include-easyblocks-from-pr: 2248 - - CMake-3.24.3-GCCcore-11.3.0.eb: - options: - include-easyblocks-from-pr: 2248 - - foss-2022a diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml new file mode 100644 index 0000000000..8de4c6169d --- /dev/null +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -0,0 +1,7 @@ +easyconfigs: + - OpenBLAS-0.3.20-GCC-11.3.0: + # use easyconfig that includes patch to fix detection of Neoverse V1 CPUs, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18870 + options: + from-pr: 18870 + - foss-2022a From c2b6abae4cc882087c4f78972bfa20426c4a11fc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Sep 2023 20:42:34 +0200 Subject: [PATCH 23/41] set shared_fs_path for bot on AWS CitC Slurm cluster --- bot/bot-eessi-aws-citc.cfg | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg index 0a91b12de2..a5e53d82c1 100644 --- a/bot/bot-eessi-aws-citc.cfg +++ b/bot/bot-eessi-aws-citc.cfg @@ -44,6 +44,10 @@ command_response_fmt = # name of the job script used for building an EESSI stack build_job_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/bot-build.slurm +# path to directory on shared filesystem that can be used for sharing data across build jobs +# (for example source tarballs used by EasyBuild) +shared_fs_path = /mnt/shared/home/bot/shared + # Path (directory) to which build logs for (only) failing builds should be copied by bot/build.sh script build_logs_dir = /mnt/shared/bot-build-logs From e092e6be82177ab6b2c89cbf2405494c7d3ef28b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Sep 2023 20:50:43 +0200 Subject: [PATCH 24/41] only create & bind build_logs_dir and shared_fs_path if they are specified --- bot/build.sh | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 1e7aac49c1..cf9d69b65f 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -57,11 +57,13 @@ echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'" BUILD_LOGS_DIR=$(cfg_get_value "site_config" "build_logs_dir") echo "bot/build.sh: BUILD_LOGS_DIR='${BUILD_LOGS_DIR}'" # if $BUILD_LOGS_DIR is set, add it to $SINGULARITY_BIND so the path is available in the build container -mkdir -p ${BUILD_LOGS_DIR} -if [[ -z ${SINGULARITY_BIND} ]]; then - export SINGULARITY_BIND="${BUILD_LOGS_DIR}" -else - export SINGULARITY_BIND="${SINGULARITY_BIND},${BUILD_LOGS_DIR}" +if [[ ! -z ${BUILD_LOGS_DIR} ]]; then + mkdir -p ${BUILD_LOGS_DIR} + if [[ -z ${SINGULARITY_BIND} ]]; then + export SINGULARITY_BIND="${BUILD_LOGS_DIR}" + else + export SINGULARITY_BIND="${SINGULARITY_BIND},${BUILD_LOGS_DIR}" + fi fi # check if path to directory on shared filesystem is specified, @@ -69,11 +71,13 @@ fi SHARED_FS_PATH=$(cfg_get_value "site_config" "shared_fs_path") echo "bot/build.sh: SHARED_FS_PATH='${SHARED_FS_PATH}'" # if $SHARED_FS_PATH is set, add it to $SINGULARITY_BIND so the path is available in the build container -mkdir -p ${SHARED_FS_PATH} -if [[ -z ${SINGULARITY_BIND} ]]; then - export SINGULARITY_BIND="${SHARED_FS_PATH}" -else - export SINGULARITY_BIND="${SINGULARITY_BIND},${SHARED_FS_PATH}" +if [[ ! -z ${SHARED_FS_PATH} ]]; then + mkdir -p ${SHARED_FS_PATH} + if [[ -z ${SINGULARITY_BIND} ]]; then + export SINGULARITY_BIND="${SHARED_FS_PATH}" + else + export SINGULARITY_BIND="${SINGULARITY_BIND},${SHARED_FS_PATH}" + fi fi SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") From 4252a16d86ce45ec76078b414e5f919a8e5297d1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Sep 2023 21:21:38 +0200 Subject: [PATCH 25/41] use EasyBuild v4.8.1 to install R 4.1.0 with foss/2021a --- .github/workflows/test_eessi.yml | 1 + eessi-2023.06-eb-4.7.2-2021a.yml | 6 ------ eessi-2023.06-eb-4.8.1-2021a.yml | 7 +++++++ 3 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 eessi-2023.06-eb-4.8.1-2021a.yml diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 1a67ad29ea..387c773885 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -30,6 +30,7 @@ jobs: - eessi-2023.06-eb-4.8.0-system.yml - eessi-2023.06-eb-4.8.0-2021b.yml - eessi-2023.06-eb-4.8.1-system.yml + - eessi-2023.06-eb-4.8.1-2021a.yml - eessi-2023.06-eb-4.8.1-2021b.yml steps: - name: Check out software-layer repository diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index aaa23207d1..42a42f2236 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -24,9 +24,3 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18348 options: from-pr: 18348 - - Xvfb-1.20.11-GCCcore-10.3.0.eb: - # fix permission issues for xvfb-run script, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18834 - options: - from-pr: 18834 - - R-4.1.0-foss-2021a.eb diff --git a/eessi-2023.06-eb-4.8.1-2021a.yml b/eessi-2023.06-eb-4.8.1-2021a.yml new file mode 100644 index 0000000000..999eb09925 --- /dev/null +++ b/eessi-2023.06-eb-4.8.1-2021a.yml @@ -0,0 +1,7 @@ +easyconfigs: + - Xvfb-1.20.11-GCCcore-10.3.0.eb: + # fix permission issues for xvfb-run script, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18834 + options: + from-pr: 18834 + - R-4.1.0-foss-2021a.eb From 96704bcb314098a602291af5b56988ca08e4586d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 Sep 2023 20:39:51 +0200 Subject: [PATCH 26/41] add more info to known issues list, and remove entries for aarch64/generic and aarch64/neoverse_n1, since there we don't actually have problems with OpenBLAS test suite at all --- eessi-2023.06-known-issues.yml | 16 ++++++++++++++++ eessi-2023.06-known_issues.yml | 13 ------------- 2 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 eessi-2023.06-known-issues.yml delete mode 100644 eessi-2023.06-known_issues.yml diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml new file mode 100644 index 0000000000..604cc57555 --- /dev/null +++ b/eessi-2023.06-known-issues.yml @@ -0,0 +1,16 @@ +- aarch64/neoverse_v1: + - OpenBLAS/0.3.18-GCC-11.2.0: + - issue: https://github.com/EESSI/software-layer/issues/314 + - info: "Increased number of non-numerical failures in OpenBLAS test suite compared (238 vs max. 150 on x86_64/*)" + - OpenBLAS/0.3.20-GCC-11.3.0: + - issue: https://github.com/EESSI/software-layer/issues/314 + - info: "Increased number of non-numerical failures in OpenBLAS test suite compared (238 vs max. 150 on x86_64/*)" + - OpenBLAS/0.3.21-GCC-12.2.0: + - issue: https://github.com/EESSI/software-layer/issues/314 + - info: "Increased number of non-numerical failures in OpenBLAS test suite compared (344 vs max. 150 on x86_64/*)" + - SciPy-bundle/2021.05-foss-2021a: + - issue: https://github.com/EESSI/software-layer/issues/318 + - info: "2 failing tests (vs 30554 passed) in scipy test suite" + - SciPy-bundle/2021.10-foss-2021b: + - issue: https://github.com/EESSI/software-layer/issues/318 + - info: "20 failing tests (vs 14429 passed) in numpy test suite + 55 failing tests (vs 32438 passed) in scipy test suite" diff --git a/eessi-2023.06-known_issues.yml b/eessi-2023.06-known_issues.yml deleted file mode 100644 index de01f1ff89..0000000000 --- a/eessi-2023.06-known_issues.yml +++ /dev/null @@ -1,13 +0,0 @@ -- aarch64/generic: - - OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 - - OpenBLAS/0.3.18-GCC-11.2.0: https://github.com/EESSI/software-layer/issues/314 - - OpenBLAS/0.3.15-GCC-10.3.0: https://github.com/EESSI/software-layer/issues/314 -- aarch64/neoverse_n1: - - OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 - - OpenBLAS/0.3.18-GCC-11.2.0: https://github.com/EESSI/software-layer/issues/314 - - OpenBLAS/0.3.15-GCC-10.3.0: https://github.com/EESSI/software-layer/issues/314 -- aarch64/neoverse_v1: - - OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 - - OpenBLAS/0.3.18-GCC-11.2.0: https://github.com/EESSI/software-layer/issues/314 - - OpenBLAS/0.3.15-GCC-10.3.0: https://github.com/EESSI/software-layer/issues/314 - - SciPy-bundle/2021.10-foss-2021b: https://github.com/EESSI/software-layer/issues/318 From 4c8745b4b29069f7f6273e9aaaed2df1dbc7d9ce Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 Sep 2023 20:44:18 +0200 Subject: [PATCH 27/41] only increase limit for numerical test failures for OpenBLAS for aarch64/neoverse_v1 --- eb_hooks.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 83bdca4cb5..004746124e 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -18,6 +18,8 @@ from distutils.version import LooseVersion +CPU_TARGET_NEOVERSE_V1 = 'aarch64/neoverse_v1' + EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs' @@ -160,13 +162,14 @@ def parse_hook_fontconfig_add_fonts(ec, eprefix): def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): - """Relax number of failing numerical LAPACK tests on Arm 64-bit systems.""" + """Relax number of failing numerical LAPACK tests on for aarch64/neoverse_v1 CPU target.""" + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if ec.name == 'OpenBLAS': - # relax maximum number of failed numerical LAPACK tests on Arm 64-bit systems, - # since the default setting of 150 that works well on x86_64 is a bit too strict + # relax maximum number of failed numerical LAPACK tests for aarch64/neoverse_v1 CPU target + # since the default setting of 150 that works well on other aarch64 target and x86_64 is a bit too strict # See https://github.com/EESSI/software-layer/issues/314 cfg_option = 'max_failing_lapack_tests_num_errors' - if get_cpu_architecture() == AARCH64: + if cpu_target == CPU_TARGET_NEOVERSE_V1: orig_value = ec[cfg_option] ec[cfg_option] = 400 print_msg("Maximum number of failing LAPACK tests with numerical errors for %s relaxed to %s (was %s)", @@ -270,7 +273,7 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): In previous versions we were not as strict yet on the numpy/SciPy tests """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if self.name == 'SciPy-bundle' and self.version == '2021.10' and cpu_target == 'aarch64/neoverse_v1': + if self.name == 'SciPy-bundle' and self.version == '2021.10' and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg['testopts'] = "|| echo ignoring failing tests" @@ -280,7 +283,7 @@ def pre_test_hook_ignore_failing_tests_FFTWMPI(self, *args, **kwargs): cfr. https://github.com/EESSI/software-layer/issues/325 """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if self.name == 'FFTW.MPI' and self.version == '3.3.10' and cpu_target == 'aarch64/neoverse_v1': + if self.name == 'FFTW.MPI' and self.version == '3.3.10' and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg['testopts'] = "|| echo ignoring failing tests" PARSE_HOOKS = { From 4a026c4e249edd877bb5e4416e3ef47456e6baca Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 Sep 2023 20:53:25 +0200 Subject: [PATCH 28/41] also mention known issue with flaky FFTW test on aarch64/neoverse_v1 --- eessi-2023.06-known-issues.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index 604cc57555..421c43f1cb 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -1,4 +1,10 @@ - aarch64/neoverse_v1: + - FFTW/3.3.10-3.3.10-GCC-11.3.0: + - issue: https://github.com/EESSI/software-layer/issues/325 + - info: "Flaky FFTW tests, random failures" + - FFTW/3.3.10-3.3.10-gompi-2021b: + - issue: https://github.com/EESSI/software-layer/issues/325 + - info: "Flaky FFTW tests, random failures" - OpenBLAS/0.3.18-GCC-11.2.0: - issue: https://github.com/EESSI/software-layer/issues/314 - info: "Increased number of non-numerical failures in OpenBLAS test suite compared (238 vs max. 150 on x86_64/*)" From 941e0d06bd32f1c7f9fc4c65dcc8ae23323f10ee Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 29 Sep 2023 08:31:45 +0200 Subject: [PATCH 29/41] install OpenBLAS for foss/2023a via easyconfigs PR #18887 --- eessi-2023.06-eb-4.8.1-2023a.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eessi-2023.06-eb-4.8.1-2023a.yml b/eessi-2023.06-eb-4.8.1-2023a.yml index 8c2313b8a4..b015676757 100644 --- a/eessi-2023.06-eb-4.8.1-2023a.yml +++ b/eessi-2023.06-eb-4.8.1-2023a.yml @@ -1,8 +1,10 @@ easyconfigs: - GCC-12.3.0 - OpenBLAS-0.3.23-GCC-12.3.0: - # add patch to avoid hang when running OpenBLAS test suite, + # add patch to disable flaky DDRGES3 LAPACK test, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18887; + # also pulls in patch to avoid hang when running OpenBLAS test suite, # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18790 options: - from-pr: 18790 + from-pr: 18887 - foss-2023a From 1fdbb42dfe74b12db717f8165c11bb0fd6c19352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 29 Sep 2023 09:23:45 +0200 Subject: [PATCH 30/41] fix typo --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 004746124e..95abb36e7b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -162,7 +162,7 @@ def parse_hook_fontconfig_add_fonts(ec, eprefix): def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): - """Relax number of failing numerical LAPACK tests on for aarch64/neoverse_v1 CPU target.""" + """Relax number of failing numerical LAPACK tests for aarch64/neoverse_v1 CPU target.""" cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if ec.name == 'OpenBLAS': # relax maximum number of failed numerical LAPACK tests for aarch64/neoverse_v1 CPU target From 40e56c005cd76caafe83be63fc7b3accd1b743f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 29 Sep 2023 09:23:54 +0200 Subject: [PATCH 31/41] fix typo --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 95abb36e7b..e5999efcb7 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -166,7 +166,7 @@ def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if ec.name == 'OpenBLAS': # relax maximum number of failed numerical LAPACK tests for aarch64/neoverse_v1 CPU target - # since the default setting of 150 that works well on other aarch64 target and x86_64 is a bit too strict + # since the default setting of 150 that works well on other aarch64 targets and x86_64 is a bit too strict # See https://github.com/EESSI/software-layer/issues/314 cfg_option = 'max_failing_lapack_tests_num_errors' if cpu_target == CPU_TARGET_NEOVERSE_V1: From 15985f950c602b24270b63a98de842e53dd5c01a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 29 Sep 2023 15:08:17 +0200 Subject: [PATCH 32/41] fix typo --- eessi-2023.06-known-issues.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index 421c43f1cb..7d20c99da4 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -7,13 +7,13 @@ - info: "Flaky FFTW tests, random failures" - OpenBLAS/0.3.18-GCC-11.2.0: - issue: https://github.com/EESSI/software-layer/issues/314 - - info: "Increased number of non-numerical failures in OpenBLAS test suite compared (238 vs max. 150 on x86_64/*)" + - info: "Increased number of non-numerical failures in OpenBLAS test suite (238 vs max. 150 on x86_64/*)" - OpenBLAS/0.3.20-GCC-11.3.0: - issue: https://github.com/EESSI/software-layer/issues/314 - - info: "Increased number of non-numerical failures in OpenBLAS test suite compared (238 vs max. 150 on x86_64/*)" + - info: "Increased number of non-numerical failures in OpenBLAS test suite (238 vs max. 150 on x86_64/*)" - OpenBLAS/0.3.21-GCC-12.2.0: - issue: https://github.com/EESSI/software-layer/issues/314 - - info: "Increased number of non-numerical failures in OpenBLAS test suite compared (344 vs max. 150 on x86_64/*)" + - info: "Increased number of non-numerical failures in OpenBLAS test suite (344 vs max. 150 on x86_64/*)" - SciPy-bundle/2021.05-foss-2021a: - issue: https://github.com/EESSI/software-layer/issues/318 - info: "2 failing tests (vs 30554 passed) in scipy test suite" From 0f7ee215e3bf43e5ce31b1d34b83c19197f08b27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 2 Oct 2023 11:51:28 +0200 Subject: [PATCH 33/41] additional build/command permissions --- bot/bot-eessi-aws-citc.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg index a5e53d82c1..8a911182ca 100644 --- a/bot/bot-eessi-aws-citc.cfg +++ b/bot/bot-eessi-aws-citc.cfg @@ -28,7 +28,7 @@ private_key = /mnt/shared/home/bot/eessi-bot-software-layer/eessi-bot-citc-aws.2 # which GH accounts have the permission to send commands to the bot # if value is left/empty everyone can send commands # value can be a space delimited list of GH accounts -command_permission = boegel trz42 bedroge laraPPr ocaisa casparvl satishskamath maxim-masterov TopRichard xinan1911 +command_permission = boegel trz42 bedroge laraPPr ocaisa casparvl satishskamath maxim-masterov TopRichard xinan1911 Neves-P # format of the response when processing bot commands command_response_fmt = @@ -100,7 +100,7 @@ submit_command = /usr/bin/sbatch # the label 'bot:build' (apparently this cannot be restricted on GitHub) # if value is left/empty everyone can trigger the build # value can be a space delimited list of GH accounts -build_permission = boegel trz42 bedroge laraPPr ocaisa casparvl satishskamath maxim-masterov TopRichard xinan1911 +build_permission = boegel trz42 bedroge laraPPr ocaisa casparvl satishskamath maxim-masterov TopRichard xinan1911 Neves-P # template for comment when user who set a label has no permission to trigger build jobs no_build_permission_comment = Label `bot:build` has been set by user `{build_labeler}`, but only users `{build_permission_users}` have permission to trigger the action From 2bed6ad9b55239122067ec93bffd86c20a8e34f1 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Tue, 3 Oct 2023 17:16:02 +0200 Subject: [PATCH 34/41] {2023.06}[foss/2022a] BAMM v2.5.0 --- eessi-2023.06-eb-4.8.1-2022a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index 8de4c6169d..d26ae75411 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -5,3 +5,4 @@ easyconfigs: options: from-pr: 18870 - foss-2022a + - BAMM-2.5.0-foss-2022a.eb From 776e3cd8359ff0dd638b029a5a818cc902e00d18 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 3 Oct 2023 19:57:22 +0200 Subject: [PATCH 35/41] determine easystack files to process via PR patch file --- EESSI-pilot-install-software.sh | 50 ++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index ed5319a6db..c6c51e7abc 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -41,7 +41,7 @@ function copy_build_log() { echo "- working directory: ${PWD}" >> ${build_log_path} echo "- Slurm job ID: ${SLURM_OUT}" >> ${build_log_path} echo "- EasyBuild version: ${eb_version}" >> ${build_log_path} - echo "- easystack file: ${es}" >> ${build_log_path} + echo "- easystack file: ${easystack_file}" >> ${build_log_path} echo "EasyBuild log file ${build_log} copied to ${build_log_path} (with context appended)" fi @@ -186,36 +186,42 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -for eb_version in '4.7.2' '4.8.0' '4.8.1'; do +# assume there's only one diff file that corresponds to the PR patch file +pr_diff=$(ls [0-9]*.diff | head -1) + +# use PR patch file to determine in which easystack files stuff was added +for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^eessi.*yml$'); do + + echo -e "Processing easystack file ${easystack_file}...\n\n" + + # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file + eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') # load EasyBuild module (will be installed if it's not available yet) source ${TOPDIR}/load_easybuild_module.sh ${eb_version} echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." - for es in $(ls eessi-${EESSI_PILOT_VERSION}-eb-${eb_version}-*.yml); do - - if [ -f ${es} ]; then - echo_green "Feeding easystack file ${es} to EasyBuild..." + if [ -f ${easystack_file} ]; then + echo_green "Feeding easystack file ${easystack_file} to EasyBuild..." - ${EB} --easystack ${TOPDIR}/${es} --robot - ec=$? + ${EB} --easystack ${TOPDIR}/${easystack_file} --robot + ec=$? - # copy EasyBuild log file if EasyBuild exited with an error - if [ ${ec} -ne 0 ]; then - eb_last_log=$(unset EB_VERBOSE; eb --last-log) - # copy to current working directory - cp -a ${eb_last_log} . - echo "Last EasyBuild log file copied from ${eb_last_log} to ${PWD}" - # copy to build logs dir (with context added) - copy_build_log "${eb_last_log}" "${build_logs_dir}" - fi - - $TOPDIR/check_missing_installations.sh ${TOPDIR}/${es} - else - fatal_error "Easystack file ${es} not found!" + # copy EasyBuild log file if EasyBuild exited with an error + if [ ${ec} -ne 0 ]; then + eb_last_log=$(unset EB_VERBOSE; eb --last-log) + # copy to current working directory + cp -a ${eb_last_log} . + echo "Last EasyBuild log file copied from ${eb_last_log} to ${PWD}" + # copy to build logs dir (with context added) + copy_build_log "${eb_last_log}" "${build_logs_dir}" fi - done + + $TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file} + else + fatal_error "Easystack file ${easystack_file} not found!" + fi done From 46573849c7030c9ec9c2d6b9baa7cc92ec48b7ab Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 4 Oct 2023 08:01:26 +0200 Subject: [PATCH 36/41] use 'git diff' to determine which easystack files were changed --- EESSI-pilot-install-software.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index c6c51e7abc..4f473eb82f 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -186,11 +186,8 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -# assume there's only one diff file that corresponds to the PR patch file -pr_diff=$(ls [0-9]*.diff | head -1) - -# use PR patch file to determine in which easystack files stuff was added -for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^eessi.*yml$'); do +# use 'git diff' to determine which easystack files were changed +for easystack_file in $(git diff --name-only | grep '^eessi.*yml$'); do echo -e "Processing easystack file ${easystack_file}...\n\n" From e588608054a35ed1dcef99bc8700119932007a0f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 4 Oct 2023 08:55:10 +0200 Subject: [PATCH 37/41] fix typo in FFTW version in known issues YAML file --- eessi-2023.06-known-issues.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index 7d20c99da4..f2f96aee34 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -1,8 +1,8 @@ - aarch64/neoverse_v1: - - FFTW/3.3.10-3.3.10-GCC-11.3.0: + - FFTW/3.3.10-GCC-11.3.0: - issue: https://github.com/EESSI/software-layer/issues/325 - info: "Flaky FFTW tests, random failures" - - FFTW/3.3.10-3.3.10-gompi-2021b: + - FFTW/3.3.10-gompi-2021b: - issue: https://github.com/EESSI/software-layer/issues/325 - info: "Flaky FFTW tests, random failures" - OpenBLAS/0.3.18-GCC-11.2.0: From 19dc39cea7f7236c586d588d73bb0b63e27b3c8d Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 4 Oct 2023 23:28:50 +0200 Subject: [PATCH 38/41] get changed/new files from PR.diff file --- EESSI-pilot-install-software.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 4f473eb82f..c6c51e7abc 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -186,8 +186,11 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -# use 'git diff' to determine which easystack files were changed -for easystack_file in $(git diff --name-only | grep '^eessi.*yml$'); do +# assume there's only one diff file that corresponds to the PR patch file +pr_diff=$(ls [0-9]*.diff | head -1) + +# use PR patch file to determine in which easystack files stuff was added +for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^eessi.*yml$'); do echo -e "Processing easystack file ${easystack_file}...\n\n" From 08420f1407fad943b6612e6c5b339fe4ee10e8c5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 5 Oct 2023 10:57:59 +0200 Subject: [PATCH 39/41] remove bot configuration (2023.06) --- bot/bot-eessi-aws-citc.cfg | 238 ------------------------------------- 1 file changed, 238 deletions(-) delete mode 100644 bot/bot-eessi-aws-citc.cfg diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg deleted file mode 100644 index 8a911182ca..0000000000 --- a/bot/bot-eessi-aws-citc.cfg +++ /dev/null @@ -1,238 +0,0 @@ -# Also see documentation at https://github.com/EESSI/eessi-bot-software-layer/blob/main/README.md#step5.5 - -[github] -# replace '123456' with the ID of your GitHub App -app_id = 281041 - -# a short (!) name for your app instance that can be used for example -# when adding/updating a comment to a PR -# (!) a short yet descriptive name is preferred because it appears in -# comments to the PR -# for example, the name could include the name of the cluster the bot -# runs on and the username which runs the bot -# NOTE avoid putting an actual username here as it will be visible on -# potentially publicly accessible GitHub pages. -app_name = eessi-bot-citc-aws - -# replace '12345678' with the ID of the installation of your GitHub App -# (can be derived by creating an event and then checking for the list -# of sent events and its payload either via the Smee channel's web page -# or via the Advanced section of your GitHub App on github.com) -installation_id = 33078935 - -# path to the private key that was generated when the GitHub App was registered -private_key = /mnt/shared/home/bot/eessi-bot-software-layer/eessi-bot-citc-aws.2023-01-12.private-key.pem - - -[bot_control] -# which GH accounts have the permission to send commands to the bot -# if value is left/empty everyone can send commands -# value can be a space delimited list of GH accounts -command_permission = boegel trz42 bedroge laraPPr ocaisa casparvl satishskamath maxim-masterov TopRichard xinan1911 Neves-P - -# format of the response when processing bot commands -command_response_fmt = -
Updates by the bot instance {app_name} - (click for details) - - {comment_response} - {comment_result} -
- - -[buildenv] -# name of the job script used for building an EESSI stack -build_job_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/bot-build.slurm - -# path to directory on shared filesystem that can be used for sharing data across build jobs -# (for example source tarballs used by EasyBuild) -shared_fs_path = /mnt/shared/home/bot/shared - -# Path (directory) to which build logs for (only) failing builds should be copied by bot/build.sh script -build_logs_dir = /mnt/shared/bot-build-logs - -# The container_cachedir may be used to reuse downloaded container image files -# across jobs. Thus, jobs can more quickly launch containers. -container_cachedir = /mnt/shared/home/bot/eessi-bot-software-layer/containers-cache-dir - -# it may happen that we need to customize some CVMFS configuration -# the value of cvmfs_customizations is a dictionary which maps a file -# name to an entry that needs to be added to that file -# cvmfs_customizations = {} - -# if compute nodes have no internet connection, we need to set http(s)_proxy -# or commands such as pip3 cannot download software from package repositories -# for example, the temporary EasyBuild is installed via pip3 first -# http_proxy = http://PROXY_DNS:3128/ -# https_proxy = http://PROXY_DNS:3128/ - -# directory under which the bot prepares directories per job -# structure created is as follows: YYYY.MM/pr_PR_NUMBER/event_EVENT_ID/run_RUN_NUMBER/OS+SUBDIR -jobs_base_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs - -# configure environment -# list of comma-separated modules to be loaded by build_job_script -# useful/needed if some tool is not provided as system-wide package -# (read by bot and handed over to build_job_script via parameter -# --load-modules) -# load_modules = - -# PATH to temporary directory on build node ... ends up being used for -# for example, EESSI_TMPDIR --> /tmp/$USER/EESSI -# escaping variables with '\' delays expansion to the start of the -# build_job_script; this can be used for referencing environment -# variables that are only set inside a Slurm job -local_tmp = /tmp/$USER/EESSI - -# parameters to be added to all job submissions -# NOTE do not quote parameter string. Quotes are retained when reading in config and -# then the whole 'string' is recognised as a single parameter. -# NOTE 2 '--get-user-env' may be needed on systems where the job's environment needs -# to be initialised as if it is for a login shell. -# note: hardcoded 24h time limit until https://github.com/EESSI/eessi-bot-software-layer/issues/146 is fixed -# note: 16 cores which corresponds to *.4xlarge node types, see also arch_target_map -slurm_params = --hold --time=24:0:0 --nodes=1 --ntasks-per-node=16 - -# full path to the job submission command -submit_command = /usr/bin/sbatch - -# which GH account has the permission to trigger the build (by setting -# the label 'bot:build' (apparently this cannot be restricted on GitHub) -# if value is left/empty everyone can trigger the build -# value can be a space delimited list of GH accounts -build_permission = boegel trz42 bedroge laraPPr ocaisa casparvl satishskamath maxim-masterov TopRichard xinan1911 Neves-P - -# template for comment when user who set a label has no permission to trigger build jobs -no_build_permission_comment = Label `bot:build` has been set by user `{build_labeler}`, but only users `{build_permission_users}` have permission to trigger the action - - -[deploycfg] -# script for uploading built software packages -tarball_upload_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/eessi-upload-to-staging - -# URL to S3/minio bucket -# if attribute is set, bucket_base will be constructed as follows -# bucket_base=${endpoint_url}/${bucket_name} -# otherwise, bucket_base will be constructed as follows -# bucket_base=https://${bucket_name}.s3.amazonaws.com -# - The former variant is used for non AWS S3 services, eg, minio, or when -# the bucket name is not provided in the hostname (see latter case). -# - The latter variant is used for AWS S3 services. -# endpoint_url = URL_TO_S3_SERVER - -# bucket name -bucket_name = eessi-staging-2023.06 - -# upload policy: defines what policy is used for uploading built artefacts -# to an S3 bucket -# 'all' ..: upload all artefacts (mulitple uploads of the same artefact possible) -# 'latest': for each build target (eessi-VERSION-{software,init,compat}-OS-ARCH) -# only upload the latest built artefact -# 'once' : only once upload any built artefact for the build target -# 'none' : do not upload any built artefacts -upload_policy = latest - -# which GH account has the permission to trigger the deployment (by setting -# the label 'bot:deploy' (apparently this cannot be restricted on GitHub) -# if value is left/empty everyone can trigger the deployment -# value can be a space delimited list of GH accounts -deploy_permission = boegel trz42 bedroge ocaisa casparvl - -# template for comment when user who set a label has no permission to trigger deploying tarballs -no_deploy_permission_comment = Label `bot:deploy` has been set by user `{deploy_labeler}`, but only users `{deploy_permission_users}` have permission to trigger the action - - -[architecturetargets] -# defines both for which architectures the bot will build -# and what submission parameters shall be used -# 5 c4.2xlarge haswell 8 vCPU, 15 GiB RAM (1 + generic) -# 2 c4.4xlarge haswell 16 vCPU, 30 GiB RAM -# 5 c5.2xlarge skylake_avx512 8 vCPU, 16 GiB RAM (1) -# 1 c5.4xlarge skylake_avx512 16 vCPU, 32 GiB RAM -# 5 c5a.2xlarge zen2 8 vCPU, 16 GiB RAM (1) -# 1 c5a.4xlarge zen2 16 vCPU, 32 GiB RAM -# 5 c5d.2xlarge skylake_avx512 8 vCPU, 16 GiB RAM + 200 GB NVMe -# 5 c6a.2xlarge zen3 8 vCPU, 16 GiB RAM (1) -# 1 c6a.4xlarge zen3 16 vCPU, 32 GiB RAM -# 5 c6g.2xlarge neoverse_n1 8 vCPU, 16 GiB RAM (1 + generic) -# 2 c6g.4xlarge neoverse_n1 16 vCPU, 32 GiB RAM -# 1 c6g.8xlarge neoverse_n1 32 vCPU, 64 GiB RAM -# 1 c6i.2xlarge cascadelake 8 vCPU, 16 GiB RAM (1) -# 5 c7g.2xlarge neoverse_v1 8 vCPU, 16 GiB RAM (1) -# 1 c7g.4xlarge neoverse_v1 16 vCPU, 32 GiB RAM -# larger instances (*.4xlarge => 16 cores, 32GB RAM) -arch_target_map = { - "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", - "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", - "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", - "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", - "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", - "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", - "linux/aarch64/neoverse_n1" : "--constraint shape=c6g.4xlarge", - "linux/aarch64/neoverse_v1" : "--constraint shape=c7g.4xlarge" } - -[repo_targets] -# defines for which repository a arch_target should be build for -# -# only building for repository EESSI-pilot -repo_target_map = { - "linux/x86_64/generic" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/x86_64/intel/haswell" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/x86_64/intel/skylake_avx512" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/x86_64/amd/zen2" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/x86_64/amd/zen3" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/aarch64/generic" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/aarch64/neoverse_n1" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/aarch64/neoverse_v1" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"] } - -# points to definition of repositories (default EESSI-pilot defined by build container) -repos_cfg_dir = /mnt/shared/home/bot/eessi-bot-software-layer/repos - -# configuration for event handler which receives events from a GitHub repository. -[event_handler] -# path to the log file to log messages for event handler -log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_event_handler.log - - -[job_manager] -# path to the log file to log messages for job manager -log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_job_manager.log - -# directory where job manager stores information about jobs to be tracked -# e.g. as symbolic link JOBID -> directory to job -job_ids_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs - -# full path to the job status checking command -poll_command = /usr/bin/squeue - -# polling interval in seconds -poll_interval = 60 - -# full path to the command for manipulating existing jobs -scontrol_command = /usr/bin/scontrol - - -# variable 'comment' under 'submitted_job_comments' should not be changed as there are regular expression patterns matching it -[submitted_job_comments] -initial_comment = New job on instance `{app_name}` for architecture `{arch_name}` for repository `{repo_id}` in job dir `{symlink}` -awaits_release = job id `{job_id}` awaits release by job manager - - -[new_job_comments] -awaits_launch = job awaits launch by Slurm scheduler - - -[running_job_comments] -running_job = job `{job_id}` is running - - -[finished_job_comments] -success = :grin: SUCCESS tarball `{tarball_name}` ({tarball_size} GiB) in job dir -failure = :cry: FAILURE -no_slurm_out = No slurm output `{slurm_out}` in job dir -slurm_out = Found slurm output `{slurm_out}` in job dir -missing_modules = Slurm output lacks message "No missing modules!". -no_tarball_message = Slurm output lacks message about created tarball. -no_matching_tarball = No tarball matching `{tarball_pattern}` found in job dir. -multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected. -job_result_unknown_fmt =
:shrug: UNKNOWN _(click triangle for detailed information)_
  • Job results file `{filename}` does not exist in job directory or reading it failed.
  • No artefacts were found/reported.
From 38e6af4b0f1009ec1997823ccaf1fa6c72587128 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 5 Oct 2023 15:49:11 +0200 Subject: [PATCH 40/41] show active EasyBuild configuration when checking for missing installations --- check_missing_installations.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 3627d1d0b5..e927f14701 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -22,6 +22,9 @@ source $TOPDIR/scripts/utils.sh source $TOPDIR/configure_easybuild +echo ">> Active EasyBuild configuration when checking for missing installations:" +${EB:-eb} --show-config + echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." eb_missing_out=$LOCAL_TMPDIR/eb_missing.out ${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out} From c244bdc92dc6b941ea661c600eb40496946d1501 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 6 Oct 2023 09:20:14 +0200 Subject: [PATCH 41/41] simplify CI workflow to check for missing installations, just loop over easystack files per CPU target --- .github/workflows/test_eessi.yml | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 69e988ed02..6dfd78e428 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -20,20 +20,6 @@ jobs: - x86_64/intel/haswell - x86_64/intel/skylake_avx512 - x86_64/generic - EASYSTACK_FILE: - - eessi-2023.06-eb-4.7.2-2021a.yml - - eessi-2023.06-eb-4.7.2-2021b.yml - - eessi-2023.06-eb-4.7.2-2022a.yml - - eessi-2023.06-eb-4.7.2-2022b.yml - - eessi-2023.06-eb-4.7.2-system.yml - - eessi-2023.06-eb-4.8.0-2021a.yml - - eessi-2023.06-eb-4.8.0-system.yml - - eessi-2023.06-eb-4.8.0-2021b.yml - - eessi-2023.06-eb-4.8.1-system.yml - - eessi-2023.06-eb-4.8.1-2021a.yml - - eessi-2023.06-eb-4.8.1-2021b.yml - - eessi-2023.06-eb-4.8.1-2022a.yml - - eessi-2023.06-eb-4.8.1-2023a.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 @@ -55,7 +41,12 @@ jobs: export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" - ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}} + for easystack_file in $(ls eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + echo "check missing installations for ${easystack_file}..." + ./check_missing_installations.sh ${easystack_file} + ec=$? + if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi + done - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) run: | @@ -66,15 +57,18 @@ jobs: export EESSI_OS_TYPE=linux export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort - echo "modify easystack file by adding a missing package (GCC/8.3.0)" - echo " - GCC-8.3.0:" >> ${{matrix.EASYSTACK_FILE}} - tail -n 5 ${{matrix.EASYSTACK_FILE}} + # create dummy easystack file with a single entry (something that is not installed in EESSI) + easystack_file="test.yml" + echo "easyconfigs:" > ${easystack_file} + echo " - GCC-8.3.0:" >> ${easystack_file} + echo "created easystack file '${easystack_file}' with a missing installation (GCC/8.3.0):" + cat ${easystack_file} # note, check_missing_installations.sh exits 1 if a package was # missing, which is intepreted as false (exit code based, not # boolean logic), hence when the script exits 0 if no package was # missing it is interpreted as true, thus the test did not capture # the missing package - if ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}}; then + if ./check_missing_installations.sh ${easystack_file}; then echo "did NOT capture missing package; test FAILED" exit 1 else