From 77920b267a69299af92359f6da2637188c20a357 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 7 Sep 2023 18:20:08 +0200 Subject: [PATCH 01/16] {2023.06}[foss/2021a] GDAL v3.3.0 --- .github/workflows/test_eessi.yml | 1 + eessi-2023.06-eb-4.8.0-2021a.yml | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 eessi-2023.06-eb-4.8.0-2021a.yml diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 7667d02c26..48c41c7ce0 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -26,6 +26,7 @@ jobs: - eessi-2023.06-eb-4.7.2-2022a.yml - eessi-2023.06-eb-4.7.2-2022b.yml - eessi-2023.06-eb-4.7.2-system.yml + - eessi-2023.06-eb-4.8.0-2021a.yml - eessi-2023.06-eb-4.8.0-system.yml steps: - name: Check out software-layer repository diff --git a/eessi-2023.06-eb-4.8.0-2021a.yml b/eessi-2023.06-eb-4.8.0-2021a.yml new file mode 100644 index 0000000000..1adf0572d3 --- /dev/null +++ b/eessi-2023.06-eb-4.8.0-2021a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - GDAL-3.3.0-foss-2021a.eb From a9caba346620846eba20fa31e8944a63ffa5a55b Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 8 Sep 2023 10:19:51 +0200 Subject: [PATCH 02/16] {2023.06}[foss/2021b] OpenFoam v2112 --- eessi-2023.06-eb-4.8.0-2021b.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 eessi-2023.06-eb-4.8.0-2021b.yml diff --git a/eessi-2023.06-eb-4.8.0-2021b.yml b/eessi-2023.06-eb-4.8.0-2021b.yml new file mode 100644 index 0000000000..b2202b5e7d --- /dev/null +++ b/eessi-2023.06-eb-4.8.0-2021b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - OpenFOAM-v2112-foss-2021b.eb From 1df48dcd78e3c166835c4b47bf70a4ee46ec2f5f Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 8 Sep 2023 10:24:45 +0200 Subject: [PATCH 03/16] add new easystack file --- .github/workflows/test_eessi.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 7667d02c26..dba7403901 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -27,6 +27,7 @@ jobs: - eessi-2023.06-eb-4.7.2-2022b.yml - eessi-2023.06-eb-4.7.2-system.yml - eessi-2023.06-eb-4.8.0-system.yml + - eessi-2023.06-eb-4.8.0-2021b.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 From 8c057bc7376d4ce201849262e0fa2a1da65104df Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 8 Sep 2023 12:48:58 +0200 Subject: [PATCH 04/16] include patch for tsprintf.c --- eessi-2023.06-eb-4.8.0-2021b.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.0-2021b.yml b/eessi-2023.06-eb-4.8.0-2021b.yml index b2202b5e7d..ea32b3acbb 100644 --- a/eessi-2023.06-eb-4.8.0-2021b.yml +++ b/eessi-2023.06-eb-4.8.0-2021b.yml @@ -1,2 +1,6 @@ easyconfigs: - - OpenFOAM-v2112-foss-2021b.eb + - OpenFOAM-v2112-foss-2021b.eb: + #fix for different output of tsprintf.c test from GNU C library <2.36 + #see https://github.com/easybuilders/easybuild-easyconfigs/pull/18746 + options: + from-pr: 18746 From 39eda7ba930570afef702cc834632b99422cd403 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 8 Sep 2023 13:12:32 +0200 Subject: [PATCH 05/16] include patch for tsprintf.c --- eessi-2023.06-eb-4.8.0-2021b.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.0-2021b.yml b/eessi-2023.06-eb-4.8.0-2021b.yml index ea32b3acbb..ff15917797 100644 --- a/eessi-2023.06-eb-4.8.0-2021b.yml +++ b/eessi-2023.06-eb-4.8.0-2021b.yml @@ -1,6 +1,7 @@ easyconfigs: - - OpenFOAM-v2112-foss-2021b.eb: + - MPFR-4.1.0-GCCcore-11.2.0.eb: #fix for different output of tsprintf.c test from GNU C library <2.36 #see https://github.com/easybuilders/easybuild-easyconfigs/pull/18746 options: from-pr: 18746 + - OpenFOAM-v2112-foss-2021b.eb From e0fe6b5c26cd9eb0442ffa5a0496d9d422f04090 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 9 Sep 2023 12:29:40 +0200 Subject: [PATCH 06/16] improve comment for use of --from-pr for MPFR 4.1.0 --- eessi-2023.06-eb-4.8.0-2021b.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi-2023.06-eb-4.8.0-2021b.yml b/eessi-2023.06-eb-4.8.0-2021b.yml index ff15917797..477ba6320c 100644 --- a/eessi-2023.06-eb-4.8.0-2021b.yml +++ b/eessi-2023.06-eb-4.8.0-2021b.yml @@ -1,7 +1,7 @@ easyconfigs: - MPFR-4.1.0-GCCcore-11.2.0.eb: - #fix for different output of tsprintf.c test from GNU C library <2.36 - #see https://github.com/easybuilders/easybuild-easyconfigs/pull/18746 + # use patch for MPFR 4.1.0 to fix failing tsprintf test with glibc >= 2.37, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18746 options: from-pr: 18746 - OpenFOAM-v2112-foss-2021b.eb From 6317155e0afa3a85b678571ea7c8e2187e85ff5b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 12 Sep 2023 14:44:28 +0200 Subject: [PATCH 07/16] {2023.06} EasyBuild v4.8.1 --- eessi-2023.06-eb-4.8.0-system.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eessi-2023.06-eb-4.8.0-system.yml b/eessi-2023.06-eb-4.8.0-system.yml index 744a52bde6..2928f110b0 100644 --- a/eessi-2023.06-eb-4.8.0-system.yml +++ b/eessi-2023.06-eb-4.8.0-system.yml @@ -5,3 +5,6 @@ easyconfigs: # + https://github.com/easybuilders/easybuild-easyblocks/pull/2995 options: include-easyblocks-from-pr: 2995 + - EasyBuild-4.8.1.eb: + options: + from-pr: 18761 From 26f6c02d6075e1079e49afe73947c0ed2e2cd5fa Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Sep 2023 11:05:47 +0200 Subject: [PATCH 08/16] trigger rebuild of Java/11 with EasyBuild v4.8.1 --- .github/workflows/test_eessi.yml | 1 + EESSI-pilot-install-software.sh | 2 +- eessi-2023.06-eb-4.8.1-system.yml | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 eessi-2023.06-eb-4.8.1-system.yml diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index da1379043a..2db4809b21 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -29,6 +29,7 @@ jobs: - eessi-2023.06-eb-4.8.0-2021a.yml - eessi-2023.06-eb-4.8.0-system.yml - eessi-2023.06-eb-4.8.0-2021b.yml + - eessi-2023.06-eb-4.8.1-system.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 29fbb7aa62..4605b77873 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -172,7 +172,7 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -for eb_version in '4.7.2' '4.8.0'; do +for eb_version in '4.7.2' '4.8.0' '4.8.1'; do # load EasyBuild module (will be installed if it's not available yet) source ${TOPDIR}/load_easybuild_module.sh ${eb_version} diff --git a/eessi-2023.06-eb-4.8.1-system.yml b/eessi-2023.06-eb-4.8.1-system.yml new file mode 100644 index 0000000000..bab3755330 --- /dev/null +++ b/eessi-2023.06-eb-4.8.1-system.yml @@ -0,0 +1,4 @@ +easyconfigs: + - Java-11.eb: + options: + rebuild: True From ea318e635ff1e6f9b8c6dc4f26003ab3bd5f720e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Sep 2023 14:38:44 +0200 Subject: [PATCH 09/16] remove 'rebuild' option for Java/11 with EasyBuild v4.8.1 --- eessi-2023.06-eb-4.8.1-system.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eessi-2023.06-eb-4.8.1-system.yml b/eessi-2023.06-eb-4.8.1-system.yml index bab3755330..8966fbfb7a 100644 --- a/eessi-2023.06-eb-4.8.1-system.yml +++ b/eessi-2023.06-eb-4.8.1-system.yml @@ -1,4 +1,3 @@ easyconfigs: - - Java-11.eb: - options: - rebuild: True + # rebuilt with EasyBuild v4.8.1 (now wraps around Java/11.0.20) + - Java-11.eb From 1a28493b4fb379198e2c113edaac07a8a869d079 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 18 Sep 2023 15:59:45 +0200 Subject: [PATCH 10/16] {2023.06}[foss/2021b] R 4.2.0 --- eessi-2023.06-eb-4.8.1-2021b.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 eessi-2023.06-eb-4.8.1-2021b.yml diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml new file mode 100644 index 0000000000..a0ef0fa303 --- /dev/null +++ b/eessi-2023.06-eb-4.8.1-2021b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - R-4.2.0-foss-2021b.eb From 57be9d9ce95a12c3a98a689d6dfcf32e324264eb Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 18 Sep 2023 16:34:38 +0200 Subject: [PATCH 11/16] update .github/workflows/test_eessi.yml --- .github/workflows/test_eessi.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 2db4809b21..1a67ad29ea 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -30,6 +30,7 @@ jobs: - eessi-2023.06-eb-4.8.0-system.yml - eessi-2023.06-eb-4.8.0-2021b.yml - eessi-2023.06-eb-4.8.1-system.yml + - eessi-2023.06-eb-4.8.1-2021b.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 From 65305199a81a5be603f37c39465e9250d045d183 Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 19 Sep 2023 15:23:50 +0200 Subject: [PATCH 12/16] add fix permission issues when copying xvfb-run script in Xvfb easyconfigs --- eessi-2023.06-eb-4.8.1-2021b.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml index a0ef0fa303..c2c50ba27b 100644 --- a/eessi-2023.06-eb-4.8.1-2021b.yml +++ b/eessi-2023.06-eb-4.8.1-2021b.yml @@ -1,2 +1,7 @@ easyconfigs: + - Xvfb-1.20.13-GCCcore-11.2.0.eb: + # enable exec permissions for xvfb-run after copying; + # need to also enable user write permissions on xvfb-run to ensure that copying with preserved permissions works + options: + from-pr: 18834 - R-4.2.0-foss-2021b.eb From b8b8c2b6e31cfaff3dbe7160148822cbb90c2878 Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 19 Sep 2023 15:59:05 +0200 Subject: [PATCH 13/16] resolve parsing error of easystack --- eessi-2023.06-eb-4.8.1-2021b.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml index c2c50ba27b..62f529563a 100644 --- a/eessi-2023.06-eb-4.8.1-2021b.yml +++ b/eessi-2023.06-eb-4.8.1-2021b.yml @@ -2,6 +2,6 @@ easyconfigs: - Xvfb-1.20.13-GCCcore-11.2.0.eb: # enable exec permissions for xvfb-run after copying; # need to also enable user write permissions on xvfb-run to ensure that copying with preserved permissions works - options: - from-pr: 18834 + options: + from-pr: 18834 - R-4.2.0-foss-2021b.eb From 74f74ae32465fd2410cde9c448cc6ccf86d01db2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Sep 2023 18:36:27 +0200 Subject: [PATCH 14/16] use easybuild/source subdirectory in shared filesystem path as EasyBuild source path, if specified via shared_fs_path in provided configuration --- EESSI-pilot-install-software.sh | 14 ++++++++++++++ bot/build.sh | 13 +++++++++++++ 2 files changed, 27 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 4605b77873..ed5319a6db 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -8,10 +8,12 @@ display_help() { echo "usage: $0 [OPTIONS]" + echo " --build-logs-dir - location to copy EasyBuild logs to for failed builds" echo " -g | --generic - instructs script to build for generic architecture target" echo " -h | --help - display this usage information" echo " -x | --http-proxy URL - provides URL for the environment variable http_proxy" echo " -y | --https-proxy URL - provides URL for the environment variable https_proxy" + echo " --shared-fs-path - path to directory on shared filesystem that can be used" } function copy_build_log() { @@ -70,6 +72,10 @@ while [[ $# -gt 0 ]]; do export build_logs_dir="${2}" shift 2 ;; + --shared-fs-path) + export shared_fs_path="${2}" + shift 2 + ;; -*|--*) echo "Error: Unknown option: $1" >&2 exit 1 @@ -160,6 +166,14 @@ fi echo ">> Configuring EasyBuild..." source $TOPDIR/configure_easybuild +if [ ! -z "${shared_fs_path}" ]; then + shared_eb_sourcepath=${shared_fs_path}/easybuild/sources + echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path" + export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH} +fi + +${EB} --show-config + echo ">> Setting up \$MODULEPATH..." # make sure no modules are loaded module --force purge diff --git a/bot/build.sh b/bot/build.sh index bd7b2fa8d8..1e7aac49c1 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -64,6 +64,18 @@ else export SINGULARITY_BIND="${SINGULARITY_BIND},${BUILD_LOGS_DIR}" fi +# check if path to directory on shared filesystem is specified, +# and use it as location for source tarballs used by EasyBuild if so +SHARED_FS_PATH=$(cfg_get_value "site_config" "shared_fs_path") +echo "bot/build.sh: SHARED_FS_PATH='${SHARED_FS_PATH}'" +# if $SHARED_FS_PATH is set, add it to $SINGULARITY_BIND so the path is available in the build container +mkdir -p ${SHARED_FS_PATH} +if [[ -z ${SINGULARITY_BIND} ]]; then + export SINGULARITY_BIND="${SHARED_FS_PATH}" +else + export SINGULARITY_BIND="${SINGULARITY_BIND},${SHARED_FS_PATH}" +fi + SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then @@ -167,6 +179,7 @@ if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then INSTALL_SCRIPT_ARGS+=("--generic") fi [[ ! -z ${BUILD_LOGS_DIR} ]] && INSTALL_SCRIPT_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}") +[[ ! -z ${SHARED_FS_PATH} ]] && INSTALL_SCRIPT_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}") # create tmp file for output of build step build_outerr=$(mktemp build.outerr.XXXX) From 24146ee35424ee5f9e3c621611a1df13118d1315 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Sep 2023 20:18:36 +0200 Subject: [PATCH 15/16] {2023.06}[system] ReFrame 4.3.3 --- eessi-2023.06-eb-4.8.1-system.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eessi-2023.06-eb-4.8.1-system.yml b/eessi-2023.06-eb-4.8.1-system.yml index 8966fbfb7a..1e9664e10f 100644 --- a/eessi-2023.06-eb-4.8.1-system.yml +++ b/eessi-2023.06-eb-4.8.1-system.yml @@ -1,3 +1,6 @@ easyconfigs: # rebuilt with EasyBuild v4.8.1 (now wraps around Java/11.0.20) - Java-11.eb + - ReFrame-4.3.3.eb: + options: + from-pr: 18851 From c2b6abae4cc882087c4f78972bfa20426c4a11fc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Sep 2023 20:42:34 +0200 Subject: [PATCH 16/16] set shared_fs_path for bot on AWS CitC Slurm cluster --- bot/bot-eessi-aws-citc.cfg | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg index 0a91b12de2..a5e53d82c1 100644 --- a/bot/bot-eessi-aws-citc.cfg +++ b/bot/bot-eessi-aws-citc.cfg @@ -44,6 +44,10 @@ command_response_fmt = # name of the job script used for building an EESSI stack build_job_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/bot-build.slurm +# path to directory on shared filesystem that can be used for sharing data across build jobs +# (for example source tarballs used by EasyBuild) +shared_fs_path = /mnt/shared/home/bot/shared + # Path (directory) to which build logs for (only) failing builds should be copied by bot/build.sh script build_logs_dir = /mnt/shared/bot-build-logs