From 793e9f8ec925aa09b020f72d4acffa1ce0b400eb Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 28 Feb 2024 13:20:54 +0000 Subject: [PATCH 01/10] {2023.06}[foss/2023a] dask v2023.9.2 --- .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 + eb_hooks.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 157a47a49e..d444b1f8ae 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -29,3 +29,4 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19554 options: from-pr: 19554 + - dask-2023.9.2-foss-2023a.eb diff --git a/eb_hooks.py b/eb_hooks.py index d29a837339..4ed72a12ac 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -204,6 +204,18 @@ def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!") +def parse_hook_Pillow_SIMD_harcoded_paths(ec, eprefix): + # patch setup.py to prefix hardcoded /usr/* and /lib paths with value of %(sysroot) template + # (which will be empty if EasyBuild is not configured to use an alternate sysroot); + # see also https://gitlab.com/eessi/support/-/issues/9 + if ec.name == 'Pillow-SIMD': + ec.update('preinstallopts', """sed -i 's@"/usr/@"%(sysroot)s/usr/@g' setup.py && """) + ec.update('preinstallopts', """sed -i 's@"/lib@"%(sysroot)s/lib@g' setup.py && """) + print_msg("Using custom configure options for %s: %s", ec.name, ec['preinstallopts']) + else: + raise EasyBuildError("Pillow-SIMD-specific hook triggered for non-Pillow-SIMD easyconfig?!") + + def parse_hook_pybind11_replace_catch2(ec, eprefix): """ Replace Catch2 build dependency in pybind11 easyconfigs with one that doesn't use system toolchain. @@ -237,7 +249,6 @@ def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix): def parse_hook_ucx_eprefix(ec, eprefix): - """Make UCX aware of compatibility layer via additional configuration options.""" if ec.name == 'UCX': ec.update('configopts', '--with-sysroot=%s' % eprefix) ec.update('configopts', '--with-rdmacm=%s' % os.path.join(eprefix, 'usr')) @@ -582,6 +593,7 @@ def inject_gpu_property(ec): 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, + 'Pillow-SIMD' : parse_hook_Pillow_SIMD_harcoded_paths, 'pybind11': parse_hook_pybind11_replace_catch2, 'Qt5': parse_hook_qt5_check_qtwebengine_disable, 'UCX': parse_hook_ucx_eprefix, From 68a766fe8637c7c90b7c667aa0f738b8807f115f Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 28 Feb 2024 13:27:38 +0000 Subject: [PATCH 02/10] fixed info --- eb_hooks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 4ed72a12ac..d30700267b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -205,6 +205,7 @@ def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): def parse_hook_Pillow_SIMD_harcoded_paths(ec, eprefix): + """Make Pillow-SIMD aware of sysroot.""" # patch setup.py to prefix hardcoded /usr/* and /lib paths with value of %(sysroot) template # (which will be empty if EasyBuild is not configured to use an alternate sysroot); # see also https://gitlab.com/eessi/support/-/issues/9 @@ -249,6 +250,7 @@ def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix): def parse_hook_ucx_eprefix(ec, eprefix): + """Make UCX aware of compatibility layer via additional configuration options.""" if ec.name == 'UCX': ec.update('configopts', '--with-sysroot=%s' % eprefix) ec.update('configopts', '--with-rdmacm=%s' % os.path.join(eprefix, 'usr')) From 3cdf80390fa3150bcde82427537776c22bf7bb25 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 28 Feb 2024 17:33:43 +0000 Subject: [PATCH 03/10] removed the hook and added --from-pr --- .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 5 ++++- eb_hooks.py | 14 -------------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index d444b1f8ae..983d59cc04 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -29,4 +29,7 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19554 options: from-pr: 19554 - - dask-2023.9.2-foss-2023a.eb + - dask-2023.9.2-foss-2023a.eb: + options: + from-pr: 19996 + diff --git a/eb_hooks.py b/eb_hooks.py index d30700267b..d29a837339 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -204,19 +204,6 @@ def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!") -def parse_hook_Pillow_SIMD_harcoded_paths(ec, eprefix): - """Make Pillow-SIMD aware of sysroot.""" - # patch setup.py to prefix hardcoded /usr/* and /lib paths with value of %(sysroot) template - # (which will be empty if EasyBuild is not configured to use an alternate sysroot); - # see also https://gitlab.com/eessi/support/-/issues/9 - if ec.name == 'Pillow-SIMD': - ec.update('preinstallopts', """sed -i 's@"/usr/@"%(sysroot)s/usr/@g' setup.py && """) - ec.update('preinstallopts', """sed -i 's@"/lib@"%(sysroot)s/lib@g' setup.py && """) - print_msg("Using custom configure options for %s: %s", ec.name, ec['preinstallopts']) - else: - raise EasyBuildError("Pillow-SIMD-specific hook triggered for non-Pillow-SIMD easyconfig?!") - - def parse_hook_pybind11_replace_catch2(ec, eprefix): """ Replace Catch2 build dependency in pybind11 easyconfigs with one that doesn't use system toolchain. @@ -595,7 +582,6 @@ def inject_gpu_property(ec): 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, - 'Pillow-SIMD' : parse_hook_Pillow_SIMD_harcoded_paths, 'pybind11': parse_hook_pybind11_replace_catch2, 'Qt5': parse_hook_qt5_check_qtwebengine_disable, 'UCX': parse_hook_ucx_eprefix, From cf7705643865f85b0e73f5185952a400f41c95f6 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Thu, 29 Feb 2024 13:39:14 +0100 Subject: [PATCH 04/10] remove-blanks --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 983d59cc04..7d9c331fcf 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -32,4 +32,3 @@ easyconfigs: - dask-2023.9.2-foss-2023a.eb: options: from-pr: 19996 - From 887e102631ef5fbfadffef744c7724e28a01217f Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 29 Feb 2024 13:50:41 +0100 Subject: [PATCH 05/10] Update easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 7d9c331fcf..d187d8e92c 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -29,6 +29,7 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19554 options: from-pr: 19554 - - dask-2023.9.2-foss-2023a.eb: + - Pillow-SIMD-9.5.0-GCCcore-12.3.0.eb: options: from-pr: 19996 + - dask-2023.9.2-foss-2023a.eb From 938e52e66b529458fad9984f79bce156dc43d023 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 29 Feb 2024 14:16:17 +0100 Subject: [PATCH 06/10] Update easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index d187d8e92c..0a0a625c35 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -31,5 +31,6 @@ easyconfigs: from-pr: 19554 - Pillow-SIMD-9.5.0-GCCcore-12.3.0.eb: options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19996 from-pr: 19996 - dask-2023.9.2-foss-2023a.eb From 301ab593867335700b576cc9949fe58c408325e7 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 29 Feb 2024 18:12:22 +0100 Subject: [PATCH 07/10] Make check_missing_installations.sh check against develop branch of EasyBuild --- check_missing_installations.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 5ea7c5a4f5..8845adb161 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -18,6 +18,13 @@ easystack=$1 LOCAL_TMPDIR=$(mktemp -d) +# Clone the develop branch of EasyBuild and use that to search for easyconfigs +git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs +EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs + +# All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr +grep -v from-pr ${easystack} > ${LOCAL_TMPDIR}/${easystack} + source $TOPDIR/scripts/utils.sh source $TOPDIR/configure_easybuild @@ -27,7 +34,7 @@ ${EB:-eb} --show-config echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." eb_missing_out=$LOCAL_TMPDIR/eb_missing.out -${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out} +${EB:-eb} --easystack ${LOCAL_TMPDIR}/${easystack} --missing 2>&1 | tee ${eb_missing_out} exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." From 93d1d7df3c0659e9886bac3cd0d4c994e7a82d09 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 29 Feb 2024 18:19:46 +0100 Subject: [PATCH 08/10] Use filename rather than full path --- check_missing_installations.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 8845adb161..6c38e8e37f 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -23,7 +23,8 @@ git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $ EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs # All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr -grep -v from-pr ${easystack} > ${LOCAL_TMPDIR}/${easystack} +tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack}) +grep -v from-pr ${easystack} > ${tmp_easystack} source $TOPDIR/scripts/utils.sh @@ -34,7 +35,7 @@ ${EB:-eb} --show-config echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." eb_missing_out=$LOCAL_TMPDIR/eb_missing.out -${EB:-eb} --easystack ${LOCAL_TMPDIR}/${easystack} --missing 2>&1 | tee ${eb_missing_out} +${EB:-eb} --easystack ${tmp_easystack} --missing 2>&1 | tee ${eb_missing_out} exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." From f6259dc563bcb1cad3ee962f56501872e53c0cba Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 29 Feb 2024 18:26:16 +0100 Subject: [PATCH 09/10] export envvar --- check_missing_installations.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 6c38e8e37f..c902fa8184 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -20,7 +20,7 @@ LOCAL_TMPDIR=$(mktemp -d) # Clone the develop branch of EasyBuild and use that to search for easyconfigs git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs -EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs +export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs # All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack}) From 78ba8d8e304ec0ae4c82721e3e5fae7e25402f91 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 1 Mar 2024 12:53:07 +0100 Subject: [PATCH 10/10] Adding OSU microbenchmarks with GPU support for 2023a, and one without GPU support for 2023b --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 0a0a625c35..0d8e71e86c 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -34,3 +34,5 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19996 from-pr: 19996 - dask-2023.9.2-foss-2023a.eb + - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb + - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb