From 9e861655020ef8abcc9233dfd69a3d26070c33d8 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Fri, 6 Oct 2023 12:18:34 +0000 Subject: [PATCH 01/10] {2023.06}[foss/2022a] ASE V3.22.1 --- eessi-2023.06-eb-4.8.1-2022a.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 eessi-2023.06-eb-4.8.1-2022a.yml diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml new file mode 100644 index 0000000000..56424735f3 --- /dev/null +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - ASE-3.22.1-foss-2022a.eb From 17936e88a127c5db3f6256a01a79075989f5aad6 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Fri, 6 Oct 2023 19:37:16 +0000 Subject: [PATCH 02/10] added from-pr to Pillow as a trial --- .github/workflows/test_eessi.yml | 40 +++++++++++++++++++++++--------- EESSI-pilot-install-software.sh | 4 ++-- check_missing_installations.sh | 3 +++ eessi-2023.06-eb-4.7.2-2021a.yml | 4 ++++ eessi-2023.06-eb-4.8.1-2022a.yml | 10 ++++++++ 5 files changed, 48 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 72dc642c7d..832d4cf74f 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -4,14 +4,14 @@ on: [push, pull_request, workflow_dispatch] permissions: contents: read # to fetch code (actions/checkout) jobs: - eessi_pilot_repo: + pilot: runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: EESSI_VERSION: - 2023.06 - EESSI_SOFTWARE_SUBDIR_OVERRIDE: + EESSI_SOFTWARE_SUBDIR: - aarch64/generic - x86_64/amd/zen2 - x86_64/intel/broadwell @@ -25,6 +25,7 @@ jobs: - eessi-2023.06-eb-4.7.2-2022b.yml - eessi-2023.06-eb-4.7.2-system.yml - eessi-2023.06-eb-4.8.0-system.yml + - eessi-2023.06-eb-4.8.1-2022a.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 @@ -36,38 +37,55 @@ jobs: cvmfs_http_proxy: DIRECT cvmfs_repositories: pilot.nessi.no - - name: Test check_missing_installations.sh script + - name: Test check_missing_installations.sh script with EESSI_SOFTWARE_SUBDIR_OVERRIDE + if: '!cancelled()' + run: | + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} + source /cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}}/init/bash + module load EasyBuild + eb --version + export EESSI_PREFIX=/cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}} + export EESSI_OS_TYPE=linux + env | grep ^EESSI | sort + echo "just run check_missing_installations.sh (should use ${{matrix.EASYSTACK_FILE}})" + ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}} + + - name: Test check_missing_installations.sh script without EESSI_SOFTWARE_SUBDIR_OVERRIDE + if: '!cancelled()' run: | - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} source /cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild - which eb eb --version export EESSI_PREFIX=/cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort - echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" + echo "just run check_missing_installations.sh (should use ${{matrix.EASYSTACK_FILE}})" ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}} - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) + if: '!cancelled()' run: | - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} source /cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild which eb eb --version export EESSI_PREFIX=/cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort - echo "modify easystack file by adding a missing package (GCC/8.3.0)" - echo " - GCC-8.3.0:" >> ${{matrix.EASYSTACK_FILE}} - tail -n 5 ${{matrix.EASYSTACK_FILE}} + # create dummy easystack file with a single entry (something that is not installed in EESSI) + easystack_file="test.yml" + echo "easyconfigs:" > ${easystack_file} + echo " - GCC-8.3.0:" >> ${easystack_file} + echo "created easystack file '${easystack_file}' with a missing installation (GCC/8.3.0):" + cat ${easystack_file} # note, check_missing_installations.sh exits 1 if a package was # missing, which is intepreted as false (exit code based, not # boolean logic), hence when the script exits 0 if no package was # missing it is interpreted as true, thus the test did not capture # the missing package - if ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}}; then + if ./check_missing_installations.sh ${easystack_file}; then echo "did NOT capture missing package; test FAILED" exit 1 else diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index c6c51e7abc..026919157f 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -172,8 +172,6 @@ if [ ! -z "${shared_fs_path}" ]; then export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH} fi -${EB} --show-config - echo ">> Setting up \$MODULEPATH..." # make sure no modules are loaded module --force purge @@ -200,6 +198,8 @@ for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[ # load EasyBuild module (will be installed if it's not available yet) source ${TOPDIR}/load_easybuild_module.sh ${eb_version} + ${EB} --show-config + echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." if [ -f ${easystack_file} ]; then diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 3627d1d0b5..e927f14701 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -22,6 +22,9 @@ source $TOPDIR/scripts/utils.sh source $TOPDIR/configure_easybuild +echo ">> Active EasyBuild configuration when checking for missing installations:" +${EB:-eb} --show-config + echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." eb_missing_out=$LOCAL_TMPDIR/eb_missing.out ${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out} diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index dfabf377ea..28612c5566 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -21,3 +21,7 @@ easyconfigs: options: from-pr: 18446 - WRF-4.3-foss-2021a-dmpar.eb + - Yambo-5.1.1-foss-2021a.eb: + options: + from-pr: 18905 + diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index 56424735f3..51ddd53302 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -1,2 +1,12 @@ easyconfigs: + - BAMM-2.5.0-foss-2022a.eb + - BamTools-2.5.2-GCC-11.3.0.eb + - BCFtools-1.15.1-GCC-11.3.0.eb + - Bowtie2-2.4.5-GCC-11.3.0.eb + - FastTree-2.1.11-GCCcore-11.3.0.eb + - SAMtools-1.16.1-GCC-11.3.0.eb + - Pillow-9.1.1-GCCcore-11.3.0.eb: + # avoid that hardcoded paths like /usr/include are used in build commands + options: + from-pr: 18881 - ASE-3.22.1-foss-2022a.eb From f8cadf0120473f145717f0c52955e8d2a51e4981 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Fri, 6 Oct 2023 19:44:14 +0000 Subject: [PATCH 03/10] sync local branch with target --- eessi-2023.06-eb-4.8.1-2022a.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index 9fdaae20b3..e6ea8299f0 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -5,14 +5,11 @@ easyconfigs: - Bowtie2-2.4.5-GCC-11.3.0.eb - FastTree-2.1.11-GCCcore-11.3.0.eb - SAMtools-1.16.1-GCC-11.3.0.eb -<<<<<<< HEAD + - gzip-1.12-GCCcore-11.3.0.eb + - lz4-1.9.3-GCCcore-11.3.0.eb + - zstd-1.5.2-GCCcore-11.3.0.eb - Pillow-9.1.1-GCCcore-11.3.0.eb: # avoid that hardcoded paths like /usr/include are used in build commands options: from-pr: 18881 - ASE-3.22.1-foss-2022a.eb -======= - - gzip-1.12-GCCcore-11.3.0.eb - - lz4-1.9.3-GCCcore-11.3.0.eb - - zstd-1.5.2-GCCcore-11.3.0.eb ->>>>>>> 181dd0ac845922c4a35db32e874c75995735dd95 From 6ad751727234b008f780f19c34b649e455347061 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 9 Oct 2023 13:24:12 +0000 Subject: [PATCH 04/10] {2023.06}[foss/2022a] MUMPS-metis V5.5.1 --- eessi-2023.06-eb-4.8.1-2022a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index 2da265489c..001207e8a6 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -50,3 +50,4 @@ easyconfigs: - PCRE2-10.40-GCCcore-11.3.0.eb - Tk-8.6.12-GCCcore-11.3.0.eb - GROMACS-2023.1-foss-2022a.eb + - MUMPS-5.5.1-foss-2022a-metis.eb From 9c5daa91e67533dc839ce82609c5496a6afcc0f2 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 11 Oct 2023 09:49:14 +0000 Subject: [PATCH 05/10] Added requested changes --- eb_hooks.py | 18 ++++++++++++++++++ eessi-2023.06-eb-4.8.1-2022a.yml | 4 ++++ 2 files changed, 22 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 084b78e5e0..46a7a5a127 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -66,6 +66,23 @@ def parse_hook(ec, *args, **kwargs): PARSE_HOOKS[ec.name](ec, eprefix) +def parse_hook_pillow_set_cpath_library_path(ec, eprefix): + """Get EESSI_CPATH environment variable from the environment""" + if ec.name == 'Pillow': + EESSI_CPATH = os.getenv('EESSI_EPREFIX') + '/usr/include' + EESSI_LIB_PATH = os.getenv('EESSI_EPREFIX') + '/usr/lib64' + print_msg("NOTE:Pillow has zlib as a dependancy,the original CPATH value: (%s) has been extended with (%s)", + os.getenv('CPATH'), EESSI_CPATH) + print_msg("NOTE:Pillow has zlib as a dependancy,the original LIBRARY_PATH value: (%s) has been extended with (%s)", + os.getenv('LIBRARY_PATH'), EESSI_LIB_PATH) + ec.log.info("NOTE:Pillow has zlib as a dependancy,the original CPATH value: (%s) has been extended with (%s)", + os.getenv('CPATH'), EESSI_CPATH) + ec.log.info("NOTE:Pillow has zlib as a dependancy,the original LIBRARY_VALUE value: (%s) has been extended with (%s)", + os.getenv('LIBRARY_PATH'), EESSI_LIB_PATH) + os.environ['CPATH'] = os.pathsep.join(filter(None,[os.environ.get('CPATH',''), EESSI_CPATH])) + os.environ['LIBRARY_PATH'] = os.pathsep.join(filter(None,[os.environ.get('LIBRARY_PATH',''), EESSI_LIB_PATH])) + + def pre_prepare_hook(self, *args, **kwargs): """Main pre-prepare hook: trigger custom functions.""" @@ -273,6 +290,7 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): 'fontconfig': parse_hook_fontconfig_add_fonts, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'UCX': parse_hook_ucx_eprefix, + 'Pillow': parse_hook_pillow_set_cpath_library_path, } POST_PREPARE_HOOKS = { diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index 001207e8a6..062f507762 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -51,3 +51,7 @@ easyconfigs: - Tk-8.6.12-GCCcore-11.3.0.eb - GROMACS-2023.1-foss-2022a.eb - MUMPS-5.5.1-foss-2022a-metis.eb + - Pillow-9.1.1-GCCcore-11.3.0.eb: + # Uses a custom hook since has zlib as dependency which has hard coded header and library path within Pillow code. + options: + from-pr: 18881 From 6ff2f289fadec8e3e97606d759b6cb11d3b4d519 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 11 Oct 2023 10:34:19 +0000 Subject: [PATCH 06/10] Added additional requested changes --- eb_hooks.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 46a7a5a127..13bbb43b3d 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -66,23 +66,6 @@ def parse_hook(ec, *args, **kwargs): PARSE_HOOKS[ec.name](ec, eprefix) -def parse_hook_pillow_set_cpath_library_path(ec, eprefix): - """Get EESSI_CPATH environment variable from the environment""" - if ec.name == 'Pillow': - EESSI_CPATH = os.getenv('EESSI_EPREFIX') + '/usr/include' - EESSI_LIB_PATH = os.getenv('EESSI_EPREFIX') + '/usr/lib64' - print_msg("NOTE:Pillow has zlib as a dependancy,the original CPATH value: (%s) has been extended with (%s)", - os.getenv('CPATH'), EESSI_CPATH) - print_msg("NOTE:Pillow has zlib as a dependancy,the original LIBRARY_PATH value: (%s) has been extended with (%s)", - os.getenv('LIBRARY_PATH'), EESSI_LIB_PATH) - ec.log.info("NOTE:Pillow has zlib as a dependancy,the original CPATH value: (%s) has been extended with (%s)", - os.getenv('CPATH'), EESSI_CPATH) - ec.log.info("NOTE:Pillow has zlib as a dependancy,the original LIBRARY_VALUE value: (%s) has been extended with (%s)", - os.getenv('LIBRARY_PATH'), EESSI_LIB_PATH) - os.environ['CPATH'] = os.pathsep.join(filter(None,[os.environ.get('CPATH',''), EESSI_CPATH])) - os.environ['LIBRARY_PATH'] = os.pathsep.join(filter(None,[os.environ.get('LIBRARY_PATH',''), EESSI_LIB_PATH])) - - def pre_prepare_hook(self, *args, **kwargs): """Main pre-prepare hook: trigger custom functions.""" @@ -194,6 +177,23 @@ def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!") +def parse_hook_pillow_set_cpath_library_path(ec, eprefix): + """Get CPATH and LIBRARY_PATH environment variables from the environment""" + if ec.name == 'Pillow': + EESSI_CPATH = os.getenv('EESSI_EPREFIX') + '/usr/include' + EESSI_LIB_PATH = os.getenv('EESSI_EPREFIX') + '/usr/lib64' + print_msg("NOTE: Pillow has zlib as a dependancy, The original CPATH value: (%s) has been extended with (%s)", + os.getenv('CPATH'), EESSI_CPATH) + print_msg("NOTE: Pillow has zlib as a dependancy, The original LIBRARY_PATH value: (%s) has been extended with (%s)", + os.getenv('LIBRARY_PATH'), EESSI_LIB_PATH) + ec.log.info("NOTE: Pillow has zlib as a dependancy, The original CPATH value: (%s) has been extended with (%s)", + os.getenv('CPATH'), EESSI_CPATH) + ec.log.info("NOTE: Pillow has zlib as a dependancy, The original LIBRARY_VALUE value: (%s) has been extended with (%s)", + os.getenv('LIBRARY_PATH'), EESSI_LIB_PATH) + os.environ['CPATH'] = os.pathsep.join(filter(None,[os.environ.get('CPATH',''), EESSI_CPATH])) + os.environ['LIBRARY_PATH'] = os.pathsep.join(filter(None,[os.environ.get('LIBRARY_PATH',''), EESSI_LIB_PATH])) + + def parse_hook_ucx_eprefix(ec, eprefix): """Make UCX aware of compatibility layer via additional configuration options.""" if ec.name == 'UCX': @@ -289,8 +289,8 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, - 'UCX': parse_hook_ucx_eprefix, 'Pillow': parse_hook_pillow_set_cpath_library_path, + 'UCX': parse_hook_ucx_eprefix, } POST_PREPARE_HOOKS = { From 6b80c9fe16449c9c600bcf28a33945a2c40a4e1c Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 11 Oct 2023 12:18:23 +0000 Subject: [PATCH 07/10] Added latest modifications --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 13bbb43b3d..9031d97831 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -178,7 +178,7 @@ def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): def parse_hook_pillow_set_cpath_library_path(ec, eprefix): - """Get CPATH and LIBRARY_PATH environment variables from the environment""" + """Extend CPATH and LIBRARY_PATH environment variables using EESSI_EPREFIX.""" if ec.name == 'Pillow': EESSI_CPATH = os.getenv('EESSI_EPREFIX') + '/usr/include' EESSI_LIB_PATH = os.getenv('EESSI_EPREFIX') + '/usr/lib64' From 30cd9d737a0e983e640e24f455c0d07f47ad2729 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Thu, 12 Oct 2023 11:57:16 +0000 Subject: [PATCH 08/10] {2023.06}[foss/2022a] ParaView-mpi V5.10.1 --- eessi-2023.06-eb-4.8.1-2022a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index 062f507762..f2d316c15b 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -55,3 +55,4 @@ easyconfigs: # Uses a custom hook since has zlib as dependency which has hard coded header and library path within Pillow code. options: from-pr: 18881 + - ParaView-5.10.1-foss-2022a-mpi.eb From 9679400e498f00765aaba13a6130d1abf887ddfe Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 17 Oct 2023 07:36:52 +0000 Subject: [PATCH 09/10] added timeout value --- eessi-2023.06-eb-4.8.1-2022a.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index f2d316c15b..862e4ac267 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -56,3 +56,5 @@ easyconfigs: options: from-pr: 18881 - ParaView-5.10.1-foss-2022a-mpi.eb + options: + download-timeout: 1000 From c25aa007814bd7db85cd1a2edb6fd6856bdb262d Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 17 Oct 2023 07:37:39 +0000 Subject: [PATCH 10/10] added timeout value --- eessi-2023.06-eb-4.8.1-2022a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index 862e4ac267..3079a58b60 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -55,6 +55,6 @@ easyconfigs: # Uses a custom hook since has zlib as dependency which has hard coded header and library path within Pillow code. options: from-pr: 18881 - - ParaView-5.10.1-foss-2022a-mpi.eb + - ParaView-5.10.1-foss-2022a-mpi.eb: options: download-timeout: 1000