From 01dfde905718a1eabad623465d52dbc7f3d8ef0c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 4 Jun 2024 12:55:18 +0200 Subject: [PATCH 01/11] add hooks for fixing sanitycheck and augmenting modluafooter --- eb_hooks.py | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index e4a957acdc..14c7c30d91 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -5,6 +5,7 @@ import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import obtain_config_guess +from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option, update_build_option @@ -349,6 +350,32 @@ def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") +def parse_hook_librosa_custom_ctypes(ec, *args, **kwargs): + """ + Add exts_filter to soundfile extension in exts_list + """ + if ec.name == 'librosa' and ec.version in ('0.10.1',): + ec_dict = ec.asdict() + eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH') + custom_ctypes_path = eessi_software_path.replace('versions', 'host_injections', 1) + custom_ctypes_path = custom_ctypes_path.replace('software', 'extra', 1) + custom_ctypes_path = os.path.join(custom_ctypes_path, "custom_ctypes") + ebpythonprefixes = "EBPYTHONPREFIXES=%s" % custom_ctypes_path + exts_list_new = [] + for item in ec_dict['exts_list']: + if item[0] == 'soundfile': + ext_dict = item[2] + ext_dict['exts_filter'] = (ebpythonprefixes + ' ' + EXTS_FILTER_PYTHON_PACKAGES[0], + EXTS_FILTER_PYTHON_PACKAGES[1]) + exts_list_new.append((item[0], item[1], ext_dict)) + else: + exts_list_new.append(item) + ec['exts_list'] = exts_list_new + print_msg("New exts_list: '%s'", ec['exts_list']) + else: + raise EasyBuildError("librosa/0.10.1-specific hook triggered for non-librosa/0.10.1 easyconfig?!") + + def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwargs): """ Solve issues with compiling or running the tests on both @@ -852,6 +879,37 @@ def inject_gpu_property(ec): return ec +def pre_module_hook(self, *args, **kwargs): + """Main pre-module-check hook: trigger custom functions based on software name.""" + if self.name in PRE_MODULE_HOOKS: + PRE_MODULE_HOOKS[self.name](self, *args, **kwargs) + + +def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs): + """ + Add EBPYTHONPREFIXES to modluafooter + """ + if self.name == 'librosa' and self.version == '0.10.1': + eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH') + custom_ctypes_path = eessi_software_path.replace('versions', 'host_injections', 1) + custom_ctypes_path = custom_ctypes_path.replace('software', 'extra', 1) + custom_ctypes_path = os.path.join(custom_ctypes_path, 'custom_ctypes') + key = 'modluafooter' + values = ['prepend_path("EBPYTHONPREFIXES","%s")' % (custom_ctypes_path)] + print_msg("Adding '%s' to modluafooter", values[0]) + if not key in self.cfg: + self.cfg[key] = '\n'.join(values) + else: + new_value = self.cfg[key] + for value in values: + if not value in new_value: + new_value = '\n'.join([new_value, value]) + self.cfg[key] = new_value + print_msg("Full modluafooter is '%s'", self.cfg[key]) + else: + raise EasyBuildError("librosa/0.10.1-specific hook triggered for non-librosa/0.10.1 easyconfig?!") + + PARSE_HOOKS = { 'casacore': parse_hook_casacore_disable_vectorize, 'CGAL': parse_hook_cgal_toolchainopts_precise, @@ -859,6 +917,7 @@ def inject_gpu_property(ec): 'GPAW': parse_hook_gpaw_harcoded_path, 'ImageMagick': parse_hook_imagemagick_add_dependency, 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, + 'librosa': parse_hook_librosa_custom_ctypes, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'Pillow-SIMD' : parse_hook_Pillow_SIMD_harcoded_paths, 'pybind11': parse_hook_pybind11_replace_catch2, @@ -909,3 +968,7 @@ def inject_gpu_property(ec): 'cuDNN': post_sanitycheck_cudnn, 'cuTENSOR': post_sanitycheck_cutensor, } + +PRE_MODULE_HOOKS = { + 'librosa': pre_module_hook_librosa_augment_modluafooter, +} From 8a6b28399898ee18853b5fa7e63744fd4ee341a4 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 4 Jun 2024 13:18:48 +0200 Subject: [PATCH 02/11] script to install custom ctypes - copies the script to a new location on CVMFS scripts/extra - calls script in EESSI-install-software.sh - script itself installs custom ctypes library under host_injections --- EESSI-install-software.sh | 4 ++ install_scripts.sh | 6 ++ scripts/extra/install_custom_ctypes.sh | 92 ++++++++++++++++++++++++++ 3 files changed, 102 insertions(+) create mode 100755 scripts/extra/install_custom_ctypes.sh diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index ca6fed71d5..fced4ee587 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -256,6 +256,10 @@ if command_exists "nvidia-smi"; then ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh fi +# Install extra software that is needed (e.g., for providing a custom ctypes +# library when needed) +${EESSI_PREFIX}/scripts/extra/install_custom_ctypes.sh --temp-dir /tmp/temp + # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') if [ -z "${changed_easystacks}" ]; then diff --git a/install_scripts.sh b/install_scripts.sh index 07643a39e6..000ad99444 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -116,6 +116,12 @@ nvidia_files=( ) copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_files[@]}" +# Copy files for the scripts/extra directory +extra_files=( + install_custom_ctypes.sh +) +copy_files_by_list ${TOPDIR}/scripts/extra ${INSTALL_PREFIX}/scripts/extra "${extra_files[@]}" + # Copy over EasyBuild hooks file used for installations hook_files=( eb_hooks.py diff --git a/scripts/extra/install_custom_ctypes.sh b/scripts/extra/install_custom_ctypes.sh new file mode 100755 index 0000000000..cd0159cf85 --- /dev/null +++ b/scripts/extra/install_custom_ctypes.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +# This script can be used to install custom ctypes under +# CVMFS_REPO/host_injections/VERSION/extra/OS_TYPE/SOFTWARE_SUBDIR/software/custom_ctypes +# and then set EESSI_USE_CUSTOM_CTYPES_DIR=CVMFS_REPO/host_injections/VERSION/extra/OS_TYPE/SOFTWARE_SUBDIR/custom_ctypes +# before loading a module that needs the custom ctypes implementation +# The custom ctypes is downloaded from https://github.com/NorESSI/custom_ctypes which is a fork of +# https://github.com/ComputeCanada/custom_ctypes + +# The `host_injections` directory is a variant symlink that by default points to +# `/opt/eessi`, unless otherwise defined in the local CVMFS configuration (see +# https://cvmfs.readthedocs.io/en/stable/cpt-repo.html#variant-symlinks). For the +# installation to be successful, this directory needs to be writeable by the user +# executing this script. + +# Initialise our bash functions +TOPDIR=$(dirname $(realpath ${BASH_SOURCE})) +source "${TOPDIR}"/utils.sh + +# Function to display help message +show_help() { + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --help Display this help message" + echo " -t, --temp-dir /path/to/tmpdir Specify a location to use for temporary" + echo " storage during the installation" +} + +# Initialize variables +TEMP_DIR= + +# Parse command-line options +while [[ $# -gt 0 ]]; do + case "$1" in + --help) + show_help + exit 0 + ;; + -t|--temp-dir) + if [ -n "$2" ]; then + TEMP_DIR="$2" + shift 2 + else + echo "Error: Argument required for $1" + show_help + exit 1 + fi + ;; + *) + show_help + fatal_error "Error: Unknown option: $1" + ;; + esac +done + +# Make sure NESSI is initialised +check_eessi_initialised + +# As an installation location just use $EESSI_SOFTWARE_PATH but replacing `versions` with `host_injections` and +# `software` with `extra` +# also append `/custom_ctypes` +NESSI_SITE_INSTALL=${EESSI_SOFTWARE_PATH/versions/host_injections} +NESSI_SITE_INSTALL=${NESSI_SITE_INSTALL/software/extra}/custom_ctypes + +# we need a directory we can use for temporary storage +if [[ -z "${TEMP_DIR}" ]]; then + tmpdir=$(mktemp -d) +else + mkdir -p ${TEMP_DIR} + tmpdir=$(mktemp -d --tmpdir=${TEMP_DIR} custom_ctypes.XXX) + if [[ ! -d "$tmpdir" ]] ; then + fatal_error "Could not create directory ${tmpdir}" + fi +fi +echo "Created temporary directory '${tmpdir}'" + +# check if custom_ctypes has already been installed +if [[ -d ${NESSI_SITE_INSTALL}/lib ]]; then + fatal_error "Error: Installation of custom_ctypes already found at '${NESSI_SITE_INSTALL}'" +fi + +# download custom_ctypes to temp directory +wget https://github.com/NorESSI/custom_ctypes/archive/refs/heads/main.tar.gz -P ${tmpdir} + +# make sure target directory exists +mkdir -p ${NESSI_SITE_INSTALL} + +# unpack custom_ctypes to target directory +tar xvfz ${tmpdir}/main.tar.gz --strip-components=1 -C ${NESSI_SITE_INSTALL} + +# clean up tmpdir +rm -rf "${tmpdir}" From a9eda018276efde4cdb64ab2f5f4629c73610219 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 4 Jun 2024 13:22:59 +0200 Subject: [PATCH 03/11] {2023.06}[foss/2023a] librosa v0.10.1 --- .../pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 4f31c4dd08..c01439e791 100644 --- a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -55,3 +55,4 @@ easyconfigs: - PyTorch-2.1.2-foss-2023a-CUDA-12.1.1.eb: options: cuda-compute-capabilities: 6.0,6.1,7.0,7.5,8.0,8.6,8.9,9.0 + - librosa-0.10.1-foss-2023a.eb From e222519fb8313daefff0236341160c37835972b8 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 4 Jun 2024 14:22:28 +0200 Subject: [PATCH 04/11] fix location of utils.sh --- scripts/extra/install_custom_ctypes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/extra/install_custom_ctypes.sh b/scripts/extra/install_custom_ctypes.sh index cd0159cf85..e546f587a7 100755 --- a/scripts/extra/install_custom_ctypes.sh +++ b/scripts/extra/install_custom_ctypes.sh @@ -15,7 +15,7 @@ # Initialise our bash functions TOPDIR=$(dirname $(realpath ${BASH_SOURCE})) -source "${TOPDIR}"/utils.sh +source "${TOPDIR}"/../utils.sh # Function to display help message show_help() { From b28c28e0dda8df37f01a70ddc9ac3b1f50e074e3 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 4 Jun 2024 15:23:04 +0200 Subject: [PATCH 05/11] add a bit debug output --- scripts/extra/install_custom_ctypes.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/extra/install_custom_ctypes.sh b/scripts/extra/install_custom_ctypes.sh index e546f587a7..16d99145c7 100755 --- a/scripts/extra/install_custom_ctypes.sh +++ b/scripts/extra/install_custom_ctypes.sh @@ -13,6 +13,9 @@ # installation to be successful, this directory needs to be writeable by the user # executing this script. +# some logging +echo ">>> Running ${BASH_SOURCE}" + # Initialise our bash functions TOPDIR=$(dirname $(realpath ${BASH_SOURCE})) source "${TOPDIR}"/../utils.sh @@ -74,6 +77,10 @@ else fi echo "Created temporary directory '${tmpdir}'" +# some logging +echo ">>> Checking contents under '${NESSI_SITE_INSTALL}'" +tree -d ${NESSI_SITE_INSTALL} + # check if custom_ctypes has already been installed if [[ -d ${NESSI_SITE_INSTALL}/lib ]]; then fatal_error "Error: Installation of custom_ctypes already found at '${NESSI_SITE_INSTALL}'" From 0839b6851401708368860ae07d6895d1f1878539 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 4 Jun 2024 17:21:19 +0200 Subject: [PATCH 06/11] replace tree with ls --- scripts/extra/install_custom_ctypes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/extra/install_custom_ctypes.sh b/scripts/extra/install_custom_ctypes.sh index 16d99145c7..2bdc34a66c 100755 --- a/scripts/extra/install_custom_ctypes.sh +++ b/scripts/extra/install_custom_ctypes.sh @@ -79,7 +79,7 @@ echo "Created temporary directory '${tmpdir}'" # some logging echo ">>> Checking contents under '${NESSI_SITE_INSTALL}'" -tree -d ${NESSI_SITE_INSTALL} +ls -lR ${NESSI_SITE_INSTALL} # check if custom_ctypes has already been installed if [[ -d ${NESSI_SITE_INSTALL}/lib ]]; then From d16182937c98ddcd343c3551ff4ef3a79970220b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 4 Jun 2024 17:47:09 +0200 Subject: [PATCH 07/11] show symlink target for host_injections --- EESSI-install-software.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index fced4ee587..bfd47a5139 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -258,6 +258,7 @@ fi # Install extra software that is needed (e.g., for providing a custom ctypes # library when needed) +ls -l ${EESSI_CVMFS_REPO}/host_injections ${EESSI_PREFIX}/scripts/extra/install_custom_ctypes.sh --temp-dir /tmp/temp # use PR patch file to determine in which easystack files stuff was added From 119a6a57463bd795de906f956fbf8a6952015554 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 4 Jun 2024 18:36:45 +0200 Subject: [PATCH 08/11] tuning script output and behaviour --- EESSI-install-software.sh | 2 +- scripts/extra/install_custom_ctypes.sh | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index bfd47a5139..7d6cad7add 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -258,7 +258,7 @@ fi # Install extra software that is needed (e.g., for providing a custom ctypes # library when needed) -ls -l ${EESSI_CVMFS_REPO}/host_injections +echo "Location of host_injections: $(ls -l ${EESSI_CVMFS_REPO}/host_injections)" ${EESSI_PREFIX}/scripts/extra/install_custom_ctypes.sh --temp-dir /tmp/temp # use PR patch file to determine in which easystack files stuff was added diff --git a/scripts/extra/install_custom_ctypes.sh b/scripts/extra/install_custom_ctypes.sh index 2bdc34a66c..6392da00e5 100755 --- a/scripts/extra/install_custom_ctypes.sh +++ b/scripts/extra/install_custom_ctypes.sh @@ -77,13 +77,10 @@ else fi echo "Created temporary directory '${tmpdir}'" -# some logging -echo ">>> Checking contents under '${NESSI_SITE_INSTALL}'" -ls -lR ${NESSI_SITE_INSTALL} - # check if custom_ctypes has already been installed if [[ -d ${NESSI_SITE_INSTALL}/lib ]]; then - fatal_error "Error: Installation of custom_ctypes already found at '${NESSI_SITE_INSTALL}'" + echo "INFO: Installation of custom_ctypes already found at '${NESSI_SITE_INSTALL}'" + exit 0 fi # download custom_ctypes to temp directory From 8514bd417d736cd55eb7d3e00e23581ab94f1b46 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 5 Jun 2024 21:14:36 +0200 Subject: [PATCH 09/11] use .eb file, easystack file and standard location for custom_ctypes --- EESSI-install-software.sh | 5 +- eb_hooks.py | 8 +- scripts/extra/custom_ctypes-1.2.eb | 29 ++++++ .../extra/eessi-2023.06-extra-packages.yml | 2 + scripts/extra/install_custom_ctypes.sh | 96 ------------------- scripts/extra/install_extra_packages.sh | 94 ++++++++++++++++++ 6 files changed, 130 insertions(+), 104 deletions(-) create mode 100644 scripts/extra/custom_ctypes-1.2.eb create mode 100644 scripts/extra/eessi-2023.06-extra-packages.yml delete mode 100755 scripts/extra/install_custom_ctypes.sh create mode 100755 scripts/extra/install_extra_packages.sh diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 7d6cad7add..c0d0ce0088 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -258,8 +258,9 @@ fi # Install extra software that is needed (e.g., for providing a custom ctypes # library when needed) -echo "Location of host_injections: $(ls -l ${EESSI_CVMFS_REPO}/host_injections)" -${EESSI_PREFIX}/scripts/extra/install_custom_ctypes.sh --temp-dir /tmp/temp +cd scripts/extra +${EESSI_PREFIX}/scripts/extra/install_extra_packages.sh --temp-dir /tmp/temp --easystack eessi-2023.06-extra-packages.yml +cd - # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') diff --git a/eb_hooks.py b/eb_hooks.py index 14c7c30d91..58523f160b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -357,9 +357,7 @@ def parse_hook_librosa_custom_ctypes(ec, *args, **kwargs): if ec.name == 'librosa' and ec.version in ('0.10.1',): ec_dict = ec.asdict() eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH') - custom_ctypes_path = eessi_software_path.replace('versions', 'host_injections', 1) - custom_ctypes_path = custom_ctypes_path.replace('software', 'extra', 1) - custom_ctypes_path = os.path.join(custom_ctypes_path, "custom_ctypes") + custom_ctypes_path = os.path.join(eessi_software_path, "software", "custom_ctypes", "1.2") ebpythonprefixes = "EBPYTHONPREFIXES=%s" % custom_ctypes_path exts_list_new = [] for item in ec_dict['exts_list']: @@ -891,9 +889,7 @@ def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs): """ if self.name == 'librosa' and self.version == '0.10.1': eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH') - custom_ctypes_path = eessi_software_path.replace('versions', 'host_injections', 1) - custom_ctypes_path = custom_ctypes_path.replace('software', 'extra', 1) - custom_ctypes_path = os.path.join(custom_ctypes_path, 'custom_ctypes') + custom_ctypes_path = os.path.join(eessi_software_path, "software", "custom_ctypes", "1.2") key = 'modluafooter' values = ['prepend_path("EBPYTHONPREFIXES","%s")' % (custom_ctypes_path)] print_msg("Adding '%s' to modluafooter", values[0]) diff --git a/scripts/extra/custom_ctypes-1.2.eb b/scripts/extra/custom_ctypes-1.2.eb new file mode 100644 index 0000000000..35be6dcc41 --- /dev/null +++ b/scripts/extra/custom_ctypes-1.2.eb @@ -0,0 +1,29 @@ +## +# This is a contribution from the NESSI project +# Homepage: https://documentation.sigma2.no +# +# Authors:: Thomas Roeblitz +# License:: GPL-2.0-only +# +## + +easyblock = 'Tarball' + +name = 'custom_ctypes' +version = '1.2' + +homepage = 'https://github.com/ComputeCanada/custom_ctypes' +description = """custum_ctypes is a small Python package to fix the discovery of libraries with Python's ctypes module. It changes the behavior of find_library to return absolute paths to shared objects rather than just the names.""" + +toolchain = SYSTEM + +source_urls = ['https://github.com/ComputeCanada/custom_ctypes/archive/refs/tags'] +sources = ['%(version)s.tar.gz'] +checksums = ['3b30ce633c6a329169f2b10ff24b8eaaeef3fa208a66cdacdb53c22f02a88d9b'] + +sanity_check_paths = { + 'files': ['README.md'], + 'dirs': ['lib'], +} + +moduleclass = 'lib' diff --git a/scripts/extra/eessi-2023.06-extra-packages.yml b/scripts/extra/eessi-2023.06-extra-packages.yml new file mode 100644 index 0000000000..22670ec7a3 --- /dev/null +++ b/scripts/extra/eessi-2023.06-extra-packages.yml @@ -0,0 +1,2 @@ +easyconfigs: + - custom_ctypes-1.2.eb diff --git a/scripts/extra/install_custom_ctypes.sh b/scripts/extra/install_custom_ctypes.sh deleted file mode 100755 index 6392da00e5..0000000000 --- a/scripts/extra/install_custom_ctypes.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash - -# This script can be used to install custom ctypes under -# CVMFS_REPO/host_injections/VERSION/extra/OS_TYPE/SOFTWARE_SUBDIR/software/custom_ctypes -# and then set EESSI_USE_CUSTOM_CTYPES_DIR=CVMFS_REPO/host_injections/VERSION/extra/OS_TYPE/SOFTWARE_SUBDIR/custom_ctypes -# before loading a module that needs the custom ctypes implementation -# The custom ctypes is downloaded from https://github.com/NorESSI/custom_ctypes which is a fork of -# https://github.com/ComputeCanada/custom_ctypes - -# The `host_injections` directory is a variant symlink that by default points to -# `/opt/eessi`, unless otherwise defined in the local CVMFS configuration (see -# https://cvmfs.readthedocs.io/en/stable/cpt-repo.html#variant-symlinks). For the -# installation to be successful, this directory needs to be writeable by the user -# executing this script. - -# some logging -echo ">>> Running ${BASH_SOURCE}" - -# Initialise our bash functions -TOPDIR=$(dirname $(realpath ${BASH_SOURCE})) -source "${TOPDIR}"/../utils.sh - -# Function to display help message -show_help() { - echo "Usage: $0 [OPTIONS]" - echo "Options:" - echo " --help Display this help message" - echo " -t, --temp-dir /path/to/tmpdir Specify a location to use for temporary" - echo " storage during the installation" -} - -# Initialize variables -TEMP_DIR= - -# Parse command-line options -while [[ $# -gt 0 ]]; do - case "$1" in - --help) - show_help - exit 0 - ;; - -t|--temp-dir) - if [ -n "$2" ]; then - TEMP_DIR="$2" - shift 2 - else - echo "Error: Argument required for $1" - show_help - exit 1 - fi - ;; - *) - show_help - fatal_error "Error: Unknown option: $1" - ;; - esac -done - -# Make sure NESSI is initialised -check_eessi_initialised - -# As an installation location just use $EESSI_SOFTWARE_PATH but replacing `versions` with `host_injections` and -# `software` with `extra` -# also append `/custom_ctypes` -NESSI_SITE_INSTALL=${EESSI_SOFTWARE_PATH/versions/host_injections} -NESSI_SITE_INSTALL=${NESSI_SITE_INSTALL/software/extra}/custom_ctypes - -# we need a directory we can use for temporary storage -if [[ -z "${TEMP_DIR}" ]]; then - tmpdir=$(mktemp -d) -else - mkdir -p ${TEMP_DIR} - tmpdir=$(mktemp -d --tmpdir=${TEMP_DIR} custom_ctypes.XXX) - if [[ ! -d "$tmpdir" ]] ; then - fatal_error "Could not create directory ${tmpdir}" - fi -fi -echo "Created temporary directory '${tmpdir}'" - -# check if custom_ctypes has already been installed -if [[ -d ${NESSI_SITE_INSTALL}/lib ]]; then - echo "INFO: Installation of custom_ctypes already found at '${NESSI_SITE_INSTALL}'" - exit 0 -fi - -# download custom_ctypes to temp directory -wget https://github.com/NorESSI/custom_ctypes/archive/refs/heads/main.tar.gz -P ${tmpdir} - -# make sure target directory exists -mkdir -p ${NESSI_SITE_INSTALL} - -# unpack custom_ctypes to target directory -tar xvfz ${tmpdir}/main.tar.gz --strip-components=1 -C ${NESSI_SITE_INSTALL} - -# clean up tmpdir -rm -rf "${tmpdir}" diff --git a/scripts/extra/install_extra_packages.sh b/scripts/extra/install_extra_packages.sh new file mode 100755 index 0000000000..e0bc583c5d --- /dev/null +++ b/scripts/extra/install_extra_packages.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +# This script can be used to install extra packages under ${EESSI_SOFTWARE_PATH} + +# some logging +echo ">>> Running ${BASH_SOURCE}" + +# Initialise our bash functions +TOPDIR=$(dirname $(realpath ${BASH_SOURCE})) +source "${TOPDIR}"/../utils.sh + +# Function to display help message +show_help() { + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --help Display this help message" + echo " -e, --easystack EASYSTACKFILE Easystack file which specifies easyconfigs to be installed." + echo " -t, --temp-dir /path/to/tmpdir Specify a location to use for temporary" + echo " storage during the installation" +} + +# Initialize variables +TEMP_DIR= +EASYSTACK_FILE= + +# Parse command-line options +while [[ $# -gt 0 ]]; do + case "$1" in + --help) + show_help + exit 0 + ;; + -e|--easystack) + if [ -n "$2" ]; then + EASYSTACK_FILE="$2" + shift 2 + else + echo "Error: Argument required for $1" + show_help + exit 1 + fi + ;; + -t|--temp-dir) + if [ -n "$2" ]; then + TEMP_DIR="$2" + shift 2 + else + echo "Error: Argument required for $1" + show_help + exit 1 + fi + ;; + *) + show_help + fatal_error "Error: Unknown option: $1" + ;; + esac +done + +if [[ -z ${EASYSTACK_FILE} ]]; then + show_help + fatal_error "Error: need to specify easystack file" +fi + +# Make sure NESSI is initialised +check_eessi_initialised + +# As an installation location just use $EESSI_SOFTWARE_PATH +NESSI_CVMFS_INSTALL=${EESSI_SOFTWARE_PATH} + +# we need a directory we can use for temporary storage +if [[ -z "${TEMP_DIR}" ]]; then + tmpdir=$(mktemp -d) +else + mkdir -p ${TEMP_DIR} + tmpdir=$(mktemp -d --tmpdir=${TEMP_DIR} custom_ctypes.XXX) + if [[ ! -d "$tmpdir" ]] ; then + fatal_error "Could not create directory ${tmpdir}" + fi +fi +echo "Created temporary directory '${tmpdir}'" + +# load EasyBuild +ml EasyBuild + +# load NESSI-extend/2023.06-easybuild +ml NESSI-extend/2023.06-easybuild + +eb --show-config + +eb --easystack ${EASYSTACK_FILE} --robot + +# clean up tmpdir +rm -rf "${tmpdir}" From 532e77412c725a4c95337974a353966aebe74ebb Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 5 Jun 2024 21:53:01 +0200 Subject: [PATCH 10/11] fix bug in call extra install script --- EESSI-install-software.sh | 6 +++--- install_scripts.sh | 6 ------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index c0d0ce0088..96978ca500 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -258,9 +258,9 @@ fi # Install extra software that is needed (e.g., for providing a custom ctypes # library when needed) -cd scripts/extra -${EESSI_PREFIX}/scripts/extra/install_extra_packages.sh --temp-dir /tmp/temp --easystack eessi-2023.06-extra-packages.yml -cd - +cd ${TOPDIR}/scripts/extra +./install_extra_packages.sh --temp-dir /tmp/temp --easystack eessi-2023.06-extra-packages.yml +cd ${TOPDIR} # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') diff --git a/install_scripts.sh b/install_scripts.sh index 000ad99444..07643a39e6 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -116,12 +116,6 @@ nvidia_files=( ) copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_files[@]}" -# Copy files for the scripts/extra directory -extra_files=( - install_custom_ctypes.sh -) -copy_files_by_list ${TOPDIR}/scripts/extra ${INSTALL_PREFIX}/scripts/extra "${extra_files[@]}" - # Copy over EasyBuild hooks file used for installations hook_files=( eb_hooks.py From 6f639761313f4383a75d287481f86c0bfbcdff63 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 5 Jun 2024 22:35:21 +0200 Subject: [PATCH 11/11] make sure env var is defined --- scripts/extra/install_extra_packages.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/extra/install_extra_packages.sh b/scripts/extra/install_extra_packages.sh index e0bc583c5d..18e4903edf 100755 --- a/scripts/extra/install_extra_packages.sh +++ b/scripts/extra/install_extra_packages.sh @@ -66,14 +66,14 @@ fi check_eessi_initialised # As an installation location just use $EESSI_SOFTWARE_PATH -NESSI_CVMFS_INSTALL=${EESSI_SOFTWARE_PATH} +export NESSI_CVMFS_INSTALL=${EESSI_SOFTWARE_PATH} # we need a directory we can use for temporary storage if [[ -z "${TEMP_DIR}" ]]; then tmpdir=$(mktemp -d) else mkdir -p ${TEMP_DIR} - tmpdir=$(mktemp -d --tmpdir=${TEMP_DIR} custom_ctypes.XXX) + tmpdir=$(mktemp -d --tmpdir=${TEMP_DIR} extra.XXX) if [[ ! -d "$tmpdir" ]] ; then fatal_error "Could not create directory ${tmpdir}" fi