From 74a9a5596596e159a73606016a25ae255201fe67 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 23 May 2024 11:09:49 +0200 Subject: [PATCH] various updates to take suggestions into account - `EESSI-install-software.sh` - use `scripts/gpu_support/nvidia/install_cuda_and_libraries.sh` with `scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml` - `create_lmodsitepackage.py` - consolidate `eessi_{cuda,cudnn}_enabled_load_hook` functions in a single one (`eessi_cuda_and_libraries_enabled_load_hook`) - the remaining hook is prepared to easily add new modules, e.g., cuTENSOR - `eb_hooks.py` - put code that iterates over all files replacing non-distributable ones with symlinks into `host_injections` with a common function (`replace_non_distributable_files_with_symlinks`) - `install_scripts.sh` - add files to copy to CVMFS (see `nvidia_files`) - `scripts/gpu_support/nvidia/install_cuda_and_libraries.sh` - improved creation of tmp directory --- EESSI-install-software.sh | 6 +- create_lmodsitepackage.py | 59 ++--- eb_hooks.py | 98 ++++---- install_scripts.sh | 6 +- .../nvidia/install_cuda_and_libraries.sh | 6 +- .../nvidia/install_cudnn_host_injections.sh | 210 ------------------ 6 files changed, 81 insertions(+), 304 deletions(-) delete mode 100755 scripts/gpu_support/nvidia/install_cudnn_host_injections.sh diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 111fb67e53..0299d5d61b 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -233,8 +233,10 @@ else fi if [ -z "${skip_cuda_install}" ] || [ ! "${skip_cuda_install}" ]; then - ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12.1.1 --accept-cuda-eula - ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cudnn_host_injections.sh -c 12.1.1 -d 8.9.2.26 + ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh \ + -e ${EESSI_PREFIX}/scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml \ + -t /tmp/temp \ + --accept-cuda-eula else echo "Skipping installation of CUDA SDK and cu* libraries in host_injections, since the --skip-cuda-install flag was passed OR no EasyBuild module was found" fi diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 5e44ba8187..33796e56cb 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -107,35 +107,41 @@ end -local function eessi_cuda_enabled_load_hook(t) +local function eessi_cuda_and_libraries_enabled_load_hook(t) local frameStk = require("FrameStk"):singleton() local mt = frameStk:mt() local simpleName = string.match(t.modFullName, "(.-)/") - -- If we try to load CUDA itself, check if the full CUDA SDK was installed on the host in host_injections. - -- This is required for end users to build additional CUDA software. If the full SDK isn't present, refuse - -- to load the CUDA module and print an informative message on how to set up GPU support for EESSI + local packagesList = { ["CUDA"] = true, ["cuDNN"] = true } + -- If we try to load any of the modules in packagesList, we check if the + -- full package was installed on the host in host_injections. + -- This is required for end users to build additional software that depends + -- on the package. If the full SDK isn't present, refuse + -- to load the module and print an informative message on how to set up GPU support for EESSI local refer_to_docs = "For more information on how to do this, see https://www.eessi.io/docs/gpu/.\\n" - if simpleName == 'CUDA' then + if packagesList[simpleName] then + -- simpleName is a module in packagesList -- get the full host_injections path local hostInjections = string.gsub(os.getenv('EESSI_SOFTWARE_PATH') or "", 'versions', 'host_injections') - -- build final path where the CUDA software should be installed - local cudaEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild" - local cudaDirExists = isDir(cudaEasyBuildDir) - if not cudaDirExists then + + -- build final path where the software should be installed + local packageEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild" + local packageDirExists = isDir(packageEasyBuildDir) + if not packageDirExists then local advice = "but while the module file exists, the actual software is not entirely shipped with EESSI " - advice = advice .. "due to licencing. You will need to install a full copy of the CUDA SDK where EESSI " + advice = advice .. "due to licencing. You will need to install a full copy of the " .. simpleName .. " package where EESSI " advice = advice .. "can find it.\\n" advice = advice .. refer_to_docs LmodError("\\nYou requested to load ", simpleName, " ", advice) end end - -- when loading CUDA enabled modules check if the necessary driver libraries are accessible to the EESSI linker, + -- when loading CUDA (and cu*) enabled modules check if the necessary driver libraries are accessible to the EESSI linker, -- otherwise, refuse to load the requested module and print error message local haveGpu = mt:haveProperty(simpleName,"arch","gpu") if haveGpu then local arch = os.getenv("EESSI_CPU_FAMILY") or "" - local cudaVersionFile = "/cvmfs/software.eessi.io/host_injections/nvidia/" .. arch .. "/latest/cuda_version.txt" - local cudaDriverFile = "/cvmfs/software.eessi.io/host_injections/nvidia/" .. arch .. "/latest/libcuda.so" + local cvmfs_repo = os.getenv("EESSI_CVMFS_REPO") or "" + local cudaVersionFile = cvmfs_repo .. "/host_injections/nvidia/" .. arch .. "/latest/cuda_version.txt" + local cudaDriverFile = cvmfs_repo .. "/host_injections/nvidia/" .. arch .. "/latest/libcuda.so" local cudaDriverExists = isFile(cudaDriverFile) local singularityCudaExists = isFile("/.singularity.d/libs/libcuda.so") if not (cudaDriverExists or singularityCudaExists) then @@ -172,38 +178,13 @@ end end -local function eessi_cudnn_enabled_load_hook(t) - local frameStk = require("FrameStk"):singleton() - local mt = frameStk:mt() - local simpleName = string.match(t.modFullName, "(.-)/") - -- If we try to load cuDNN itself, check if the full cuDNN package was installed on the host in host_injections. - -- This is required for end users to build additional cuDNN dependent software. If the full SDK isn't present, refuse - -- to load the cuDNN module and print an informative message on how to set up GPU support for EESSI - local refer_to_docs = "For more information on how to do this, see https://www.eessi.io/docs/gpu/.\\n" - if simpleName == 'cuDNN' then - -- get the full host_injections path - local hostInjections = string.gsub(os.getenv('EESSI_SOFTWARE_PATH') or "", 'versions', 'host_injections') - -- build final path where the cuDNN software should be installed - local cudnnEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild" - local cudnnDirExists = isDir(cudnnEasyBuildDir) - if not cudnnDirExists then - local advice = "but while the module file exists, the actual software is not entirely shipped with EESSI " - advice = advice .. "due to licencing. You will need to install a full copy of the cuDNN package where EESSI " - advice = advice .. "can find it.\\n" - advice = advice .. refer_to_docs - LmodError("\\nYou requested to load ", simpleName, " ", advice) - end - end -end - -- Combine both functions into a single one, as we can only register one function as load hook in lmod -- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed function eessi_load_hook(t) -- Only apply CUDA and cuDNN hooks if the loaded module is in the EESSI prefix -- This avoids getting an Lmod Error when trying to load a CUDA and cuDNN module from a local software stack if from_eessi_prefix(t) then - eessi_cuda_enabled_load_hook(t) - eessi_cudnn_enabled_load_hook(t) + eessi_cuda_and_libraries_enabled_load_hook(t) end end diff --git a/eb_hooks.py b/eb_hooks.py index 46e79810d9..d3fefe2558 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -567,6 +567,47 @@ def post_sanitycheck_hook(self, *args, **kwargs): POST_SANITYCHECK_HOOKS[self.name](self, *args, **kwargs) +def replace_non_distributable_files_with_symlinks(log, install_dir, package, allowlist): + """ + Replace files that cannot be distributed with symlinks into host_injections + """ + extension_based = { "CUDA": False, "cuDNN": True } + if not package in extension_based: + raise EasyBuildError("Don't know how to strip non-distributable files from package %s.", package) + + # iterate over all files in the package installation directory + for dir_path, _, files in os.walk(install_dir): + for filename in files: + full_path = os.path.join(dir_path, filename) + # we only really care about real files, i.e. not symlinks + if not os.path.islink(full_path): + # check if the current file name stub is part of the allowlist + basename = filename.split('.')[0] + if extension_based[package]: + if '.' in filename: + extension = '.' + filename.split('.')[1] + if basename in allowlist: + log.debug("%s is found in allowlist, so keeping it: %s", basename, full_path) + elif extension_based[package] and '.' in filename and extension in allowlist: + log.debug("%s is found in allowlist, so keeping it: %s", extension, full_path) + else: + if extension_based[package]: + print_name = filename + else: + print_name = basename + log.debug("%s is not found in allowlist, so replacing it with symlink: %s", + print_name, full_path) + # if it is not in the allowlist, delete the file and create a symlink to host_injections + host_inj_path = full_path.replace('versions', 'host_injections') + # make sure source and target of symlink are not the same + if full_path == host_inj_path: + raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " + "are using this hook for an EESSI installation?", + full_path, host_inj_path) + remove_file(full_path) + symlink(host_inj_path, full_path) + + def post_sanitycheck_cuda(self, *args, **kwargs): """ Remove files from CUDA installation that we are not allowed to ship, @@ -606,28 +647,9 @@ def post_sanitycheck_cuda(self, *args, **kwargs): if 'libcudart' not in allowlist: raise EasyBuildError("Did not find 'libcudart' in allowlist: %s" % allowlist) - # iterate over all files in the CUDA installation directory - for dir_path, _, files in os.walk(self.installdir): - for filename in files: - full_path = os.path.join(dir_path, filename) - # we only really care about real files, i.e. not symlinks - if not os.path.islink(full_path): - # check if the current file name stub is part of the allowlist - basename = filename.split('.')[0] - if basename in allowlist: - self.log.debug("%s is found in allowlist, so keeping it: %s", basename, full_path) - else: - self.log.debug("%s is not found in allowlist, so replacing it with symlink: %s", - basename, full_path) - # if it is not in the allowlist, delete the file and create a symlink to host_injections - host_inj_path = full_path.replace('versions', 'host_injections') - # make sure source and target of symlink are not the same - if full_path == host_inj_path: - raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " - "are using this hook for an EESSI installation?", - full_path, host_inj_path) - remove_file(full_path) - symlink(host_inj_path, full_path) + # replace files that are not distributable with symlinks into + # host_injections + replace_non_distributable_files_with_symlinks(self.log, self.installdir, self.name, allowlist) else: raise EasyBuildError("CUDA-specific hook triggered for non-CUDA easyconfig?!") @@ -643,8 +665,7 @@ def post_sanitycheck_cudnn(self, *args, **kwargs): allowlist = ['LICENSE'] - # read cuDNN LICENSE, construct allowlist based on section 2. Distribution - # that specifies list of files that can be shipped + # read cuDNN LICENSE, construct allowlist based on section "2. Distribution" that specifies list of files that can be shipped license_path = os.path.join(self.installdir, 'LICENSE') search_string = "2. Distribution. The following portions of the SDK are distributable under the Agreement:" with open(license_path) as infile: @@ -660,32 +681,9 @@ def post_sanitycheck_cudnn(self, *args, **kwargs): allowlist = sorted(set(allowlist)) self.log.info("Allowlist for files in cuDNN installation that can be redistributed: " + ', '.join(allowlist)) - # iterate over all files in the cuDNN installation directory - for dir_path, _, files in os.walk(self.installdir): - for filename in files: - full_path = os.path.join(dir_path, filename) - # we only really care about real files, i.e. not symlinks - if not os.path.islink(full_path): - # check if the current file is part of the allowlist - basename = filename.split('.')[0] - if '.' in filename: - extension = '.' + filename.split('.')[1] - if basename in allowlist: - self.log.debug("%s is found in allowlist, so keeping it: %s", basename, full_path) - elif '.' in filename and extension in allowlist: - self.log.debug("%s is found in allowlist, so keeping it: %s", extension, full_path) - else: - self.log.debug("%s is not found in allowlist, so replacing it with symlink: %s", - filename, full_path) - # if it is not in the allowlist, delete the file and create a symlink to host_injections - host_inj_path = full_path.replace('versions', 'host_injections') - # make sure source and target of symlink are not the same - if full_path == host_inj_path: - raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " - "are using this hook for a EESSI installation?", - full_path, host_inj_path) - remove_file(full_path) - symlink(host_inj_path, full_path) + # replace files that are not distributable with symlinks into + # host_injections + replace_non_distributable_files_with_symlinks(self.log, self.installdir, self.name, allowlist) else: raise EasyBuildError("cuDNN-specific hook triggered for non-cuDNN easyconfig?!") diff --git a/install_scripts.sh b/install_scripts.sh index 05891d2551..8c435cef80 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -110,7 +110,11 @@ copy_files_by_list ${TOPDIR}/scripts ${INSTALL_PREFIX}/scripts "${script_files[@ # Copy files for the scripts/gpu_support/nvidia directory nvidia_files=( - install_cuda_host_injections.sh install_cudnn_host_injections.sh link_nvidia_host_libraries.sh + eessi-2023.06-cuda-and-libraries.yml + install_cuda_and_libraries.sh + install_cuda_host_injections.sh + install_cudnn_host_injections.sh + link_nvidia_host_libraries.sh ) copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_files[@]}" diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index 140fd8d4e1..f04aa1aff6 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -93,11 +93,13 @@ export EESSI_SITE_INSTALL=${EESSI_SOFTWARE_PATH/versions/host_injections} if [[ -z "${TEMP_DIR}" ]]; then tmpdir=$(mktemp -d) else - tmpdir="${TEMP_DIR}"/temp - if ! mkdir "$tmpdir" ; then + mkdir -p ${TEMP_DIR} + tmpdir=$(mktemp -d --tmpdir=${TEMP_DIR} cuda_n_co.XXX) + if [[ ! -d "$tmpdir" ]] ; then fatal_error "Could not create directory ${tmpdir}" fi fi +echo "Created temporary directory '${tmpdir}'" # workaround for EasyBuild not being found when loading "extend" module module load EasyBuild/4.9.1 diff --git a/scripts/gpu_support/nvidia/install_cudnn_host_injections.sh b/scripts/gpu_support/nvidia/install_cudnn_host_injections.sh deleted file mode 100755 index d23cd5ebb7..0000000000 --- a/scripts/gpu_support/nvidia/install_cudnn_host_injections.sh +++ /dev/null @@ -1,210 +0,0 @@ -#!/usr/bin/env bash - -# This script can be used to install cuDNN under the `.../host_injections` directory. -# This provides the parts of the cuDNN installation that cannot be redistributed as -# part of EESSI due to license limitations. While GPU-based software from EESSI will -# _run_ without these, installation of additional software that requires the cuDNN -# installation(s) under `host_injections` to be present. -# -# The `host_injections` directory is a variant symlink that by default points to -# `/opt/eessi`, unless otherwise defined in the local CVMFS configuration (see -# https://cvmfs.readthedocs.io/en/stable/cpt-repo.html#variant-symlinks). For the -# installation to be successful, this directory needs to be writeable by the user -# executing this script. - -# Initialise our bash functions -TOPDIR=$(dirname $(realpath $BASH_SOURCE)) -source "$TOPDIR"/../../utils.sh - -# Function to display help message -show_help() { - echo "Usage: $0 [OPTIONS]" - echo "Options:" - echo " --help Display this help message" - echo " -c, --cuda-version CUDA_VERSION Specify a version of CUDA to be used" - echo " when installing cuDNN (must" - echo " have a corresponding easyconfig in the" - echo " EasyBuild release)" - echo " -d, --cudnn-version CUDNN_VERSION Specify a version of cuDNN to install (must" - echo " have a corresponding easyconfig in the" - echo " EasyBuild release)" - echo " -t, --temp-dir /path/to/tmpdir Specify a location to use for temporary" - echo " storage during the cuDNN install" - echo " (must have >10GB available)" -} - -# Initialize variables -cuda_version="" -cudnn_version="" - -# Parse command-line options -while [[ $# -gt 0 ]]; do - case "$1" in - --help) - show_help - exit 0 - ;; - -c|--cuda-version) - if [ -n "$2" ]; then - cuda_version="$2" - shift 2 - else - echo "Error: Argument required for $1" - show_help - exit 1 - fi - ;; - -d|--cudnn-version) - if [ -n "$2" ]; then - cudnn_version="$2" - shift 2 - else - echo "Error: Argument required for $1" - show_help - exit 1 - fi - ;; - -t|--temp-dir) - if [ -n "$2" ]; then - CUDA_TEMP_DIR="$2" - shift 2 - else - echo "Error: Argument required for $1" - show_help - exit 1 - fi - ;; - *) - show_help - fatal_error "Error: Unknown option: $1" - ;; - esac -done - -# Make sure EESSI is initialised -check_eessi_initialised - -# Make sure the CUDA version supplied is a semantic version -is_semantic_version() { - local version=$1 - local regex='^[0-9]+\.[0-9]+\.[0-9]+$' - - if [[ $version =~ $regex ]]; then - return 0 # Return success (0) if it's a semantic version - else - return 1 # Return failure (1) if it's not a semantic version - fi -} -if ! is_semantic_version "$cuda_version"; then - show_help - error="\nYou must provide a semantic version for CUDA (e.g., 12.1.1) via the appropriate\n" - error="${error}command line option. This script is intended for use with EESSI so the 'correct'\n" - error="${error}version to provide is probably one of those available under\n" - error="${error}$EESSI_SOFTWARE_PATH/software/cuDNN\n" - fatal_error "${error}" -fi - -# As an installation location just use $EESSI_SOFTWARE_PATH but replacing `versions` with `host_injections` -cudnn_install_parent=${EESSI_SOFTWARE_PATH/versions/host_injections} - -# Only install cuDNN if specified version is not found. -# (existence of easybuild subdir implies a successful install) -if [ -d "${cudnn_install_parent}"/software/cuDNN/*-CUDA-"${cuda_version}"/easybuild ]; then - echo_green "cuDNN software found! No need to install cuDNN again." -else - # We need to be able write to the installation space so let's make sure we can - if ! create_directory_structure "${cudnn_install_parent}"/software/cuDNN ; then - fatal_error "No write permissions to directory ${cudnn_install_parent}/software/cuDNN" - fi - - # we need a directory we can use for temporary storage - if [[ -z "${CUDA_TEMP_DIR}" ]]; then - tmpdir=$(mktemp -d) - else - tmpdir="${CUDA_TEMP_DIR}"/temp - if ! mkdir "$tmpdir" ; then - fatal_error "Could not create directory ${tmpdir}" - fi - fi - - required_space_in_tmpdir=50000 - # Let's see if we have sources and build locations defined if not, we use the temporary space - if [[ -z "${EASYBUILD_BUILDPATH}" ]]; then - export EASYBUILD_BUILDPATH=${tmpdir}/build - required_space_in_tmpdir=$((required_space_in_tmpdir + 5000000)) - fi - if [[ -z "${EASYBUILD_SOURCEPATH}" ]]; then - export EASYBUILD_SOURCEPATH=${tmpdir}/sources - required_space_in_tmpdir=$((required_space_in_tmpdir + 5000000)) - fi - - # The install is pretty fat, you need lots of space for download/unpack/install (~3*5GB), - # need to do a space check before we proceed - avail_space=$(df --output=avail "${cudnn_install_parent}"/ | tail -n 1 | awk '{print $1}') - if (( avail_space < 5000000 )); then - fatal_error "Need at least 5GB disk space to install cuDNN under ${cudnn_install_parent}, exiting now..." - fi - avail_space=$(df --output=avail "${tmpdir}"/ | tail -n 1 | awk '{print $1}') - if (( avail_space < required_space_in_tmpdir )); then - error="Need at least ${required_space_in_tmpdir} disk space under ${tmpdir}.\n" - error="${error}Set the environment variable CUDA_TEMP_DIR to a location with adequate space to pass this check." - error="${error}You can alternatively set EASYBUILD_BUILDPATH and/or EASYBUILD_SOURCEPATH " - error="${error}to reduce this requirement. Exiting now..." - fatal_error "${error}" - fi - - if ! command -v "eb" &>/dev/null; then - echo_yellow "Attempting to load an EasyBuild module to do actual install" - module load EasyBuild - # There are some scenarios where this may fail - if [ $? -ne 0 ]; then - error="'eb' command not found in your environment and\n" - error="${error} module load EasyBuild\n" - error="${error}failed for some reason.\n" - error="${error}Please re-run this script with the 'eb' command available." - fatal_error "${error}" - fi - fi - - cudnn_easyconfig="cuDNN-${cudnn_version}-CUDA-${cuda_version}.eb" - - # Check the easyconfig file is available in the release - # (eb search always returns 0, so we need a grep to ensure a usable exit code) - eb --search ^${cudnn_easyconfig}|grep cuDNN > /dev/null 2>&1 - # Check the exit code - if [ $? -ne 0 ]; then - eb_version=$(eb --version) - available_cudnn_easyconfigs=$(eb --search ^cuDNN-*.eb|grep cuDNN) - - error="The easyconfig ${cudnn_easyconfig} was not found in EasyBuild version:\n" - error="${error} ${eb_version}\n" - error="${error}You either need to give a different version of CUDA to install _or_ \n" - error="${error}use a different version of EasyBuild for the installation.\n" - error="${error}\nThe versions of available with the current eb command are:\n" - error="${error}${available_cudnn_easyconfigs}" - fatal_error "${error}" - fi - - # We need the --rebuild option, as the cuDNN module may or may not be on the - # `MODULEPATH` yet. Even if it is, we still want to redo this installation - # since it will provide the symlinked targets for the parts of the cuDNN - # installation in the `.../versions/...` prefix - # We install the module in our `tmpdir` since we do not need the modulefile, - # we only care about providing the targets for the symlinks. - extra_args="--rebuild --installpath-modules=${tmpdir}" - - # We don't want hooks used in this install, we need a vanilla cuDNN installation - touch "$tmpdir"/none.py - # shellcheck disable=SC2086 # Intended splitting of extra_args - eb --prefix="$tmpdir" ${extra_args} --hooks="$tmpdir"/none.py --installpath="${cudnn_install_parent}"/ "${cudnn_easyconfig}" - ret=$? - if [ $ret -ne 0 ]; then - eb_last_log=$(unset EB_VERBOSE; eb --last-log) - cp -a ${eb_last_log} . - fatal_error "cuDNN installation failed, please check EasyBuild logs $(basename ${eb_last_log})..." - else - echo_green "cuDNN installation at ${cudnn_install_parent}/software/cuDNN/${cudnn_version}-CUDA-${cuda_version} succeeded!" - fi - # clean up tmpdir - rm -rf "${tmpdir}" -fi