diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index d840910516..6c59f46570 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -204,8 +204,10 @@ ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX} # TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install # Allow skipping CUDA SDK install in e.g. CI environments if [ -z "${skip_cuda_install}" ] || [ ! "${skip_cuda_install}" ]; then - ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12.1.1 --accept-cuda-eula - ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuDNN_host_injections.sh -c 12.1.1 -d 8.9.2.26 + ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh \ + -e ${EESSI_PREFIX}/scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml \ + -t /tmp/temp \ + --accept-cuda-eula else echo "Skipping installation of CUDA SDK and cu* libraries in host_injections, since the --skip-cuda-install flag was passed" fi diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index bafbb63414..20c4098b8f 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -105,37 +105,41 @@ end - -local function eessi_cuda_enabled_load_hook(t) +local function eessi_cuda_and_libraries_enabled_load_hook(t) local frameStk = require("FrameStk"):singleton() local mt = frameStk:mt() local simpleName = string.match(t.modFullName, "(.-)/") - -- If we try to load CUDA itself, check if the full CUDA SDK was installed on the host in host_injections. - -- This is required for end users to build additional CUDA software. If the full SDK isn't present, refuse - -- to load the CUDA module and print an informative message on how to set up GPU support for NESSI + local packagesList = { ["CUDA"] = true, ["cuDNN"] = true, ["cuTENSOR"] = true } + -- If we try to load any of the modules in packagesList, we check if the + -- full package was installed on the host in host_injections. + -- This is required for end users to build additional software that depends + -- on the package. If the full SDK isn't present, refuse + -- to load the module and print an informative message on how to set up GPU support for NESSI local refer_to_docs = "For more information on how to do this, see https://www.eessi.io/docs/gpu/.\\n" - if simpleName == 'CUDA' then + if packagesList[simpleName] then + -- simpleName is a module in packagesList -- get the full host_injections path local hostInjections = string.gsub(os.getenv('EESSI_SOFTWARE_PATH') or "", 'versions', 'host_injections') - -- build final path where the CUDA software should be installed - local cudaEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild" - local cudaDirExists = isDir(cudaEasyBuildDir) - if not cudaDirExists then + -- build final path where the software should be installed + local packageEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild" + local packageDirExists = isDir(packageEasyBuildDir) + if not packageDirExists then local advice = "but while the module file exists, the actual software is not entirely shipped with NESSI " - advice = advice .. "due to licencing. You will need to install a full copy of the CUDA SDK where NESSI " + advice = advice .. "due to licencing. You will need to install a full copy of the " .. simpleName .. " package where NESSI " advice = advice .. "can find it.\\n" advice = advice .. refer_to_docs LmodError("\\nYou requested to load ", simpleName, " ", advice) end end - -- when loading CUDA enabled modules check if the necessary driver libraries are accessible to the NESSI linker, + -- when loading CUDA (and cu*) enabled modules check if the necessary driver libraries are accessible to the NESSI linker, -- otherwise, refuse to load the requested module and print error message local checkGpu = mt:haveProperty(simpleName,"arch","gpu") local overrideGpuCheck = os.getenv("EESSI_OVERRIDE_GPU_CHECK") if checkGpu and (overrideGpuCheck == nil) then local arch = os.getenv("EESSI_CPU_FAMILY") or "" - local cudaVersionFile = "/cvmfs/pilot.nessi.no/host_injections/nvidia/" .. arch .. "/latest/cuda_version.txt" - local cudaDriverFile = "/cvmfs/pilot.nessi.no/host_injections/nvidia/" .. arch .. "/latest/libcuda.so" + local cvmfs_repo = os.getenv("EESSI_CVMFS_REPO") or "" + local cudaVersionFile = cvmfs_repo .. "/host_injections/nvidia/" .. arch .. "/latest/cuda_version.txt" + local cudaDriverFile = cvmfs_repo .. "/host_injections/nvidia/" .. arch .. "/latest/libcuda.so" local cudaDriverExists = isFile(cudaDriverFile) local singularityCudaExists = isFile("/.singularity.d/libs/libcuda.so") if not (cudaDriverExists or singularityCudaExists) then @@ -174,38 +178,13 @@ end end -local function eessi_cudnn_enabled_load_hook(t) - local frameStk = require("FrameStk"):singleton() - local mt = frameStk:mt() - local simpleName = string.match(t.modFullName, "(.-)/") - -- If we try to load cuDNN itself, check if the full cuDNN package was installed on the host in host_injections. - -- This is required for end users to build additional cuDNN dependent software. If the full SDK isn't present, refuse - -- to load the cuDNN module and print an informative message on how to set up GPU support for NESSI - local refer_to_docs = "For more information on how to do this, see https://www.eessi.io/docs/gpu/.\\n" - if simpleName == 'cuDNN' then - -- get the full host_injections path - local hostInjections = string.gsub(os.getenv('EESSI_SOFTWARE_PATH') or "", 'versions', 'host_injections') - -- build final path where the cuDNN software should be installed - local cudnnEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild" - local cudnnDirExists = isDir(cudnnEasyBuildDir) - if not cudnnDirExists then - local advice = "but while the module file exists, the actual software is not entirely shipped with NESSI " - advice = advice .. "due to licencing. You will need to install a full copy of the cuDNN package where NESSI " - advice = advice .. "can find it.\\n" - advice = advice .. refer_to_docs - LmodError("\\nYou requested to load ", simpleName, " ", advice) - end - end -end - -- Combine both functions into a single one, as we can only register one function as load hook in lmod -- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed function eessi_load_hook(t) - -- Only apply CUDA and cuDNN hooks if the loaded module is in the NESSI prefix - -- This avoids getting an Lmod Error when trying to load a CUDA or cuDNN module from a local software stack + -- Only apply CUDA and libraries hook if the loaded module is in the NESSI prefix + -- This avoids getting an Lmod Error when trying to load a CUDA or library module from a local software stack if from_eessi_prefix(t) then - eessi_cuda_enabled_load_hook(t) - eessi_cudnn_enabled_load_hook(t) + eessi_cuda_and_libraries_enabled_load_hook(t) end end diff --git a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml index 63f8804820..8e850e19b9 100644 --- a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml +++ b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml @@ -3,4 +3,6 @@ easyconfigs: options: from-pr: 20299 - EESSI-extend-2023.06-easybuild.eb -# comment to trigger rebuild + - cuDNN-8.9.2.26-CUDA-12.1.1.eb + - cuTENSOR-2.0.1.2-CUDA-12.1.1.eb + - Nextflow-23.10.0.eb diff --git a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml index 142928809e..ecfa63223f 100644 --- a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml +++ b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml @@ -3,3 +3,7 @@ easyconfigs: - waLBerla-6.1-foss-2022b.eb - R-4.2.2-foss-2022b.eb - WRF-4.4.1-foss-2022b-dmpar.eb + - bokeh-3.2.1-foss-2022b.eb + - MDAnalysis-2.4.2-foss-2022b.eb + - arrow-R-11.0.0.3-foss-2022b-R-4.2.2.eb + - biom-format-2.1.15-foss-2022b.eb diff --git a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index fecdbf39bf..2081e8833e 100644 --- a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -11,7 +11,15 @@ easyconfigs: - R-bundle-CRAN-2023.12-foss-2023a.eb - OpenFOAM-10-foss-2023a.eb - PyOpenGL-3.1.7-GCCcore-12.3.0.eb - - ipympl-0.9.3-foss-2023a.eb + # replaced by ipympl-0.9.3-gfbf-2023a.eb + # - ipympl-0.9.3-foss-2023a.eb + - ipympl-0.9.3-gfbf-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20586 + # which removed the foss/2023a ec file + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18852 + # which added/updated the gfbf/2023a ec file + from-pr: 18852 - SuperLU_DIST-8.1.2-foss-2023a.eb: options: from-pr: 20162 @@ -34,7 +42,13 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19451; options: from-pr: 19451 - - cuDNN-8.9.2.26-CUDA-12.1.1.eb - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb - ABySS-2.3.7-foss-2023a.eb - - BLAST+-2.14.1-gompi-2023a.eb + - Perl-bundle-CPAN-5.36.1-GCCcore-12.3.0.eb: + options: + from-pr: 20540 + - WhatsHap-2.2-foss-2023a.eb + - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb + - BLAST+-2.14.1-gompi-2023a.eb: + options: + from-pr: 20674 diff --git a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml index 12564d9cb9..5a7453a9c0 100644 --- a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml +++ b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml @@ -23,3 +23,4 @@ easyconfigs: options: from-pr: 20439 - GDB-13.2-GCCcore-13.2.0.eb + - IPython-8.17.2-GCCcore-13.2.0.eb diff --git a/eb_hooks.py b/eb_hooks.py index cf1c911b23..c7b2a62f64 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -623,6 +623,47 @@ def post_sanitycheck_hook(self, *args, **kwargs): POST_SANITYCHECK_HOOKS[self.name](self, *args, **kwargs) +def replace_non_distributable_files_with_symlinks(log, install_dir, package, allowlist): + """ + Replace files that cannot be distributed with symlinks into host_injections + """ + extension_based = { "CUDA": False, "cuDNN": True, "cuTENSOR": True } + if not package in extension_based: + raise EasyBuildError("Don't know how to strip non-distributable files from package %s.", package) + + # iterate over all files in the package installation directory + for dir_path, _, files in os.walk(install_dir): + for filename in files: + full_path = os.path.join(dir_path, filename) + # we only really care about real files, i.e. not symlinks + if not os.path.islink(full_path): + # check if the current file name stub is part of the allowlist + basename = filename.split('.')[0] + if extension_based[package]: + if '.' in filename: + extension = '.' + filename.split('.')[1] + if basename in allowlist: + log.debug("%s is found in allowlist, so keeping it: %s", basename, full_path) + elif extension_based[package] and '.' in filename and extension in allowlist: + log.debug("%s is found in allowlist, so keeping it: %s", extension, full_path) + else: + if extension_based[package]: + print_name = filename + else: + print_name = basename + log.debug("%s is not found in allowlist, so replacing it with symlink: %s", + print_name, full_path) + # if it is not in the allowlist, delete the file and create a symlink to host_injections + host_inj_path = full_path.replace('versions', 'host_injections') + # make sure source and target of symlink are not the same + if full_path == host_inj_path: + raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " + "are using this hook for an EESSI installation?", + full_path, host_inj_path) + remove_file(full_path) + symlink(host_inj_path, full_path) + + def post_sanitycheck_cuda(self, *args, **kwargs): """ Remove files from CUDA installation that we are not allowed to ship, @@ -662,33 +703,14 @@ def post_sanitycheck_cuda(self, *args, **kwargs): if 'libcudart' not in allowlist: raise EasyBuildError("Did not find 'libcudart' in allowlist: %s" % allowlist) - # iterate over all files in the CUDA installation directory - for dir_path, _, files in os.walk(self.installdir): - for filename in files: - full_path = os.path.join(dir_path, filename) - # we only really care about real files, i.e. not symlinks - if not os.path.islink(full_path): - # check if the current file name stub is part of the allowlist - basename = filename.split('.')[0] - if basename in allowlist: - self.log.debug("%s is found in allowlist, so keeping it: %s", basename, full_path) - else: - self.log.debug("%s is not found in allowlist, so replacing it with symlink: %s", - basename, full_path) - # if it is not in the allowlist, delete the file and create a symlink to host_injections - host_inj_path = full_path.replace('versions', 'host_injections') - # make sure source and target of symlink are not the same - if full_path == host_inj_path: - raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " - "are using this hook for an EESSI installation?", - full_path, host_inj_path) - remove_file(full_path) - symlink(host_inj_path, full_path) + # replace files that are not distributable with symlinks into + # host_injections + replace_non_distributable_files_with_symlinks(self.log, self.installdir, self.name, allowlist) else: raise EasyBuildError("CUDA-specific hook triggered for non-CUDA easyconfig?!") -def post_sanitycheck_cuDNN(self, *args, **kwargs): +def post_sanitycheck_cudnn(self, *args, **kwargs): """ Remove files from cuDNN installation that we are not allowed to ship, and replace them with a symlink to a corresponding installation under host_injections. @@ -698,7 +720,7 @@ def post_sanitycheck_cuDNN(self, *args, **kwargs): allowlist = ['LICENSE'] - # read cuDNN LICENSE, construct allowlist based on section 2.6 that specifies list of files that can be shipped + # read cuDNN LICENSE, construct allowlist based on section "2. Distribution" that specifies list of files that can be shipped license_path = os.path.join(self.installdir, 'LICENSE') search_string = "2. Distribution. The following portions of the SDK are distributable under the Agreement:" with open(license_path) as infile: @@ -714,79 +736,90 @@ def post_sanitycheck_cuDNN(self, *args, **kwargs): allowlist = sorted(set(allowlist)) self.log.info("Allowlist for files in cuDNN installation that can be redistributed: " + ', '.join(allowlist)) - # iterate over all files in the CUDA installation directory - for dir_path, _, files in os.walk(self.installdir): - for filename in files: - full_path = os.path.join(dir_path, filename) - # we only really care about real files, i.e. not symlinks - if not os.path.islink(full_path): - # check if the current file is part of the allowlist - basename = filename.split('.')[0] - if '.' in filename: - extension = '.' + filename.split('.')[1] - if basename in allowlist: - self.log.debug("%s is found in allowlist, so keeping it: %s", basename, full_path) - elif '.' in filename and extension in allowlist: - self.log.debug("%s is found in allowlist, so keeping it: %s", extension, full_path) - else: - self.log.debug("%s is not found in allowlist, so replacing it with symlink: %s", - filename, full_path) - # if it is not in the allowlist, delete the file and create a symlink to host_injections - host_inj_path = full_path.replace('versions', 'host_injections') - # make sure source and target of symlink are not the same - if full_path == host_inj_path: - raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " - "are using this hook for a NESSI installation?", - full_path, host_inj_path) - remove_file(full_path) - symlink(host_inj_path, full_path) + # replace files that are not distributable with symlinks into + # host_injections + replace_non_distributable_files_with_symlinks(self.log, self.installdir, self.name, allowlist) else: raise EasyBuildError("cuDNN-specific hook triggered for non-cuDNN easyconfig?!") +def post_sanitycheck_cutensor(self, *args, **kwargs): + """ + Remove files from cuTENSOR installation that we are not allowed to ship, + and replace them with a symlink to a corresponding installation under host_injections. + """ + if self.name == 'cuTENSOR': + print_msg("Replacing files in cuTENSOR installation that we can not ship with symlinks to host_injections...") + + allowlist = ['LICENSE'] + + # read cuTENSOR LICENSE, construct allowlist based on section "2. Distribution" that specifies list of files that can be shipped + license_path = os.path.join(self.installdir, 'LICENSE') + search_string = "2. Distribution. The following portions of the SDK are distributable under the Agreement:" + with open(license_path) as infile: + for line in infile: + if line.strip().startswith(search_string): + # remove search string, split into words, remove trailing + # dots '.' and only retain words starting with a dot '.' + distributable = line[len(search_string):] + for word in distributable.split(): + if word[0] == '.': + allowlist.append(word.rstrip('.')) + + allowlist = sorted(set(allowlist)) + self.log.info("Allowlist for files in cuTENSOR installation that can be redistributed: " + ', '.join(allowlist)) + + # replace files that are not distributable with symlinks into + # host_injections + replace_non_distributable_files_with_symlinks(self.log, self.installdir, self.name, allowlist) + else: + raise EasyBuildError("cuTENSOR-specific hook triggered for non-cuTENSOR easyconfig?!") + + def inject_gpu_property(ec): """ - Add 'gpu' property, via modluafooter easyconfig parameter + Add 'gpu' property EESSIVERSION envvars and drop dependencies to + build dependencies, via modluafooter easyconfig parameter """ ec_dict = ec.asdict() - # Check if CUDA is in the dependencies, if so add the 'gpu' Lmod property - if ('CUDA' in [dep[0] for dep in iter(ec_dict['dependencies'])]): - ec.log.info("Injecting gpu as Lmod arch property and envvar with CUDA version") - key = 'modluafooter' - value = 'add_property("arch","gpu")' - cuda_version = 0 - for dep in iter(ec_dict['dependencies']): - # Make CUDA a build dependency only (rpathing saves us from link errors) - if 'CUDA' in dep[0]: - cuda_version = dep[1] - ec_dict['dependencies'].remove(dep) - if dep not in ec_dict['builddependencies']: - ec_dict['builddependencies'].append(dep) - value = '\n'.join([value, 'setenv("EESSICUDAVERSION","%s")' % cuda_version]) - if key in ec_dict: - if not value in ec_dict[key]: - ec[key] = '\n'.join([ec_dict[key], value]) - else: - ec[key] = value - # Check if cuDNN is in the dependencies, if so add the 'gpu' Lmod property - if ('cuDNN' in [dep[0] for dep in iter(ec_dict['dependencies'])]): - ec.log.info("Injecting gpu as Lmod arch property and envvar with cuDNN version") + # check if CUDA, cuDNN, you-name-it is in the dependencies, if so + # - drop dependency to build dependency + # - add 'gpu' Lmod property + # - add envvar with package version + packages_list = ( "CUDA", "cuDNN", "cuTENSOR" ) + packages_version = { } + add_gpu_property = '' + + for package in packages_list: + # Check if package is in the dependencies, if so drop dependency to build + # dependency and set variable for later adding the 'gpu' Lmod property + if (package in [dep[0] for dep in iter(ec_dict['dependencies'])]): + add_gpu_property = 'add_property("arch","gpu")' + for dep in iter(ec_dict['dependencies']): + if package in dep[0]: + # make package a build dependency only (rpathing saves us from link errors) + ec.log.info("Dropping dependency on %s to build dependency" % package) + ec_dict['dependencies'].remove(dep) + if dep not in ec_dict['builddependencies']: + ec_dict['builddependencies'].append(dep) + # take note of version for creating the modluafooter + packages_version[package] = dep[1] + if add_gpu_property: + ec.log.info("Injecting gpu as Lmod arch property and envvars for dependencies with their version") key = 'modluafooter' - value = 'add_property("arch","gpu")' - cudnn_version = 0 - for dep in iter(ec_dict['dependencies']): - # Make cuDNN a build dependency only (rpathing saves us from link errors) - if 'cuDNN' in dep[0]: - cudnn_version = dep[1] - ec_dict['dependencies'].remove(dep) - if dep not in ec_dict['builddependencies']: - ec_dict['builddependencies'].append(dep) - value = '\n'.join([value, 'setenv("EESSICUDNNVERSION","%s")' % cudnn_version]) - if key in ec_dict: - if not value in ec_dict[key]: - ec[key] = '\n'.join([ec_dict[key], value]) + values = [add_gpu_property] + for package, version in packages_version.items(): + envvar = "EESSI%sVERSION" % package.upper() + values.append('setenv("%s","%s")' % (envvar, version)) + if not key in ec_dict: + ec[key] = '\n'.join(values) else: - ec[key] = value + new_value = ec_dict[key] + for value in values: + if not value in new_value: + new_value = '\n'.join([new_value, value]) + ec[key] = new_value + return ec @@ -843,5 +876,6 @@ def inject_gpu_property(ec): POST_SANITYCHECK_HOOKS = { 'CUDA': post_sanitycheck_cuda, - 'cuDNN': post_sanitycheck_cuDNN, + 'cuDNN': post_sanitycheck_cudnn, + 'cuTENSOR': post_sanitycheck_cutensor, } diff --git a/install_scripts.sh b/install_scripts.sh index 8bbcb6a7bf..17712a0ae7 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -110,7 +110,11 @@ copy_files_by_list ${TOPDIR}/scripts ${INSTALL_PREFIX}/scripts "${script_files[@ # Copy files for the scripts/gpu_support/nvidia directory nvidia_files=( - install_cuda_host_injections.sh install_cuDNN_host_injections.sh link_nvidia_host_libraries.sh + eessi-2023.06-cuda-and-libraries.yml + install_cuda_and_libraries.sh + install_cuda_host_injections.sh + install_cuDNN_host_injections.sh + link_nvidia_host_libraries.sh ) copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_files[@]}" diff --git a/scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml b/scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml new file mode 100644 index 0000000000..74615872d4 --- /dev/null +++ b/scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml @@ -0,0 +1,4 @@ +easyconfigs: + - CUDA-12.1.1.eb + - cuDNN-8.9.2.26-CUDA-12.1.1.eb + - cuTENSOR-2.0.1.2-CUDA-12.1.1.eb diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh new file mode 100755 index 0000000000..ba3416c6c5 --- /dev/null +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash + +# This script can be used to install CUDA and other libraries by NVIDIA under +# the `.../host_injections` directory. +# +# This provides the parts of the CUDA installation and other libriaries that +# cannot be redistributed as part of NESSI due to license limitations. While +# GPU-based software from NESSI will _run_ without these, installation of +# additional software that builds upon CUDA or other libraries requires that +# these installation are present under `host_injections`. +# +# The `host_injections` directory is a variant symlink that by default points to +# `/opt/eessi`, unless otherwise defined in the local CVMFS configuration (see +# https://cvmfs.readthedocs.io/en/stable/cpt-repo.html#variant-symlinks). For the +# installation to be successful, this directory needs to be writeable by the user +# executing this script. + +# Initialise our bash functions +TOPDIR=$(dirname $(realpath $BASH_SOURCE)) +source "$TOPDIR"/../../utils.sh + +# Function to display help message +show_help() { + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --help Display this help message" + echo " --accept-cuda-eula You _must_ accept the CUDA EULA to install" + echo " CUDA, see the EULA at" + echo " https://docs.nvidia.com/cuda/eula/index.html" + echo " -e, --easystack EASYSTACKFILE Path to easystack file that defines which" + echo " packages shall be installed" + echo " -t, --temp-dir /path/to/tmpdir Specify a location to use for temporary" + echo " storage during the installation of CUDA" + echo " and/or other libraries (must have" + echo " several GB available; depends on the number of installations)" +} + +# Initialize variables +eula_accepted=0 +EASYSTACKFILE= +TEMP_DIR= + +# Parse command-line options +while [[ $# -gt 0 ]]; do + case "$1" in + --help) + show_help + exit 0 + ;; + --accept-cuda-eula) + eula_accepted=1 + shift 1 + ;; + -e|--easystack) + if [ -n "$2" ]; then + EASYSTACKFILE="$2" + shift 2 + else + echo "Error: Argument required for $1" + show_help + exit 1 + fi + ;; + -t|--temp-dir) + if [ -n "$2" ]; then + TEMP_DIR="$2" + shift 2 + else + echo "Error: Argument required for $1" + show_help + exit 1 + fi + ;; + *) + show_help + fatal_error "Error: Unknown option: $1" + ;; + esac +done + +if [[ -z "${EASYSTACKFILE}" ]]; then + fatal_error "Need the name/path to an easystack file. See command line options\n" +fi + +# Make sure NESSI is initialised +check_eessi_initialised + +# As an installation location just use $EESSI_SOFTWARE_PATH but replacing `versions` with `host_injections` +# (CUDA is a binary installation so no need to worry too much about the EasyBuild setup) +export NESSI_SITE_INSTALL=${EESSI_SOFTWARE_PATH/versions/host_injections} + +# we need a directory we can use for temporary storage +if [[ -z "${TEMP_DIR}" ]]; then + tmpdir=$(mktemp -d) +else + mkdir -p ${TEMP_DIR} + tmpdir=$(mktemp -d --tmpdir=${TEMP_DIR} cuda_n_co.XXX) + if [[ ! -d "$tmpdir" ]] ; then + fatal_error "Could not create directory ${tmpdir}" + fi +fi +echo "Created temporary directory '${tmpdir}'" + +# workaround for EasyBuild not being found when loading "extend" module +module load EasyBuild/4.9.1 + +# load NESSI-extend/2023.06-easybuild module && verify that it is loaded +NESSI_EXTEND_MODULE="NESSI-extend/2023.06-easybuild" +module load ${NESSI_EXTEND_MODULE} +ret=$? +if [ "${ret}" -ne 0 ]; then + fatal_error "An error occured while trying to load ${NESSI_EXTEND_MODULE}\n" +fi + +# do a 'eb --dry-run-short' with the EASYSTACKFILE and determine list of packages +# to be installed +echo ">> Determining if packages specified in ${EASYSTACKFILE} are missing under ${NESSI_SITE_INSTALL}" +eb_dry_run_short_out=${tmpdir}/eb_dry_run_short.out +eb --dry-run-short --rebuild --easystack ${EASYSTACKFILE} 2>&1 | tee ${eb_dry_run_short_out} +ret=$? + +# Check if CUDA shall be installed +cuda_install_needed=0 +cat ${eb_dry_run_short_out} | grep "^ \* \[[xR]\]" | grep "module: CUDA/" +ret=$? +if [ "${ret}" -eq 0 ]; then + cuda_install_needed=1 +fi + +# Make sure the CUDA EULA is accepted if it shall be installed +if [ "${cuda_install_needed}" -eq 1 ] && [ "${eula_accepted}" -ne 1 ]; then + show_help + error="\nCUDA shall be installed. However, the CUDA EULA has not been accepted\nYou _must_ accept the CUDA EULA via the appropriate command line option.\n" + fatal_error "${error}" +fi + +# determine the number of packages to be installed (assume 5 GB + num_packages * +# 3GB space needed) +number_of_packages=$(cat ${eb_dry_run_short_out} | grep "^ \* \[[xR]\]" | sed -e 's/^.*module: //' | uniq | wc -l) +echo "number of packages to be (re-)installed: '${number_of_packages}'" +base_storage_space=$((5000000 + ${number_of_packages} * 3000000)) + +required_space_in_tmpdir=${base_storage_space} +# Let's see if we have sources and build locations defined if not, we use the temporary space +if [[ -z "${EASYBUILD_BUILDPATH}" ]]; then + export EASYBUILD_BUILDPATH=${tmpdir}/build + required_space_in_tmpdir=$((required_space_in_tmpdir + ${base_storage_space})) +fi +if [[ -z "${EASYBUILD_SOURCEPATH}" ]]; then + export EASYBUILD_SOURCEPATH=${tmpdir}/sources + required_space_in_tmpdir=$((required_space_in_tmpdir + ${base_storage_space})) +fi + +# The install is pretty fat, you need lots of space for download/unpack/install +# (~3*${base_storage_space}*1000 Bytes), +# need to do a space check before we proceed +avail_space=$(df --output=avail "${NESSI_SITE_INSTALL}"/ | tail -n 1 | awk '{print $1}') +min_disk_storage=$((3 * ${base_storage_space})) +if (( avail_space < ${min_disk_storage} )); then + fatal_error "Need at least $(echo "${min_disk_storage} / 1000000" | bc) GB disk space to install CUDA and other libraries under ${NESSI_SITE_INSTALL}, exiting now..." +fi +avail_space=$(df --output=avail "${tmpdir}"/ | tail -n 1 | awk '{print $1}') +if (( avail_space < required_space_in_tmpdir )); then + error="Need at least $(echo "${required_space_in_tmpdir} / 1000000" | bc) temporary disk space under ${tmpdir}.\n" + error="${error}Set the environment variable TEMP_DIR to a location with adequate space to pass this check." + error="${error}You can alternatively set EASYBUILD_BUILDPATH and/or EASYBUILD_SOURCEPATH" + error="${error}to reduce this requirement. Exiting now..." + fatal_error "${error}" +fi + +# Brief explanation of parameters: +# - prefix: using $tmpdir as default base directory for several EB settings +# - rebuild: we need the --rebuild option, as the CUDA module may or may not be on the +# `MODULEPATH` yet. Even if it is, we still want to redo this installation +# since it will provide the symlinked targets for the parts of the CUDA +# and/or other installation in the `.../versions/...` prefix +# - installpath-modules: We install the module in our `tmpdir` since we do not need the modulefile, +# we only care about providing the targets for the symlinks. +# - ${cuda_arg}: We only set the --accept-eula-for=CUDA option if CUDA will be installed and if +# this script was called with the argument --accept-cuda-eula. +# - hooks: We don't want hooks used in this install, we need vanilla +# installations of CUDA and/or other libraries +# - easystack: Path to easystack file that defines which packages shall be +# installed +cuda_arg= +if [[ ${eula_accepted} -eq 1 ]]; then + cuda_arg="--accept-eula-for=CUDA" +fi +touch "$tmpdir"/none.py +eb --prefix="$tmpdir" \ + --rebuild \ + --installpath-modules=${tmpdir} \ + "${cuda_arg}" \ + --hooks="$tmpdir"/none.py \ + --easystack ${EASYSTACKFILE} +ret=$? +if [ $ret -ne 0 ]; then + eb_last_log=$(unset EB_VERBOSE; eb --last-log) + cp -a ${eb_last_log} . + fatal_error "some installation failed, please check EasyBuild logs $(basename ${eb_last_log})..." +else + echo_green "all installations at ${NESSI_SITE_INSTALL}/software/... succeeded!" +fi +# clean up tmpdir +rm -rf "${tmpdir}" diff --git a/test_suite.sh b/test_suite.sh index f213ad68a3..2f304dd9bc 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -183,7 +183,16 @@ else fi # List the tests we want to run -export REFRAME_ARGS='--tag CI --tag 1_node --nocolor --exclude=TensorFlow' +# export REFRAME_ARGS='--tag CI --tag 1_node --nocolor --exclude=TensorFlow' +export REFRAME_FLAGS='--nocolor' +export REFRAME_TAGS='--tag CI --tag 1_node' +# Limit test names to relevant tests +# For now, we limit to OSU. +# In the future, we should create some mapping between module names of modules that are deployed. +# E.g. if TensorFlow was deployed, we want to run with `-n TensorFlow`. If OpenMPI was deployed +# we want to run with `-n OSU`, or maybe even `-n OSU -n GROMACS.*foss` to also test _one_ OpenMPI application +export REFRAME_INCLUDE_PATTERNS='-n OSU' +export REFRAME_ARGS="${REFRAME_FLAGS} ${REFRAME_TAGS} ${REFRAME_INCLUDE_PATTERNS}" echo "Listing tests: reframe ${REFRAME_ARGS} --list" reframe ${REFRAME_ARGS} --list if [[ $? -eq 0 ]]; then