diff --git a/easybuild/easyblocks/c/clang.py b/easybuild/easyblocks/c/clang.py index d6528aa553..ddefa7cbff 100644 --- a/easybuild/easyblocks/c/clang.py +++ b/easybuild/easyblocks/c/clang.py @@ -49,13 +49,13 @@ from easybuild.tools.filetools import apply_regex_substitutions, change_dir, mkdir, symlink, which from easybuild.tools.modules import get_software_root from easybuild.tools.run import run_shell_cmd -from easybuild.tools.systemtools import AARCH32, AARCH64, POWER, X86_64 +from easybuild.tools.systemtools import AARCH32, AARCH64, POWER, RISCV64, X86_64 from easybuild.tools.systemtools import get_cpu_architecture, get_os_name, get_os_version, get_shared_lib_ext from easybuild.tools.environment import setvar # List of all possible build targets for Clang CLANG_TARGETS = ["all", "AArch64", "AMDGPU", "ARM", "CppBackend", "Hexagon", "Mips", - "MBlaze", "MSP430", "NVPTX", "PowerPC", "R600", "Sparc", + "MBlaze", "MSP430", "NVPTX", "PowerPC", "R600", "RISCV", "Sparc", "SystemZ", "X86", "XCore"] # Mapping of EasyBuild CPU architecture names to list of default LLVM target names @@ -63,6 +63,7 @@ AARCH32: ['ARM'], AARCH64: ['AArch64'], POWER: ['PowerPC'], + RISCV64: ['RISCV'], X86_64: ['X86'], } diff --git a/easybuild/easyblocks/g/gamess_us.py b/easybuild/easyblocks/g/gamess_us.py index 0220bc8c0c..ca8985bfcc 100644 --- a/easybuild/easyblocks/g/gamess_us.py +++ b/easybuild/easyblocks/g/gamess_us.py @@ -33,12 +33,12 @@ @author: Toon Willems (Ghent University) @author: Pablo Escobar (sciCORE, SIB, University of Basel) @author: Benjamin Roberts (The University of Auckland) +@author: Alex Domingo (Vrije Universiteit Brussel) """ import fileinput import glob import os import re -import shutil import sys import tempfile @@ -46,11 +46,22 @@ from easybuild.framework.easyconfig import CUSTOM, MANDATORY from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option -from easybuild.tools.filetools import read_file, write_file +from easybuild.tools.filetools import change_dir, copy_file, mkdir, read_file, write_file, remove_dir from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd, run_cmd_qa -from easybuild.tools.systemtools import get_platform_name -from easybuild.tools import toolchain +from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import POWER, X86_64 +from easybuild.tools.systemtools import get_cpu_architecture +from easybuild.tools import LooseVersion, toolchain + +GAMESS_INSTALL_INFO = 'install.info' +GAMESS_SERIAL_TESTS = [ + 'exam05', # only the gradients for CITYP=CIS run in parallel + 'exam32', # only CCTYP=CCSD or CCTYP=CCSD(T) can run in parallel + 'exam42', # ROHF'S CCTYP must be CCSD or CR-CCL, with serial execution + 'exam45', # only CCTYP=CCSD or CCTYP=CCSD(T) can run in parallel + 'exam46', # ROHF'S CCTYP must be CCSD or CR-CCL, with serial execution + 'exam47', # ROHF'S CCTYP must be CCSD or CR-CCL, with serial execution +] class EB_GAMESS_minus_US(EasyBlock): @@ -63,8 +74,10 @@ def extra_options(): 'ddi_comm': ['mpi', "DDI communication layer to use", CUSTOM], 'maxcpus': [None, "Maximum number of cores per node", MANDATORY], 'maxnodes': [None, "Maximum number of nodes", MANDATORY], + 'hyperthreading': [True, "Enable support for hyperthreading (2 threads per core)", CUSTOM], 'runtest': [True, "Run GAMESS-US tests", CUSTOM], - 'scratch_dir': ['$TMPDIR', "Scratch dir to be used in rungms script", CUSTOM], + 'scratch_dir': ['$TMPDIR', "dir for temporary binary files", CUSTOM], + 'user_scratch_dir': ['$TMPDIR', "dir for supplementary output files", CUSTOM], } return EasyBlock.extra_options(extra_vars) @@ -72,14 +85,32 @@ def __init__(self, *args, **kwargs): """Easyblock constructor, enable building in installation directory.""" super(EB_GAMESS_minus_US, self).__init__(*args, **kwargs) self.build_in_installdir = True + + # resolve path to scratch dir and make it + scratch_dir_resolved = os.path.expandvars(self.cfg['scratch_dir']) + if "$" in scratch_dir_resolved: + error_msg = "Provided scratch directory '%s' does not resolve into a path." + raise EasyBuildError(error_msg, self.cfg['scratch_dir']) + + mkdir(scratch_dir_resolved, parents=True) + self.testdir = None if self.cfg['runtest']: - self.testdir = tempfile.mkdtemp() + # run tests in scratch dir + self.testdir = tempfile.mkdtemp(dir=scratch_dir_resolved) + # make sure test dir doesn't contain [ or ], rungms csh script doesn't handle that well ("set: No match") if re.search(r'[\[\]]', self.testdir): error_msg = "Temporary dir for tests '%s' will cause problems with rungms csh script" raise EasyBuildError(error_msg, self.testdir) + # OpenMP cannot be enabled across the board, activate it only through + # GMS_OPENMP so that it gets enabled by the build where needed + self.omp_enabled = False + if self.toolchain.options.get('openmp', None): + self.omp_enabled = True + self.toolchain.options['openmp'] = False + def extract_step(self): """Extract sources.""" # strip off 'gamess' part to avoid having everything in a 'gamess' subdirectory @@ -87,107 +118,215 @@ def extract_step(self): super(EB_GAMESS_minus_US, self).extract_step() def configure_step(self): - """Configure GAMESS-US build via provided interactive 'config' script.""" + """Configure GAMESS-US via install.info file""" + installinfo_opts = {} + + # installation paths + installinfo_opts["GMS_PATH"] = self.installdir + installinfo_opts["GMS_BUILD_DIR"] = self.builddir # machine type - platform_name = get_platform_name() - x86_64_linux_re = re.compile('^x86_64-.*$') - if x86_64_linux_re.match(platform_name): + cpu_arch = get_cpu_architecture() + if cpu_arch == X86_64: machinetype = "linux64" + elif cpu_arch == POWER: + machinetype = "ibm64" else: - raise EasyBuildError("Build target %s currently unsupported", platform_name) + raise EasyBuildError("Build target %s currently unsupported", cpu_arch) + + installinfo_opts["GMS_TARGET"] = machinetype + installinfo_opts["GMS_HPC_SYSTEM_TARGET"] = "generic" - # compiler config + # Compiler config comp_fam = self.toolchain.comp_family() - fortran_comp, fortran_ver = None, None + fortran_comp, fortran_version = None, None if comp_fam == toolchain.INTELCOMP: fortran_comp = 'ifort' (out, _) = run_cmd("ifort -v", simple=False) res = re.search(r"^ifort version ([0-9]+)\.[0-9.]+$", out) - if res: - fortran_ver = res.group(1) - else: + try: + version_num = res.group(1) + except (AttributeError, IndexError): raise EasyBuildError("Failed to determine ifort major version number") + fortran_version = {"GMS_IFORT_VERNO": version_num} elif comp_fam == toolchain.GCC: fortran_comp = 'gfortran' - fortran_ver = '.'.join(get_software_version('GCC').split('.')[:2]) + version_num = '.'.join(get_software_version('GCC').split('.')[:2]) + fortran_version = {"GMS_GFORTRAN_VERNO": version_num} else: raise EasyBuildError("Compiler family '%s' currently unsupported.", comp_fam) - # math library config + installinfo_opts["GMS_FORTRAN"] = fortran_comp + installinfo_opts.update(fortran_version) + + # OpenMP config + installinfo_opts["GMS_OPENMP"] = self.omp_enabled + + # Math library config known_mathlibs = ['imkl', 'OpenBLAS', 'ATLAS', 'ACML'] - mathlib, mathlib_root = None, None - for mathlib in known_mathlibs: - mathlib_root = get_software_root(mathlib) - if mathlib_root is not None: - break - if mathlib_root is None: + loaded_mathlib = [mathlib for mathlib in known_mathlibs if get_software_root(mathlib)] + + # math library: default settings + try: + mathlib = loaded_mathlib[0].lower() + mathlib_root = get_software_root(loaded_mathlib[0]) + mathlib_subfolder = '' + mathlib_flags = '' + except IndexError: raise EasyBuildError("None of the known math libraries (%s) available, giving up.", known_mathlibs) + + # math library: special cases if mathlib == 'imkl': mathlib = 'mkl' - mathlib_root = os.path.join(mathlib_root, 'mkl') + mathlib_subfolder = 'mkl/lib/intel64' + mkl_version = '12' + imkl_ver = get_software_version('imkl') + if LooseVersion(imkl_ver) >= LooseVersion("2021"): + # OneAPI version + mathlib_subfolder = 'mkl/latest/lib/intel64' + mkl_version = 'oneapi' + installinfo_opts["GMS_MKL_VERNO"] = mkl_version + + if LooseVersion(self.version) >= LooseVersion('20230601'): + installinfo_opts["GMS_THREADED_BLAS"] = self.omp_enabled + + elif mathlib == 'openblas': + mathlib_flags = "-lopenblas -lgfortran" + if LooseVersion(self.version) >= LooseVersion('20210101'): + mathlib_subfolder = 'lib' + + if mathlib_root is not None: + mathlib_path = os.path.join(mathlib_root, mathlib_subfolder) + self.log.debug("Software root of math libraries set to: %s", mathlib_path) else: - mathlib = mathlib.lower() + raise EasyBuildError("Software root of math libraries (%s) not found", mathlib) + + installinfo_opts["GMS_MATHLIB"] = mathlib + installinfo_opts["GMS_MATHLIB_PATH"] = mathlib_path + installinfo_opts["GMS_LAPACK_LINK_LINE"] = '"%s"' % mathlib_flags # verify selected DDI communication layer known_ddi_comms = ['mpi', 'mixed', 'shmem', 'sockets'] if not self.cfg['ddi_comm'] in known_ddi_comms: - raise EasyBuildError("Unsupported DDI communication layer specified (known: %s): %s", - known_ddi_comms, self.cfg['ddi_comm']) + raise EasyBuildError( + "Unsupported DDI communication layer specified (known: %s): %s", known_ddi_comms, self.cfg['ddi_comm'] + ) + + installinfo_opts["GMS_DDI_COMM"] = self.cfg['ddi_comm'] # MPI library config - mpilib, mpilib_root, mpilib_path = None, None, None + mpilib, mpilib_path = '', '' if self.cfg['ddi_comm'] == 'mpi': - known_mpilibs = ['impi', 'OpenMPI', 'MVAPICH2', 'MPICH2'] - for mpilib in known_mpilibs: - mpilib_root = get_software_root(mpilib) - if mpilib_root is not None: - break - if mpilib_root is None: + loaded_mpilib = [mpilib for mpilib in known_mpilibs if get_software_root(mpilib)] + + # mpi library: default settings + try: + mpilib = loaded_mpilib[0].lower() + mpilib_root = get_software_root(loaded_mpilib[0]) + mpilib_subfolder = '' + except IndexError: raise EasyBuildError("None of the known MPI libraries (%s) available, giving up.", known_mpilibs) - mpilib_path = mpilib_root + if mpilib == 'impi': - mpilib_path = os.path.join(mpilib_root, 'intel64') + impi_ver = get_software_version('impi') + if LooseVersion(impi_ver) >= LooseVersion("2021"): + mpilib_subfolder = "mpi/latest" + else: + mpilib_subfolder = "intel64" + + if LooseVersion(impi_ver) >= LooseVersion("2019"): + # fix rungms settings for newer versions of Intel MPI + print("PATCH IMPI") + rungms = os.path.join(self.builddir, 'rungms') + try: + for line in fileinput.input(rungms, inplace=1, backup='.orig'): + line = re.sub(r"^(\s*setenv\s*I_MPI_STATS).*", r"# \1", line) + line = re.sub(r"^(\s*setenv\s*I_MPI_WAIT_MODE)\s*enable.*", r"\1 1", line) + sys.stdout.write(line) + except IOError as err: + raise EasyBuildError("Failed to patch Intel MPI settings in %s: %s", rungms, err) + + if mpilib_root is not None: + mpilib_path = os.path.join(mpilib_root, mpilib_subfolder) + self.log.debug("Software root of MPI libraries set to: %s", mpilib_path) else: - mpilib = mpilib.lower() - else: - mpilib, mpilib_root = '', '' - - # run interactive 'config' script to generate install.info file - cmd = "%(preconfigopts)s ./config %(configopts)s" % { - 'preconfigopts': self.cfg['preconfigopts'], - 'configopts': self.cfg['configopts'], - } - qa = { - "After the new window is open, please hit to go on.": '', - "please enter your target machine name: ": machinetype, - "Version? [00] ": self.version, - "Please enter your choice of FORTRAN: ": fortran_comp, - "hit to continue to the math library setup.": '', - "MKL pathname? ": mathlib_root, - "MKL version (or 'skip')? ": 'skip', - "MKL version (or 'proceed')? ": 'proceed', # changed in gamess-20170420R1 - "please hit to compile the GAMESS source code activator": '', - "please hit to set up your network for Linux clusters.": '', - "communication library ('sockets' or 'mpi')? ": self.cfg['ddi_comm'], - "Enter MPI library (impi, mvapich2, mpt, sockets):": mpilib, - "Enter MPI library (impi, mpich, mpich2, mvapich2, mpt, sockets):": mpilib, # changed in gamess-20170420R1 - "Please enter your %s's location: " % mpilib: mpilib_root, - "Do you want to try LIBCCHEM? (yes/no): ": 'no', - "Enter full path to OpenBLAS libraries (without 'lib' subdirectory):": mathlib_root, - "Optional: Build Michigan State University CCT3 & CCSD3A methods? (yes/no): ": 'no', - } - stdqa = { - r"GAMESS directory\? \[.*\] ": self.builddir, - r"GAMESS build directory\? \[.*\] ": self.installdir, # building in install directory - r"Enter only the main version number, such as .*\nVersion\? ": fortran_ver, - r".+gfortran version.\n( \n)?Please enter only the first decimal place, such as .*:": fortran_ver, - "Enter your choice of 'mkl' or .* 'none': ": mathlib, - } - run_cmd_qa(cmd, qa=qa, std_qa=stdqa, log_all=True, simple=True) - - self.log.debug("Contents of install.info:\n%s" % read_file(os.path.join(self.builddir, 'install.info'))) + raise EasyBuildError("Software root of MPI libraries (%s) not found", mpilib) + + installinfo_opts["GMS_MPI_LIB"] = mpilib + installinfo_opts["GMS_MPI_PATH"] = mpilib_path + + # Accelerators (disabled) + # TODO: add support for GPUs + if LooseVersion(self.version) >= LooseVersion('20230601'): + # offloading onto Intel GPUs + installinfo_opts["GMS_OPENMP_OFFLOAD"] = False + + # These are extra programs which for now we simply set all to FALSE + installinfo_opts["GMS_MSUCC"] = False + installinfo_opts["GMS_PHI"] = "none" + installinfo_opts["GMS_SHMTYPE"] = "sysv" + installinfo_opts["GMS_LIBCCHEM"] = False # libcchem + if LooseVersion(self.version) >= LooseVersion('20230601'): + # install options for libcchem2 + installinfo_opts["GMS_HPCCHEM"] = False + installinfo_opts["GMS_HPCCHEM_USE_DATA_SERVERS"] = False + # build Michigan State University code + installinfo_opts["GMS_MSUAUTO"] = False + + # Optional plug-ins and interfaces + # libXC + if LooseVersion(self.version) >= LooseVersion('20200101'): + installinfo_opts['GMS_LIBXC'] = False + if get_software_root('libxc'): + installinfo_opts['GMS_LIBXC'] = True + # the linker needs to be patched to use external libXC + lixc_libs = [os.path.join(os.environ['EBROOTLIBXC'], 'lib', lib) for lib in ['libxcf03.a', 'libxc.a']] + libxc_linker_flags = ' '.join(lixc_libs) + try: + lked = os.path.join(self.builddir, 'lked') + for line in fileinput.input(lked, inplace=1, backup='.orig'): + line = re.sub(r"^(\s*set\sLIBXC_FLAGS)=.*GMS_PATH.*", r'\1="%s"' % libxc_linker_flags, line) + sys.stdout.write(line) + except IOError as err: + raise EasyBuildError("Failed to patch %s: %s", lked, err) + # MDI + # needs https://github.com/MolSSI-MDI/MDI_Library + installinfo_opts['GMS_MDI'] = False + # NBO + installinfo_opts['NBO'] = False + if get_software_root('NBO'): + installinfo_opts['NBO'] = True + # NEO + installinfo_opts['NEO'] = False + # RISM + if LooseVersion(self.version) >= LooseVersion('20230601'): + installinfo_opts['RISM'] = False + # TINKER + installinfo_opts['TINKER'] = False + if get_software_root('TINKER'): + installinfo_opts['TINKER'] = True + # VB2000 + installinfo_opts['VB2000'] = False + # VM2 + installinfo_opts['GMS_VM2'] = False + # XMVB + installinfo_opts['XMVB'] = False + + # add include paths from dependencies + installinfo_opts["GMS_FPE_FLAGS"] = '"%s"' % os.environ['CPPFLAGS'] + # might be useful for debugging + # installinfo_opts["GMS_FPE_FLAGS"] = '"%s"' % os.environ['CPPFLAGS'] + "-ffpe-trap=invalid,zero,overflow" + + # write install.info file with configuration settings + installinfo_file = os.path.join(self.builddir, GAMESS_INSTALL_INFO) + # replace boolean options with their string representation + boolean_opts = {opt: str(val).lower() for opt, val in installinfo_opts.items() if val in [True, False]} + installinfo_opts.update(boolean_opts) + # format: setenv KEY VALUE + installinfo_txt = '\n'.join(["setenv %s %s" % (k, installinfo_opts[k]) for k in installinfo_opts]) + write_file(installinfo_file, installinfo_txt) + self.log.debug("Contents of %s:\n%s" % (installinfo_file, read_file(installinfo_file))) # patch hardcoded settings in rungms to use values specified in easyconfig file rungms = os.path.join(self.builddir, 'rungms') @@ -202,13 +341,45 @@ def configure_step(self): line = re.sub(r"^(\s*set GA_MPI_ROOT)=.*%s.*" % mpilib.lower(), r"\1=%s" % mpilib_path, line) # comment out all adjustments to $LD_LIBRARY_PATH that involves hardcoded paths line = re.sub(r"^(\s*)(setenv\s*LD_LIBRARY_PATH\s*/.*)", r"\1#\2", line) - if self.cfg['scratch_dir']: - line = re.sub(r"^(\s*set\s*SCR)=.*", r"\1=%s" % self.cfg['scratch_dir'], line) - line = re.sub(r"^(\s*set\s*USERSCR)=.*", r"\1=%s" % self.cfg['scratch_dir'], line) + # scratch directory paths + line = re.sub(r"^(\s*set\s*SCR)=.*", r"if ( ! $?SCR ) \1=%s" % self.cfg['scratch_dir'], line) + line = re.sub( + r"^(\s*set\s*USERSCR)=.*", r"if ( ! $?USERSCR ) \1=%s" % self.cfg['user_scratch_dir'], line + ) + line = re.sub(r"^(df -k \$SCR)$", r"mkdir -p $SCR && mkdir -p $USERSCR && \1", line) + if self.cfg['hyperthreading'] is False: + # disable hyperthreading (1 thread per core) + line = re.sub(r"\$PPN \+ \$PPN", r"$PPN", line) + line = re.sub(r"\$NCPUS \+ \$NCPUS", r"$NCPUS", line) sys.stdout.write(line) except IOError as err: raise EasyBuildError("Failed to patch %s: %s", rungms, err) + # Replacing the MAXCPUS and MAXNODES in compddi to a value from the EasyConfig file + compddi = os.path.join(self.builddir, 'ddi/compddi') + try: + for line in fileinput.input(compddi, inplace=1, backup='.orig'): + line = re.sub(r"^(\s*set MAXCPUS)=.*", r"\1=%s" % self.cfg['maxcpus'], line, 1) + line = re.sub(r"^(\s*set MAXNODES)=.*", r"\1=%s" % self.cfg['maxnodes'], line, 1) + sys.stdout.write(line) + except IOError as err: + raise EasyBuildError("Failed to patch compddi", compddi, err) + + # for GAMESS-US 20200630-R1 we need to build the actvte.x program + if self.version == "20200630-R1": + actvte = os.path.join(self.builddir, 'tools/actvte.code') + try: + for line in fileinput.input(actvte, inplace=1, backup='.orig'): + line = re.sub("[*]UNX", " ", line) + sys.stdout.write(line) + except IOError as err: + raise EasyBuildError("Failed to patch actvte.code", actvte, err) + # compiling + run_cmd("mv %s/tools/actvte.code" % self.builddir + " %s/tools/actvte.f" % self.builddir) + run_cmd( + "%s -o " % fortran_comp + " %s/tools/actvte.x" % self.builddir + " %s/tools/actvte.f" % self.builddir + ) + def build_step(self): """Custom build procedure for GAMESS-US: using compddi, compall and lked scripts.""" compddi = os.path.join(self.cfg['start_dir'], 'ddi', 'compddi') @@ -240,54 +411,81 @@ def build_step(self): def test_step(self): """Run GAMESS-US tests (if 'runtest' easyconfig parameter is set to True).""" - # don't use provided 'runall' script for tests, since that only runs the tests single-core if self.cfg['runtest']: - - if not build_option('mpi_tests'): - self.log.info("Skipping testing of GAMESS-US since MPI testing is disabled") - return - + # Avoid provided 'runall' script for tests, since that only runs the tests in serial + # Tests must be run in parallel for MPI builds and can be run serial for other build types + + target_tests = [ + # (test name, path to test input) + (os.path.splitext(os.path.basename(exam_file))[0], exam_file) + for exam_file in glob.glob(os.path.join(self.installdir, 'tests', 'standard', 'exam*.inp')) + ] + test_procs = "1" + test_env_vars = ['export OMP_NUM_THREADS=1'] + + if self.cfg['ddi_comm'] == 'mpi': + if not build_option('mpi_tests'): + self.log.info("Skipping tests of MPI build of GAMESS-US by user request ('mpi_tests' is disabled)") + return + + # MPI builds can only run tests that support parallel execution + if int(self.cfg['parallel']) < 2: + self.log.info("Skipping testing of GAMESS-US as MPI tests need at least 2 CPU cores to run") + return + + test_procs = str(self.cfg['parallel']) + target_tests = [exam for exam in target_tests if exam[0] not in GAMESS_SERIAL_TESTS] + + if self.toolchain.mpi_family() == toolchain.INTELMPI: + test_env_vars.extend([ + # enable fallback in case first fabric fails (see $I_MPI_FABRICS_LIST) + 'export I_MPI_FALLBACK=enable', + # tests are only run locally (single node), so no SSH required + 'export I_MPI_HYDRA_BOOTSTRAP=fork', + ]) + + # Prepare test directory to run tests try: cwd = os.getcwd() - os.chdir(self.testdir) + change_dir(self.testdir) except OSError as err: raise EasyBuildError("Failed to move to temporary directory for running tests: %s", err) - # copy input files for exam standard tests - for test_input in glob.glob(os.path.join(self.installdir, 'tests', 'standard', 'exam*.inp')): + for exam, exam_file in target_tests: try: - shutil.copy2(test_input, os.getcwd()) + copy_file(exam_file, self.testdir) except OSError as err: - raise EasyBuildError("Failed to copy %s to %s: %s", test_input, os.getcwd(), err) + raise EasyBuildError("Failed to copy test '%s' to %s: %s", exam, self.testdir, err) + test_env_vars.append('SCR=%s' % self.testdir) + + # run target exam tests, dump output to exam.log rungms = os.path.join(self.installdir, 'rungms') - test_env_vars = ['TMPDIR=%s' % self.testdir] - if self.toolchain.mpi_family() == toolchain.INTELMPI: - test_env_vars.extend([ - 'I_MPI_FALLBACK=enable', # enable fallback in case first fabric fails (see $I_MPI_FABRICS_LIST) - 'I_MPI_HYDRA_BOOTSTRAP=fork', # tests are only run locally (2 processes), so no SSH required - ]) - - # run all exam tests, dump output to exam.log - n_tests = 47 - for i in range(1, n_tests + 1): - test_cmd = ' '.join(test_env_vars + [rungms, 'exam%02d' % i, self.version, '1', '2']) - (out, _) = run_cmd(test_cmd, log_all=True, simple=False) - write_file('exam%02d.log' % i, out) + for exam, exam_file in target_tests: + rungms_prefix = ' && '.join(test_env_vars) + test_cmd = [rungms_prefix, rungms, exam_file, self.version, test_procs, test_procs] + (out, _) = run_cmd(' '.join(test_cmd), log_all=True, simple=False) + write_file('%s.log' % exam, out) - # verify output of tests check_cmd = os.path.join(self.installdir, 'tests', 'standard', 'checktst') (out, _) = run_cmd(check_cmd, log_all=True, simple=False) - success_regex = re.compile("^All %d test results are correct" % n_tests, re.M) - if success_regex.search(out): - self.log.info("All tests ran successfully!") + + # verify output of tests + failed_regex = re.compile(r"^.*!!FAILED\.$", re.M) + failed_tests = set([exam[0:6] for exam in failed_regex.findall(out)]) + done_tests = set([exam[0] for exam in target_tests]) + if done_tests - failed_tests == done_tests: + info_msg = "All target tests ran successfully!" + if self.cfg['ddi_comm'] == 'mpi': + info_msg += " (serial tests ignored: %s)" % ", ".join(GAMESS_SERIAL_TESTS) + self.log.info(info_msg) else: - raise EasyBuildError("Not all tests ran successfully...") + raise EasyBuildError("ERROR: Not all target tests ran successfully") # cleanup - os.chdir(cwd) + change_dir(cwd) try: - shutil.rmtree(self.testdir) + remove_dir(self.testdir) except OSError as err: raise EasyBuildError("Failed to remove test directory %s: %s", self.testdir, err) diff --git a/easybuild/easyblocks/j/java.py b/easybuild/easyblocks/j/java.py index 3916b734a9..a835f671e2 100644 --- a/easybuild/easyblocks/j/java.py +++ b/easybuild/easyblocks/j/java.py @@ -38,7 +38,7 @@ from easybuild.tools.config import build_option from easybuild.tools.filetools import adjust_permissions, change_dir, copy_dir, copy_file, remove_dir, which from easybuild.tools.run import run_shell_cmd -from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_shared_lib_ext +from easybuild.tools.systemtools import AARCH64, POWER, RISCV64, X86_64, get_cpu_architecture, get_shared_lib_ext from easybuild.tools.utilities import nub @@ -54,6 +54,8 @@ def __init__(self, *args, **kwargs): jdkarch = 'aarch64' elif myarch == POWER: jdkarch = 'ppc64le' + elif myarch == RISCV64: + jdkarch = 'riscv64' elif myarch == X86_64: jdkarch = 'x64' else: diff --git a/easybuild/easyblocks/m/mesa.py b/easybuild/easyblocks/m/mesa.py index 4a73625185..e482ac9c13 100644 --- a/easybuild/easyblocks/m/mesa.py +++ b/easybuild/easyblocks/m/mesa.py @@ -35,7 +35,7 @@ from easybuild.easyblocks.generic.mesonninja import MesonNinja from easybuild.tools.filetools import copy_dir -from easybuild.tools.systemtools import POWER, X86_64, AARCH64 +from easybuild.tools.systemtools import POWER, X86_64, AARCH64, RISCV64 from easybuild.tools.systemtools import get_cpu_architecture, get_cpu_features, get_shared_lib_ext @@ -69,6 +69,7 @@ def __init__(self, *args, **kwargs): X86_64: ['swrast'], POWER: ['swrast'], AARCH64: ['swrast'], + RISCV64: ['swrast'], } if LooseVersion(self.version) < LooseVersion('22'): # swr driver support removed in Mesa 22.0 diff --git a/easybuild/easyblocks/m/metalwalls.py b/easybuild/easyblocks/m/metalwalls.py new file mode 100644 index 0000000000..1241249a7a --- /dev/null +++ b/easybuild/easyblocks/m/metalwalls.py @@ -0,0 +1,175 @@ +## +# Copyright 2009-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for `metalwalls`, implemented as an easyblock + +@author: Davide Grassano (CECAM, EPFL) +""" +import os + +import easybuild.tools.environment as env +import easybuild.tools.toolchain as toolchain +from easybuild.tools import LooseVersion +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.modules import get_software_root, get_software_version +from easybuild.tools.run import run_cmd + +from easybuild.easyblocks.generic.makecp import MakeCp + + +class EB_MetalWalls(MakeCp): + """Support for building and installing `metalwalls`.""" + + def __init__(self, *args, **kwargs): + """Add extra config options specific to `metalwalls`.""" + super(EB_MetalWalls, self).__init__(*args, **kwargs) + + self._build_python_interface = False + self._extra_schecks = [] + + def configure_step(self): + """Custom configuration procedure for `metalwalls`.""" + comp_fam = self.toolchain.comp_family() + + f90flags = os.getenv('F90FLAGS', '').split(' ') + fppflags = os.getenv('FPPFLAGS', '').split(' ') + ldflags = os.getenv('LDFLAGS', '').split(' ') + f90wrap = '' + f2py = '' + fcompiler = '' + + if comp_fam == toolchain.INTELCOMP: + jflag = '-module' + fppflags = ['-fpp'] + elif comp_fam == toolchain.GCC: + jflag = '-J' + fppflags = ['-cpp'] + elif comp_fam == toolchain.NVHPC: + if self.toolchain.options.get('usempi', False): + raise EasyBuildError('NVHPC compilation does not support MPI') + jflag = '-module ' # Space is needed + fppflags = ['-DMW_SERIAL', '-Mpreprocess'] + f90flags += ['-Minline', '-acc', '-gpu=managed'] + else: + raise EasyBuildError('Unsupported compiler family: %s' % comp_fam) + + # https://gitlab.com/ampere2/metalwalls/-/wikis/install#plumed + plumed = get_software_root('PLUMED') + f90wrap = get_software_root('f90wrap') + f90wrap_version = get_software_version('f90wrap') + + if LooseVersion(self.version) <= LooseVersion('21.06.1'): + if LooseVersion(f90wrap_version) > LooseVersion('0.2.13'): + raise EasyBuildError('MetalWalls version %s requires f90wrap <= 0.2.13' % self.version) + + tpl_rgx = 'alltests\\.append(suite_%s)' + if plumed: + f90flags += ['-fallow-argument-mismatch'] # Code inside ifdef causes mismatch errors + fppflags += ['-DMW_USE_PLUMED'] + cmd = ['touch', 'mw2.diff'] + run_cmd(' '.join(cmd), log_all=False, log_ok=False, simple=False, regexp=False) + cmd = ['plumed', 'patch', '-d mw2.diff', '--patch', '--shared', '--engine', 'mw2'] + run_cmd(' '.join(cmd), log_all=True, simple=False) + else: + self.log.info('PLUMED not found, excluding from test-suite') + rgx = tpl_rgx % 'plumed' + cmd = ['sed', '-i', "'s/^\\( \\+\\)%s$/\\1pass # %s/'" % (rgx, rgx), 'tests/regression_tests.py'] + cmd = ['sed', '-i', "'s/%s/pass/'" % rgx, 'tests/regression_tests.py'] + run_cmd(' '.join(cmd), log_all=True, simple=False) + + if f90wrap: + if not get_software_root('mpi4py'): + raise EasyBuildError('Building the Python interface requires mpi4py') + self._build_python_interface = True + f90wrap = 'f90wrap' + f2py = 'f2py' + fcompiler = os.getenv('F90_SEQ') + f90flags += ['-fPIC'] + self.cfg.update('build_cmd_targets', 'python') + self.cfg.update('files_to_copy', 'build/python') + self._extra_schecks += ['python/mw.py', 'python/metalwalls.py'] + + else: + self.log.info('f90wrap not found, excluding python interface from test-suite') + rgx = tpl_rgx % 'python_interface' + cmd = ['sed', '-i', "'s/%s/pass/'" % rgx, 'tests/regression_tests.py'] + run_cmd(' '.join(cmd), log_all=True, simple=False) + + # Add libraries with LAPACK support + lapack_shared_libs = os.getenv('LAPACK_SHARED_LIBS', None) + if not lapack_shared_libs: + raise EasyBuildError('Must use a toolchain with LAPACK support') + extra_libs = lapack_shared_libs.replace('.so', '').split(',') + extra_libs = [_[3:] for _ in extra_libs if _.startswith('lib')] + ldflags += ['-l%s' % lib for lib in extra_libs] + + # Write the `config.mk` file + configmk = os.path.join(self.cfg['start_dir'], 'config.mk') + with open(configmk, 'w') as f: + f.write('F90FLAGS := %s\n' % ' '.join(f90flags)) + f.write('FPPFLAGS := %s\n' % ' '.join(fppflags)) + f.write('LDFLAGS := %s\n' % ' '.join(ldflags)) + f.write('F90WRAP := %s\n' % f90wrap) + f.write('F2PY := %s\n' % f2py) + f.write('FCOMPILER := %s\n' % fcompiler) + f.write('J := %s\n' % jflag) + + with open(configmk, 'r') as f: + self.log.info('Contents of generated `config.mk`:\n%s' % f.read()) + + def test_step(self): + """ + Test the compilation using `metalwalls`'s test suite. + """ + + if self._build_python_interface: + ppath = os.getenv('PYTHONPATH', '') + ppath = os.path.join(self.cfg['start_dir'], 'build', 'python') + ':' + ppath + self.log.info('Setting PYTHONPATH for testing to %s' % ppath) + env.setvar('PYTHONPATH', ppath) + + super(EB_MetalWalls, self).test_step() + + def make_module_extra(self, extra=None): + """Add custom entries to module.""" + + txt = super(EB_MetalWalls, self).make_module_extra() + + if self._build_python_interface: + txt += self.module_generator.prepend_paths('PYTHONPATH', 'python') + + return txt + + def sanity_check_step(self): + """Custom sanity check for Quantum ESPRESSO.""" + + bins = ['mw'] + + custom_paths = { + 'files': [os.path.join('bin', x) for x in bins] + self._extra_schecks, + 'dirs': [] + } + + super(EB_MetalWalls, self).sanity_check_step(custom_paths=custom_paths)