From 6c04a20b7bed3a2531751b4152dfd2169348686c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 18 Dec 2024 15:21:55 +0100 Subject: [PATCH 1/5] remove unused custom easyblock for TAU --- easybuild/easyblocks/t/tau.py | 280 ---------------------------------- 1 file changed, 280 deletions(-) delete mode 100644 easybuild/easyblocks/t/tau.py diff --git a/easybuild/easyblocks/t/tau.py b/easybuild/easyblocks/t/tau.py deleted file mode 100644 index 2cb7598656..0000000000 --- a/easybuild/easyblocks/t/tau.py +++ /dev/null @@ -1,280 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing TAU, implemented as an easyblock - -@author Kenneth Hoste (Ghent University) -@author Markus Geimer (Juelich Supercomputing Centre) -@author Bernd Mohr (Juelich Supercomputing Centre) -""" -import os - -from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.easyblocks.pdt import find_arch_dir -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools import toolchain -from easybuild.tools.build_log import EasyBuildError, print_msg -from easybuild.tools.filetools import symlink -from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.systemtools import get_shared_lib_ext - - -KNOWN_BACKENDS = { - 'scalasca': 'Scalasca', - 'scorep': 'Score-P', - 'vampirtrace': 'Vampirtrace', -} - - -class EB_TAU(ConfigureMake): - """Support for building/installing TAU.""" - - @staticmethod - def extra_options(): - """Custom easyconfig parameters for TAU.""" - backends = "Extra TAU backends to build and install; possible values: %s" % ','.join(sorted(KNOWN_BACKENDS)) - extra_vars = { - 'extra_backends': [None, backends, CUSTOM], - 'tau_makefile': ['Makefile.tau-papi-mpi-pdt', "Name of Makefile to use in $TAU_MAKEFILE", CUSTOM], - } - return ConfigureMake.extra_options(extra_vars) - - def __init__(self, *args, **kwargs): - """Initialize TAU easyblock.""" - super(EB_TAU, self).__init__(*args, **kwargs) - - self.variant_index = 0 - self.cc, self.cxx, self.fortran = None, None, None - self.mpi_inc_dir, self.mpi_lib_dir = None, None - self.opt_pkgs_opts = None - self.variant_labels = None - - def run_all_steps(self, *args, **kwargs): - """ - Put configure options in place for the different selected backends of TAU, - for the MPI, OpenMP, and hybrid variants. - """ - if self.cfg['extra_backends'] is None: - self.cfg['extra_backends'] = [] - - # make sure selected extra backends are known - unknown_backends = [] - for backend in self.cfg['extra_backends']: - if backend not in KNOWN_BACKENDS: - unknown_backends.append(backend) - if unknown_backends: - raise EasyBuildError("Encountered unknown backends: %s", ', '.join(unknown_backends)) - - # compiler options - comp_opts = "-cc=%(cc)s -c++=%(cxx)s -fortran=%(fortran)s" - - # variant-specific options - openmp_opts = " -opari" - mpi_opts = " -mpiinc=%(mpi_inc_dir)s -mpilib=%(mpi_lib_dir)s" - - # options for optional packages - opt_pkgs_opts = " %(opt_pkgs_opts)s" - - # backend option - backend_opt = " %(backend_opt)s" - - # compose templates - mpi_tmpl = comp_opts + mpi_opts + opt_pkgs_opts + backend_opt - openmp_tmpl = comp_opts + openmp_opts + opt_pkgs_opts + backend_opt - hybrid_tmpl = comp_opts + openmp_opts + mpi_opts + opt_pkgs_opts + backend_opt - - # number of iterations: # backends + 1 (for basic) - iter_cnt = len(self.cfg['extra_backends']) + 1 - - # define list of configure options to iterate over - if self.cfg['configopts']: - raise EasyBuildError("Specifying additional configure options for TAU is not supported (yet)") - - self.cfg['configopts'] = [mpi_tmpl, openmp_tmpl, hybrid_tmpl] * iter_cnt - self.log.debug("List of configure options to iterate over: %s", self.cfg['configopts']) - - # custom prefix option for configure command - self.cfg['prefix_opt'] = '-prefix=' - - # installation command is 'make install clean' - self.cfg.update('installopts', 'clean') - - return super(EB_TAU, self).run_all_steps(*args, **kwargs) - - def prepare_step(self, *args, **kwargs): - """Custom prepare step for Tau: check required dependencies and collect information on them.""" - super(EB_TAU, self).prepare_step(*args, **kwargs) - - # install prefixes for selected backends - self.backend_opts = {'tau': ''} - for backend_name, dep in KNOWN_BACKENDS.items(): - root = get_software_root(dep) - if backend_name in self.cfg['extra_backends']: - if root: - self.backend_opts[backend_name] = "-%s=%s" % (backend_name, root) - else: - raise EasyBuildError("%s is listed in extra_backends, but not available as a dependency", dep) - elif root: - raise EasyBuildError("%s included as dependency, but '%s' not in extra_backends", dep, backend_name) - - # make sure Scalasca v1.x is used as a dependency (if it's there) - if 'scalasca' in self.backend_opts and get_software_version('Scalasca').split('.')[0] != '1': - raise EasyBuildError("Scalasca v1.x must be used when scalasca backend is enabled") - - # determine values for compiler flags to use - known_compilers = { - toolchain.CLANGGCC: ['clang', 'clang++', 'gfortran'], - toolchain.GCC: ['gcc', 'g++', 'gfortran'], - toolchain.INTELCOMP: ['icc', 'icpc', 'intel'], - } - comp_fam = self.toolchain.comp_family() - if comp_fam in known_compilers: - self.cc, self.cxx, self.fortran = known_compilers[comp_fam] - - # determine values for MPI flags - self.mpi_inc_dir, self.mpi_lib_dir = os.getenv('MPI_INC_DIR'), os.getenv('MPI_LIB_DIR') - - # determine value for optional packages option template - self.opt_pkgs_opts = '' - for dep, opt in [('OTF', 'otf'), ('PAPI', 'papi'), ('PDT', 'pdt'), ('binutils', 'bfd')]: - root = get_software_root(dep) - if root: - self.opt_pkgs_opts += ' -%s=%s' % (opt, root) - - # determine list of labels, based on selected (extra) backends, variants and optional packages - self.variant_labels = [] - backend_labels = ['', '-epilog-scalasca-trace', '-scorep', '-vampirtrace-trace'] - for backend, backend_label in zip(['tau'] + sorted(KNOWN_BACKENDS.keys()), backend_labels): - if backend in ['tau'] + self.cfg['extra_backends']: - for pref, suff in [('-mpi', ''), ('', '-openmp-opari'), ('-mpi', '-openmp-opari')]: - - variant_label = 'tau' - # For non-GCC builds, the compiler name is encoded in the variant - if self.cxx and self.cxx != 'g++': - variant_label += '-' + self.cxx - if get_software_root('PAPI'): - variant_label += '-papi' - variant_label += pref - if get_software_root('PDT'): - variant_label += '-pdt' - variant_label += suff + backend_label - - self.variant_labels.append(variant_label) - - def make_installdir(self): - """Skip make install dir 'step', install dir is already created in prepare_step.""" - pass - - def configure_step(self): - """Custom configuration procedure for TAU: template configuration options before using them.""" - if self.cc is None or self.cxx is None or self.fortran is None: - raise EasyBuildError("Compiler family not supported yet: %s", self.toolchain.comp_family()) - - if self.mpi_inc_dir is None or self.mpi_lib_dir is None: - raise EasyBuildError("Failed to determine MPI include/library paths, no MPI available in toolchain?") - - # make sure selected default TAU makefile will be available - avail_makefiles = ['Makefile.' + x for x in self.variant_labels] - if self.cfg['tau_makefile'] not in avail_makefiles: - raise EasyBuildError("Specified tau_makefile %s will not be available (only: %s)", - self.cfg['tau_makefile'], avail_makefiles) - - # inform which backend/variant is being handled - backend = (['tau'] + self.cfg['extra_backends'])[self.variant_index // 3] - variant = ['mpi', 'openmp', 'hybrid'][self.variant_index % 3] - print_msg("starting with %s backend (%s variant)" % (backend, variant), log=self.log, silent=self.silent) - - self.cfg['configopts'] = self.cfg['configopts'] % { - 'backend_opt': self.backend_opts[backend], - 'cc': self.cc, - 'cxx': self.cxx, - 'fortran': self.fortran, - 'mpi_inc_dir': self.mpi_inc_dir, - 'mpi_lib_dir': self.mpi_lib_dir, - 'opt_pkgs_opts': self.opt_pkgs_opts, - } - - for key in ['preconfigopts', 'configopts', 'prebuildopts', 'preinstallopts']: - self.log.debug("%s for TAU (variant index: %s): %s", key, self.variant_index, self.cfg[key]) - - # Configure creates required subfolders in installdir, so create first (but only once, during first iteration) - if self.iter_idx == 0: - super(EB_TAU, self).make_installdir() - - super(EB_TAU, self).configure_step() - - self.variant_index += 1 - - def build_step(self): - """No custom build procedure for TAU.""" - pass - - def install_step(self): - """Create symlinks into arch-specific directories""" - super(EB_TAU, self).install_step() - # Link arch-specific directories into prefix - arch_dir = find_arch_dir(self.installdir) - self.log.info('Found %s as architecture specific directory. Creating symlinks...', arch_dir) - for subdir in ('bin', 'lib'): - src = os.path.join(arch_dir, subdir) - dst = os.path.join(self.installdir, subdir) - if os.path.lexists(dst): - self.log.info('Skipping creation of symlink %s as it already exists', dst) - else: - symlink(os.path.relpath(src, self.installdir), dst, use_abspath_source=False) - - def sanity_check_step(self): - """Custom sanity check for TAU.""" - custom_paths = { - 'files': - [os.path.join('bin', 'pprof'), os.path.join('include', 'TAU.h'), - os.path.join('lib', 'libTAU.%s' % get_shared_lib_ext())] + - [os.path.join('lib', 'lib%s.a' % x) for x in self.variant_labels] + - [os.path.join('lib', 'Makefile.' + x) for x in self.variant_labels], - 'dirs': [], - } - super(EB_TAU, self).sanity_check_step(custom_paths=custom_paths) - - def make_module_extra(self): - """Custom extra module file entries for TAU.""" - txt = super(EB_TAU, self).make_module_extra() - - txt += self.module_generator.prepend_paths('TAU_MF_DIR', 'lib') - - tau_makefile = os.path.join(self.installdir, 'lib', self.cfg['tau_makefile']) - txt += self.module_generator.set_environment('TAU_MAKEFILE', tau_makefile) - - # default measurement settings - tau_vars = { - 'TAU_CALLPATH': '1', - 'TAU_CALLPATH_DEPTH': '10', - 'TAU_COMM_MATRIX': '1', - 'TAU_PROFILE': '1', - 'TAU_TRACE': '0', - } - for key in tau_vars: - txt += self.module_generator.set_environment(key, tau_vars[key]) - - return txt From 9cf1f2f52ec4474fbf5bfeff9ad3cc22e9edf956 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 18 Dec 2024 15:30:18 +0100 Subject: [PATCH 2/5] clean up custom easyblock for Paraver, only support Paravar >= v4.7 --- easybuild/easyblocks/p/paraver.py | 90 +++++-------------------------- 1 file changed, 13 insertions(+), 77 deletions(-) diff --git a/easybuild/easyblocks/p/paraver.py b/easybuild/easyblocks/p/paraver.py index 4e4b9c6390..6c1f8c3971 100644 --- a/easybuild/easyblocks/p/paraver.py +++ b/easybuild/easyblocks/p/paraver.py @@ -34,46 +34,19 @@ from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.tools.build_log import EasyBuildError, print_msg -from easybuild.tools.filetools import change_dir +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_libdir, get_software_root class EB_Paraver(ConfigureMake): """Support for building/installing Paraver.""" - def run_all_steps(self, *args, **kwargs): - """ - Put configure/build/install options in place for the 3 different components of Paraver. - Each component lives in a separate subdirectory. - """ - if LooseVersion(self.version) < LooseVersion('4.7'): - - # leverage support for iterated installation for older Paraver versions - self.components = ['ptools_common_files', 'paraver-kernel', 'wxparaver'] - self.current_component = 0 # index in list above - - # initiate configopts with empty list - self.cfg['configopts'] = [] - - # first phase: build and install ptools - # no specific configure options for the ptools component (but configopts list element must be there) - self.cfg.update('configopts', ['']) - - # second phase: build and install paraver-kernel - self.cfg.update('configopts', ["--with-boost=%(boost)s --with-ptools-common-files=%(installdir)s"]) + def __init__(self, *args, **kwargs): + """Constructor for custom easyblock for Paraver.""" + super(EB_Paraver, self).__init__(*args, **kwargs) - # third phase: build and install wxparaver - wxparaver_configopts = ' '.join([ - '--with-boost=%(boost)s', - '--with-wxpropgrid=%(wxpropgrid)s', - '--with-paraver=%(installdir)s', - ]) - self.cfg.update('configopts', [wxparaver_configopts]) - else: - self.components, self.current_component = None, None - - return super(EB_Paraver, self).run_all_steps(*args, **kwargs) + if LooseVersion(self.version) < LooseVersion('4.7'): + raise EasyBuildError("Custom easyblock for Paraver only supports Paraver versions >= 4.7") def configure_step(self): """Custom configuration procedure for Paraver: template configuration options before using them.""" @@ -88,8 +61,6 @@ def configure_step(self): wxwidgets = get_software_root('wxWidgets') if wxwidgets: wx_config = os.path.join(wxwidgets, 'bin', 'wx-config') - elif LooseVersion(self.version) >= LooseVersion('4.7'): - raise EasyBuildError("wxWidgets is not available as a dependency") # determine value to pass to --with-wxpropgrid (library name) wxpropgrid = None @@ -107,53 +78,18 @@ def configure_step(self): else: self.log.info("wxPropertyGrid not included as dependency, assuming that's OK...") - if LooseVersion(self.version) < LooseVersion('4.7'): - component = self.components[self.current_component] - change_dir(component) - self.log.info("Customized start directory for component %s: %s", component, os.getcwd()) - - print_msg("starting with component %s" % component, log=self.log) - - self.cfg['configopts'] = self.cfg['configopts'] % { - 'boost': boost_root, - 'installdir': self.installdir, - 'wxpropgrid': wxpropgrid, - } - else: - self.cfg.update('configopts', '--with-boost=%s' % boost_root) - self.cfg.update('configopts', '--with-paraver=%s' % self.installdir) - self.cfg.update('configopts', '--with-wx-config=%s' % wx_config) - # wxPropertyGrid is not required with recent wxWidgets - if wxpropgrid: - self.cfg.update('configopts', '--with-wxpropgrid=%s' % wxpropgrid) + self.cfg.update('configopts', '--with-boost=%s' % boost_root) + self.cfg.update('configopts', '--with-paraver=%s' % self.installdir) + self.cfg.update('configopts', '--with-wx-config=%s' % wx_config) + # wxPropertyGrid is not required with recent wxWidgets + if wxpropgrid: + self.cfg.update('configopts', '--with-wxpropgrid=%s' % wxpropgrid) super(EB_Paraver, self).configure_step() def build_step(self): """Custom build procedure for Paraver: skip 'make' for recent versions.""" - - if LooseVersion(self.version) < LooseVersion('4.7'): - super(EB_Paraver, self).build_step() - - def install_step(self): - """Custom installation procedure for Paraver: put symlink in place for library subdirectory.""" - super(EB_Paraver, self).install_step() - - if LooseVersion(self.version) < LooseVersion('4.7'): - # link lib to lib64 if needed - # this is a workaround for an issue with libtool which sometimes creates lib64 rather than lib - if self.components[self.current_component] == self.components[0]: - lib64dir = os.path.join(self.installdir, 'lib64') - libdir = os.path.join(self.installdir, 'lib') - - if os.path.exists(lib64dir): - try: - self.log.debug("Symlinking %s to %s", lib64dir, libdir) - os.symlink(lib64dir, libdir) - except OSError as err: - raise EasyBuildError("Symlinking lib64 to lib failed: %s" % err) - - self.current_component += 1 + pass def sanity_check_step(self): """Custom sanity check for Paraver.""" From b875ce7e9f8f0cdba5360a48b0e73139fa29d326 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 18 Dec 2024 15:52:00 +0100 Subject: [PATCH 3/5] fix docstring for `build_step` method in custom easyblock for Paraver Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com> --- easybuild/easyblocks/p/paraver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/p/paraver.py b/easybuild/easyblocks/p/paraver.py index 6c1f8c3971..a7f0a453a4 100644 --- a/easybuild/easyblocks/p/paraver.py +++ b/easybuild/easyblocks/p/paraver.py @@ -88,7 +88,7 @@ def configure_step(self): super(EB_Paraver, self).configure_step() def build_step(self): - """Custom build procedure for Paraver: skip 'make' for recent versions.""" + """No build ('make') required for recent versions.""" pass def sanity_check_step(self): From ee1ee1473d45b16e2f63d97f9864ac0f5db017e0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 18 Dec 2024 15:53:10 +0100 Subject: [PATCH 4/5] add back raising of an error if wxWidgets is not a dependency for Paraver --- easybuild/easyblocks/p/paraver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/easyblocks/p/paraver.py b/easybuild/easyblocks/p/paraver.py index a7f0a453a4..83631eb1d1 100644 --- a/easybuild/easyblocks/p/paraver.py +++ b/easybuild/easyblocks/p/paraver.py @@ -61,6 +61,8 @@ def configure_step(self): wxwidgets = get_software_root('wxWidgets') if wxwidgets: wx_config = os.path.join(wxwidgets, 'bin', 'wx-config') + else: + raise EasyBuildError("wxWidgets is not available as a dependency") # determine value to pass to --with-wxpropgrid (library name) wxpropgrid = None From f0254270237f4034b07b1cd12b24085914bf587a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 18 Dec 2024 16:16:25 +0100 Subject: [PATCH 5/5] take into account that custom easyblock for Paraver requires v4.7+ in easyblocks tests --- test/easyblocks/init_easyblocks.py | 3 +++ test/easyblocks/module.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/test/easyblocks/init_easyblocks.py b/test/easyblocks/init_easyblocks.py index 68331218bd..0514ef8f34 100644 --- a/test/easyblocks/init_easyblocks.py +++ b/test/easyblocks/init_easyblocks.py @@ -237,6 +237,9 @@ def innertest(self): elif easyblock_fn == 'openssl_wrapper.py': # easyblock to create OpenSSL wrapper expects an OpenSSL version innertest = make_inner_test(easyblock, version='1.1') + elif easyblock_fn == 'paraver.py': + # custom easyblock for Paraver requires version >= 4.7 + innertest = make_inner_test(easyblock, version='4.8') elif easyblock_fn == 'torchvision.py': # torchvision easyblock requires that PyTorch is listed as dependency innertest = make_inner_test(easyblock, name='torchvision', deps=[('PyTorch', '1.12.1')]) diff --git a/test/easyblocks/module.py b/test/easyblocks/module.py index 7b9c6d612f..173581f7de 100644 --- a/test/easyblocks/module.py +++ b/test/easyblocks/module.py @@ -464,6 +464,9 @@ def innertest(self): elif eb_fn == 'openssl_wrapper.py': # easyblock to create OpenSSL wrapper expects an OpenSSL version innertest = make_inner_test(easyblock, name='OpenSSL-wrapper', version='1.1') + elif eb_fn == 'paraver.py': + # custom easyblock for Paraver requires version >= 4.7 + innertest = make_inner_test(easyblock, name='Paraver', version='4.8') elif eb_fn == 'torchvision.py': # torchvision easyblock requires that PyTorch is listed as dependency extra_txt = "dependencies = [('PyTorch', '1.12.1')]"