From c1be18d496c9d15a118e3bfaabfd6c68261b131c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Mon, 14 Oct 2024 11:15:53 +0200 Subject: [PATCH 1/3] Enhance AOCC EasyBlock to use config files for GCC toolchain when required MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With AOCC 4.2.0 and 5.0.0, it was observed that wrappers for `clang++` and `flang` will cause compile errors since some libraries are not linked anymore. To work around this, create config files for `clang` and `clang++` instead. Since `flang` doesn't understand config files yet, keep the old behavior. Also add a sanity check to ensure that the correct toolchain is selected. Signed-off-by: Jan André Reuter Co-authored-by: crivella Signed-off-by: Jan André Reuter --- easybuild/easyblocks/a/aocc.py | 146 +++++++++++++++++++++++++++------ 1 file changed, 119 insertions(+), 27 deletions(-) diff --git a/easybuild/easyblocks/a/aocc.py b/easybuild/easyblocks/a/aocc.py index 66fd2f40ba..0b31db9ae8 100644 --- a/easybuild/easyblocks/a/aocc.py +++ b/easybuild/easyblocks/a/aocc.py @@ -31,7 +31,9 @@ @author: Sebastian Achilles (Forschungszentrum Juelich GmbH) """ +import glob import os +import re import stat from easybuild.tools import LooseVersion @@ -40,7 +42,9 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, move_file, write_file -from easybuild.tools.systemtools import get_shared_lib_ext +from easybuild.tools.modules import get_software_root, get_software_version +from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import get_shared_lib_ext, get_cpu_architecture # Wrapper script definition WRAPPER_TEMPLATE = """#!/bin/sh @@ -48,7 +52,6 @@ %(compiler_name)s --gcc-toolchain=$EBROOTGCCCORE "$@" """ - class EB_AOCC(PackedBinary): """ Support for installing the AOCC compilers @@ -66,6 +69,7 @@ def __init__(self, *args, **kwargs): super(EB_AOCC, self).__init__(*args, **kwargs) self.clangversion = self.cfg['clangversion'] + self.gcc_prefix = None def _aocc_guess_clang_version(self): map_aocc_to_clang_ver = { @@ -76,6 +80,7 @@ def _aocc_guess_clang_version(self): '4.0.0': '14.0.6', '4.1.0': '16.0.3', '4.2.0': '16.0.3', + '5.0.0': '17.0.6', } if self.version in map_aocc_to_clang_ver: @@ -89,43 +94,25 @@ def _aocc_guess_clang_version(self): ] raise EasyBuildError('\n'.join(error_lines)) - def install_step(self): - # EULA for AOCC must be accepted via --accept-eula-for EasyBuild configuration option, - # or via 'accept_eula = True' in easyconfig file - self.check_accepted_eula(more_info='http://developer.amd.com/wordpress/media/files/AOCC_EULA.pdf') - - # AOCC is based on Clang. Try to guess the clangversion from the AOCC version - # if clangversion is not specified in the easyconfig - if self.clangversion is None: - self.clangversion = self._aocc_guess_clang_version() - - super(EB_AOCC, self).install_step() - - def post_install_step(self): - """Create wrappers for the compilers to make sure compilers picks up GCCcore as GCC toolchain""" + def _create_compiler_wrappers(self, compilers_to_wrap): + if not compilers_to_wrap: + return - orig_compiler_tmpl = '%s.orig' + orig_compiler_tmpl = '%s/%s.orig' def create_wrapper(wrapper_comp): """Create for a particular compiler, with a particular name""" wrapper_f = os.path.join(self.installdir, 'bin', wrapper_comp) - write_file(wrapper_f, WRAPPER_TEMPLATE % {'compiler_name': orig_compiler_tmpl % wrapper_comp}) + write_file(wrapper_f, WRAPPER_TEMPLATE % {'compiler_name': orig_compiler_tmpl % + (os.path.join(self.installdir, 'bin'), wrapper_comp)}) perms = stat.S_IXUSR | stat.S_IRUSR | stat.S_IXGRP | stat.S_IRGRP | stat.S_IXOTH | stat.S_IROTH adjust_permissions(wrapper_f, perms) - compilers_to_wrap = [ - 'clang', - 'clang++', - 'clang-%s' % LooseVersion(self.clangversion).version[0], - 'clang-cpp', - 'flang', - ] - # Rename original compilers and prepare wrappers to pick up GCCcore as GCC toolchain for the compilers for comp in compilers_to_wrap: actual_compiler = os.path.join(self.installdir, 'bin', comp) if os.path.isfile(actual_compiler): - move_file(actual_compiler, orig_compiler_tmpl % actual_compiler) + move_file(actual_compiler, orig_compiler_tmpl % (os.path.join(self.installdir, 'bin'), comp)) else: err_str = "Tried to move '%s' to '%s', but it does not exist!" raise EasyBuildError(err_str, actual_compiler, '%s.orig' % actual_compiler) @@ -137,6 +124,109 @@ def create_wrapper(wrapper_comp): err_str = "Creating wrapper for '%s' not possible, since original compiler was not renamed!" raise EasyBuildError(err_str, actual_compiler) + def _create_compiler_config_files(self, compilers_to_add_config_file): + if not compilers_to_add_config_file: + return + + # For each of the compiler suites, add a .cfg file which points to the correct GCCcore as the GCC toolchain. + # We need the GCC prefix for this. + self._set_gcc_prefix() + bin_dir = os.path.join(self.installdir, 'bin') + prefix_str = '--gcc-install-dir=%s' % self.gcc_prefix + for comp in compilers_to_add_config_file: + with open(os.path.join(bin_dir, '%s.cfg' % comp), 'w') as f: + f.write(prefix_str) + + def _sanity_check_gcc_prefix(self): + """Check if the GCC prefix is correct.""" + compilers_to_check = [ + 'clang', + 'clang++', + 'clang-%s' % LooseVersion(self.clangversion).version[0], + 'clang-cpp', + 'flang', + ] + + # Set prefix if not done during installation. + self._set_gcc_prefix() + rgx = re.compile('Selected GCC installation: (.*)') + for comp in compilers_to_check: + cmd = "%s -v" % os.path.join(self.installdir, 'bin', comp) + out, _ = run_cmd(cmd, log_all=False, log_ok=False, simple=False, regexp=False) + mch = rgx.search(out) + if mch is None: + self.log.debug(out) + raise EasyBuildError("Failed to extract GCC installation path from output of `%s`", cmd) + gcc_prefix = mch.group(1) + if gcc_prefix != self.gcc_prefix: + raise EasyBuildError( + "GCC installation path `%s` does not match expected path `%s`", gcc_prefix, self.gcc_prefix + ) + + def _set_gcc_prefix(self): + """Set the GCC prefix for the build.""" + if not self.gcc_prefix: + arch = get_cpu_architecture() + gcc_root = get_software_root('GCCcore') + gcc_version = get_software_version('GCCcore') + # If that doesn't work, try with GCC + if gcc_root is None: + gcc_root = get_software_root('GCC') + gcc_version = get_software_version('GCC') + # If that doesn't work either, print error and exit + if gcc_root is None: + raise EasyBuildError("Can't find GCC or GCCcore to use") + + pattern = os.path.join(gcc_root, 'lib', 'gcc', '%s-*' % arch, '%s' % gcc_version) + matches = glob.glob(pattern) + if not matches: + raise EasyBuildError("Can't find GCC version %s for architecture %s in %s", gcc_version, arch, pattern) + self.gcc_prefix = os.path.abspath(matches[0]) + self.log.debug("Using %s as the gcc install location", self.gcc_prefix) + + def install_step(self): + # EULA for AOCC must be accepted via --accept-eula-for EasyBuild configuration option, + # or via 'accept_eula = True' in easyconfig file + self.check_accepted_eula(more_info='http://developer.amd.com/wordpress/media/files/AOCC_EULA.pdf') + + # AOCC is based on Clang. Try to guess the clangversion from the AOCC version + # if clangversion is not specified in the easyconfig + if self.clangversion is None: + self.clangversion = self._aocc_guess_clang_version() + + super(EB_AOCC, self).install_step() + + def post_install_step(self): + """ + For AOCC <5.0.0: + Create wrappers for the compilers to make sure compilers picks up GCCcore as GCC toolchain. + For AOCC >= 5.0.0: + Create [compiler-name].cfg files to point the compiler to the correct GCCcore as GCC toolchain. + For compilers not supporting this option, wrap the compilers using the old method. + """ + compilers_to_wrap = [] + compilers_to_add_config_files = [] + + if LooseVersion(self.version) < LooseVersion("5.0.0"): + compilers_to_wrap += [ + 'clang', + 'clang++', + 'clang-%s' % LooseVersion(self.clangversion).version[0], + 'clang-cpp', + 'flang', + ] + else: + compilers_to_add_config_files += [ + 'clang', + 'clang++', + 'clang-cpp' + ] + compilers_to_wrap += [ + 'flang' + ] + + self._create_compiler_config_files(compilers_to_add_config_files) + self._create_compiler_wrappers(compilers_to_wrap) super(EB_AOCC, self).post_install_step() def sanity_check_step(self): @@ -167,6 +257,8 @@ def sanity_check_step(self): "flang --help", "llvm-config --cxxflags", ] + + self._sanity_check_gcc_prefix() super(EB_AOCC, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) def make_module_extra(self): From 1a5b97fd0204964a131cda086455c57df1504eaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Mon, 14 Oct 2024 14:18:53 +0200 Subject: [PATCH 2/3] Patch AOCC compiler wrapper to transfer original compiler name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While the previous commit fixed the observed issues for `clang++`, `flang` continued to be broken, as the driver did not detect a Fortran compiler due to the renaming. To work around this, call the compiler via `exec` to fix the naming, which causes the driver to work correctly. Signed-off-by: Jan André Reuter --- easybuild/easyblocks/a/aocc.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/a/aocc.py b/easybuild/easyblocks/a/aocc.py index 0b31db9ae8..3cf442e29e 100644 --- a/easybuild/easyblocks/a/aocc.py +++ b/easybuild/easyblocks/a/aocc.py @@ -49,7 +49,8 @@ # Wrapper script definition WRAPPER_TEMPLATE = """#!/bin/sh -%(compiler_name)s --gcc-toolchain=$EBROOTGCCCORE "$@" +# Patch argv[0] to the actual compiler so that the correct driver is used internally +(exec -a "%(actual_compiler_name)s" %(compiler_name)s --gcc-toolchain=$EBROOTGCCCORE "$@") """ class EB_AOCC(PackedBinary): @@ -104,7 +105,8 @@ def create_wrapper(wrapper_comp): """Create for a particular compiler, with a particular name""" wrapper_f = os.path.join(self.installdir, 'bin', wrapper_comp) write_file(wrapper_f, WRAPPER_TEMPLATE % {'compiler_name': orig_compiler_tmpl % - (os.path.join(self.installdir, 'bin'), wrapper_comp)}) + (os.path.join(self.installdir, 'bin'), wrapper_comp), + 'actual_compiler_name': wrapper_comp}) perms = stat.S_IXUSR | stat.S_IRUSR | stat.S_IXGRP | stat.S_IRGRP | stat.S_IXOTH | stat.S_IROTH adjust_permissions(wrapper_f, perms) From 4085c51a12b93b92e7f844ed2b65b43eda5051e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Mon, 14 Oct 2024 14:30:27 +0200 Subject: [PATCH 3/3] Add compile tests to AOCC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan André Reuter --- easybuild/easyblocks/a/aocc.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/easybuild/easyblocks/a/aocc.py b/easybuild/easyblocks/a/aocc.py index 3cf442e29e..cf81f144f7 100644 --- a/easybuild/easyblocks/a/aocc.py +++ b/easybuild/easyblocks/a/aocc.py @@ -35,6 +35,7 @@ import os import re import stat +import tempfile from easybuild.tools import LooseVersion @@ -53,6 +54,18 @@ (exec -a "%(actual_compiler_name)s" %(compiler_name)s --gcc-toolchain=$EBROOTGCCCORE "$@") """ +AOCC_MINIMAL_CPP_EXAMPLE = """ +#include + +int main(){ std::cout << "It works!" << std::endl; } +""" + +AOCC_MINIMAL_FORTRAN_EXAMPLE = """ +program main +end program main +""" + + class EB_AOCC(PackedBinary): """ Support for installing the AOCC compilers @@ -261,6 +274,17 @@ def sanity_check_step(self): ] self._sanity_check_gcc_prefix() + + # Check if clang++ can actually compile programs. This can fail if the wrong driver is picked up by LLVM. + tmpdir = tempfile.mkdtemp() + write_file(os.path.join(tmpdir, 'minimal.cpp'), AOCC_MINIMAL_CPP_EXAMPLE) + minimal_cpp_compiler_cmd = "cd %s && clang++ minimal.cpp -o minimal_cpp" % tmpdir + custom_commands.append(minimal_cpp_compiler_cmd) + # Check if flang can actually compile programs. This can fail if the wrong driver is picked up by LLVM. + write_file(os.path.join(tmpdir, 'minimal.f90'), AOCC_MINIMAL_FORTRAN_EXAMPLE) + minimal_f90_compiler_cmd = "cd %s && flang minimal.f90 -o minimal_f90" % tmpdir + custom_commands.append(minimal_f90_compiler_cmd) + super(EB_AOCC, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) def make_module_extra(self):