From 50d99c38d0bcfd1d6b65a04335cfd082d6079bd1 Mon Sep 17 00:00:00 2001 From: Georgios Kafanas Date: Sat, 3 Aug 2024 01:48:34 +0200 Subject: [PATCH] [feature] Copy build log and artifacts to a permanent location after failures The files can be build in some selected build path (--buildpath), and the logs of successful compilation are then concentrated to some other location for permanent storage (--logfile-format). Logs of failed builds remain in the build path location so that they can be inspected. However, this setup is problematic when building software in HPC jobs. Quite often in HPC systems the build path is set to some fast storage local to the node, like NVME raid mounted on `/tmp` or `/dev/shm` (as suggested in the documentation: https://docs.easybuild.io/configuration/#buildpath). The node storage is often wiped out after the end of a job, so the log files and the artifacts are no longer available after the termination of the job. This commit adds an option (--errorlogpath)to accumulate errors in some more permanent location, so that the can be easily inspected after a failed build. --- easybuild/framework/easyblock.py | 28 +++++++++++++++++++++++++--- easybuild/tools/config.py | 21 +++++++++++++++++++++ easybuild/tools/options.py | 8 +++++--- 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index ce436f1841..0cdb39354c 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -73,13 +73,13 @@ from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath -from easybuild.tools.config import install_path, log_path, package_path, source_paths +from easybuild.tools.config import install_path, log_path, package_path, source_paths, error_log_path from easybuild.tools.environment import restore_env, sanitize_env from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock -from easybuild.tools.filetools import compute_checksum, convert_name, copy_file, create_lock, create_patch_info +from easybuild.tools.filetools import convert_name, copy_file, copy_dir, create_lock, create_patch_info, is_readable from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file -from easybuild.tools.filetools import encode_class_name, extract_file +from easybuild.tools.filetools import encode_class_name, extract_file, compute_checksum from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink @@ -4445,6 +4445,28 @@ def ensure_writable_log_dir(log_dir): # there may be multiple log files, or the file name may be different due to zipping logs = glob.glob('%s*' % application_log) print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent) + err_log_path = error_log_path(ec=ecdict['ec']) + if err_log_path and not(success): + print_msg("Build log and artifacts copied to permanent storage: %s" % err_log_path, log=_log, silent=silent) + for log_file in logs: + target_file = os.path.join(err_log_path, os.path.basename(log_file)) + copy_file(log_file, target_file) + + name = ecdict['ec'].name + version = ecdict['ec'].version + + toolchain_dict = ecdict['ec'].toolchain.as_dict() + toolchain_components = [ + toolchain_dict['name'], + toolchain_dict['version'], + toolchain_dict['versionsuffix'], + ] + toolchain_components = [s for s in toolchain_components if len(s) > 0] + toolchain = '-'.join(toolchain_components) + + dest_build_path = os.path.join(err_log_path, name, version, toolchain) + if is_readable(app.builddir): + copy_dir(app.builddir, dest_build_path) del app diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 6bec64764c..42b5b16621 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -106,6 +106,7 @@ DEFAULT_PATH_SUBDIRS = { 'buildpath': 'build', 'containerpath': 'containers', + 'errorlogpath': 'error_log', 'installpath': '', 'packagepath': 'packages', 'repositorypath': 'ebfiles_repo', @@ -471,6 +472,7 @@ class ConfigurationVariables(BaseConfigurationVariables): 'buildpath', 'config', 'containerpath', + 'errorlogpath', 'installpath', 'installpath_modules', 'installpath_software', @@ -836,6 +838,25 @@ def log_path(ec=None): return log_file_format(return_directory=True, ec=ec, date=date, timestamp=timestamp) +def error_log_path(ec=None): + """ + Return the default error log path + + This is a path where file from the build_log_path can be stored permanently + :param ec: dict-like value that provides values for %(name)s and %(version)s template values + """ + error_log_path = ConfigurationVariables()['errorlogpath'] + + if ec is None: + ec = {} + + name, version = ec.get('name', '%(name)s'), ec.get('version', '%(version)s') + date = time.strftime("%Y%m%d") + timestamp = time.strftime("%H%M%S") + + return '/'.join([error_log_path, name + '-' + version, date + '-' + timestamp]) + + def get_build_log_path(): """ Return (temporary) directory for build log diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index df10ec859e..e3ee764e3f 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -562,6 +562,8 @@ def config_options(self): 'envvars-user-modules': ("List of environment variables that hold the base paths for which user-specific " "modules will be installed relative to", 'strlist', 'store', [DEFAULT_ENVVAR_USERS_MODULES]), + 'errorlogpath': ("Location where logs and artifacts are copied in case of an error", + None, 'store', mk_full_default_path('errorlogpath')), 'external-modules-metadata': ("List of (glob patterns for) paths to files specifying metadata " "for external modules (INI format)", 'strlist', 'store', None), 'hooks': ("Location of Python module with hook implementations", 'str', 'store', None), @@ -1137,7 +1139,7 @@ def _postprocess_config(self): # - the could also specify the location of a *remote* (Git( repository, # which can be done in variety of formats (git@:/), https://, etc.) # (see also https://github.com/easybuilders/easybuild-framework/issues/3892); - path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath', + path_opt_names = ['buildpath', 'containerpath', 'errorlogpath', 'git_working_dirs_path', 'installpath', 'installpath_modules', 'installpath_software', 'prefix', 'packagepath', 'robot_paths', 'sourcepath'] @@ -1147,8 +1149,8 @@ def _postprocess_config(self): if self.options.prefix is not None: # prefix applies to all paths, and repository has to be reinitialised to take new repositorypath in account # in the legacy-style configuration, repository is initialised in configuration file itself - path_opts = ['buildpath', 'containerpath', 'installpath', 'packagepath', 'repository', 'repositorypath', - 'sourcepath'] + path_opts = ['buildpath', 'containerpath', 'errorlogpath', 'installpath', 'packagepath', 'repository', + 'repositorypath', 'sourcepath'] for dest in path_opts: if not self.options._action_taken.get(dest, False): if dest == 'repository':