From e30df76f2c28ca50ac99de4b3f9fc603112008ec Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 27 Jan 2020 15:58:51 +0800 Subject: [PATCH 001/864] advise pr labels in --review-pr --- easybuild/framework/easyconfig/tools.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 7d717d6258..45b1c85048 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -47,7 +47,8 @@ from easybuild.base import fancylogger from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR, ActiveMNS, EasyConfig -from easybuild.framework.easyconfig.easyconfig import create_paths, get_easyblock_class, process_easyconfig +from easybuild.framework.easyconfig.easyconfig import create_paths, det_file_info, get_easyblock_class +from easybuild.framework.easyconfig.easyconfig import process_easyconfig from easybuild.framework.easyconfig.format.yeb import quote_yaml_special_chars from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning @@ -538,6 +539,16 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): else: lines.extend(['', "(no related easyconfigs found for %s)\n" % os.path.basename(ec['spec'])]) + labels = [] + file_info = det_file_info(pr_files, download_repo_path) + if any(file_info['new_folder']): + labels.append('new') + if any(file_info['new_file_in_existing_folder']): + labels.append('update') + + if labels: # TODO: compare to actual PR labels? + lines.extend(['', "This PR should be labeled %s" % ', '.join(labels)]) + return '\n'.join(lines) From 0eb57e93108e1d09e2f85fe718c6e92a1227215c Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 18 Feb 2020 12:33:47 +0800 Subject: [PATCH 002/864] compare with actual PR labels --- easybuild/framework/easyconfig/tools.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 45b1c85048..2d0d090b07 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -55,7 +55,7 @@ from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env from easybuild.tools.filetools import find_easyconfigs, is_patch_file, read_file, resolve_path, which, write_file -from easybuild.tools.github import fetch_easyconfigs_from_pr, download_repo +from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_pr_data, download_repo from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.toolchain.toolchain import is_system_toolchain @@ -546,8 +546,19 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): if any(file_info['new_file_in_existing_folder']): labels.append('update') - if labels: # TODO: compare to actual PR labels? - lines.extend(['', "This PR should be labeled %s" % ', '.join(labels)]) + github_account = build_option('pr_target_account') + github_repo = build_option('pr_target_repo') + github_user = build_option('github_user') + pr_data, _ = fetch_pr_data(pr, github_account, github_repo, github_user) + pr_labels = [label['name'] for label in pr_data['labels']] + + missing_labels = [] + for label in labels: + if label not in pr_labels: + missing_labels.append(label) + + if missing_labels: + lines.extend(['', "This PR should be labeled %s" % ', '.join(missing_labels)]) return '\n'.join(lines) From 0e5beaf0eccc7f4102cc1b18f09dc574cb186840 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 19 Feb 2020 12:01:29 +0800 Subject: [PATCH 003/864] try to set missing labels --- easybuild/framework/easyconfig/tools.py | 5 +-- easybuild/tools/github.py | 45 ++++++++++++++++++++----- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 2d0d090b07..df4bfc722f 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -55,7 +55,7 @@ from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env from easybuild.tools.filetools import find_easyconfigs, is_patch_file, read_file, resolve_path, which, write_file -from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_pr_data, download_repo +from easybuild.tools.github import download_repo, fetch_easyconfigs_from_pr, fetch_pr_data, post_pr_labels from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.toolchain.toolchain import is_system_toolchain @@ -558,7 +558,8 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): missing_labels.append(label) if missing_labels: - lines.extend(['', "This PR should be labeled %s" % ', '.join(missing_labels)]) + if not post_pr_labels(pr, labels): + lines.extend(['', "This PR should be labeled %s" % ', '.join(missing_labels)]) return '\n'.join(lines) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 77dd8da0f5..68962c4983 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1271,6 +1271,41 @@ def merge_url(gh): print_warning("Review indicates this PR should not be merged (use -f/--force to do so anyway)") +@only_if_module_is_available('git', pkgname='GitPython') +def post_pr_labels(pr, labels): + """ + Update PR labels + """ + pr_target_account = build_option('pr_target_account') + pr_target_repo = build_option('pr_target_repo') + + # fetch GitHub token (required to perform actions on GitHub) + github_user = build_option('github_user') + if github_user is None: + raise EasyBuildError("GitHub user must be specified to open a pull request") + + github_token = fetch_github_token(github_user) + if github_token is None: + raise EasyBuildError("GitHub token for user '%s' must be available to open a pull request", github_user) + + dry_run = build_option('dry_run') or build_option('extended_dry_run') + + if not dry_run: + g = RestClient(GITHUB_API_URL, username=github_user, token=github_token) + + pr_url = g.repos[pr_target_account][pr_target_repo].issues[pr] + try: + status, data = pr_url.labels.post(body=labels) + if status == HTTP_STATUS_OK: + print_msg("Added labels %s to PR#%s" % (', '.join(labels), pr), log=_log, prefix=False) + return True + except HTTPError as err: + _log.info("Failed to add labels to PR# %s: %s." % (pr, err)) + return False + else: + return True + + @only_if_module_is_available('git', pkgname='GitPython') def new_branch_github(paths, ecs, commit_msg=None): """ @@ -1479,15 +1514,9 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_metadata=None): print_msg("Opened pull request: %s" % data['html_url'], log=_log, prefix=False) if labels: - # post labels pr = data['html_url'].split('/')[-1] - pr_url = g.repos[pr_target_account][pr_target_repo].issues[pr] - try: - status, data = pr_url.labels.post(body=labels) - if status == HTTP_STATUS_OK: - print_msg("Added labels %s to PR#%s" % (', '.join(labels), pr), log=_log, prefix=False) - except HTTPError as err: - _log.info("Failed to add labels to PR# %s: %s." % (pr, err)) + if not post_pr_labels(pr, labels): + print_msg("This PR should be labeled %s" % ', '.join(labels), log=_log, prefix=False) def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): From 1b990bb304c4ca2bba625729d55a2b7556fd68c4 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 30 Mar 2020 15:55:28 +0800 Subject: [PATCH 004/864] remove unnecessary decorator and don't raise error if github user or token not set in post_pr_labels --- easybuild/tools/github.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 68962c4983..67dea5bffd 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1271,7 +1271,6 @@ def merge_url(gh): print_warning("Review indicates this PR should not be merged (use -f/--force to do so anyway)") -@only_if_module_is_available('git', pkgname='GitPython') def post_pr_labels(pr, labels): """ Update PR labels @@ -1279,14 +1278,16 @@ def post_pr_labels(pr, labels): pr_target_account = build_option('pr_target_account') pr_target_repo = build_option('pr_target_repo') - # fetch GitHub token (required to perform actions on GitHub) + # fetch GitHub token if available github_user = build_option('github_user') if github_user is None: - raise EasyBuildError("GitHub user must be specified to open a pull request") + _log.info("GitHub user not specified, not adding labels to PR# %s" % pr) + return False github_token = fetch_github_token(github_user) if github_token is None: - raise EasyBuildError("GitHub token for user '%s' must be available to open a pull request", github_user) + _log.info("GitHub token for user '%s' not found, not adding labels to PR# %s" % (github_user, pr)) + return False dry_run = build_option('dry_run') or build_option('extended_dry_run') From 97e6644e015b23acd2065b51e39f7ebb296d1969 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 8 Apr 2020 16:52:44 +0800 Subject: [PATCH 005/864] flesh out code to determine labels into separate function --- easybuild/framework/easyconfig/easyconfig.py | 12 ++++++++++++ easybuild/framework/easyconfig/tools.py | 7 ++----- easybuild/tools/github.py | 8 ++------ 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index b3e8af1cb8..723b5e4f69 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -2145,6 +2145,18 @@ def det_file_info(paths, target_dir): return file_info +def det_labels(file_info): + """Determine labels from file_info. + Currently only detects whether easyconfig is for a new software or an update. + """ + labels = [] + if any(file_info['new_folder']): + labels.append('new') + if any(file_info['new_file_in_existing_folder']): + labels.append('update') + return labels + + def copy_easyconfigs(paths, target_dir): """ Copy easyconfig files to specified directory, in the 'right' location and using the filename expected by robot. diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index df4bfc722f..522153c873 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -47,7 +47,7 @@ from easybuild.base import fancylogger from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR, ActiveMNS, EasyConfig -from easybuild.framework.easyconfig.easyconfig import create_paths, det_file_info, get_easyblock_class +from easybuild.framework.easyconfig.easyconfig import create_paths, det_file_info, det_labels, get_easyblock_class from easybuild.framework.easyconfig.easyconfig import process_easyconfig from easybuild.framework.easyconfig.format.yeb import quote_yaml_special_chars from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check @@ -541,10 +541,7 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): labels = [] file_info = det_file_info(pr_files, download_repo_path) - if any(file_info['new_folder']): - labels.append('new') - if any(file_info['new_file_in_existing_folder']): - labels.append('update') + labels = det_labels(file_info) github_account = build_option('pr_target_account') github_repo = build_option('pr_target_repo') diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 67dea5bffd..6c1f8f9bc4 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -45,7 +45,7 @@ from easybuild.base import fancylogger from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR -from easybuild.framework.easyconfig.easyconfig import copy_easyconfigs, copy_patch_files, det_file_info +from easybuild.framework.easyconfig.easyconfig import copy_easyconfigs, copy_patch_files, det_file_info, det_labels from easybuild.framework.easyconfig.easyconfig import process_easyconfig from easybuild.framework.easyconfig.parser import EasyConfigParser from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning @@ -1428,11 +1428,7 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_metadata=None): file_info = det_file_info(ec_paths, target_dir) # label easyconfigs for new software and/or new easyconfigs for existing software - labels = [] - if any(file_info['new_folder']): - labels.append('new') - if any(file_info['new_file_in_existing_folder']): - labels.append('update') + labels = det_labels(file_info) # only use most common toolchain(s) in toolchain label of PR title toolchains = ['%(name)s/%(version)s' % ec['toolchain'] for ec in file_info['ecs']] From 24207e8c27d3a9a30228f47b9cbc889466c5d062 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 8 Apr 2020 16:54:56 +0800 Subject: [PATCH 006/864] revert posting labels in --review-pr --- easybuild/framework/easyconfig/tools.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 522153c873..4c682cc9cc 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -555,8 +555,7 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): missing_labels.append(label) if missing_labels: - if not post_pr_labels(pr, labels): - lines.extend(['', "This PR should be labeled %s" % ', '.join(missing_labels)]) + lines.extend(['', "This PR should be labeled %s" % ', '.join(missing_labels)]) return '\n'.join(lines) From 430cbe5757f505f9d0e8b7b53be02c09fcbe5ee3 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 8 Apr 2020 17:14:04 +0800 Subject: [PATCH 007/864] also consider easyblocks in new det_labels function --- easybuild/framework/easyconfig/easyconfig.py | 16 +++++++++----- easybuild/framework/easyconfig/tools.py | 6 ++--- easybuild/tools/github.py | 23 ++++++++++---------- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 723b5e4f69..118541af9a 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -2145,15 +2145,19 @@ def det_file_info(paths, target_dir): return file_info -def det_labels(file_info): +def det_labels(file_info, pr_target_repo): """Determine labels from file_info. - Currently only detects whether easyconfig is for a new software or an update. + Currently only detects whether easyconfig is for a new software or an update or an easyblock is new. """ labels = [] - if any(file_info['new_folder']): - labels.append('new') - if any(file_info['new_file_in_existing_folder']): - labels.append('update') + if pr_target_repo == GITHUB_EASYCONFIGS_REPO: + if any(file_info['new_folder']): + labels.append('new') + if any(file_info['new_file_in_existing_folder']): + labels.append('update') + elif pr_target_repo == GITHUB_EASYBLOCKS_REPO: + if any(file_info['new']): + labels.append('new') return labels diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 4c682cc9cc..e2eac64dce 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -55,7 +55,7 @@ from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env from easybuild.tools.filetools import find_easyconfigs, is_patch_file, read_file, resolve_path, which, write_file -from easybuild.tools.github import download_repo, fetch_easyconfigs_from_pr, fetch_pr_data, post_pr_labels +from easybuild.tools.github import download_repo, fetch_easyconfigs_from_pr, fetch_pr_data from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.toolchain.toolchain import is_system_toolchain @@ -539,9 +539,7 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): else: lines.extend(['', "(no related easyconfigs found for %s)\n" % os.path.basename(ec['spec'])]) - labels = [] file_info = det_file_info(pr_files, download_repo_path) - labels = det_labels(file_info) github_account = build_option('pr_target_account') github_repo = build_option('pr_target_repo') @@ -549,6 +547,8 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): pr_data, _ = fetch_pr_data(pr, github_account, github_repo, github_user) pr_labels = [label['name'] for label in pr_data['labels']] + labels = det_labels(file_info, github_repo) + missing_labels = [] for label in labels: if label not in pr_labels: diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 6c1f8f9bc4..b66a0bfe4c 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1428,17 +1428,18 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_metadata=None): file_info = det_file_info(ec_paths, target_dir) # label easyconfigs for new software and/or new easyconfigs for existing software - labels = det_labels(file_info) - - # only use most common toolchain(s) in toolchain label of PR title - toolchains = ['%(name)s/%(version)s' % ec['toolchain'] for ec in file_info['ecs']] - toolchains_counted = sorted([(toolchains.count(tc), tc) for tc in nub(toolchains)]) - toolchain_label = ','.join([tc for (cnt, tc) in toolchains_counted if cnt == toolchains_counted[-1][0]]) - - # only use most common module class(es) in moduleclass label of PR title - classes = [ec['moduleclass'] for ec in file_info['ecs']] - classes_counted = sorted([(classes.count(c), c) for c in nub(classes)]) - class_label = ','.join([tc for (cnt, tc) in classes_counted if cnt == classes_counted[-1][0]]) + labels = det_labels(file_info, pr_target_repo) + + if pr_target_repo == GITHUB_EASYCONFIGS_REPO: + # only use most common toolchain(s) in toolchain label of PR title + toolchains = ['%(name)s/%(version)s' % ec['toolchain'] for ec in file_info['ecs']] + toolchains_counted = sorted([(toolchains.count(tc), tc) for tc in nub(toolchains)]) + toolchain_label = ','.join([tc for (cnt, tc) in toolchains_counted if cnt == toolchains_counted[-1][0]]) + + # only use most common module class(es) in moduleclass label of PR title + classes = [ec['moduleclass'] for ec in file_info['ecs']] + classes_counted = sorted([(classes.count(c), c) for c in nub(classes)]) + class_label = ','.join([tc for (cnt, tc) in classes_counted if cnt == classes_counted[-1][0]]) if title is None: From 7ab4be89079bc07ab9f318a0c8199af7f5f396da Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 8 Apr 2020 17:20:06 +0800 Subject: [PATCH 008/864] add missing import --- easybuild/framework/easyconfig/easyconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 118541af9a..3b861969b2 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -62,6 +62,7 @@ from easybuild.tools.config import Singleton, build_option, get_module_naming_scheme from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX, copy_file, decode_class_name, encode_class_name from easybuild.tools.filetools import find_backup_name_candidate, find_easyconfigs, read_file, write_file +from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO from easybuild.tools.hooks import PARSE, load_hooks, run_hook from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version From 413871547afb0a6bc6065097958662a6e5d68cfd Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 8 Apr 2020 22:52:09 +0800 Subject: [PATCH 009/864] move det_labels to github.py --- easybuild/framework/easyconfig/easyconfig.py | 17 ----------------- easybuild/framework/easyconfig/tools.py | 4 ++-- easybuild/tools/github.py | 19 ++++++++++++++++++- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 4b351f3ee7..4bb72d7252 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -64,7 +64,6 @@ from easybuild.tools.config import Singleton, build_option, get_module_naming_scheme from easybuild.tools.filetools import copy_file, decode_class_name, encode_class_name from easybuild.tools.filetools import find_backup_name_candidate, find_easyconfigs, read_file, write_file -from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO from easybuild.tools.hooks import PARSE, load_hooks, run_hook from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version @@ -2172,22 +2171,6 @@ def det_file_info(paths, target_dir): return file_info -def det_labels(file_info, pr_target_repo): - """Determine labels from file_info. - Currently only detects whether easyconfig is for a new software or an update or an easyblock is new. - """ - labels = [] - if pr_target_repo == GITHUB_EASYCONFIGS_REPO: - if any(file_info['new_folder']): - labels.append('new') - if any(file_info['new_file_in_existing_folder']): - labels.append('update') - elif pr_target_repo == GITHUB_EASYBLOCKS_REPO: - if any(file_info['new']): - labels.append('new') - return labels - - def copy_easyconfigs(paths, target_dir): """ Copy easyconfig files to specified directory, in the 'right' location and using the filename expected by robot. diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 3992a23675..0c01957faf 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -47,7 +47,7 @@ from easybuild.base import fancylogger from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR, ActiveMNS, EasyConfig -from easybuild.framework.easyconfig.easyconfig import create_paths, det_file_info, det_labels, get_easyblock_class +from easybuild.framework.easyconfig.easyconfig import create_paths, det_file_info, get_easyblock_class from easybuild.framework.easyconfig.easyconfig import process_easyconfig from easybuild.framework.easyconfig.format.yeb import quote_yaml_special_chars from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check @@ -55,7 +55,7 @@ from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env from easybuild.tools.filetools import find_easyconfigs, is_patch_file, read_file, resolve_path, which, write_file -from easybuild.tools.github import download_repo, fetch_easyconfigs_from_pr, fetch_pr_data +from easybuild.tools.github import det_labels, download_repo, fetch_easyconfigs_from_pr, fetch_pr_data from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.toolchain.toolchain import is_system_toolchain diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index d509d84df0..b1df75b711 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -45,7 +45,7 @@ from easybuild.base import fancylogger from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR -from easybuild.framework.easyconfig.easyconfig import copy_easyconfigs, copy_patch_files, det_file_info, det_labels +from easybuild.framework.easyconfig.easyconfig import copy_easyconfigs, copy_patch_files, det_file_info from easybuild.framework.easyconfig.easyconfig import process_easyconfig from easybuild.framework.easyconfig.parser import EasyConfigParser from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning @@ -1306,6 +1306,23 @@ def merge_url(gh): print_warning("Review indicates this PR should not be merged (use -f/--force to do so anyway)") +def det_labels(file_info, pr_target_repo): + """ + Determine labels from file_info. + Currently only detects whether easyconfig is for a new software or an update or an easyblock is new. + """ + labels = [] + if pr_target_repo == GITHUB_EASYCONFIGS_REPO: + if any(file_info['new_folder']): + labels.append('new') + if any(file_info['new_file_in_existing_folder']): + labels.append('update') + elif pr_target_repo == GITHUB_EASYBLOCKS_REPO: + if any(file_info['new']): + labels.append('new') + return labels + + def post_pr_labels(pr, labels): """ Update PR labels From 947fb7253f332ba7c4600b94f9f41ddeace2ea57 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 9 Apr 2020 11:26:11 +0800 Subject: [PATCH 010/864] fix wrong indentation --- easybuild/tools/github.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index b1df75b711..b984c8dabe 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1315,8 +1315,8 @@ def det_labels(file_info, pr_target_repo): if pr_target_repo == GITHUB_EASYCONFIGS_REPO: if any(file_info['new_folder']): labels.append('new') - if any(file_info['new_file_in_existing_folder']): - labels.append('update') + if any(file_info['new_file_in_existing_folder']): + labels.append('update') elif pr_target_repo == GITHUB_EASYBLOCKS_REPO: if any(file_info['new']): labels.append('new') From 056dd34a2f42436b840b324c085e5a7ac3447144 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 9 Apr 2020 11:28:58 +0800 Subject: [PATCH 011/864] get correct pr_target_repo for review_pr and exit if it is not easyconfigs --- easybuild/framework/easyconfig/tools.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 0c01957faf..e26532ee4c 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -55,6 +55,7 @@ from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env from easybuild.tools.filetools import find_easyconfigs, is_patch_file, read_file, resolve_path, which, write_file +from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO from easybuild.tools.github import det_labels, download_repo, fetch_easyconfigs_from_pr, fetch_pr_data from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import OrderedDict @@ -513,6 +514,10 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): :param colored: boolean indicating whether a colored multi-diff should be generated :param branch: easybuild-easyconfigs branch to compare with """ + pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO + if pr_target_repo != GITHUB_EASYCONFIGS_REPO: + raise EasyBuildError("Reviewing PRs for repositories other than easyconfigs hasn't been implemented yet") + tmpdir = tempfile.mkdtemp() download_repo_path = download_repo(branch=branch, path=tmpdir) @@ -541,13 +546,12 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): file_info = det_file_info(pr_files, download_repo_path) - github_account = build_option('pr_target_account') - github_repo = build_option('pr_target_repo') + pr_target_account = build_option('pr_target_account') github_user = build_option('github_user') - pr_data, _ = fetch_pr_data(pr, github_account, github_repo, github_user) + pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user) pr_labels = [label['name'] for label in pr_data['labels']] - labels = det_labels(file_info, github_repo) + labels = det_labels(file_info, pr_target_repo) missing_labels = [] for label in labels: From e23f00ce42c856d9509484e5d1ca0c7aad892ff9 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 9 Apr 2020 12:08:13 +0800 Subject: [PATCH 012/864] also test for label advice in test_review_pr --- test/framework/options.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 84409ff8ea..7e52096a27 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2901,6 +2901,20 @@ def test_review_pr(self): regex = re.compile(r"^Comparing gzip-1.10-\S* with gzip-1.10-") self.assertTrue(regex.search(txt), "Pattern '%s' not found in: %s" % (regex.pattern, txt)) + self.mock_stdout(True) + self.mock_stderr(True) + # closed, unlabeled PR for gzip 1.2.8 easyconfig, see https://github.com/easybuilders/easybuild-easyconfigs/pull/5365 + args = [ + '--color=never', + '--github-user=%s' % GITHUB_TEST_ACCOUNT, + '--review-pr=5365', + ] + self.eb_main(args, raise_error=True) + txt = self.get_stdout() + self.mock_stdout(False) + self.mock_stderr(False) + self.assertTrue("This PR should be labeled update" in txt) + def test_set_tmpdir(self): """Test set_tmpdir config function.""" self.purge_environment() From ace7b59388f4badf792689d0db45fcb35dc51fcf Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 9 Apr 2020 12:08:34 +0800 Subject: [PATCH 013/864] add test for det_labels --- test/framework/github.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/test/framework/github.py b/test/framework/github.py index bd1e7cecd4..30500e0b08 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -42,7 +42,7 @@ from easybuild.tools.config import build_option, module_classes from easybuild.tools.configobj import ConfigObj from easybuild.tools.filetools import read_file, write_file -from easybuild.tools.github import VALID_CLOSE_PR_REASONS +from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, VALID_CLOSE_PR_REASONS from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters import easybuild.tools.github as gh @@ -565,6 +565,38 @@ def run_check(expected_result=False): expected_warning = '' self.assertEqual(run_check(True), '') + def test_det_labels(self): + """Test for det_labels function.""" + + file_info = {'new_folder': [False], 'new_file_in_existing_folder': [True]} + self.mock_stdout(True) + res = gh.det_labels(file_info, GITHUB_EASYCONFIGS_REPO) + self.mock_stdout(False) + self.assertEqual(len(res), 1) + self.assertEqual(res[0], 'update') + + file_info = {'new_folder': [True], 'new_file_in_existing_folder': [False]} + self.mock_stdout(True) + res = gh.det_labels(file_info, GITHUB_EASYCONFIGS_REPO) + self.mock_stdout(False) + self.assertEqual(len(res), 1) + self.assertEqual(res[0], 'new') + + file_info = {'new_folder': [True, False], 'new_file_in_existing_folder': [False, True]} + self.mock_stdout(True) + res = gh.det_labels(file_info, GITHUB_EASYCONFIGS_REPO) + self.mock_stdout(False) + self.assertEqual(len(res), 2) + self.assertTrue('new' in res) + self.assertTrue('update' in res) + + file_info = {'new': [True]} + self.mock_stdout(True) + res = gh.det_labels(file_info, GITHUB_EASYBLOCKS_REPO) + self.mock_stdout(False) + self.assertEqual(len(res), 1) + self.assertEqual(res[0], 'new') + def test_det_patch_specs(self): """Test for det_patch_specs function.""" From ed887fed59f5355c03fbe6f1bbe1f1713960114e Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 9 Apr 2020 12:10:26 +0800 Subject: [PATCH 014/864] appease the hound --- test/framework/options.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 7e52096a27..f5fecf7a4f 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2903,7 +2903,8 @@ def test_review_pr(self): self.mock_stdout(True) self.mock_stderr(True) - # closed, unlabeled PR for gzip 1.2.8 easyconfig, see https://github.com/easybuilders/easybuild-easyconfigs/pull/5365 + # closed, unlabeled PR for gzip 1.2.8 easyconfig, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/5365 args = [ '--color=never', '--github-user=%s' % GITHUB_TEST_ACCOUNT, From 3d3229631af9a0e178b3815efdee6ac1aaaa5977 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Sat, 11 Apr 2020 12:52:09 +0800 Subject: [PATCH 015/864] cleanup code --- easybuild/framework/easyconfig/tools.py | 8 +++--- easybuild/tools/github.py | 8 +++--- test/framework/github.py | 33 ++++++++----------------- test/framework/options.py | 2 +- 4 files changed, 18 insertions(+), 33 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index e26532ee4c..2df64507e7 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -56,7 +56,7 @@ from easybuild.tools.environment import restore_env from easybuild.tools.filetools import find_easyconfigs, is_patch_file, read_file, resolve_path, which, write_file from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO -from easybuild.tools.github import det_labels, download_repo, fetch_easyconfigs_from_pr, fetch_pr_data +from easybuild.tools.github import det_pr_labels, download_repo, fetch_easyconfigs_from_pr, fetch_pr_data from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.toolchain.toolchain import is_system_toolchain @@ -551,15 +551,15 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user) pr_labels = [label['name'] for label in pr_data['labels']] - labels = det_labels(file_info, pr_target_repo) + expected_labels = det_pr_labels(file_info, pr_target_repo) missing_labels = [] - for label in labels: + for label in expected_labels: if label not in pr_labels: missing_labels.append(label) if missing_labels: - lines.extend(['', "This PR should be labeled %s" % ', '.join(missing_labels)]) + lines.extend(['', "This PR should be labeled with %s" % ', '.join(["'%s'" % l for l in missing_labels])]) return '\n'.join(lines) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index c6c056f8d5..5f5d78b768 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1313,10 +1313,9 @@ def merge_url(gh): print_warning("Review indicates this PR should not be merged (use -f/--force to do so anyway)") -def det_labels(file_info, pr_target_repo): +def det_pr_labels(file_info, pr_target_repo): """ - Determine labels from file_info. - Currently only detects whether easyconfig is for a new software or an update or an easyblock is new. + Determine labels for a pull request based on provided information on files changed by that pull request. """ labels = [] if pr_target_repo == GITHUB_EASYCONFIGS_REPO: @@ -1486,8 +1485,7 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None, file_info = det_file_info(ec_paths, target_dir) - # label easyconfigs for new software and/or new easyconfigs for existing software - labels = det_labels(file_info, pr_target_repo) + labels = det_pr_labels(file_info, pr_target_repo) if pr_target_repo == GITHUB_EASYCONFIGS_REPO: # only use most common toolchain(s) in toolchain label of PR title diff --git a/test/framework/github.py b/test/framework/github.py index 5b94ce6f6b..56355a0fb3 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -566,37 +566,24 @@ def run_check(expected_result=False): expected_warning = '' self.assertEqual(run_check(True), '') - def test_det_labels(self): - """Test for det_labels function.""" + def test_det_pr_labels(self): + """Test for det_pr_labels function.""" file_info = {'new_folder': [False], 'new_file_in_existing_folder': [True]} - self.mock_stdout(True) - res = gh.det_labels(file_info, GITHUB_EASYCONFIGS_REPO) - self.mock_stdout(False) - self.assertEqual(len(res), 1) - self.assertEqual(res[0], 'update') + res = gh.det_pr_labels(file_info, GITHUB_EASYCONFIGS_REPO) + self.assertEqual(res, ['update']) file_info = {'new_folder': [True], 'new_file_in_existing_folder': [False]} - self.mock_stdout(True) - res = gh.det_labels(file_info, GITHUB_EASYCONFIGS_REPO) - self.mock_stdout(False) - self.assertEqual(len(res), 1) - self.assertEqual(res[0], 'new') + res = gh.det_pr_labels(file_info, GITHUB_EASYCONFIGS_REPO) + self.assertEqual(res, ['new']) file_info = {'new_folder': [True, False], 'new_file_in_existing_folder': [False, True]} - self.mock_stdout(True) - res = gh.det_labels(file_info, GITHUB_EASYCONFIGS_REPO) - self.mock_stdout(False) - self.assertEqual(len(res), 2) - self.assertTrue('new' in res) - self.assertTrue('update' in res) + res = gh.det_pr_labels(file_info, GITHUB_EASYCONFIGS_REPO) + self.assertTrue(sorted(res), ['new', 'update']) file_info = {'new': [True]} - self.mock_stdout(True) - res = gh.det_labels(file_info, GITHUB_EASYBLOCKS_REPO) - self.mock_stdout(False) - self.assertEqual(len(res), 1) - self.assertEqual(res[0], 'new') + res = gh.det_pr_labels(file_info, GITHUB_EASYBLOCKS_REPO) + self.assertEqual(res, ['new']) def test_det_patch_specs(self): """Test for det_patch_specs function.""" diff --git a/test/framework/options.py b/test/framework/options.py index 6179b46f8c..0230bf53e5 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2955,7 +2955,7 @@ def test_review_pr(self): txt = self.get_stdout() self.mock_stdout(False) self.mock_stderr(False) - self.assertTrue("This PR should be labeled update" in txt) + self.assertTrue("This PR should be labeled with 'update'" in txt) def test_set_tmpdir(self): """Test set_tmpdir config function.""" From 88c6d1418b887f2a0a61db7c65d3fbac7684e703 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Sun, 12 Apr 2020 11:01:48 +0800 Subject: [PATCH 016/864] ignore PR labels when testing --review-pr label suggestions --- easybuild/framework/easyconfig/tools.py | 5 +++-- easybuild/main.py | 2 +- test/framework/options.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 2df64507e7..db8fa121bf 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -506,13 +506,14 @@ def find_related_easyconfigs(path, ec): return sorted(res) -def review_pr(paths=None, pr=None, colored=True, branch='develop'): +def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False): """ Print multi-diff overview between specified easyconfigs or PR and specified branch. :param pr: pull request number in easybuild-easyconfigs repo to review :param paths: path tuples (path, generated) of easyconfigs to review :param colored: boolean indicating whether a colored multi-diff should be generated :param branch: easybuild-easyconfigs branch to compare with + :param testing: whether to ignore PR labels (used in test_review_pr) """ pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO if pr_target_repo != GITHUB_EASYCONFIGS_REPO: @@ -549,7 +550,7 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): pr_target_account = build_option('pr_target_account') github_user = build_option('github_user') pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user) - pr_labels = [label['name'] for label in pr_data['labels']] + pr_labels = [label['name'] for label in pr_data['labels']] if not testing else [] expected_labels = det_pr_labels(file_info, pr_target_repo) diff --git a/easybuild/main.py b/easybuild/main.py index 415321dc9a..8bd9ed5d07 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -247,7 +247,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): merge_pr(options.merge_pr) elif options.review_pr: - print(review_pr(pr=options.review_pr, colored=use_color(options.color))) + print(review_pr(pr=options.review_pr, colored=use_color(options.color), testing=testing)) elif options.list_installed_software: detailed = options.list_installed_software == 'detailed' diff --git a/test/framework/options.py b/test/framework/options.py index 0230bf53e5..c76841c82b 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2944,14 +2944,14 @@ def test_review_pr(self): self.mock_stdout(True) self.mock_stderr(True) - # closed, unlabeled PR for gzip 1.2.8 easyconfig, + # closed PR for gzip 1.2.8 easyconfig, # see https://github.com/easybuilders/easybuild-easyconfigs/pull/5365 args = [ '--color=never', '--github-user=%s' % GITHUB_TEST_ACCOUNT, '--review-pr=5365', ] - self.eb_main(args, raise_error=True) + self.eb_main(args, raise_error=True, testing=True) txt = self.get_stdout() self.mock_stdout(False) self.mock_stderr(False) From ef3979ed4c8883832048a29b99d29e016c0e7f71 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Sun, 12 Apr 2020 14:58:18 +0800 Subject: [PATCH 017/864] add support for --add-pr-labels --- easybuild/framework/easyconfig/tools.py | 25 +++++++------- easybuild/main.py | 6 +++- easybuild/tools/github.py | 43 +++++++++++++++++++++++-- easybuild/tools/options.py | 3 +- test/framework/options.py | 2 +- 5 files changed, 62 insertions(+), 17 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index db8fa121bf..32f7728ce3 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -545,22 +545,23 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False else: lines.extend(['', "(no related easyconfigs found for %s)\n" % os.path.basename(ec['spec'])]) - file_info = det_file_info(pr_files, download_repo_path) + if pr: + file_info = det_file_info(pr_files, download_repo_path) - pr_target_account = build_option('pr_target_account') - github_user = build_option('github_user') - pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user) - pr_labels = [label['name'] for label in pr_data['labels']] if not testing else [] + pr_target_account = build_option('pr_target_account') + github_user = build_option('github_user') + pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user) + pr_labels = [label['name'] for label in pr_data['labels']] if not testing else [] - expected_labels = det_pr_labels(file_info, pr_target_repo) + expected_labels = det_pr_labels(file_info, pr_target_repo) - missing_labels = [] - for label in expected_labels: - if label not in pr_labels: - missing_labels.append(label) + missing_labels = [] + for label in expected_labels: + if label not in pr_labels: + missing_labels.append(label) - if missing_labels: - lines.extend(['', "This PR should be labeled with %s" % ', '.join(["'%s'" % l for l in missing_labels])]) + if missing_labels: + lines.extend(['', "This PR should be labelled with %s" % ', '.join(["'%s'" % l for l in missing_labels])]) return '\n'.join(lines) diff --git a/easybuild/main.py b/easybuild/main.py index 8bd9ed5d07..96b6074f80 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -59,7 +59,7 @@ from easybuild.tools.filetools import adjust_permissions, cleanup, copy_file, copy_files, dump_index, load_index from easybuild.tools.filetools import read_file, write_file from easybuild.tools.github import check_github, close_pr, new_branch_github, find_easybuild_easyconfig -from easybuild.tools.github import install_github_token, list_prs, new_pr, new_pr_from_branch, merge_pr +from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, new_pr, new_pr_from_branch, merge_pr from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr from easybuild.tools.hooks import START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool @@ -249,6 +249,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): elif options.review_pr: print(review_pr(pr=options.review_pr, colored=use_color(options.color), testing=testing)) + elif options.add_pr_labels: + add_pr_labels(options.add_pr_labels) + elif options.list_installed_software: detailed = options.list_installed_software == 'detailed' print(list_software(output_format=options.output_format, detailed=detailed, only_installed=True)) @@ -264,6 +267,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ + options.add_pr_labels, options.check_github, options.create_index, options.install_github_token, diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 5f5d78b768..271c1e27bd 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1334,7 +1334,7 @@ def post_pr_labels(pr, labels): Update PR labels """ pr_target_account = build_option('pr_target_account') - pr_target_repo = build_option('pr_target_repo') + pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO # fetch GitHub token if available github_user = build_option('github_user') @@ -1365,6 +1365,45 @@ def post_pr_labels(pr, labels): return True +def add_pr_labels(pr, branch='develop'): + """ + Try to determine and add labels to PR. + :param pr: pull request number in easybuild-easyconfigs repo + :param branch: easybuild-easyconfigs branch to compare with + """ + pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO + if pr_target_repo != GITHUB_EASYCONFIGS_REPO: + raise EasyBuildError("Adding labels to PRs for repositories other than easyconfigs hasn't been implemented yet") + + tmpdir = tempfile.mkdtemp() + + download_repo_path = download_repo(branch=branch, path=tmpdir) + + pr_files = [path for path in fetch_easyconfigs_from_pr(pr) if path.endswith('.eb')] + + file_info = det_file_info(pr_files, download_repo_path) + + pr_target_account = build_option('pr_target_account') + github_user = build_option('github_user') + pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user) + pr_labels = [label['name'] for label in pr_data['labels']] + + expected_labels = det_pr_labels(file_info, pr_target_repo) + + missing_labels = [] + for label in expected_labels: + if label not in pr_labels: + missing_labels.append(label) + + if missing_labels: + missing_labels_txt = ', '.join(["'%s'" % l for l in missing_labels]) + print_msg("PR #%s should be labelled %s" % (pr, missing_labels_txt), log=_log, prefix=False) + if not post_pr_labels(pr, missing_labels): + print_msg("Could not add labels %s to PR #%s" % (missing_labels_txt, pr), log=_log, prefix=False) + else: + print_msg("Could not determine any missing labels for PR #%s" % pr, log=_log, prefix=False) + + @only_if_module_is_available('git', pkgname='GitPython') def new_branch_github(paths, ecs, commit_msg=None): """ @@ -1574,7 +1613,7 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None, if labels: pr = data['html_url'].split('/')[-1] if not post_pr_labels(pr, labels): - print_msg("This PR should be labeled %s" % ', '.join(labels), log=_log, prefix=False) + print_msg("This PR should be labelled %s" % ', '.join(labels), log=_log, prefix=False) def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 89af72c9f6..e4deed61ca 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -76,7 +76,7 @@ from easybuild.tools.docs import list_easyblocks, list_toolchains from easybuild.tools.environment import restore_env, unset_env_vars from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, install_fake_vsc, move_file, which -from easybuild.tools.github import GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO +from easybuild.tools.github import GITHUB_EB_MAIN from easybuild.tools.github import GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED, GITHUB_PR_STATE_OPEN from easybuild.tools.github import GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS @@ -591,6 +591,7 @@ def github_options(self): descr = ("GitHub integration options", "Integration with GitHub") opts = OrderedDict({ + 'add-pr-labels': ("Try to add labels to PR based on files changed", int, 'store', None, {'metavar': 'PR#'}), 'check-github': ("Check status of GitHub integration, and report back", None, 'store_true', False), 'check-contrib': ("Runs checks to see whether the given easyconfigs are ready to be contributed back", None, 'store_true', False), diff --git a/test/framework/options.py b/test/framework/options.py index c76841c82b..7c32891e86 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2955,7 +2955,7 @@ def test_review_pr(self): txt = self.get_stdout() self.mock_stdout(False) self.mock_stderr(False) - self.assertTrue("This PR should be labeled with 'update'" in txt) + self.assertTrue("This PR should be labelled with 'update'" in txt) def test_set_tmpdir(self): """Test set_tmpdir config function.""" From 3683bb1a9712d9e46c719d65cce1d70b86176e81 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 29 Jun 2020 18:03:49 +0200 Subject: [PATCH 018/864] Also dump environment to reprod directory --- easybuild/framework/easyblock.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 46490ef823..7f8f8ebd8b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -59,7 +59,7 @@ from easybuild.framework.easyconfig.format.format import SANITY_CHECK_PATHS_DIRS, SANITY_CHECK_PATHS_FILES from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig from easybuild.framework.easyconfig.style import MAX_LINE_LENGTH -from easybuild.framework.easyconfig.tools import get_paths_for +from easybuild.framework.easyconfig.tools import dump_env_script, get_paths_for, parse_easyconfigs from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.framework.extension import resolve_exts_filter_template from easybuild.tools import config, run @@ -3437,6 +3437,13 @@ def reproduce_build(app, reprod_dir_root): except NotImplementedError as err: _log.warning("Unable to dump easyconfig instance to %s: %s", reprod_spec, err) + # Add the build environment dump to the reprod directory + try: + dump_env_script(parse_easyconfigs([reprod_spec])) + _log.debug("Created build environment dump for easyconfig %s", reprod_spec) + except EasyBuildError as err: + _log.warning("Failed to create build environment dump for easyconfig %s: %s", reprod_spec, err) + # also archive the relevant easyblocks reprod_easyblock_dir = os.path.join(reprod_dir, 'easyblocks') for easyblock_class in inspect.getmro(type(app)): From ba3347a6b06728aa0999fe805c819c232fec4ade Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 29 Jun 2020 18:12:00 +0200 Subject: [PATCH 019/864] Add test --- test/framework/toy_build.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 0092a5c617..433d00b6dd 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1815,6 +1815,11 @@ def test_reproducability(self): self.assertTrue(os.path.exists(reprod_ec)) + # Also check that the dumpenv script is placed alongside it + dumpenv_script = '%s.env' % os.path.splitext(os.path.basename(reprod_ec))[0] + reprod_dumpenv = os.path.join(reprod_dir, dumpenv_script) + self.assertTrue(os.path.exists(reprod_dumpenv)) + # Check that the toytoy easyblock is recorded in the reprod easyconfig ec = EasyConfig(reprod_ec) self.assertEqual(ec.parser.get_config_dict()['easyblock'], 'EB_toytoy') From 428def9e6ece5452457464c6d50761c74afc659f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 29 Jun 2020 18:52:48 +0200 Subject: [PATCH 020/864] Only do a simple process on easyconfig for dumping --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 7f8f8ebd8b..97244798d2 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -59,7 +59,7 @@ from easybuild.framework.easyconfig.format.format import SANITY_CHECK_PATHS_DIRS, SANITY_CHECK_PATHS_FILES from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig from easybuild.framework.easyconfig.style import MAX_LINE_LENGTH -from easybuild.framework.easyconfig.tools import dump_env_script, get_paths_for, parse_easyconfigs +from easybuild.framework.easyconfig.tools import dump_env_script, get_paths_for, process_easyconfig from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.framework.extension import resolve_exts_filter_template from easybuild.tools import config, run @@ -3439,7 +3439,7 @@ def reproduce_build(app, reprod_dir_root): # Add the build environment dump to the reprod directory try: - dump_env_script(parse_easyconfigs([reprod_spec])) + dump_env_script([process_easyconfig(reprod_spec)]) _log.debug("Created build environment dump for easyconfig %s", reprod_spec) except EasyBuildError as err: _log.warning("Failed to create build environment dump for easyconfig %s: %s", reprod_spec, err) From a19b15ac4c2cbe541bc5444ab6ca684481291eb1 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 29 Jun 2020 19:14:25 +0200 Subject: [PATCH 021/864] Fix env duping and make silent --- easybuild/framework/easyblock.py | 2 +- easybuild/framework/easyconfig/tools.py | 13 ++++++++++--- test/framework/toy_build.py | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 97244798d2..68d7aca9bc 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3439,7 +3439,7 @@ def reproduce_build(app, reprod_dir_root): # Add the build environment dump to the reprod directory try: - dump_env_script([process_easyconfig(reprod_spec)]) + dump_env_script(process_easyconfig(reprod_spec), relative_path=True, silent=True) _log.debug("Created build environment dump for easyconfig %s", reprod_spec) except EasyBuildError as err: _log.warning("Failed to create build environment dump for easyconfig %s: %s", reprod_spec, err) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 2a3260ae80..0daa56c1b2 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -541,7 +541,7 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): return '\n'.join(lines) -def dump_env_script(easyconfigs): +def dump_env_script(easyconfigs, silent=False, relative_path=False): """ Dump source scripts that set up build environment for specified easyconfigs. @@ -549,7 +549,10 @@ def dump_env_script(easyconfigs): """ ecs_and_script_paths = [] for easyconfig in easyconfigs: - script_path = '%s.env' % os.path.splitext(os.path.basename(easyconfig['spec']))[0] + if relative_path: + script_path = '%s.env' % os.path.splitext(easyconfig['spec'])[0] + else: + script_path = '%s.env' % os.path.splitext(os.path.basename(easyconfig['spec']))[0] ecs_and_script_paths.append((easyconfig['ec'], script_path)) # don't just overwrite existing scripts @@ -597,7 +600,11 @@ def dump_env_script(easyconfigs): script_lines.append("# (no build environment defined)") write_file(script_path, '\n'.join(script_lines)) - print_msg("Script to set up build environment for %s dumped to %s" % (ecfile, script_path), prefix=False) + msg = "Script to set up build environment for %s dumped to %s" % (ecfile, script_path) + if silent: + _log.info(msg) + else: + print_msg(msg, prefix=False) restore_env(orig_env) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 433d00b6dd..6b91f1d03e 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1816,7 +1816,7 @@ def test_reproducability(self): self.assertTrue(os.path.exists(reprod_ec)) # Also check that the dumpenv script is placed alongside it - dumpenv_script = '%s.env' % os.path.splitext(os.path.basename(reprod_ec))[0] + dumpenv_script = '%s.env' % os.path.splitext(reprod_ec)[0] reprod_dumpenv = os.path.join(reprod_dir, dumpenv_script) self.assertTrue(os.path.exists(reprod_dumpenv)) From acdb9654de015643e43cc459220d4bcbe68706e3 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 30 Jun 2020 10:04:24 +0200 Subject: [PATCH 022/864] Refactor dump_env_script rather than create new easyblock instance --- easybuild/framework/easyblock.py | 4 +- easybuild/framework/easyconfig/tools.py | 77 ++++++++++++++----------- 2 files changed, 45 insertions(+), 36 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 68d7aca9bc..c811d55319 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -59,7 +59,7 @@ from easybuild.framework.easyconfig.format.format import SANITY_CHECK_PATHS_DIRS, SANITY_CHECK_PATHS_FILES from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig from easybuild.framework.easyconfig.style import MAX_LINE_LENGTH -from easybuild.framework.easyconfig.tools import dump_env_script, get_paths_for, process_easyconfig +from easybuild.framework.easyconfig.tools import dump_env_easyblock, get_paths_for, process_easyconfig from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.framework.extension import resolve_exts_filter_template from easybuild.tools import config, run @@ -3439,7 +3439,7 @@ def reproduce_build(app, reprod_dir_root): # Add the build environment dump to the reprod directory try: - dump_env_script(process_easyconfig(reprod_spec), relative_path=True, silent=True) + dump_env_easyblock(app, ec_path=reprod_spec, silent=True) _log.debug("Created build environment dump for easyconfig %s", reprod_spec) except EasyBuildError as err: _log.warning("Failed to create build environment dump for easyconfig %s: %s", reprod_spec, err) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 0daa56c1b2..edc21cea2e 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -541,7 +541,46 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): return '\n'.join(lines) -def dump_env_script(easyconfigs, silent=False, relative_path=False): +def dump_env_easyblock(app, orig_env=None, ec_path=None, script_path=None, silent=False): + if orig_env is None: + orig_env = copy.deepcopy(os.environ) + if ec_path is None: + raise EasyBuildError("The path to the easyconfig relevant to this environment dump is required") + if script_path is None: + # Assume we are placing it alongside the easyconfig path + script_path = '%s.env' % os.path.splitext(ec_path)[0] + # Compose script + ecfile = os.path.basename(ec_path) + script_lines = [ + "#!/bin/bash", + "# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile), + "# usage: source %s" % os.path.basename(script_path), + ] + + script_lines.extend(['', "# toolchain & dependency modules"]) + if app.toolchain.modules: + script_lines.extend(["module load %s" % mod for mod in app.toolchain.modules]) + else: + script_lines.append("# (no modules loaded)") + + script_lines.extend(['', "# build environment"]) + if app.toolchain.vars: + env_vars = sorted(app.toolchain.vars.items()) + script_lines.extend(["export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars]) + else: + script_lines.append("# (no build environment defined)") + + write_file(script_path, '\n'.join(script_lines)) + msg = "Script to set up build environment for %s dumped to %s" % (ecfile, script_path) + if silent: + _log.info(msg) + else: + print_msg(msg, prefix=False) + + restore_env(orig_env) + + +def dump_env_script(easyconfigs): """ Dump source scripts that set up build environment for specified easyconfigs. @@ -549,10 +588,7 @@ def dump_env_script(easyconfigs, silent=False, relative_path=False): """ ecs_and_script_paths = [] for easyconfig in easyconfigs: - if relative_path: - script_path = '%s.env' % os.path.splitext(easyconfig['spec'])[0] - else: - script_path = '%s.env' % os.path.splitext(os.path.basename(easyconfig['spec']))[0] + script_path = '%s.env' % os.path.splitext(os.path.basename(easyconfig['spec']))[0] ecs_and_script_paths.append((easyconfig['ec'], script_path)) # don't just overwrite existing scripts @@ -578,35 +614,8 @@ def dump_env_script(easyconfigs, silent=False, relative_path=False): app.check_readiness_step() app.prepare_step(start_dir=False) - # compose script - ecfile = os.path.basename(ec.path) - script_lines = [ - "#!/bin/bash", - "# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile), - "# usage: source %s" % os.path.basename(script_path), - ] - - script_lines.extend(['', "# toolchain & dependency modules"]) - if app.toolchain.modules: - script_lines.extend(["module load %s" % mod for mod in app.toolchain.modules]) - else: - script_lines.append("# (no modules loaded)") - - script_lines.extend(['', "# build environment"]) - if app.toolchain.vars: - env_vars = sorted(app.toolchain.vars.items()) - script_lines.extend(["export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars]) - else: - script_lines.append("# (no build environment defined)") - - write_file(script_path, '\n'.join(script_lines)) - msg = "Script to set up build environment for %s dumped to %s" % (ecfile, script_path) - if silent: - _log.info(msg) - else: - print_msg(msg, prefix=False) - - restore_env(orig_env) + # create the environment dump + dump_env_easyblock(app, ec_path=ec.path, script_path=script_path) def categorize_files_by_type(paths): From d66b0cce380390072e9042566dae277949aba583 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 30 Jun 2020 10:05:54 +0200 Subject: [PATCH 023/864] Remove unused import --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index c811d55319..d5054a69b7 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -59,7 +59,7 @@ from easybuild.framework.easyconfig.format.format import SANITY_CHECK_PATHS_DIRS, SANITY_CHECK_PATHS_FILES from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig from easybuild.framework.easyconfig.style import MAX_LINE_LENGTH -from easybuild.framework.easyconfig.tools import dump_env_easyblock, get_paths_for, process_easyconfig +from easybuild.framework.easyconfig.tools import dump_env_easyblock, get_paths_for from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.framework.extension import resolve_exts_filter_template from easybuild.tools import config, run From 4a023f090c05b1ec2f96f571d6fb91957789795f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 30 Jun 2020 10:23:50 +0200 Subject: [PATCH 024/864] Missed a kwarg in use of refactored function --- easybuild/framework/easyconfig/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index edc21cea2e..8d3f5fe337 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -615,7 +615,7 @@ def dump_env_script(easyconfigs): app.prepare_step(start_dir=False) # create the environment dump - dump_env_easyblock(app, ec_path=ec.path, script_path=script_path) + dump_env_easyblock(app, orig_env=orig_env, ec_path=ec.path, script_path=script_path) def categorize_files_by_type(paths): From e0033da32f881398a16ef35f4826f01dd34ded2a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 30 Jun 2020 10:48:02 +0200 Subject: [PATCH 025/864] Correct name on test --- test/framework/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 1ff9f33d08..315c5e7fce 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4143,8 +4143,8 @@ def test_prefix(self): expected = ['buildpath', 'containerpath', 'installpath', 'packagepath', 'prefix', 'repositorypath'] self.assertEqual(sorted(regex.findall(txt)), expected) - def test_dump_env_config(self): - """Test for --dump-env-config.""" + def test_dump_env_script(self): + """Test for --dump-env-script.""" fftw = 'FFTW-3.3.7-gompic-2018a' gcc = 'GCC-4.9.2' From 1fe810dd1b19e36a1fe3e9c353866bef5d5251ed Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 30 Jun 2020 11:20:13 +0200 Subject: [PATCH 026/864] Add an additional test for the contents of the dumped env script in the reprod dir --- test/framework/toy_build.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 6b91f1d03e..616ea6221b 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1820,6 +1820,18 @@ def test_reproducability(self): reprod_dumpenv = os.path.join(reprod_dir, dumpenv_script) self.assertTrue(os.path.exists(reprod_dumpenv)) + # Check contents of the dumpenv script + patterns = [ + """#!/bin/bash""", + """# usage: source toy-0.0.env""", + # defining build env + """# (no modules loaded)""", + """# (no build environment defined)""", + ] + env_file = open(reprod_dumpenv, "r").read() + for pattern in patterns: + self.assertTrue(pattern in env_file) + # Check that the toytoy easyblock is recorded in the reprod easyconfig ec = EasyConfig(reprod_ec) self.assertEqual(ec.parser.get_config_dict()['easyblock'], 'EB_toytoy') From 8d1730f2dfa87b4456e8ed934387093911462563 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 4 Jul 2020 13:28:37 +0200 Subject: [PATCH 027/864] remove duplicate (merge conflict due to #3375 was not resolved correctly) + fix comment --- easybuild/framework/easyblock.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 4a3d371995..8b625cfc56 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3475,20 +3475,7 @@ def reproduce_build(app, reprod_dir_root): except EasyBuildError as err: _log.warning("Failed to create build environment dump for easyconfig %s: %s", reprod_spec, err) - # also archive the relevant easyblocks - reprod_easyblock_dir = os.path.join(reprod_dir, 'easyblocks') - for easyblock_class in inspect.getmro(type(app)): - easyblock_path = inspect.getsourcefile(easyblock_class) - easyblock_basedir, easyblock_filename = os.path.split(easyblock_path) - # if we reach EasyBlock or ExtensionEasyBlock class, we are done - # (ExtensionEasyblock is hardcoded to avoid a cyclical import) - if easyblock_class.__name__ in [EasyBlock.__name__, 'ExtensionEasyBlock']: - break - else: - copy_file(easyblock_path, os.path.join(reprod_easyblock_dir, easyblock_filename)) - _log.info("Dumped easyblock %s required for reproduction to %s", easyblock_filename, reprod_easyblock_dir) - - # also archive all the relevant easyblocks (including any used by extensions) + # also archive all the relevant easyblocks copy_easyblocks_for_reprod([app], reprod_dir) # if there is a hook file we should also archive it From 78d24d735fddadef935b1677dadc25cbbfce4507 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 6 Jul 2020 17:21:30 +0200 Subject: [PATCH 028/864] Do the environment dump after the easyblock was used --- easybuild/framework/easyblock.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index c2523e2b8f..be54e65908 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3293,6 +3293,14 @@ def build_and_install_one(ecdict, init_env): result = app.run_all_steps(run_test_cases=run_test_cases) if not dry_run: + # Copy over the build environment used during the configuraton + reprod_spec = os.path.join(reprod_dir, app.cfg.filename()) + try: + dump_env_easyblock(app, ec_path=reprod_spec, silent=True) + _log.debug("Created build environment dump for easyconfig %s", reprod_spec) + except EasyBuildError as err: + _log.warning("Failed to create build environment dump for easyconfig %s: %s", reprod_spec, err) + # also add any extension easyblocks used during the build for reproducability if app.ext_instances: copy_easyblocks_for_reprod(app.ext_instances, reprod_dir) @@ -3469,13 +3477,6 @@ def reproduce_build(app, reprod_dir_root): except NotImplementedError as err: _log.warning("Unable to dump easyconfig instance to %s: %s", reprod_spec, err) - # Add the build environment dump to the reprod directory - try: - dump_env_easyblock(app, ec_path=reprod_spec, silent=True) - _log.debug("Created build environment dump for easyconfig %s", reprod_spec) - except EasyBuildError as err: - _log.warning("Failed to create build environment dump for easyconfig %s: %s", reprod_spec, err) - # also archive all the relevant easyblocks copy_easyblocks_for_reprod([app], reprod_dir) From c2d4361aa53143179fcc75efe6b6fe48bfb85380 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 6 Jul 2020 17:52:19 +0200 Subject: [PATCH 029/864] Add test for dumping of buildenv (to check if it includes module load statements) Include Extension as an easyblock that should not be dumped to the reprod dir (since it is part of the framework) --- easybuild/framework/easyblock.py | 6 +++--- test/framework/toy_build.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index be54e65908..1543536076 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3444,9 +3444,9 @@ def copy_easyblocks_for_reprod(easyblock_instances, reprod_dir): for easyblock_instance in easyblock_instances: for easyblock_class in inspect.getmro(type(easyblock_instance)): easyblock_path = inspect.getsourcefile(easyblock_class) - # if we reach EasyBlock or ExtensionEasyBlock class, we are done - # (ExtensionEasyblock is hardcoded to avoid a cyclical import) - if easyblock_class.__name__ in [EasyBlock.__name__, 'ExtensionEasyBlock']: + # if we reach EasyBlock, Extension or ExtensionEasyBlock class, we are done + # (Extension and ExtensionEasyblock are hardcoded to avoid a cyclical import) + if easyblock_class.__name__ in [EasyBlock.__name__, 'Extension', 'ExtensionEasyBlock']: break else: easyblock_paths.add(easyblock_path) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index f84d8da761..a33faca4a6 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1930,6 +1930,24 @@ def test_toy_toy(self): load1_regex = re.compile('load.*toy/0.0-one', re.M) self.assertTrue(load1_regex.search(mod2_txt), "Pattern '%s' found in: %s" % (load1_regex.pattern, mod2_txt)) + # Check the contents of the dumped env in the reprod dir to ensure it contains the dependency load + reprod_dir = os.path.join(self.test_installpath, 'software', 'toy', '0.0-two', 'easybuild', 'reprod') + dumpenv_script = os.path.join(reprod_dir, 'toy-0.0-two.env') + reprod_dumpenv = os.path.join(reprod_dir, dumpenv_script) + self.assertTrue(os.path.exists(reprod_dumpenv)) + + # Check contents of the dumpenv script + patterns = [ + """#!/bin/bash""", + """# usage: source toy-0.0-two.env""", + # defining build env + """module load toy/0.0-one""", + """# (no build environment defined)""", + ] + env_file = open(reprod_dumpenv, "r").read() + for pattern in patterns: + self.assertTrue(pattern in env_file) + def test_toy_sanity_check_commands(self): """Test toy build with extra sanity check commands.""" From 043e65c3e23f26ac07d29b636679a4a65200ba97 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 14 Sep 2020 13:10:36 +0800 Subject: [PATCH 030/864] add test for add_pr_labels --- easybuild/framework/easyconfig/tools.py | 6 +---- easybuild/tools/github.py | 8 +++--- test/framework/github.py | 34 +++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 32f7728ce3..25803a16d6 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -554,11 +554,7 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False pr_labels = [label['name'] for label in pr_data['labels']] if not testing else [] expected_labels = det_pr_labels(file_info, pr_target_repo) - - missing_labels = [] - for label in expected_labels: - if label not in pr_labels: - missing_labels.append(label) + missing_labels = [label for label in expected_labels if label not in pr_labels] if missing_labels: lines.extend(['', "This PR should be labelled with %s" % ', '.join(["'%s'" % l for l in missing_labels])]) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 96530f2c0c..7b5f334beb 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1392,16 +1392,14 @@ def add_pr_labels(pr, branch='develop'): pr_labels = [label['name'] for label in pr_data['labels']] expected_labels = det_pr_labels(file_info, pr_target_repo) + missing_labels = [label for label in expected_labels if label not in pr_labels] - missing_labels = [] - for label in expected_labels: - if label not in pr_labels: - missing_labels.append(label) + dry_run = build_option('dry_run') or build_option('extended_dry_run') if missing_labels: missing_labels_txt = ', '.join(["'%s'" % l for l in missing_labels]) print_msg("PR #%s should be labelled %s" % (pr, missing_labels_txt), log=_log, prefix=False) - if not post_pr_labels(pr, missing_labels): + if not dry_run and not post_pr_labels(pr, missing_labels): print_msg("Could not add labels %s to PR #%s" % (missing_labels_txt, pr), log=_log, prefix=False) else: print_msg("Could not determine any missing labels for PR #%s" % pr, log=_log, prefix=False) diff --git a/test/framework/github.py b/test/framework/github.py index 06ed040981..d90ed1babc 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -120,6 +120,40 @@ def test_read(self): except (IOError, OSError): pass + def test_add_pr_labels(self): + """Test add_pr_labels function.""" + if self.skip_github_tests: + print("Skipping test_add_pr_labels, no GitHub token available?") + return + + build_options = { + 'pr_target_account': GITHUB_USER, + 'pr_target_repo': GITHUB_EASYBLOCKS_REPO, + 'github_user': GITHUB_TEST_ACCOUNT, + 'dry_run': True, + } + init_config(build_options=build_options) + + self.mock_stdout(True) + error_pattern = "Adding labels to PRs for repositories other than easyconfigs hasn't been implemented yet" + self.assertErrorRegex(EasyBuildError, error_pattern, gh.add_pr_labels, 1) + self.mock_stdout(False) + + build_options['pr_target_repo'] = GITHUB_EASYCONFIGS_REPO + init_config(build_options=build_options) + + self.mock_stdout(True) + gh.add_pr_labels(11262) + stdout = self.get_stdout() + self.mock_stdout(False) + self.assertTrue("Could not determine any missing labels for PR 11262" in stdout) + + self.mock_stdout(True) + gh.add_pr_labels(11250) + stdout = self.get_stdout() + self.mock_stdout(False) + self.assertTrue("PR #11250 should be labelled 'update'" in stdout) + def test_fetch_pr_data(self): """Test fetch_pr_data function.""" if self.skip_github_tests: From 31bd93810c6f803708cbb24571b8cb4fe96eeeb8 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 14 Sep 2020 13:17:19 +0800 Subject: [PATCH 031/864] fix ambiguous variable name --- easybuild/framework/easyconfig/tools.py | 2 +- easybuild/tools/github.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 25803a16d6..ee4db69257 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -557,7 +557,7 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False missing_labels = [label for label in expected_labels if label not in pr_labels] if missing_labels: - lines.extend(['', "This PR should be labelled with %s" % ', '.join(["'%s'" % l for l in missing_labels])]) + lines.extend(['', "This PR should be labelled with %s" % ', '.join(["'%s'" % ml for ml in missing_labels])]) return '\n'.join(lines) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 7b5f334beb..1badac7880 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1397,7 +1397,7 @@ def add_pr_labels(pr, branch='develop'): dry_run = build_option('dry_run') or build_option('extended_dry_run') if missing_labels: - missing_labels_txt = ', '.join(["'%s'" % l for l in missing_labels]) + missing_labels_txt = ', '.join(["'%s'" % ml for ml in missing_labels]) print_msg("PR #%s should be labelled %s" % (pr, missing_labels_txt), log=_log, prefix=False) if not dry_run and not post_pr_labels(pr, missing_labels): print_msg("Could not add labels %s to PR #%s" % (missing_labels_txt, pr), log=_log, prefix=False) From d9f272acc442ba1220514ad7e9d1e49e2fe9fa8f Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 16 Sep 2020 10:10:22 +0800 Subject: [PATCH 032/864] fix test for add_pr_labels --- test/framework/github.py | 14 ++++-- .../easybuild/easyblocks/e/__init__.py | 0 .../easybuild/easyblocks/e/easybuildmeta.py | 34 +++++++++++++ .../easybuild/easyblocks/generic/makecp.py | 49 +++++++++++++++++++ 4 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 test/framework/sandbox/easybuild/easyblocks/e/__init__.py create mode 100644 test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py create mode 100644 test/framework/sandbox/easybuild/easyblocks/generic/makecp.py diff --git a/test/framework/github.py b/test/framework/github.py index d90ed1babc..f5d1457635 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -142,17 +142,25 @@ def test_add_pr_labels(self): build_options['pr_target_repo'] = GITHUB_EASYCONFIGS_REPO init_config(build_options=build_options) + # PR #11262 includes easyconfigs that use 'dummy' toolchain, + # so we need to allow triggering deprecated behaviour + self.allow_deprecated_behaviour() + self.mock_stdout(True) + self.mock_stderr(True) gh.add_pr_labels(11262) stdout = self.get_stdout() self.mock_stdout(False) - self.assertTrue("Could not determine any missing labels for PR 11262" in stdout) + self.mock_stderr(False) + self.assertTrue("Could not determine any missing labels for PR #11262" in stdout) self.mock_stdout(True) - gh.add_pr_labels(11250) + self.mock_stderr(True) + gh.add_pr_labels(8006) # closed, unmerged, unlabeled PR stdout = self.get_stdout() self.mock_stdout(False) - self.assertTrue("PR #11250 should be labelled 'update'" in stdout) + self.mock_stderr(False) + self.assertTrue("PR #8006 should be labelled 'update'" in stdout) def test_fetch_pr_data(self): """Test fetch_pr_data function.""" diff --git a/test/framework/sandbox/easybuild/easyblocks/e/__init__.py b/test/framework/sandbox/easybuild/easyblocks/e/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py b/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py new file mode 100644 index 0000000000..e27c0c66d0 --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py @@ -0,0 +1,34 @@ +## +# Copyright 2009-2020 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Dummy easyblock for EasyBuildMeta + +@author: Miguel Dias Costa (National University of Singapore) +""" +from easybuild.framework.easyblock import EasyBlock + + +class EB_EasyBuildMeta(EasyBlock): + pass diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py b/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py new file mode 100644 index 0000000000..6b258c87d6 --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py @@ -0,0 +1,49 @@ +## +# Copyright 2009-2020 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Dummy easyblock for Makecp. + +@author: Miguel Dias Costa (National University of Singapore) +""" +from easybuild.easyblocks.generic.configuremake import ConfigureMake +from easybuild.framework.easyconfig import BUILD, MANDATORY + + +class MakeCp(ConfigureMake): + """Dummy support for software with no configure and no make install step.""" + + @staticmethod + def extra_options(extra_vars=None): + """ + Define list of files or directories to be copied after make + """ + extra = { + 'files_to_copy': [None, "List of files or dirs to copy", MANDATORY], + 'with_configure': [False, "Run configure script before building", BUILD], + } + if extra_vars is None: + extra_vars = {} + extra.update(extra_vars) + return ConfigureMake.extra_options(extra_vars=extra) From 3d4fc6410d3a12b60a2c8ee6f07f3defaffc1940 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 16 Sep 2020 10:27:27 +0800 Subject: [PATCH 033/864] add new sandboxed easyblocks to test_list_easyblocks --- test/framework/options.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 7bc7e513a0..eae1768436 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -695,6 +695,8 @@ def test_list_easyblocks(self): r'EasyBlock', r'\|-- bar', r'\|-- ConfigureMake', + r'\| \|-- MakeCp', + r'\|-- EB_EasyBuildMeta', r'\|-- EB_FFTW', r'\|-- EB_foo', r'\| \|-- EB_foofoo', From cec696c7a9ca340725c40fb0ba2d5bd4598be03e Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Fri, 16 Oct 2020 16:13:02 +0200 Subject: [PATCH 034/864] Add --http-header-fields option --- easybuild/tools/config.py | 1 + easybuild/tools/filetools.py | 19 +++++++++++++++++++ easybuild/tools/options.py | 1 + 3 files changed, 21 insertions(+) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 10d5a19add..4407710d0c 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -176,6 +176,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'filter_env_vars', 'hide_deps', 'hide_toolchains', + 'http_header_fields', 'force_download', 'from_pr', 'git_working_dirs_path', diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 1c3dbf5f5b..d3c7db04f4 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -579,6 +579,12 @@ def download_file(filename, url, path, forced=False): timeout = 10 _log.debug("Using timeout of %s seconds for initiating download" % timeout) + # extra custom HTTP header fields specified by user string + http_header_fields = build_option('http_header_fields') + # if value is a file path, read that instead (in case of sensitive data) + if http_header_fields is not None and os.path.isfile(http_header_fields): + http_header_fields = read_file(http_header_fields) + # make sure directory exists basedir = os.path.dirname(path) mkdir(basedir, parents=True) @@ -590,6 +596,19 @@ def download_file(filename, url, path, forced=False): # use custom HTTP header headers = {'User-Agent': 'EasyBuild', 'Accept': '*/*'} + + # permit additional or override headers via http_headers_fields option + # whose string value contains header-fields grammar: (see rfc7230) + # header-fields = *( header-field *(CR) LF ) + # header-field = field-name ":" OWS field-value OWS + # OWS = ( optional white space ) + # Note CR may be omitted for convenience (it is absorbed in OWS and stripped) + if http_header_fields is not None: + extraheaders = dict(hf.split(':') for hf in http_header_fields.split('\n') if hf.count(':') == 1) + for key, val in extraheaders.items(): + headers[key] = val + _log.debug('Setting custom HTTP header field: %s (not logging the value)' % (key)) + # for backward compatibility, and to avoid relying on 3rd party Python library 'requests' url_req = std_urllib.Request(url, headers=headers) used_urllib = std_urllib diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 026ae528b1..eefaca924a 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -397,6 +397,7 @@ def override_options(self): "(e.g. --hide-deps=zlib,ncurses)", 'strlist', 'extend', None), 'hide-toolchains': ("Comma separated list of toolchains that you want automatically hidden, " "(e.g. --hide-toolchains=GCCcore)", 'strlist', 'extend', None), + 'http-header-fields': ("Set extra HTTP header fields", 'str', 'store', None), 'ignore-checksums': ("Ignore failing checksum verification", None, 'store_true', False), 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), 'install-latest-eb-release': ("Install latest known version of easybuild", None, 'store_true', False), From 6dc4d866dcf7ef2d57d268bb4163d0c9b4c9f95f Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Fri, 23 Oct 2020 14:03:49 +0200 Subject: [PATCH 035/864] capitalize _T_emporary log file in case of crash... --- easybuild/tools/build_log.py | 2 +- test/framework/build_log.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index ba45075069..736e2963b1 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -215,7 +215,7 @@ def init_logging(logfile, logtostdout=False, silent=False, colorize=fancylogger. os.close(fd) fancylogger.logToFile(logfile, max_bytes=0) - print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=silent) + print_msg('Temporary log file in case of crash %s' % (logfile), log=None, silent=silent) log = fancylogger.getLogger(fname=False) diff --git a/test/framework/build_log.py b/test/framework/build_log.py index 46faa31bcc..416193dd00 100644 --- a/test/framework/build_log.py +++ b/test/framework/build_log.py @@ -419,7 +419,7 @@ def test_init_logging(self): self.assertTrue(os.path.exists(logfile)) self.assertEqual(os.path.dirname(logfile), tmpdir) self.assertTrue(isinstance(log, EasyBuildLog)) - self.assertTrue(stdout.startswith("== temporary log file in case of crash")) + self.assertTrue(stdout.startswith("== Temporary log file in case of crash")) stop_logging(logfile) From 98d8d04f425c9852429d01c8a7d36e48e2294744 Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Thu, 22 Oct 2020 08:07:58 +0200 Subject: [PATCH 036/864] replace with --http-header-fields-urlpat option --- easybuild/tools/config.py | 2 +- easybuild/tools/filetools.py | 43 +++++++++++++++++++++++++++++++----- easybuild/tools/options.py | 4 +++- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 4407710d0c..eed3d2475f 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -176,7 +176,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'filter_env_vars', 'hide_deps', 'hide_toolchains', - 'http_header_fields', + 'http_header_fields_urlpat', 'force_download', 'from_pr', 'git_working_dirs_path', diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index d3c7db04f4..a672d663fd 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -579,11 +579,41 @@ def download_file(filename, url, path, forced=False): timeout = 10 _log.debug("Using timeout of %s seconds for initiating download" % timeout) - # extra custom HTTP header fields specified by user string - http_header_fields = build_option('http_header_fields') - # if value is a file path, read that instead (in case of sensitive data) - if http_header_fields is not None and os.path.isfile(http_header_fields): - http_header_fields = read_file(http_header_fields) + # apply extra custom HTTP header fields for URLs containing a pattern + http_header_fields_urlpat = build_option('http_header_fields_urlpat') + extra_http_header_fields = list() + if isinstance(http_header_fields_urlpat, (list, tuple)): + prev_urlpat = None + header_urlpat = None + for argument in http_header_fields_urlpat: + _log.debug("Got build option http header fields urlpat = %s" % argument) + # if argument is actually a file path, read that instead (useful with sensitive data) + if os.path.isfile(argument): + argument = read_file(argument) + # use '::' as a delimeter between URL pattern and the header field + if header_urlpat is not None: + # remember previous urlpat + prev_urlpat = header_urlpat + if '::' in argument: + _log.debug("It contains ::") + [header_urlpat, header_field] = argument.split('::', 1) + elif prev_urlpat is not None: + header_urlpat = prev_urlpat # reuse previous urlpat + header_field = argument # whole argument only contains header info + else: + # ignore the argument entirely if the URL pattern isn't (or wasn't) given + _log.debug("no urlpat given, giving up") + continue + _log.debug("urlpat = %s" % header_urlpat) + _log.debug("header = %s" % header_field) + _log.debug("url = %s" % url) + if re.search(header_urlpat, url): + _log.debug("url matched!") + # if header is actually a file path, read that instead (useful with sensitive data) + if os.path.isfile(header_field): + header_field = read_file(header_field) + _log.debug("header from file = %s" % header_field) + extra_http_header_fields.append(header_field) # make sure directory exists basedir = os.path.dirname(path) @@ -603,7 +633,8 @@ def download_file(filename, url, path, forced=False): # header-field = field-name ":" OWS field-value OWS # OWS = ( optional white space ) # Note CR may be omitted for convenience (it is absorbed in OWS and stripped) - if http_header_fields is not None: + # Note field-value may not not contain ":" + for http_header_fields in extra_http_header_fields: extraheaders = dict(hf.split(':') for hf in http_header_fields.split('\n') if hf.count(':') == 1) for key, val in extraheaders.items(): headers[key] = val diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index eefaca924a..bae75c91ec 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -397,7 +397,9 @@ def override_options(self): "(e.g. --hide-deps=zlib,ncurses)", 'strlist', 'extend', None), 'hide-toolchains': ("Comma separated list of toolchains that you want automatically hidden, " "(e.g. --hide-toolchains=GCCcore)", 'strlist', 'extend', None), - 'http-header-fields': ("Set extra HTTP header fields", 'str', 'store', None), + 'http-header-fields-urlpat': (("Set extra HTTP header fields (or file) for URL patterns;" + "(e.g. ^https://www.example.com::/path/to/headers.txt)"), + None, 'append', None, {'metavar':'PAT::FIELD[,[PAT::]FIELDS..]'}), 'ignore-checksums': ("Ignore failing checksum verification", None, 'store_true', False), 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), 'install-latest-eb-release': ("Install latest known version of easybuild", None, 'store_true', False), From 5e4427dea6f0f74badd3cc0f9470a322e15f42b8 Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Fri, 23 Oct 2020 17:35:10 +0200 Subject: [PATCH 037/864] split off parse_http_header_fields_urlpat() to process header files recursively --- easybuild/tools/filetools.py | 111 ++++++++++++++++++++--------------- easybuild/tools/options.py | 4 +- 2 files changed, 67 insertions(+), 48 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index a672d663fd..3503127323 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -567,6 +567,56 @@ def derive_alt_pypi_url(url): return alt_pypi_url +def parse_http_header_fields_urlpat(arg, urlpat=None, urlpat_headers={}, maxdepth=3): + """ + Recurse into PAT::[PAT::FILE|PAT::HEADER: FIELD|HEADER: FIELD] where FILE may be a + file containing lines matching the same format, and flatten the result as a dict + e.g. {'^example.com': ['Authorization: Basic token', 'User-Agent: Special Agent']} + """ + # stop infinite recursion that might happen if a file.txt refers to itself + if maxdepth < 0: + _log.debug("Failed to parse_http_header_fields_urlpat (recursion limit)") + return urlpat_headers + + if not isinstance(arg, str): + _log.debug("Failed to parse_http_header_fields_urlpat (argument not a string)") + return urlpat_headers + + # HTTP header fields are separated by CRLF but splitting on LF is more convenient + for argline in arg.split('\n'): + argline = argline.strip() # remove optional whitespace (e.g. remaining CR) + if argline == '' or '#' in argline[0]: + continue # permit comment lines: ignore them + + if os.path.isfile(os.path.join(os.getcwd(), argline)): + # expand existing relative path to absolute + argline = os.path.join(os.path.join(os.getcwd(), argline)) + if os.path.isfile(argline): + # argline is a file path, so read that instead + _log.debug('File included in parse_http_header_fields_urlpat: %s' % argline) + argline = read_file(argline) + urlpat_headers = parse_http_header_fields_urlpat(argline, urlpat, urlpat_headers, maxdepth-1) + continue + + # URL pattern is separated by '::' from a HTTP header field + if '::' in argline: + [urlpat, argline] = argline.split('::', 1) # get the urlpat + # the remainder may be another parseable argument, recurse with same depth + urlpat_headers = parse_http_header_fields_urlpat(argline, urlpat, urlpat_headers, maxdepth) + continue + + if urlpat is not None: + if urlpat in urlpat_headers.keys(): + urlpat_headers[urlpat].append(argline) # add headers to the list + else: + urlpat_headers[urlpat] = list([argline]) # new list headers for this urlpat + else: + _log.warning("Non-empty argument to http-header-fields-urlpat ignored (missing URL pattern)") + + # return a dict full of {urlpat: [list, of, headers]} + return urlpat_headers + + def download_file(filename, url, path, forced=False): """Download a file from the given URL, to the specified path.""" @@ -579,41 +629,14 @@ def download_file(filename, url, path, forced=False): timeout = 10 _log.debug("Using timeout of %s seconds for initiating download" % timeout) - # apply extra custom HTTP header fields for URLs containing a pattern + # parse option HTTP header fields for URLs containing a pattern http_header_fields_urlpat = build_option('http_header_fields_urlpat') - extra_http_header_fields = list() - if isinstance(http_header_fields_urlpat, (list, tuple)): - prev_urlpat = None - header_urlpat = None - for argument in http_header_fields_urlpat: - _log.debug("Got build option http header fields urlpat = %s" % argument) - # if argument is actually a file path, read that instead (useful with sensitive data) - if os.path.isfile(argument): - argument = read_file(argument) - # use '::' as a delimeter between URL pattern and the header field - if header_urlpat is not None: - # remember previous urlpat - prev_urlpat = header_urlpat - if '::' in argument: - _log.debug("It contains ::") - [header_urlpat, header_field] = argument.split('::', 1) - elif prev_urlpat is not None: - header_urlpat = prev_urlpat # reuse previous urlpat - header_field = argument # whole argument only contains header info - else: - # ignore the argument entirely if the URL pattern isn't (or wasn't) given - _log.debug("no urlpat given, giving up") - continue - _log.debug("urlpat = %s" % header_urlpat) - _log.debug("header = %s" % header_field) - _log.debug("url = %s" % url) - if re.search(header_urlpat, url): - _log.debug("url matched!") - # if header is actually a file path, read that instead (useful with sensitive data) - if os.path.isfile(header_field): - header_field = read_file(header_field) - _log.debug("header from file = %s" % header_field) - extra_http_header_fields.append(header_field) + # compile a dict full of {urlpat: [header, list]} + urlpat_headers = dict() + if http_header_fields_urlpat is not None: + # there may be multiple options given, parse them all, while updating urlpat_headers + for arg in http_header_fields_urlpat: + urlpat_headers = parse_http_header_fields_urlpat(arg, None, urlpat_headers) # make sure directory exists basedir = os.path.dirname(path) @@ -627,18 +650,14 @@ def download_file(filename, url, path, forced=False): # use custom HTTP header headers = {'User-Agent': 'EasyBuild', 'Accept': '*/*'} - # permit additional or override headers via http_headers_fields option - # whose string value contains header-fields grammar: (see rfc7230) - # header-fields = *( header-field *(CR) LF ) - # header-field = field-name ":" OWS field-value OWS - # OWS = ( optional white space ) - # Note CR may be omitted for convenience (it is absorbed in OWS and stripped) - # Note field-value may not not contain ":" - for http_header_fields in extra_http_header_fields: - extraheaders = dict(hf.split(':') for hf in http_header_fields.split('\n') if hf.count(':') == 1) - for key, val in extraheaders.items(): - headers[key] = val - _log.debug('Setting custom HTTP header field: %s (not logging the value)' % (key)) + # permit additional or override headers via http_headers_fields_urlpat option + # only append/override HTTP header fields that match current url + for urlpatkey, http_header_fields in urlpat_headers.items(): + if re.search(urlpatkey, url): + extraheaders = dict(hf.split(':', 1) for hf in http_header_fields) + for key, val in extraheaders.items(): + headers[key] = val + _log.debug('Custom HTTP header field set: %s (value omitted from log)' % (key)) # for backward compatibility, and to avoid relying on 3rd party Python library 'requests' url_req = std_urllib.Request(url, headers=headers) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index bae75c91ec..00860c4dd9 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -398,8 +398,8 @@ def override_options(self): 'hide-toolchains': ("Comma separated list of toolchains that you want automatically hidden, " "(e.g. --hide-toolchains=GCCcore)", 'strlist', 'extend', None), 'http-header-fields-urlpat': (("Set extra HTTP header fields (or file) for URL patterns;" - "(e.g. ^https://www.example.com::/path/to/headers.txt)"), - None, 'append', None, {'metavar':'PAT::FIELD[,[PAT::]FIELDS..]'}), + "(e.g. ^https://www.example.com::/path/to/headers.txt)"), + None, 'append', None, {'metavar': 'PAT::FIELD[,[PAT::]FIELDS..]'}), 'ignore-checksums': ("Ignore failing checksum verification", None, 'store_true', False), 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), 'install-latest-eb-release': ("Install latest known version of easybuild", None, 'store_true', False), From fd9b7e34d64b361c950931294de4feb2ad47fdf8 Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Wed, 28 Oct 2020 12:36:26 +0100 Subject: [PATCH 038/864] add test_http_header_fields_urlpat --- easybuild/tools/filetools.py | 2 +- test/framework/options.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 3503127323..6189e9dc71 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -595,7 +595,7 @@ def parse_http_header_fields_urlpat(arg, urlpat=None, urlpat_headers={}, maxdept # argline is a file path, so read that instead _log.debug('File included in parse_http_header_fields_urlpat: %s' % argline) argline = read_file(argline) - urlpat_headers = parse_http_header_fields_urlpat(argline, urlpat, urlpat_headers, maxdepth-1) + urlpat_headers = parse_http_header_fields_urlpat(argline, urlpat, urlpat_headers, maxdepth - 1) continue # URL pattern is separated by '::' from a HTTP header field diff --git a/test/framework/options.py b/test/framework/options.py index e2c90a22be..96c9612a48 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2380,6 +2380,41 @@ def test_hide_toolchains(self): self.assertTrue(re.search(r'module: GCC/\.4\.9\.2', outtxt)) self.assertTrue(re.search(r'module: gzip/1\.6-GCC-4\.9\.2', outtxt)) + def test_http_header_fields_urlpat(self): + """Test use of --http-header-fields-urlpat.""" + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + ec_file = os.path.join(test_ecs_dir, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb') + + # define header fields:values that should (not) show up in the logs + test_applied_hdr = 'HeaderAPPLIED' + test_applied_hdr_regex = re.compile(test_applied_hdr) + test_applied_value = 'SECRETvalue' + test_applied_value_regex = re.compile(test_applied_value) + test_ignored_hdr = 'HeaderIGNORED' + test_ignored_hdr_regex = re.compile(test_ignored_hdr) + test_ignored_value = 'BOGUSvalue' + test_ignored_value_regex = re.compile(test_ignored_value) + + args = [ + ec_file, + '--dry-run', + '--http-header-fields-urlpat="gnu.org::%s: %s"' % (test_applied_hdr, test_applied_value), + '--http-header-fields-urlpat="nomatch.com::%s: %s"' % (test_ignored_hdr, test_ignored_value), + '--stop=fetch', + '--debug', + '--force', + ] + + stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) + + self.assertFalse(stderr) + + # Expect to find only the header key (not value) for the appropriate url pattern. + self.assertTrue(test_applied_hdr_regex.search(stdout)) + self.assertFalse(test_applied_value_regex.search(stdout)) + self.assertFalse(test_ignored_hdr_regex.search(stdout)) + self.assertFalse(test_ignored_value_regex.search(stdout)) + def test_test_report_env_filter(self): """Test use of --test-report-env-filter.""" From b939fc46d046ce375df66658fe6542c58fe050bb Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Fri, 20 Nov 2020 20:30:15 +0100 Subject: [PATCH 039/864] debug and expand test_http_header_fields_urlpat --- test/framework/options.py | 112 ++++++++++++++++++++++++++++++-------- 1 file changed, 90 insertions(+), 22 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 96c9612a48..996803396f 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2384,36 +2384,104 @@ def test_http_header_fields_urlpat(self): """Test use of --http-header-fields-urlpat.""" test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') ec_file = os.path.join(test_ecs_dir, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb') - - # define header fields:values that should (not) show up in the logs - test_applied_hdr = 'HeaderAPPLIED' - test_applied_hdr_regex = re.compile(test_applied_hdr) - test_applied_value = 'SECRETvalue' - test_applied_value_regex = re.compile(test_applied_value) - test_ignored_hdr = 'HeaderIGNORED' - test_ignored_hdr_regex = re.compile(test_ignored_hdr) - test_ignored_value = 'BOGUSvalue' - test_ignored_value_regex = re.compile(test_ignored_value) - - args = [ + common_args = [ ec_file, - '--dry-run', - '--http-header-fields-urlpat="gnu.org::%s: %s"' % (test_applied_hdr, test_applied_value), - '--http-header-fields-urlpat="nomatch.com::%s: %s"' % (test_ignored_hdr, test_ignored_value), '--stop=fetch', '--debug', '--force', + '--force-download', + '--logtostdout', ] - stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) + # define header fields:values that should (not) show up in the logs, either + # because they are secret or because they are not matched for the url + test_applied_hdr = 'HeaderAPPLIED' + test_applied_val = 'SECRETvalue' + test_nomatch_hdr = 'HeaderIGNORED' + test_nomatch_val = 'BOGUSvalue' + + # header fields (or its values) could be files to be read instead of literals + testcmdfile = os.path.join(self.test_prefix, 'testhttpheaderscmdline.txt') + testincfile = os.path.join(self.test_prefix, 'testhttpheadersvalinc.txt') + testexcfile = os.path.join(self.test_prefix, 'testhttpheadersvalexc.txt') + testinchdrfile = os.path.join(self.test_prefix, 'testhttpheadershdrinc.txt') + testexchdrfile = os.path.join(self.test_prefix, 'testhttpheadershdrexc.txt') + testurlpatfile = os.path.join(self.test_prefix, 'testhttpheadersurlpat.txt') + + def run_and_assert(args, msg, words_expected=None, words_unexpected=None): + stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) + if words_expected is not None: + for thestring in words_expected: + self.assertTrue(re.compile(thestring).search(stdout), "Pattern '%s' missing from log (%s)" % + (thestring, msg) ) + if words_unexpected is not None: + for thestring in words_unexpected: + self.assertFalse(re.compile(thestring).search(stdout), "Pattern '%s' leaked into log (%s)" % + (thestring, msg) ) + + # A: simple direct case (all is logged) + args = [ + *common_args, + '--http-header-fields-urlpat=gnu.org::%s:%s' % (test_applied_hdr, test_applied_val), + '--http-header-fields-urlpat=nomatch.com::%s:%s' % (test_nomatch_hdr, test_nomatch_val), + ] + # expect to find everything passed on cmdline + run_and_assert(args, 'case A', [test_applied_hdr, test_applied_val, test_nomatch_hdr, test_nomatch_val]) + + # B: simple file case (secrets in file are not logged) + args = [*common_args, '--http-header-fields-urlpat=%s' % (testcmdfile)] + write_file(testcmdfile, '\n'.join([ + 'gnu.org::%s: %s' % (test_applied_hdr, test_applied_val), + 'nomatch.com::%s: %s' % (test_nomatch_hdr, test_nomatch_val), + '' + ])) + # expect to find only the header key (not its value) and only for the appropriate url + run_and_assert(args, 'case B', + [test_applied_hdr, testcmdfile], + [test_applied_val, test_nomatch_hdr, test_nomatch_val] + ) - self.assertFalse(stderr) + # C: recursion one: header value is another file + args = [*common_args, '--http-header-fields-urlpat=%s' % (testcmdfile)] + write_file(testcmdfile, '\n'.join([ + 'gnu.org::%s: %s' % (test_applied_hdr, testincfile), + 'nomatch.com::%s: %s' % (test_nomatch_hdr, testexcfile), + '' + ])) + write_file(testincfile, '%s\n' % (test_applied_val)) + write_file(testexcfile, '%s\n' % (test_nomatch_val)) + # expect to find only the header key (not its value and not the filename) and only for the appropriate url + run_and_assert(args, 'case C', + [test_applied_hdr, testcmdfile], + [test_applied_val, test_nomatch_hdr, test_nomatch_val, testincfile, testexcfile]) + + # D: recursion two: header field+value is another file, + args = [*common_args, '--http-header-fields-urlpat=%s' % (testcmdfile)] + write_file(testcmdfile, '\n'.join([ + 'gnu.org::%s' % (testinchdrfile), + 'nomatch.com::%s' % (testexchdrfile), + '' + ])) + write_file(testinchdrfile, '%s: %s\n' % (test_applied_hdr, test_applied_val)) + write_file(testexchdrfile, '%s: %s\n' % (test_nomatch_hdr, test_nomatch_val)) + # expect to find only the header key (and the literal filename) and only for the appropriate url + run_and_assert(args, 'case D', + [test_applied_hdr, testcmdfile, testinchdrfile, testexchdrfile], + [test_applied_val, test_nomatch_hdr, test_nomatch_val]) + + # E: recursion three: url pattern + header field + value in another file + args = [*common_args, '--http-header-fields-urlpat=%s' % (testcmdfile)] + write_file(testcmdfile, '%s\n' % (testurlpatfile)) + write_file(testurlpatfile, '\n'.join([ + 'gnu.org::%s: %s' % (test_applied_hdr, test_applied_val), + 'nomatch.com::%s: %s' % (test_nomatch_hdr, test_nomatch_val), + '' + ])) + # expect to find only the header key (but not the literal filename) and only for the appropriate url + run_and_assert(args, 'case E', + [test_applied_hdr, testcmdfile, testurlpatfile], + [test_applied_val, test_nomatch_hdr, test_nomatch_val]) - # Expect to find only the header key (not value) for the appropriate url pattern. - self.assertTrue(test_applied_hdr_regex.search(stdout)) - self.assertFalse(test_applied_value_regex.search(stdout)) - self.assertFalse(test_ignored_hdr_regex.search(stdout)) - self.assertFalse(test_ignored_value_regex.search(stdout)) def test_test_report_env_filter(self): """Test use of --test-report-env-filter.""" From bd9ccaa1ee8c44d60dcdf46037e03d581a2c6e1f Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Fri, 20 Nov 2020 20:57:08 +0100 Subject: [PATCH 040/864] fixup for python-linting --- test/framework/options.py | 78 ++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 30 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 996803396f..575879211e 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2430,58 +2430,76 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None): # B: simple file case (secrets in file are not logged) args = [*common_args, '--http-header-fields-urlpat=%s' % (testcmdfile)] - write_file(testcmdfile, '\n'.join([ - 'gnu.org::%s: %s' % (test_applied_hdr, test_applied_val), - 'nomatch.com::%s: %s' % (test_nomatch_hdr, test_nomatch_val), - '' - ])) + write_file( + testcmdfile, + '\n'.join( + [ + 'gnu.org::%s: %s' % (test_applied_hdr, test_applied_val), + 'nomatch.com::%s: %s' % (test_nomatch_hdr, test_nomatch_val), + '', + ] + ), + ) # expect to find only the header key (not its value) and only for the appropriate url - run_and_assert(args, 'case B', - [test_applied_hdr, testcmdfile], - [test_applied_val, test_nomatch_hdr, test_nomatch_val] + run_and_assert( + args, 'case B', [test_applied_hdr, testcmdfile], [test_applied_val, test_nomatch_hdr, test_nomatch_val] ) # C: recursion one: header value is another file args = [*common_args, '--http-header-fields-urlpat=%s' % (testcmdfile)] - write_file(testcmdfile, '\n'.join([ - 'gnu.org::%s: %s' % (test_applied_hdr, testincfile), - 'nomatch.com::%s: %s' % (test_nomatch_hdr, testexcfile), - '' - ])) + write_file( + testcmdfile, + '\n'.join( + [ + 'gnu.org::%s: %s' % (test_applied_hdr, testincfile), + 'nomatch.com::%s: %s' % (test_nomatch_hdr, testexcfile), + '', + ] + ), + ) write_file(testincfile, '%s\n' % (test_applied_val)) write_file(testexcfile, '%s\n' % (test_nomatch_val)) # expect to find only the header key (not its value and not the filename) and only for the appropriate url - run_and_assert(args, 'case C', + run_and_assert( + args, + 'case C', [test_applied_hdr, testcmdfile], - [test_applied_val, test_nomatch_hdr, test_nomatch_val, testincfile, testexcfile]) + [test_applied_val, test_nomatch_hdr, test_nomatch_val, testincfile, testexcfile], + ) # D: recursion two: header field+value is another file, args = [*common_args, '--http-header-fields-urlpat=%s' % (testcmdfile)] - write_file(testcmdfile, '\n'.join([ - 'gnu.org::%s' % (testinchdrfile), - 'nomatch.com::%s' % (testexchdrfile), - '' - ])) + write_file(testcmdfile, '\n'.join(['gnu.org::%s' % (testinchdrfile), 'nomatch.com::%s' % (testexchdrfile), ''])) write_file(testinchdrfile, '%s: %s\n' % (test_applied_hdr, test_applied_val)) write_file(testexchdrfile, '%s: %s\n' % (test_nomatch_hdr, test_nomatch_val)) # expect to find only the header key (and the literal filename) and only for the appropriate url - run_and_assert(args, 'case D', + run_and_assert( + args, + 'case D', [test_applied_hdr, testcmdfile, testinchdrfile, testexchdrfile], - [test_applied_val, test_nomatch_hdr, test_nomatch_val]) + [test_applied_val, test_nomatch_hdr, test_nomatch_val], + ) # E: recursion three: url pattern + header field + value in another file args = [*common_args, '--http-header-fields-urlpat=%s' % (testcmdfile)] write_file(testcmdfile, '%s\n' % (testurlpatfile)) - write_file(testurlpatfile, '\n'.join([ - 'gnu.org::%s: %s' % (test_applied_hdr, test_applied_val), - 'nomatch.com::%s: %s' % (test_nomatch_hdr, test_nomatch_val), - '' - ])) + write_file( + testurlpatfile, + '\n'.join( + [ + 'gnu.org::%s: %s' % (test_applied_hdr, test_applied_val), + 'nomatch.com::%s: %s' % (test_nomatch_hdr, test_nomatch_val), + '', + ] + ), + ) # expect to find only the header key (but not the literal filename) and only for the appropriate url - run_and_assert(args, 'case E', + run_and_assert( + args, + 'case E', [test_applied_hdr, testcmdfile, testurlpatfile], - [test_applied_val, test_nomatch_hdr, test_nomatch_val]) - + [test_applied_val, test_nomatch_hdr, test_nomatch_val], + ) def test_test_report_env_filter(self): """Test use of --test-report-env-filter.""" From c6853f421a3a0bc202cd8c31121ff6ed3d3b8b16 Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Tue, 24 Nov 2020 13:54:17 +0100 Subject: [PATCH 041/864] appease the hound --- test/framework/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 575879211e..4ba6287a08 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2413,11 +2413,11 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None): if words_expected is not None: for thestring in words_expected: self.assertTrue(re.compile(thestring).search(stdout), "Pattern '%s' missing from log (%s)" % - (thestring, msg) ) + (thestring, msg) ) if words_unexpected is not None: for thestring in words_unexpected: self.assertFalse(re.compile(thestring).search(stdout), "Pattern '%s' leaked into log (%s)" % - (thestring, msg) ) + (thestring, msg) ) # A: simple direct case (all is logged) args = [ From 867aa488294786a2288ce0cb01ad7af43ba08670 Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Tue, 24 Nov 2020 16:30:10 +0100 Subject: [PATCH 042/864] rewrite for py2 compatibility --- test/framework/options.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 4ba6287a08..b9d2a8cd2c 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2413,23 +2413,26 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None): if words_expected is not None: for thestring in words_expected: self.assertTrue(re.compile(thestring).search(stdout), "Pattern '%s' missing from log (%s)" % - (thestring, msg) ) + (thestring, msg)) if words_unexpected is not None: for thestring in words_unexpected: self.assertFalse(re.compile(thestring).search(stdout), "Pattern '%s' leaked into log (%s)" % - (thestring, msg) ) + (thestring, msg)) # A: simple direct case (all is logged) - args = [ - *common_args, + args = list(common_args) + args.extend([ '--http-header-fields-urlpat=gnu.org::%s:%s' % (test_applied_hdr, test_applied_val), '--http-header-fields-urlpat=nomatch.com::%s:%s' % (test_nomatch_hdr, test_nomatch_val), - ] + ]) # expect to find everything passed on cmdline run_and_assert(args, 'case A', [test_applied_hdr, test_applied_val, test_nomatch_hdr, test_nomatch_val]) + # all subsequent tests share this argument list + args = common_args + args.append('--http-header-fields-urlpat=%s' % (testcmdfile)) + # B: simple file case (secrets in file are not logged) - args = [*common_args, '--http-header-fields-urlpat=%s' % (testcmdfile)] write_file( testcmdfile, '\n'.join( @@ -2446,7 +2449,6 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None): ) # C: recursion one: header value is another file - args = [*common_args, '--http-header-fields-urlpat=%s' % (testcmdfile)] write_file( testcmdfile, '\n'.join( @@ -2468,7 +2470,6 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None): ) # D: recursion two: header field+value is another file, - args = [*common_args, '--http-header-fields-urlpat=%s' % (testcmdfile)] write_file(testcmdfile, '\n'.join(['gnu.org::%s' % (testinchdrfile), 'nomatch.com::%s' % (testexchdrfile), ''])) write_file(testinchdrfile, '%s: %s\n' % (test_applied_hdr, test_applied_val)) write_file(testexchdrfile, '%s: %s\n' % (test_nomatch_hdr, test_nomatch_val)) @@ -2481,7 +2482,6 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None): ) # E: recursion three: url pattern + header field + value in another file - args = [*common_args, '--http-header-fields-urlpat=%s' % (testcmdfile)] write_file(testcmdfile, '%s\n' % (testurlpatfile)) write_file( testurlpatfile, From 708e131a1c32d58449efa1b0ad0107987733dba2 Mon Sep 17 00:00:00 2001 From: edmondac-admin Date: Fri, 27 Nov 2020 11:49:44 +0000 Subject: [PATCH 043/864] Specifying False for a version removes the dep --- easybuild/framework/easyconfig/easyconfig.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 3843aa22e1..ee2bb8657c 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -702,10 +702,10 @@ def parse(self): # parse dependency specifications # it's important that templating is still disabled at this stage! self.log.info("Parsing dependency specifications...") - self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']] - self['hiddendependencies'] = [ - self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies'] - ] + deps = [self._parse_dependency(dep) for dep in self['dependencies']] + self['dependencies'] = [x for x in deps if x['version'] is not False] + hiddendeps = [self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']] + self['hiddendependencies'] = [x for x in hiddendeps if x['version'] is not False] # need to take into account that builddependencies may need to be iterated over, # i.e. when the value is a list of lists of tuples @@ -715,7 +715,7 @@ def parse(self): builddeps = [[self._parse_dependency(dep, build_only=True) for dep in x] for x in builddeps] else: builddeps = [self._parse_dependency(dep, build_only=True) for dep in builddeps] - self['builddependencies'] = builddeps + self['builddependencies'] = [x for x in builddeps if x['version'] is not False] # keep track of parsed multi deps, they'll come in handy during sanity check & module steps... self.multi_deps = self.get_parsed_multi_deps() From 81bcf138c04767f6351e3ea4f1c4139d58007a7a Mon Sep 17 00:00:00 2001 From: edmondac-admin Date: Fri, 27 Nov 2020 13:35:57 +0000 Subject: [PATCH 044/864] handle different types of dependecy (tuple, dict, ...) --- easybuild/framework/easyconfig/easyconfig.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index ee2bb8657c..d49d63320f 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -702,10 +702,17 @@ def parse(self): # parse dependency specifications # it's important that templating is still disabled at this stage! self.log.info("Parsing dependency specifications...") - deps = [self._parse_dependency(dep) for dep in self['dependencies']] - self['dependencies'] = [x for x in deps if x['version'] is not False] - hiddendeps = [self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']] - self['hiddendependencies'] = [x for x in hiddendeps if x['version'] is not False] + def remove_false_versions(deps): + ret = [] + for dep in deps: + if isinstance(dep, dict) and dep['version'] == False: + continue + ret.append(dep) + return ret + + self['dependencies'] = remove_false_versions(self._parse_dependency(dep) for dep in self['dependencies']) + self['hiddendependencies'] = remove_false_versions(self._parse_dependency(dep, hidden=True) for dep in + self['hiddendependencies']) # need to take into account that builddependencies may need to be iterated over, # i.e. when the value is a list of lists of tuples @@ -715,7 +722,7 @@ def parse(self): builddeps = [[self._parse_dependency(dep, build_only=True) for dep in x] for x in builddeps] else: builddeps = [self._parse_dependency(dep, build_only=True) for dep in builddeps] - self['builddependencies'] = [x for x in builddeps if x['version'] is not False] + self['builddependencies'] = remove_false_versions(builddeps) # keep track of parsed multi deps, they'll come in handy during sanity check & module steps... self.multi_deps = self.get_parsed_multi_deps() From 11aa3feda36345f699ed9f356385de3737ffbb02 Mon Sep 17 00:00:00 2001 From: edmondac-admin Date: Fri, 27 Nov 2020 13:38:02 +0000 Subject: [PATCH 045/864] tame hound --- easybuild/framework/easyconfig/easyconfig.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index d49d63320f..8195b4fc4e 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -702,10 +702,11 @@ def parse(self): # parse dependency specifications # it's important that templating is still disabled at this stage! self.log.info("Parsing dependency specifications...") + def remove_false_versions(deps): ret = [] for dep in deps: - if isinstance(dep, dict) and dep['version'] == False: + if isinstance(dep, dict) and dep['version'] is False: continue ret.append(dep) return ret From 52bc90e85e75c9d312d7223c1931410bac563cc6 Mon Sep 17 00:00:00 2001 From: Ed Date: Mon, 4 Jan 2021 13:53:21 +0000 Subject: [PATCH 046/864] Update easybuild/framework/easyconfig/easyconfig.py Co-authored-by: Alexander Grund --- easybuild/framework/easyconfig/easyconfig.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 8195b4fc4e..b826f44382 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -704,12 +704,7 @@ def parse(self): self.log.info("Parsing dependency specifications...") def remove_false_versions(deps): - ret = [] - for dep in deps: - if isinstance(dep, dict) and dep['version'] is False: - continue - ret.append(dep) - return ret + return [dep for dep in deps if not (isinstance(dep, dict) and dep['version'] is False)] self['dependencies'] = remove_false_versions(self._parse_dependency(dep) for dep in self['dependencies']) self['hiddendependencies'] = remove_false_versions(self._parse_dependency(dep, hidden=True) for dep in From 4456b50afe2e1ded2cf5986452209c5f3120bdb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Wed, 6 Jan 2021 22:49:53 +0100 Subject: [PATCH 047/864] Add toolchain deps to template resolution --- easybuild/framework/easyconfig/templates.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 9af16131d1..782959eca9 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -80,6 +80,7 @@ TEMPLATE_SOFTWARE_VERSIONS = [ # software name, prefix for *ver and *shortver ('CUDA', 'cuda'), + ('CUDAcore', 'cuda'), ('Java', 'java'), ('Perl', 'perl'), ('Python', 'py'), @@ -242,6 +243,10 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) else: deps += config.get('builddependencies', []) + # Include all toolchain deps (e.g. CUDAcore template in fosscuda) + if config.toolchain.tcdeps is not None: + deps += config.toolchain.tcdeps + for dep in deps: if isinstance(dep, dict): dep_name, dep_version = dep['name'], dep['version'] From ab71762fea664f0be5a991ac06cbe28d52c3e60a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Thu, 7 Jan 2021 00:41:09 +0100 Subject: [PATCH 048/864] Only add tcdeps to easyconfigs and not dictionaries used in tests --- easybuild/framework/easyconfig/templates.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 782959eca9..d3674243eb 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -243,9 +243,9 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) else: deps += config.get('builddependencies', []) - # Include all toolchain deps (e.g. CUDAcore template in fosscuda) - if config.toolchain.tcdeps is not None: - deps += config.toolchain.tcdeps + # Include all toolchain deps (e.g. CUDAcore template in fosscuda) + if config.toolchain.tcdeps is not None: + deps += config.toolchain.tcdeps for dep in deps: if isinstance(dep, dict): From b2dd2604f639399d9678a901e4b76296727ee523 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Sat, 9 Jan 2021 22:15:11 +0100 Subject: [PATCH 049/864] Use extend rather than equivalent __iadd__ for template deps --- easybuild/framework/easyconfig/templates.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index d3674243eb..67e4122b8c 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -239,13 +239,13 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # only consider build dependencies when we're actually in iterative mode! if 'builddependencies' in config.iterate_options: if config.iterating: - deps += config.get('builddependencies', []) + deps.extend(config.get('builddependencies', [])) else: - deps += config.get('builddependencies', []) + deps.extend(config.get('builddependencies', [])) # Include all toolchain deps (e.g. CUDAcore template in fosscuda) if config.toolchain.tcdeps is not None: - deps += config.toolchain.tcdeps + deps.extend(config.toolchain.tcdeps) for dep in deps: if isinstance(dep, dict): From 6af422ac82eabf1d86eb97761a5787c44e634e81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Sun, 10 Jan 2021 17:10:56 +0100 Subject: [PATCH 050/864] Drop template check in toolchain string (not necessary for test) --- test/framework/robot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/robot.py b/test/framework/robot.py index dc540674ec..2cc02ba526 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -436,7 +436,7 @@ def test_resolve_dependencies_minimal(self): " ('SQLite', '3.8.10.2'),", "]", # toolchain as list line, for easy modification later; - "toolchain = {'name': 'foss', 'version': '%(version_minor)s018a'}", + "toolchain = {'name': 'foss', 'version': '2018a'}", ] write_file(barec, '\n'.join(barec_lines)) bar = process_easyconfig(barec)[0] From dc9d1d73fea84e79932407418a630c8584a2fbb5 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 22 Jan 2021 13:21:49 +0100 Subject: [PATCH 051/864] Add create_unused_dir to create a directory which does not yet exist This will append _%d until a new folder could be created. If the base folder name does not exists, it is used directly, so the simple, most common case is the cleanest --- easybuild/tools/filetools.py | 40 ++++++++++++++++++++++++++++++++++++ test/framework/filetools.py | 18 ++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 51e5bc9d03..ef90af4c58 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2473,3 +2473,43 @@ def copy_framework_files(paths, target_dir): raise EasyBuildError("Couldn't find parent folder of updated file: %s", path) return file_info + + +def create_unused_dir(parent_folder, name): + """ + Create a new folder in parent_folder using name as the name. + When a folder of that name already exists, '_0' is appended which is retried for increasing numbers until + an unused name was found + """ + if not os.path.isabs(parent_folder): + parent_folder = os.path.abspath(parent_folder) + + start_path = os.path.join(parent_folder, name) + number = None + while True: + if number is None: + path = start_path + number = 0 + else: + path = start_path + '_' + str(number) + number += 1 + try: + os.mkdir(path) + break + except OSError as err: + # Distinguish between error due to existing folder and anything else + if not os.path.exists(path): + raise EasyBuildError("Failed to create directory %s: %s", path, err) + + # set group ID and sticky bits, if desired + bits = 0 + if build_option('set_gid_bit'): + bits |= stat.S_ISGID + if build_option('sticky_bit'): + bits |= stat.S_ISVTX + if bits: + try: + adjust_permissions(path, bits, add=True, relative=True, recursive=True, onlydirs=True) + except OSError as err: + raise EasyBuildError("Failed to set group ID/sticky bit: %s", err) + return path diff --git a/test/framework/filetools.py b/test/framework/filetools.py index def99f7550..6a7abf8ef6 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2888,6 +2888,24 @@ def test_locate_files(self): error_pattern = r"One or more files not found: 2\.txt \(search paths: \)" self.assertErrorRegex(EasyBuildError, error_pattern, ft.locate_files, ['2.txt'], []) + def test_create_unused_dir(self): + path = ft.create_unused_dir(self.test_prefix, 'folder') + self.assertEqual(path, os.path.join(self.test_prefix, 'folder')) + self.assertTrue(os.path.exists(path)) + # Repeat with existing folder(s) should create new ones + for i in range(10): + path = ft.create_unused_dir(self.test_prefix, 'folder') + self.assertEqual(path, os.path.join(self.test_prefix, 'folder_%s' % i)) + self.assertTrue(os.path.exists(path)) + # Not influenced by similar folder + path = ft.create_unused_dir(self.test_prefix, 'folder2') + self.assertEqual(path, os.path.join(self.test_prefix, 'folder2')) + self.assertTrue(os.path.exists(path)) + for i in range(10): + path = ft.create_unused_dir(self.test_prefix, 'folder2') + self.assertEqual(path, os.path.join(self.test_prefix, 'folder2_%s' % i)) + self.assertTrue(os.path.exists(path)) + def suite(): """ returns all the testcases in this module """ From b603941a08a95e5842dddc06120e85783d2e68a3 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 27 Jan 2021 17:38:17 +0100 Subject: [PATCH 052/864] Avoid no-op path changes in ModulesTool LD_ENV_VAR_KEYS variables are set and restore prior and after running any module command. Often they don't have any change leading to many messages like: > INFO Environment variable LD_LIBRARY_PATH set to (previous value: '') This just clutters the log without any benefit. This change skips such messages and changes --- easybuild/tools/modules.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index a2a7739222..4cb1b21520 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -793,9 +793,12 @@ def run_module(self, *args, **kwargs): # restore selected original environment variables before running module command environ = os.environ.copy() for key in LD_ENV_VAR_KEYS: - environ[key] = ORIG_OS_ENVIRON.get(key, '') - self.log.debug("Changing %s from '%s' to '%s' in environment for module command", - key, os.environ.get(key, ''), environ[key]) + old_value = environ.get(key, '') + new_value = ORIG_OS_ENVIRON.get(key, '') + if old_value != new_value: + environ[key] = new_value + self.log.debug("Changing %s from '%s' to '%s' in environment for module command", + key, old_value, new_value) cmd_list = self.compose_cmd_list(args) full_cmd = ' '.join(cmd_list) @@ -842,11 +845,13 @@ def run_module(self, *args, **kwargs): # correct values of selected environment variables as yielded by the adjustments made # make sure we get the order right (reverse lists with [::-1]) for key in LD_ENV_VAR_KEYS: - curr_ld_val = os.environ.get(key, '').split(os.pathsep) + curr_ld_val = os.environ.get(key, '') + curr_ld_val = curr_ld_val.split(os.pathsep) if curr_ld_val else [] # Take care of empty/unset values new_ld_val = [x for x in nub(prev_ld_values[key] + curr_ld_val[::-1]) if x][::-1] - self.log.debug("Correcting paths in $%s from %s to %s" % (key, curr_ld_val, new_ld_val)) - self.set_path_env_var(key, new_ld_val) + if new_ld_val != curr_ld_val: + self.log.debug("Correcting paths in $%s from %s to %s" % (key, curr_ld_val, new_ld_val)) + self.set_path_env_var(key, new_ld_val) # Process stderr result = [] From 73055d093c02f1531d006162971c2d4eb20cf2d1 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 2 Feb 2021 11:33:12 +0100 Subject: [PATCH 053/864] Avoid module use in LMod if possible to allow faster execution --- easybuild/tools/modules.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index a2a7739222..547c2388a4 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1428,7 +1428,20 @@ def use(self, path, priority=None): if priority: self.run_module(['use', '--priority', str(priority), path]) else: - self.run_module(['use', path]) + # LMod allows modifying MODULEPATH directly. So do that to avoid the costly module use + # unless priorities are in use already + if os.environ.get('__LMOD_Priority_MODULEPATH'): + self.run_module(['use', path]) + else: + cur_mod_path = os.environ.get('MODULEPATH') + if cur_mod_path is None: + new_mod_path = path + else: + new_mod_path = [path] + [p for p in cur_mod_path.split(':') if p != path] + new_mod_path = ':'.join(new_mod_path) + self.log.debug('Changing MODULEPATH from %s to %s' % + ('' if cur_mod_path is None else cur_mod_path, new_mod_path)) + os.environ['MODULEPATH'] = new_mod_path def prepend_module_path(self, path, set_mod_paths=True, priority=None): """ From cc0a77e0296dbdc107ed2fc528a7ff527dfc577a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 2 Feb 2021 14:26:41 +0100 Subject: [PATCH 054/864] Allow use of alternate envvar to HOME for user modules --- easybuild/framework/easyblock.py | 6 +++-- easybuild/tools/config.py | 4 ++++ easybuild/tools/options.py | 9 ++++++-- test/framework/easyblock.py | 38 ++++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index acc2b64bfc..d7b65a59ae 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1345,10 +1345,12 @@ def make_module_extend_modpath(self): # add user-specific module path; use statement will be guarded so no need to create the directories user_modpath = build_option('subdir_user_modules') if user_modpath: + user_envvar = build_option('envvar_user_modules') user_modpath_exts = ActiveMNS().det_user_modpath_extensions(self.cfg) self.log.debug("Including user module path extensions returned by naming scheme: %s", user_modpath_exts) - txt += self.module_generator.use(user_modpath_exts, prefix=self.module_generator.getenv_cmd('HOME'), - guarded=True, user_modpath=user_modpath) + txt += self.module_generator.use(user_modpath_exts, + prefix=self.module_generator.getenv_cmd(user_envvar), guarded=True, + user_modpath=user_modpath) else: self.log.debug("Not including module path extensions, as specified.") return txt diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index b97186f3c5..b3e4e5b9ff 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -79,6 +79,7 @@ DEFAULT_CONT_TYPE = CONT_TYPE_SINGULARITY DEFAULT_BRANCH = 'develop' +DEFAULT_ENVVAR_USER_MODULES = 'HOME' DEFAULT_INDEX_MAX_AGE = 7 * 24 * 60 * 60 # 1 week (in seconds) DEFAULT_JOB_BACKEND = 'GC3Pie' DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") @@ -291,6 +292,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_BRANCH: [ 'pr_target_branch', ], + DEFAULT_ENVVAR_USER_MODULES: [ + 'envvar_user_modules', + ], DEFAULT_INDEX_MAX_AGE: [ 'index_max_age', ], diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index a363cde325..d712b83e75 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -60,7 +60,8 @@ from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError from easybuild.tools.build_log import init_logging, log_start, print_msg, print_warning, raise_easybuilderror from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE, DEFAULT_ALLOW_LOADED_MODULES -from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE +from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_ENVVAR_USER_MODULES +from easybuild.tools.config import DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL @@ -489,6 +490,9 @@ def config_options(self): 'buildpath': ("Temporary build path", None, 'store', mk_full_default_path('buildpath')), 'containerpath': ("Location where container recipe & image will be stored", None, 'store', mk_full_default_path('containerpath')), + 'envvar-user-modules': ("Environment variable that holds the base path for which user-specific modules are " + "installed relative to (defaults to 'HOME')", None, 'store', + DEFAULT_ENVVAR_USER_MODULES), 'external-modules-metadata': ("List of (glob patterns for) paths to files specifying metadata " "for external modules (INI format)", 'strlist', 'store', None), 'hooks': ("Location of Python module with hook implementations", 'str', 'store', None), @@ -547,7 +551,8 @@ def config_options(self): 'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']), 'subdir-software': ("Installpath subdir for software", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_software']), - 'subdir-user-modules': ("Base path of user-specific modules relative to their $HOME", None, 'store', None), + 'subdir-user-modules': ("Base path of user-specific modules relative to --envvar-user-modules (which " + "defaults to $HOME)", None, 'store', None), 'suffix-modules-path': ("Suffix for module files install path", None, 'store', GENERAL_CLASS), # this one is sort of an exception, it's something jobscripts can set, # has no real meaning for regular eb usage diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 399a269597..88a6d5d62e 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -301,6 +301,44 @@ def test_make_module_extend_modpath(self): regex = re.compile(regex, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + # Repeat this but using an alternate envvar (instead of $HOME) + build_options = { + 'envvar_user_modules': 'TURKEY', + 'subdir_user_modules': usermodsdir, + 'valid_module_classes': modclasses, + 'suffix_modules_path': 'funky', + } + init_config(build_options=build_options) + eb = EasyBlock(EasyConfig(self.eb_file)) + eb.installdir = config.install_path() + + txt = eb.make_module_extend_modpath() + if get_module_syntax() == 'Tcl': + regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses] + home = r'\$::env\(TURKEY\)' + fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir + regexs.extend([ + # extension for user modules is guarded + r'if { \[ file isdirectory \[ file join %s \[ %s \] \] \] } {$' % (home, fj_usermodsdir), + # no per-moduleclass extension for user modules + r'^\s+module use \[ file join %s \[ %s \] \]$' % (home, fj_usermodsdir), + ]) + elif get_module_syntax() == 'Lua': + regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c for c in modclasses] + home = r'os.getenv\("TURKEY"\)' + pj_usermodsdir = r'pathJoin\("%s", "funky", "Compiler/pi/3.14"\)' % usermodsdir + regexs.extend([ + # extension for user modules is guarded + r'if isDir\(pathJoin\(%s, %s\)\) then' % (home, pj_usermodsdir), + # no per-moduleclass extension for user modules + r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, %s\)\)' % (home, pj_usermodsdir), + ]) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + for regex in regexs: + regex = re.compile(regex, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + def test_make_module_req(self): """Testcase for make_module_req""" self.contents = '\n'.join([ From 91e46fd667b9199d513113c52655891531aa2565 Mon Sep 17 00:00:00 2001 From: Victor Holanda Rusu Date: Tue, 2 Feb 2021 14:43:12 +0100 Subject: [PATCH 055/864] Avoid metadata greedy behaviour --- easybuild/framework/easyconfig/easyconfig.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 3843aa22e1..7b48744382 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1289,12 +1289,16 @@ def mk_var_name_pair(var_name_pair, name): # if a version is already set in the available metadata, we retain it if 'version' not in existing_metadata: - res['version'] = [version] + # should not use [version], because this is a greedy behaviour + # using version_var_name instead + res['version'] = [version_var_name] self.log.info('setting external module %s version to be %s', mod_name, version) # if a prefix is already set in the available metadata, we retain it if 'prefix' not in existing_metadata: - res['prefix'] = prefix + # should not use prefix, because this is a greedy behaviour + # using prefix_var_name instead + res['prefix'] = prefix_var_name self.log.info('setting external module %s prefix to be %s', mod_name, prefix_var_name) break From 9fd885cb47507e6e9c9a4cff43dc0e38c13e144d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 2 Feb 2021 14:59:23 +0100 Subject: [PATCH 056/864] add test for defining %(cudaver)s template via toolchain --- test/framework/easyconfig.py | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 4e47085936..b75a0db7ed 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1050,6 +1050,51 @@ def test_templating(self): ec = EasyConfig(test_ec) self.assertEqual(ec['sanity_check_commands'], ['mpiexec -np 1 -- toy']) + def test_templating_cuda_toolchain(self): + """Test templates via toolchain component, like setting %(cudaver)s with fosscuda toolchain.""" + + build_options = {'robot_path': [self.test_prefix]} + init_config(build_options=build_options) + + # create fake easyconfig files, good enough to test with + cuda_ec = os.path.join(self.test_prefix, 'CUDA-10.1.243') + cuda_ec_txt = '\n'.join([ + "easyblock = 'Toolchain'", + "name = 'CUDA'", + "version = '10.1.243'", + "homepage = 'https://example.com'", + "description = 'CUDA'", + "toolchain = SYSTEM", + ]) + write_file(cuda_ec, cuda_ec_txt) + + fosscuda_ec = os.path.join(self.test_prefix, 'fosscuda-2021.02.eb') + fosscuda_ec_txt = '\n'.join([ + "easyblock = 'Toolchain'", + "name = 'fosscuda'", + "version = '2021.02'", + "homepage = 'https://example.com'", + "description = 'fosscuda toolchain'", + "toolchain = SYSTEM", + "dependencies = [('CUDA', '10.1.243')]", + ]) + write_file(fosscuda_ec, fosscuda_ec_txt) + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = '\n'.join([ + "easyblock = 'Toolchain'", + "name = 'test'", + "version = '1.0'", + "homepage = 'https://example.com'", + "description = 'just a test'", + "toolchain = {'name': 'fosscuda', 'version': '2021.02'}", + ]) + write_file(test_ec, test_ec_txt) + ec = EasyConfig(test_ec) + self.assertEqual(ec.template_values['cudaver'], '10.1.243') + self.assertEqual(ec.template_values['cudamajver'], '10') + self.assertEqual(ec.template_values['cudashortver'], '10.1') + def test_java_wrapper_templating(self): """test templating when the Java wrapper is a dep""" self.contents = '\n'.join([ From 6b119a686fa99591ea23a7af107084c4c15e14b8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 2 Feb 2021 15:14:02 +0100 Subject: [PATCH 057/864] avoid initializing Toolchain instance when taking into account toolchain dependencies for templates + revert change from #3541 to test from which started failing because of triggering early Toolchain initialization --- easybuild/framework/easyconfig/templates.py | 9 +++++---- test/framework/robot.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 840266a98a..610aa3291e 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -233,7 +233,6 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # a cyclic import...); # we need to know to determine whether we're iterating over a list of build dependencies is_easyconfig = hasattr(config, 'iterating') and hasattr(config, 'iterate_options') - if is_easyconfig: # if we're iterating over different lists of build dependencies, # only consider build dependencies when we're actually in iterative mode! @@ -243,9 +242,11 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) else: deps.extend(config.get('builddependencies', [])) - # Include all toolchain deps (e.g. CUDAcore template in fosscuda) - if config.toolchain.tcdeps is not None: - deps.extend(config.toolchain.tcdeps) + # include all toolchain deps (e.g. CUDAcore component in fosscuda); + # access Toolchain instance via _toolchain to avoid triggering initialization of the toolchain! + if config._toolchain is not None: + if config._toolchain.tcdeps is not None: + deps.extend(config._toolchain.tcdeps) for dep in deps: if isinstance(dep, dict): diff --git a/test/framework/robot.py b/test/framework/robot.py index ccdb320d9c..9c29cb25cd 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -436,7 +436,7 @@ def test_resolve_dependencies_minimal(self): " ('SQLite', '3.8.10.2'),", "]", # toolchain as list line, for easy modification later; - "toolchain = {'name': 'foss', 'version': '2018a'}", + "toolchain = {'name': 'foss', 'version': '%(version_minor)s018a'}", ] write_file(barec, '\n'.join(barec_lines)) bar = process_easyconfig(barec)[0] From ca8af709d544da16538345f5c213c8102916638b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 2 Feb 2021 17:44:24 +0100 Subject: [PATCH 058/864] rename EasyBlock._skip_step to EasyBlock.skip_step, to make it part of the public API --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index acc2b64bfc..776fedc96d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3044,7 +3044,7 @@ def update_config_template_run_step(self): self.cfg.template_values[name[0]] = str(getattr(self, name[0], None)) self.cfg.generate_template_values() - def _skip_step(self, step, skippable): + def skip_step(self, step, skippable): """Dedice whether or not to skip the specified step.""" module_only = build_option('module_only') force = build_option('force') or build_option('rebuild') @@ -3243,7 +3243,7 @@ def run_all_steps(self, run_test_cases): try: for (step_name, descr, step_methods, skippable) in steps: - if self._skip_step(step_name, skippable): + if self.skip_step(step_name, skippable): print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent) else: if self.dry_run: From 0ac6d016b649131f1bbadb912ba7c70ba3f150f4 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 3 Feb 2021 11:15:34 +0100 Subject: [PATCH 059/864] Allow use of a list of environment variables --- easybuild/framework/easyblock.py | 12 ++++--- easybuild/tools/config.py | 6 ++-- easybuild/tools/options.py | 9 +++-- test/framework/easyblock.py | 60 ++++++++++++++++++-------------- 4 files changed, 47 insertions(+), 40 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index d7b65a59ae..bf7721378b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1345,12 +1345,16 @@ def make_module_extend_modpath(self): # add user-specific module path; use statement will be guarded so no need to create the directories user_modpath = build_option('subdir_user_modules') if user_modpath: - user_envvar = build_option('envvar_user_modules') + user_envvars = build_option('envvars_user_modules') or ['HOME'] user_modpath_exts = ActiveMNS().det_user_modpath_extensions(self.cfg) self.log.debug("Including user module path extensions returned by naming scheme: %s", user_modpath_exts) - txt += self.module_generator.use(user_modpath_exts, - prefix=self.module_generator.getenv_cmd(user_envvar), guarded=True, - user_modpath=user_modpath) + for user_envvar in user_envvars: + if not os.getenv(user_envvar): + raise EasyBuildError("Requested environment variable %s as an additional branch for user" + "modules does not exist in current environment", user_envvar) + txt += self.module_generator.use(user_modpath_exts, + prefix=self.module_generator.getenv_cmd(user_envvar), guarded=True, + user_modpath=user_modpath) else: self.log.debug("Not including module path extensions, as specified.") return txt diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index b3e4e5b9ff..66ae496b38 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -79,7 +79,7 @@ DEFAULT_CONT_TYPE = CONT_TYPE_SINGULARITY DEFAULT_BRANCH = 'develop' -DEFAULT_ENVVAR_USER_MODULES = 'HOME' +DEFAULT_ENVVAR_USERS_MODULES = 'HOME' DEFAULT_INDEX_MAX_AGE = 7 * 24 * 60 * 60 # 1 week (in seconds) DEFAULT_JOB_BACKEND = 'GC3Pie' DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") @@ -174,6 +174,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'download_timeout', 'dump_test_report', 'easyblock', + 'envvars_user_modules', 'extra_modules', 'filter_deps', 'filter_env_vars', @@ -292,9 +293,6 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_BRANCH: [ 'pr_target_branch', ], - DEFAULT_ENVVAR_USER_MODULES: [ - 'envvar_user_modules', - ], DEFAULT_INDEX_MAX_AGE: [ 'index_max_age', ], diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index d712b83e75..6d3d1c4b64 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -60,8 +60,7 @@ from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError from easybuild.tools.build_log import init_logging, log_start, print_msg, print_warning, raise_easybuilderror from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE, DEFAULT_ALLOW_LOADED_MODULES -from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_ENVVAR_USER_MODULES -from easybuild.tools.config import DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE +from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL @@ -490,9 +489,9 @@ def config_options(self): 'buildpath': ("Temporary build path", None, 'store', mk_full_default_path('buildpath')), 'containerpath': ("Location where container recipe & image will be stored", None, 'store', mk_full_default_path('containerpath')), - 'envvar-user-modules': ("Environment variable that holds the base path for which user-specific modules are " - "installed relative to (defaults to 'HOME')", None, 'store', - DEFAULT_ENVVAR_USER_MODULES), + 'envvars-user-modules': ("List of environment variables that hold the base paths for which user-specific " + "modules will be installed relative to", 'strlist', 'store', + ['HOME']), 'external-modules-metadata': ("List of (glob patterns for) paths to files specifying metadata " "for external modules (INI format)", 'strlist', 'store', None), 'hooks': ("Location of Python module with hook implementations", 'str', 'store', None), diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 88a6d5d62e..02c93a1038 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -301,9 +301,13 @@ def test_make_module_extend_modpath(self): regex = re.compile(regex, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) - # Repeat this but using an alternate envvar (instead of $HOME) + # Repeat this but using an alternate envvars (instead of $HOME) + list_of_envvars = ['TURKEY', 'HAM'] + os.environ['TURKEY'] = "1" + os.environ['HAM'] = "1" + build_options = { - 'envvar_user_modules': 'TURKEY', + 'envvars_user_modules': list_of_envvars, 'subdir_user_modules': usermodsdir, 'valid_module_classes': modclasses, 'suffix_modules_path': 'funky', @@ -313,31 +317,33 @@ def test_make_module_extend_modpath(self): eb.installdir = config.install_path() txt = eb.make_module_extend_modpath() - if get_module_syntax() == 'Tcl': - regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses] - home = r'\$::env\(TURKEY\)' - fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir - regexs.extend([ - # extension for user modules is guarded - r'if { \[ file isdirectory \[ file join %s \[ %s \] \] \] } {$' % (home, fj_usermodsdir), - # no per-moduleclass extension for user modules - r'^\s+module use \[ file join %s \[ %s \] \]$' % (home, fj_usermodsdir), - ]) - elif get_module_syntax() == 'Lua': - regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c for c in modclasses] - home = r'os.getenv\("TURKEY"\)' - pj_usermodsdir = r'pathJoin\("%s", "funky", "Compiler/pi/3.14"\)' % usermodsdir - regexs.extend([ - # extension for user modules is guarded - r'if isDir\(pathJoin\(%s, %s\)\) then' % (home, pj_usermodsdir), - # no per-moduleclass extension for user modules - r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, %s\)\)' % (home, pj_usermodsdir), - ]) - else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) - for regex in regexs: - regex = re.compile(regex, re.M) - self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + for envvar in list_of_envvars: + if get_module_syntax() == 'Tcl': + regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses] + home = r'\$::env\(%s\)' % envvar + fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir + regexs.extend([ + # extension for user modules is guarded + r'if { \[ file isdirectory \[ file join %s \[ %s \] \] \] } {$' % (home, fj_usermodsdir), + # no per-moduleclass extension for user modules + r'^\s+module use \[ file join %s \[ %s \] \]$' % (home, fj_usermodsdir), + ]) + elif get_module_syntax() == 'Lua': + regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c for c in modclasses] + home = r'os.getenv\("%s"\)' % envvar + pj_usermodsdir = r'pathJoin\("%s", "funky", "Compiler/pi/3.14"\)' % usermodsdir + regexs.extend([ + # extension for user modules is guarded + r'if isDir\(pathJoin\(%s, %s\)\) then' % (home, pj_usermodsdir), + # no per-moduleclass extension for user modules + r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, %s\)\)' % (home, pj_usermodsdir), + ]) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + for regex in regexs: + regex = re.compile(regex, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + os.unsetenv(envvar) def test_make_module_req(self): """Testcase for make_module_req""" From ab957ab25b6fe158f3b6d644f7852b479932012d Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 3 Feb 2021 11:17:36 +0100 Subject: [PATCH 060/864] Appease hound --- test/framework/easyblock.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 02c93a1038..e45ebc9b1f 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -329,7 +329,8 @@ def test_make_module_extend_modpath(self): r'^\s+module use \[ file join %s \[ %s \] \]$' % (home, fj_usermodsdir), ]) elif get_module_syntax() == 'Lua': - regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c for c in modclasses] + regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c + for c in modclasses] home = r'os.getenv\("%s"\)' % envvar pj_usermodsdir = r'pathJoin\("%s", "funky", "Compiler/pi/3.14"\)' % usermodsdir regexs.extend([ From 83cc2e59ff593ee7d9842d0a96fe6398b4a22cae Mon Sep 17 00:00:00 2001 From: Victor Holanda Rusu Date: Wed, 3 Feb 2021 15:20:18 +0100 Subject: [PATCH 061/864] Attempt to fix unit tests --- test/framework/easyconfig.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 2e278bbd32..b32e50722b 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1548,24 +1548,24 @@ def test_external_dependencies(self): self.assertEqual(deps[3]['full_mod_name'], 'foobar/1.2.3') foobar_metadata = { 'name': ['foobar'], - 'prefix': '/software/foobar/2.3.4', - 'version': ['2.3.4'], + 'prefix': 'CRAY_FOOBAR_DIR', + 'version': ['CRAY_FOOBAR_VERSION'], } self.assertEqual(deps[3]['external_module_metadata'], foobar_metadata) self.assertEqual(deps[5]['full_mod_name'], 'pi/3.14') pi_metadata = { 'name': ['pi'], - 'prefix': '/software/pi/3.14', - 'version': ['3.14'], + 'prefix': 'PI_ROOT', + 'version': ['PI_VERSION'], } self.assertEqual(deps[5]['external_module_metadata'], pi_metadata) self.assertEqual(deps[7]['full_mod_name'], 'cray-netcdf-hdf5parallel/1.10.6') cray_netcdf_metadata = { 'name': ['netcdf-hdf5parallel'], - 'prefix': '/software/cray-netcdf-hdf5parallel/1.10.6', - 'version': ['1.10.6'], + 'prefix': 'CRAY_NETCDF_HDF5PARALLEL_PREFIX', + 'version': ['CRAY_NETCDF_HDF5PARALLEL_VERSION'], } self.assertEqual(deps[7]['external_module_metadata'], cray_netcdf_metadata) @@ -1603,8 +1603,8 @@ def test_external_dependencies(self): self.assertEqual(deps[3]['full_mod_name'], 'foobar/1.2.3') foobar_metadata = { 'name': ['foobar'], # probed from 'foobar' module - 'prefix': '/software/foobar/2.3.4', # probed from 'foobar' module - 'version': ['1.2.3'], # from [foobar/1.2.3] entry in metadata file + 'prefix': 'CRAY_FOOBAR_DIR', # probed from 'foobar' module + 'version': ['CRAY_FOOBAR_VERSION'], # from [foobar/1.2.3] entry in metadata file } self.assertEqual(deps[3]['external_module_metadata'], foobar_metadata) @@ -1617,16 +1617,16 @@ def test_external_dependencies(self): self.assertEqual(deps[5]['full_mod_name'], 'pi/3.14') pi_metadata = { 'name': ['PI'], # from [pi/3.14] entry in metadata file - 'prefix': '/software/pi/3.14', # probed from 'pi/3.14' module - 'version': ['3.14.0'], # from [pi/3.14] entry in metadata file + 'prefix': 'PI_ROOT', # probed from 'pi/3.14' module + 'version': ['PI_VERSION'], # from [pi/3.14] entry in metadata file } self.assertEqual(deps[5]['external_module_metadata'], pi_metadata) self.assertEqual(deps[7]['full_mod_name'], 'cray-netcdf-hdf5parallel/1.10.6') cray_netcdf_metadata = { 'name': ['HDF5'], - 'prefix': '/software/cray-netcdf-hdf5parallel/1.10.6', - 'version': ['1.10.6'], + 'prefix': 'CRAY_NETCDF_HDF5PARALLEL_PREFIX', + 'version': ['CRAY_NETCDF_HDF5PARALLEL_VERSION'], } self.assertEqual(deps[7]['external_module_metadata'], cray_netcdf_metadata) From 3098b8a4c84b10dacdced1f2a882e492b9df8ea2 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 4 Feb 2021 17:54:54 +0100 Subject: [PATCH 062/864] Create the lib64 symlink as a relative symlink Closes #3564 --- easybuild/framework/easyblock.py | 2 +- test/framework/toy_build.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 776fedc96d..44b1eeb16e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2405,7 +2405,7 @@ def post_install_step(self): lib_dir = os.path.join(self.installdir, 'lib') lib64_dir = os.path.join(self.installdir, 'lib64') if os.path.exists(lib_dir) and not os.path.exists(lib64_dir): - symlink(lib_dir, lib64_dir) + symlink('lib', lib64_dir, use_abspath_source=False) def sanity_check_step(self, *args, **kwargs): """ diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 1282f7db57..5d02387f26 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3151,6 +3151,8 @@ def test_test_toy_build_lib64_symlink(self): self.assertFalse(os.path.islink(lib_path)) self.assertTrue(os.path.islink(lib64_path)) self.assertTrue(os.path.samefile(lib_path, lib64_path)) + # Need relative path: https://github.com/easybuilders/easybuild-framework/issues/3564 + self.assertFalse(os.path.isabs(os.readlink(lib64_path))) # cleanup and try again with --disable-lib64-lib-symlink remove_dir(self.test_installpath) From 3dcce959e93587a56ea87eaba2b44f69cad11c0d Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Fri, 5 Feb 2021 19:50:23 +0100 Subject: [PATCH 063/864] bugfix http_header_fields_urlpat to include file from header value add true positive tests on header and file inclusion --- easybuild/tools/filetools.py | 24 ++++++++++--- test/framework/options.py | 69 +++++++++++++++++++++++------------- 2 files changed, 64 insertions(+), 29 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 6957f5254e..2363878ae1 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -569,9 +569,9 @@ def derive_alt_pypi_url(url): return alt_pypi_url -def parse_http_header_fields_urlpat(arg, urlpat=None, urlpat_headers={}, maxdepth=3): +def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_headers={}, maxdepth=3): """ - Recurse into PAT::[PAT::FILE|PAT::HEADER: FIELD|HEADER: FIELD] where FILE may be a + Recurse into [URLPAT::][HEADER: ]FILE|FIELD where FILE may be another such string or file containing lines matching the same format, and flatten the result as a dict e.g. {'^example.com': ['Authorization: Basic token', 'User-Agent: Special Agent']} """ @@ -597,16 +597,30 @@ def parse_http_header_fields_urlpat(arg, urlpat=None, urlpat_headers={}, maxdept # argline is a file path, so read that instead _log.debug('File included in parse_http_header_fields_urlpat: %s' % argline) argline = read_file(argline) - urlpat_headers = parse_http_header_fields_urlpat(argline, urlpat, urlpat_headers, maxdepth - 1) + urlpat_headers = parse_http_header_fields_urlpat(argline, urlpat, header, urlpat_headers, maxdepth - 1) continue # URL pattern is separated by '::' from a HTTP header field if '::' in argline: [urlpat, argline] = argline.split('::', 1) # get the urlpat # the remainder may be another parseable argument, recurse with same depth - urlpat_headers = parse_http_header_fields_urlpat(argline, urlpat, urlpat_headers, maxdepth) + urlpat_headers = parse_http_header_fields_urlpat(argline, urlpat, header, urlpat_headers, maxdepth) continue + # Header field has format HEADER: FIELD, and FIELD may be another parseable argument + # except if FIELD contains colons, then argline is the final HEADER: FIELD to be returned + if ':' in argline and argline.count(':') == 1: + [argheader, argline] = argline.split(':', 1) # get the header and the remainder + # the remainder may be another parseable argument, recurse with same depth + # note that argheader would be forgotten in favor of the urlpat_headers returned by recursion, + # so pass on the header for reconstruction just in case there was nothing to recurse in + urlpat_headers = parse_http_header_fields_urlpat(argline, urlpat, argheader, urlpat_headers, maxdepth) + continue + + if header is not None: + # parent caller didn't want to forget about the header, reconstruct as recursion stops here. + argline = header.strip() + ': ' + argline + if urlpat is not None: if urlpat in urlpat_headers.keys(): urlpat_headers[urlpat].append(argline) # add headers to the list @@ -638,7 +652,7 @@ def download_file(filename, url, path, forced=False): if http_header_fields_urlpat is not None: # there may be multiple options given, parse them all, while updating urlpat_headers for arg in http_header_fields_urlpat: - urlpat_headers = parse_http_header_fields_urlpat(arg, None, urlpat_headers) + urlpat_headers = parse_http_header_fields_urlpat(arg, urlpat_headers) # make sure directory exists basedir = os.path.dirname(path) diff --git a/test/framework/options.py b/test/framework/options.py index 5ca68d3507..ca5e7f0e25 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2578,10 +2578,10 @@ def test_http_header_fields_urlpat(self): # define header fields:values that should (not) show up in the logs, either # because they are secret or because they are not matched for the url - test_applied_hdr = 'HeaderAPPLIED' - test_applied_val = 'SECRETvalue' - test_nomatch_hdr = 'HeaderIGNORED' - test_nomatch_val = 'BOGUSvalue' + testdohdr = 'HeaderAPPLIED' + testdoval = 'SECRETvalue' + testdonthdr = 'HeaderIGNORED' + testdontval = 'BOGUSvalue' # header fields (or its values) could be files to be read instead of literals testcmdfile = os.path.join(self.test_prefix, 'testhttpheaderscmdline.txt') @@ -2591,6 +2591,10 @@ def test_http_header_fields_urlpat(self): testexchdrfile = os.path.join(self.test_prefix, 'testhttpheadershdrexc.txt') testurlpatfile = os.path.join(self.test_prefix, 'testhttpheadersurlpat.txt') + # log mention format upon header or file inclusion + mentionhdr = 'Custom HTTP header field set: %s' + mentionfile = 'File included in parse_http_header_fields_urlpat: %s' + def run_and_assert(args, msg, words_expected=None, words_unexpected=None): stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) if words_expected is not None: @@ -2605,11 +2609,15 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None): # A: simple direct case (all is logged) args = list(common_args) args.extend([ - '--http-header-fields-urlpat=gnu.org::%s:%s' % (test_applied_hdr, test_applied_val), - '--http-header-fields-urlpat=nomatch.com::%s:%s' % (test_nomatch_hdr, test_nomatch_val), + '--http-header-fields-urlpat=gnu.org::%s:%s' % (testdohdr, testdoval), + '--http-header-fields-urlpat=nomatch.com::%s:%s' % (testdonthdr, testdontval), ]) # expect to find everything passed on cmdline - run_and_assert(args, 'case A', [test_applied_hdr, test_applied_val, test_nomatch_hdr, test_nomatch_val]) + run_and_assert( + args, + 'case A', + [mentionhdr % (testdohdr), testdoval, testdonthdr, testdontval] + ) # all subsequent tests share this argument list args = common_args @@ -2620,15 +2628,18 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None): testcmdfile, '\n'.join( [ - 'gnu.org::%s: %s' % (test_applied_hdr, test_applied_val), - 'nomatch.com::%s: %s' % (test_nomatch_hdr, test_nomatch_val), + 'gnu.org::%s: %s' % (testdohdr, testdoval), + 'nomatch.com::%s: %s' % (testdonthdr, testdontval), '', ] ), ) # expect to find only the header key (not its value) and only for the appropriate url run_and_assert( - args, 'case B', [test_applied_hdr, testcmdfile], [test_applied_val, test_nomatch_hdr, test_nomatch_val] + args, + 'case B', + [mentionhdr % (testdohdr), mentionfile % (testcmdfile)], + [testdoval, testdonthdr, testdontval], ) # C: recursion one: header value is another file @@ -2636,32 +2647,42 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None): testcmdfile, '\n'.join( [ - 'gnu.org::%s: %s' % (test_applied_hdr, testincfile), - 'nomatch.com::%s: %s' % (test_nomatch_hdr, testexcfile), + 'gnu.org::%s: %s' % (testdohdr, testincfile), + 'nomatch.com::%s: %s' % (testdonthdr, testexcfile), '', ] ), ) - write_file(testincfile, '%s\n' % (test_applied_val)) - write_file(testexcfile, '%s\n' % (test_nomatch_val)) + write_file(testincfile, '%s\n' % (testdoval)) + write_file(testexcfile, '%s\n' % (testdontval)) # expect to find only the header key (not its value and not the filename) and only for the appropriate url run_and_assert( args, 'case C', - [test_applied_hdr, testcmdfile], - [test_applied_val, test_nomatch_hdr, test_nomatch_val, testincfile, testexcfile], + [ + mentionhdr % (testdohdr), + mentionfile % (testcmdfile), + mentionfile % (testincfile), + mentionfile % (testexcfile), + ], + [testdoval, testdonthdr, testdontval], ) # D: recursion two: header field+value is another file, write_file(testcmdfile, '\n'.join(['gnu.org::%s' % (testinchdrfile), 'nomatch.com::%s' % (testexchdrfile), ''])) - write_file(testinchdrfile, '%s: %s\n' % (test_applied_hdr, test_applied_val)) - write_file(testexchdrfile, '%s: %s\n' % (test_nomatch_hdr, test_nomatch_val)) + write_file(testinchdrfile, '%s: %s\n' % (testdohdr, testdoval)) + write_file(testexchdrfile, '%s: %s\n' % (testdonthdr, testdontval)) # expect to find only the header key (and the literal filename) and only for the appropriate url run_and_assert( args, 'case D', - [test_applied_hdr, testcmdfile, testinchdrfile, testexchdrfile], - [test_applied_val, test_nomatch_hdr, test_nomatch_val], + [ + mentionhdr % (testdohdr), + mentionfile % (testcmdfile), + mentionfile % (testinchdrfile), + mentionfile % (testexchdrfile), + ], + [testdoval, testdonthdr, testdontval], ) # E: recursion three: url pattern + header field + value in another file @@ -2670,8 +2691,8 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None): testurlpatfile, '\n'.join( [ - 'gnu.org::%s: %s' % (test_applied_hdr, test_applied_val), - 'nomatch.com::%s: %s' % (test_nomatch_hdr, test_nomatch_val), + 'gnu.org::%s: %s' % (testdohdr, testdoval), + 'nomatch.com::%s: %s' % (testdonthdr, testdontval), '', ] ), @@ -2680,8 +2701,8 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None): run_and_assert( args, 'case E', - [test_applied_hdr, testcmdfile, testurlpatfile], - [test_applied_val, test_nomatch_hdr, test_nomatch_val], + [mentionhdr % (testdohdr), mentionfile % (testcmdfile), mentionfile % (testurlpatfile)], + [testdoval, testdonthdr, testdontval], ) def test_test_report_env_filter(self): From b70fd0390e139aab39d7be0101d82836550eaca8 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 8 Feb 2021 12:10:25 +0100 Subject: [PATCH 064/864] Add comment explaining the reasoning Co-authored-by: Kenneth Hoste --- easybuild/framework/easyblock.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 44b1eeb16e..983478d255 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2405,6 +2405,8 @@ def post_install_step(self): lib_dir = os.path.join(self.installdir, 'lib') lib64_dir = os.path.join(self.installdir, 'lib64') if os.path.exists(lib_dir) and not os.path.exists(lib64_dir): + # create *relative* 'lib64' symlink to 'lib'; + # see https://github.com/easybuilders/easybuild-framework/issues/3564 symlink('lib', lib64_dir, use_abspath_source=False) def sanity_check_step(self, *args, **kwargs): From 91ce621f3606191cbe257c332a53cc8e46a3aeb5 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 4 Feb 2021 11:22:39 +0100 Subject: [PATCH 065/864] Add tests reproducing failure in handling UTF-8 files When the system encoding is not UTF-8 Python 3 will try to decode files as ASCII which fails This adds a test to CI with LC_ALL=C to enforce this situation and test cases reproducing failures in read_file, write_file and apply_regex_substitutions as seen in the wild --- .github/workflows/unit_tests.yml | 6 ++++++ test/framework/filetools.py | 34 +++++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 326810b96d..87da44909c 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -169,6 +169,12 @@ jobs: PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1) + # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set + # Hence run the tests again with LC_ALL=C + if [[ ${{matrix.python}} =~ "3." ]]; then + LC_ALL=C python -O -m test.framework.suite + fi + - name: test bootstrap script run: | # (re)initialize environment for modules tool diff --git a/test/framework/filetools.py b/test/framework/filetools.py index def99f7550..62716f17ad 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -603,11 +603,34 @@ def test_remove_symlinks(self): def test_read_write_file(self): """Test reading/writing files.""" + # Test different "encodings" + ascii_file = os.path.join(self.test_prefix, 'ascii.txt') + txt = 'Hello World\nFoo bar' + ft.write_file(ascii_file, txt) + self.assertEqual(ft.read_file(ascii_file), txt) + + binary_file = os.path.join(self.test_prefix, 'binary.txt') + txt = b'Hello World\x12\x00\x01\x02\x03\nFoo bar' + ft.write_file(binary_file, txt) + self.assertEqual(ft.read_file(binary_file, mode='rb'), txt) + + utf8_file = os.path.join(self.test_prefix, 'utf8.txt') + txt = b'Hyphen: \xe2\x80\x93\nEuro sign: \xe2\x82\xac\na with dots: \xc3\xa4' + if sys.version_info[0] == 3: + txt_decoded = txt.decode('utf-8') + else: + txt_decoded = txt + # Must work as binary and string + ft.write_file(utf8_file, txt) + self.assertEqual(ft.read_file(utf8_file), txt_decoded) + ft.write_file(utf8_file, txt_decoded) + self.assertEqual(ft.read_file(utf8_file), txt_decoded) + + # Test append fp = os.path.join(self.test_prefix, 'test.txt') txt = "test123" ft.write_file(fp, txt) self.assertEqual(ft.read_file(fp), txt) - txt2 = '\n'.join(['test', '123']) ft.write_file(fp, txt2, append=True) self.assertEqual(ft.read_file(fp), txt + txt2) @@ -1212,6 +1235,15 @@ def test_apply_regex_substitutions(self): path = os.path.join(self.test_prefix, 'nosuchfile.txt') self.assertErrorRegex(EasyBuildError, error_pat, ft.apply_regex_substitutions, path, regex_subs) + # make sure apply_regex_substitutions can patch files that include UTF-8 characters + testtxt = b"foo \xe2\x80\x93 bar" # This is an UTF-8 "-" + ft.write_file(testfile, testtxt) + ft.apply_regex_substitutions(testfile, [('foo', 'FOO')]) + txt = ft.read_file(testfile) + if sys.version_info[0] == 3: + testtxt = testtxt.decode('utf-8') + self.assertEqual(txt, testtxt.replace('foo', 'FOO')) + # make sure apply_regex_substitutions can patch files that include non-UTF-8 characters testtxt = b"foo \xe2 bar" ft.write_file(testfile, testtxt) From c276626e5e2a8ab5fd9317f1da2ef14c342cbcdf Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 4 Feb 2021 15:33:28 +0100 Subject: [PATCH 066/864] Fix failures on handling UTF-8 files in read_file, write_file and apply_regex_substitutions --- easybuild/tools/filetools.py | 44 ++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 51e5bc9d03..968a8ae1f3 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -40,7 +40,6 @@ """ import datetime import difflib -import fileinput import glob import hashlib import imp @@ -188,12 +187,18 @@ def is_readable(path): except OSError as err: raise EasyBuildError("Failed to check whether %s is readable: %s", path, err) +def _open(path, mode): + """Open a file. If mode is not binary, then utf-8 encoding will be selected for Python3""" + if sys.version_info[0] >= 3 and 'b' not in mode: + return open(path, mode, encoding='utf-8') + else: + return open(path, mode) def read_file(path, log_error=True, mode='r'): """Read contents of file at given path, in a robust way.""" txt = None try: - with open(path, mode) as handle: + with _open(path, mode) as handle: txt = handle.read() except IOError as err: if log_error: @@ -244,7 +249,7 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over # note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block try: mkdir(os.path.dirname(path), parents=True) - with open(path, mode) as handle: + with _open(path, mode) as handle: handle.write(data) except IOError as err: raise EasyBuildError("Failed to write to %s: %s", path, err) @@ -1360,19 +1365,13 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb'): for regex, subtxt in regex_subs: compiled_regex_subs.append((re.compile(regex), subtxt)) - if backup: - backup_ext = backup - else: - # no (persistent) backup file is created if empty string value is passed to 'backup' in fileinput.input - backup_ext = '' - for path in paths: try: # make sure that file can be opened in text mode; # it's possible this fails with UnicodeDecodeError when running EasyBuild with Python 3 try: - with open(path, 'r') as fp: - _ = fp.read() + with _open(path, 'r') as fp: + fp.read() except UnicodeDecodeError as err: _log.info("Encountered UnicodeDecodeError when opening %s in text mode: %s", path, err) path_backup = back_up_file(path) @@ -1381,14 +1380,21 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb'): txt_utf8 = txt.decode(encoding='utf-8', errors='replace') write_file(path, txt_utf8) - for line_id, line in enumerate(fileinput.input(path, inplace=1, backup=backup_ext)): - for regex, subtxt in compiled_regex_subs: - match = regex.search(line) - if match: - origtxt = match.group(0) - _log.info("Replacing line %d in %s: '%s' -> '%s'", (line_id + 1), path, origtxt, subtxt) - line = regex.sub(subtxt, line) - sys.stdout.write(line) + backup_path = path + (backup or '.bak') + copy_file(path, backup_path) + with _open(backup_path, 'r') as in_file: + with _open(path, 'w') as out_file: + for line_id, line in enumerate(in_file): + for regex, subtxt in compiled_regex_subs: + match = regex.search(line) + if match: + origtxt = match.group(0) + _log.info("Replacing line %d in %s: '%s' -> '%s'", + (line_id + 1), path, origtxt, subtxt) + line = regex.sub(subtxt, line) + out_file.write(line) + if not backup: + remove_file(backup_path) except (IOError, OSError) as err: raise EasyBuildError("Failed to patch %s: %s", path, err) From 64451f41368d4f20be2094847999ad4e284372e9 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 4 Feb 2021 16:39:30 +0100 Subject: [PATCH 067/864] Make sure all calls to open are correct and close is always called --- easybuild/base/optcomplete.py | 5 +- easybuild/scripts/fix_docs.py | 17 ++++--- easybuild/scripts/mk_tmpl_easyblock_for.py | 5 +- easybuild/tools/configobj.py | 15 +++--- easybuild/tools/environment.py | 12 ++--- easybuild/tools/filetools.py | 21 +++++---- easybuild/tools/jenkins.py | 10 ++-- easybuild/tools/modules.py | 5 +- test/framework/build_log.py | 2 +- test/framework/easyblock.py | 18 ++++---- test/framework/filetools.py | 4 +- test/framework/github.py | 2 +- test/framework/modules.py | 2 +- test/framework/modulestool.py | 10 ++-- test/framework/options.py | 46 ++++++++----------- test/framework/package.py | 36 +++++++-------- .../sandbox/easybuild/easyblocks/t/toy.py | 6 +-- test/framework/utilities.py | 5 +- 18 files changed, 91 insertions(+), 130 deletions(-) diff --git a/easybuild/base/optcomplete.py b/easybuild/base/optcomplete.py index f0c172d9f3..0d80e2953e 100644 --- a/easybuild/base/optcomplete.py +++ b/easybuild/base/optcomplete.py @@ -594,9 +594,8 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete if isinstance(debugfn, logging.Logger): debugfn.debug(txt) else: - f = open(debugfn, 'a') - f.write(txt) - f.close() + with open(debugfn, 'a') as f: + f.write(txt) # Exit with error code (we do not let the caller continue on purpose, this # is a run for completions only.) diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py index 8c40e34d4f..dcf46cfff4 100644 --- a/easybuild/scripts/fix_docs.py +++ b/easybuild/scripts/fix_docs.py @@ -57,13 +57,12 @@ continue with open(tmp) as f: temp = "tmp_file.py" - out = open(temp, 'w') - for line in f: - if "@author" in line: - out.write(re.sub(r"@author: (.*)", r":author: \1", line)) - elif "@param" in line: - out.write(re.sub(r"@param ([^:]*):", r":param \1:", line)) - else: - out.write(line) - out.close() + with open(temp, 'w') as out: + for line in f: + if "@author" in line: + out.write(re.sub(r"@author: (.*)", r":author: \1", line)) + elif "@param" in line: + out.write(re.sub(r"@param ([^:]*):", r":param \1:", line)) + else: + out.write(line) os.rename(temp, tmp) diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py index 34cdbf8b0d..ec4251153a 100644 --- a/easybuild/scripts/mk_tmpl_easyblock_for.py +++ b/easybuild/scripts/mk_tmpl_easyblock_for.py @@ -225,9 +225,8 @@ def make_module_extra(self): dirpath = os.path.dirname(easyblock_path) if not os.path.exists(dirpath): os.makedirs(dirpath) - f = open(easyblock_path, "w") - f.write(txt) - f.close() + with open(easyblock_path, "w") as f: + f.write(txt) except (IOError, OSError) as err: sys.stderr.write("ERROR! Writing template easyblock for %s to %s failed: %s" % (name, easyblock_path, err)) sys.exit(1) diff --git a/easybuild/tools/configobj.py b/easybuild/tools/configobj.py index ea418ce750..c42618739f 100644 --- a/easybuild/tools/configobj.py +++ b/easybuild/tools/configobj.py @@ -1213,9 +1213,8 @@ def _load(self, infile, configspec): if isinstance(infile, string_type): self.filename = infile if os.path.isfile(infile): - h = open(infile, 'r') - infile = h.read() or [] - h.close() + with open(infile, 'r') as h: + infile = h.read() or [] elif self.file_error: # raise an error if the file doesn't exist raise IOError('Config file not found: "%s".' % self.filename) @@ -1224,9 +1223,8 @@ def _load(self, infile, configspec): if self.create_empty: # this is a good test that the filename specified # isn't impossible - like on a non-existent device - h = open(infile, 'w') - h.write('') - h.close() + with open(infile, 'w') as h: + h.write('') infile = [] elif isinstance(infile, (list, tuple)): @@ -2052,9 +2050,8 @@ def write(self, outfile=None, section=None): if outfile is not None: outfile.write(output) else: - h = open(self.filename, 'wb') - h.write(output) - h.close() + with open(self.filename, 'wb') as h: + h.write(output) def validate(self, validator, preserve_errors=False, copy=False, section=None): diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py index 7e5eb636ef..f3329eb5bb 100644 --- a/easybuild/tools/environment.py +++ b/easybuild/tools/environment.py @@ -50,17 +50,11 @@ def write_changes(filename): """ Write current changes to filename and reset environment afterwards """ - script = None try: - script = open(filename, 'w') - - for key in _changes: - script.write('export %s=%s\n' % (key, shell_quote(_changes[key]))) - - script.close() + with open(filename, 'w') as script: + for key in _changes: + script.write('export %s=%s\n' % (key, shell_quote(_changes[key]))) except IOError as err: - if script is not None: - script.close() raise EasyBuildError("Failed to write to %s: %s", filename, err) reset_changes() diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 968a8ae1f3..ae0543697f 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -187,6 +187,7 @@ def is_readable(path): except OSError as err: raise EasyBuildError("Failed to check whether %s is readable: %s", path, err) + def _open(path, mode): """Open a file. If mode is not binary, then utf-8 encoding will be selected for Python3""" if sys.version_info[0] >= 3 and 'b' not in mode: @@ -194,6 +195,7 @@ def _open(path, mode): else: return open(path, mode) + def read_file(path, log_error=True, mode='r'): """Read contents of file at given path, in a robust way.""" txt = None @@ -1016,10 +1018,9 @@ def calc_block_checksum(path, algorithm): _log.debug("Using blocksize %s for calculating the checksum" % blocksize) try: - f = open(path, 'rb') - for block in iter(lambda: f.read(blocksize), b''): - algorithm.update(block) - f.close() + with open(path, 'rb') as f: + for block in iter(lambda: f.read(blocksize), b''): + algorithm.update(block) except IOError as err: raise EasyBuildError("Failed to read %s: %s", path, err) @@ -1569,16 +1570,16 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): :param sticky: set the sticky bit on this directory (a.k.a. the restricted deletion flag), to avoid users can removing/renaming files in this directory """ - if set_gid is None: - set_gid = build_option('set_gid_bit') - if sticky is None: - sticky = build_option('sticky_bit') - if not os.path.isabs(path): path = os.path.abspath(path) # exit early if path already exists if not os.path.exists(path): + if set_gid is None: + set_gid = build_option('set_gid_bit') + if sticky is None: + sticky = build_option('sticky_bit') + _log.info("Creating directory %s (parents: %s, set_gid: %s, sticky: %s)", path, parents, set_gid, sticky) # set_gid and sticky bits are only set on new directories, so we need to determine the existing parent path existing_parent_path = os.path.dirname(path) @@ -2048,7 +2049,7 @@ def find_flexlm_license(custom_env_vars=None, lic_specs=None): if lic_files: for lic_file in lic_files: try: - open(lic_file, 'r') + open(lic_file, 'r').close() valid_lic_specs.append(lic_file) except IOError as err: _log.warning("License file %s found, but failed to open it for reading: %s", lic_file, err) diff --git a/easybuild/tools/jenkins.py b/easybuild/tools/jenkins.py index 859e0030a5..40a6fe16e3 100644 --- a/easybuild/tools/jenkins.py +++ b/easybuild/tools/jenkins.py @@ -105,9 +105,8 @@ def create_success(name, stats): root.firstChild.appendChild(el) try: - output_file = open(filename, "w") - root.writexml(output_file) - output_file.close() + with open(filename, "w") as output_file: + root.writexml(output_file) except IOError as err: raise EasyBuildError("Failed to write out XML file %s: %s", filename, err) @@ -162,9 +161,8 @@ def aggregate_xml_in_dirs(base_dir, output_filename): comment = root.createComment("%s out of %s builds succeeded" % (succes, total)) root.firstChild.insertBefore(comment, properties) try: - output_file = open(output_filename, "w") - root.writexml(output_file, addindent="\t", newl="\n") - output_file.close() + with open(output_filename, "w") as output_file: + root.writexml(output_file, addindent="\t", newl="\n") except IOError as err: raise EasyBuildError("Failed to write out XML file %s: %s", output_filename, err) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index a2a7739222..3ceefc44ac 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1409,9 +1409,8 @@ def update(self): cache_dir = os.path.dirname(cache_fp) if not os.path.exists(cache_dir): mkdir(cache_dir, parents=True) - cache_file = open(cache_fp, 'w') - cache_file.write(stdout) - cache_file.close() + with open(cache_fp, 'w') as cache_file: + cache_file.write(stdout) except (IOError, OSError) as err: raise EasyBuildError("Failed to update Lmod spider cache %s: %s", cache_fp, err) diff --git a/test/framework/build_log.py b/test/framework/build_log.py index 32aefc7b74..19812d7373 100644 --- a/test/framework/build_log.py +++ b/test/framework/build_log.py @@ -70,7 +70,7 @@ def test_easybuilderror(self): logToFile(tmplog, enable=False) log_re = re.compile(r"^fancyroot ::.* BOOM \(at .*:[0-9]+ in [a-z_]+\)$", re.M) - logtxt = open(tmplog, 'r').read() + logtxt = read_file(tmplog, 'r') self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt)) # test formatting of message diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 399a269597..70b6ef7793 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -317,8 +317,8 @@ def test_make_module_req(self): # create fake directories and files that should be guessed os.makedirs(eb.installdir) - open(os.path.join(eb.installdir, 'foo.jar'), 'w').write('foo.jar') - open(os.path.join(eb.installdir, 'bla.jar'), 'w').write('bla.jar') + write_file(os.path.join(eb.installdir, 'foo.jar'), 'foo.jar') + write_file(os.path.join(eb.installdir, 'bla.jar'), 'bla.jar') for path in ('bin', ('bin', 'testdir'), 'sbin', 'share', ('share', 'man'), 'lib', 'lib64'): if isinstance(path, string_type): path = (path, ) @@ -352,7 +352,7 @@ def test_make_module_req(self): self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) # check that bin is only added to PATH if there are files in there - open(os.path.join(eb.installdir, 'bin', 'test'), 'w').write('test') + write_file(os.path.join(eb.installdir, 'bin', 'test'), 'test') guess = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertTrue(re.search(r"^prepend-path\s+PATH\s+\$root/bin$", guess, re.M)) @@ -371,14 +371,14 @@ def test_make_module_req(self): elif get_module_syntax() == 'Lua': self.assertFalse('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))' in guess) # -- With files - open(os.path.join(eb.installdir, 'lib64', 'libfoo.so'), 'w').write('test') + write_file(os.path.join(eb.installdir, 'lib64', 'libfoo.so'), 'test') guess = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertTrue(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib64$", guess, re.M)) elif get_module_syntax() == 'Lua': self.assertTrue('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))' in guess) # -- With files in lib and lib64 symlinks to lib - open(os.path.join(eb.installdir, 'lib', 'libfoo.so'), 'w').write('test') + write_file(os.path.join(eb.installdir, 'lib', 'libfoo.so'), 'test') shutil.rmtree(os.path.join(eb.installdir, 'lib64')) os.symlink('lib', os.path.join(eb.installdir, 'lib64')) guess = eb.make_module_req() @@ -425,7 +425,7 @@ def test_make_module_req(self): eb.make_module_req_guess = lambda: {'LD_LIBRARY_PATH': ['lib/pathC', 'lib/pathA', 'lib/pathB', 'lib/pathA']} for path in ['pathA', 'pathB', 'pathC']: os.mkdir(os.path.join(eb.installdir, 'lib', path)) - open(os.path.join(eb.installdir, 'lib', path, 'libfoo.so'), 'w').write('test') + write_file(os.path.join(eb.installdir, 'lib', path, 'libfoo.so'), 'test') txt = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertTrue(re.search(r"\nprepend-path\s+LD_LIBRARY_PATH\s+\$root/lib/pathC\n" + @@ -1388,7 +1388,7 @@ def test_obtain_file(self): loc = os.path.join(tmpdir, 't', 'toy', fn) self.assertEqual(res, loc) self.assertTrue(os.path.exists(loc), "%s file is found at %s" % (fn, loc)) - txt = open(loc, 'r').read() + txt = read_file(loc) eb_regex = re.compile("EasyBuild: building software with ease") self.assertTrue(eb_regex.search(txt), "Pattern '%s' found in: %s" % (eb_regex.pattern, txt)) else: @@ -1430,9 +1430,7 @@ def test_check_readiness(self): tmpdir = tempfile.mkdtemp() shutil.copy2(ec_path, tmpdir) ec_path = os.path.join(tmpdir, ec_file) - f = open(ec_path, 'a') - f.write("\ndependencies += [('nosuchsoftware', '1.2.3')]\n") - f.close() + write_file(ec_path, "\ndependencies += [('nosuchsoftware', '1.2.3')]\n", append=True) ec = EasyConfig(ec_path) eb = EasyBlock(ec) try: diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 62716f17ad..b195ae58a5 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -737,9 +737,7 @@ def test_det_patched_files(self): def test_guess_patch_level(self): "Test guess_patch_level.""" # create dummy toy.source file so guess_patch_level can work - f = open(os.path.join(self.test_buildpath, 'toy.source'), 'w') - f.write("This is toy.source") - f.close() + ft.write_file(os.path.join(self.test_buildpath, 'toy.source'), "This is toy.source") for patched_file, correct_patch_level in [ ('toy.source', 0), diff --git a/test/framework/github.py b/test/framework/github.py index fa3d7e5022..7281a46002 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -115,7 +115,7 @@ def test_read(self): try: fp = self.ghfs.read("a_directory/a_file.txt", api=False) - self.assertEqual(open(fp, 'r').read().strip(), "this is a line of text") + self.assertEqual(read_file(fp).strip(), "this is a line of text") os.remove(fp) except (IOError, OSError): pass diff --git a/test/framework/modules.py b/test/framework/modules.py index f9b6ef6238..e370b1a88f 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -678,7 +678,7 @@ def test_get_software_root_version_libdir(self): self.assertEqual(get_software_libdir(name, only_one=False), ['lib', 'lib64']) # only directories containing files in specified list should be retained - open(os.path.join(root, 'lib64', 'foo'), 'w').write('foo') + write_file(os.path.join(root, 'lib64', 'foo'), 'foo') self.assertEqual(get_software_libdir(name, fs=['foo']), 'lib64') # duplicate paths due to symlink get filtered diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index 9f9dc9ec48..4358134883 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -39,7 +39,7 @@ from easybuild.base import fancylogger from easybuild.tools import modules from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import which, write_file +from easybuild.tools.filetools import which, write_file, read_file from easybuild.tools.modules import Lmod from test.framework.utilities import init_config @@ -123,9 +123,7 @@ def test_module_mismatch(self): fancylogger.logToFile(self.logfile) mt = MockModulesTool(testing=True) - f = open(self.logfile, 'r') - logtxt = f.read() - f.close() + logtxt = read_file(self.logfile) warn_regex = re.compile("WARNING .*pattern .* not found in defined 'module' function") self.assertTrue(warn_regex.search(logtxt), "Found pattern '%s' in: %s" % (warn_regex.pattern, logtxt)) @@ -137,9 +135,7 @@ def test_module_mismatch(self): # a warning should be logged if the 'module' function is undefined del os.environ['module'] mt = MockModulesTool(testing=True) - f = open(self.logfile, 'r') - logtxt = f.read() - f.close() + logtxt = read_file(self.logfile) warn_regex = re.compile("WARNING No 'module' function defined, can't check if it matches .*") self.assertTrue(warn_regex.search(logtxt), "Pattern %s found in %s" % (warn_regex.pattern, logtxt)) diff --git a/test/framework/options.py b/test/framework/options.py index cbdd57cc71..eb99577c05 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1341,7 +1341,7 @@ def test_dry_run_short(self): robot_decoy = os.path.join(self.test_prefix, 'robot_decoy') mkdir(robot_decoy) for dry_run_arg in ['-D', '--dry-run-short']: - open(self.logfile, 'w').write('') + write_file(self.logfile, '') args = [ os.path.join(tmpdir, 'easybuild', 'easyconfigs', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb'), dry_run_arg, @@ -2160,9 +2160,7 @@ def test_try(self): ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') copy_file(os.path.join(ecs_path, 't', 'toy', 'toy-0.0.eb'), tweaked_toy_ec) - f = open(tweaked_toy_ec, 'a') - f.write("easyblock = 'ConfigureMake'") - f.close() + write_file(tweaked_toy_ec, "easyblock = 'ConfigureMake'", append=True) args = [ tweaked_toy_ec, @@ -2223,9 +2221,7 @@ def test_try_with_copy(self): ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') copy_file(os.path.join(ecs_path, 't', 'toy', 'toy-0.0.eb'), tweaked_toy_ec) - f = open(tweaked_toy_ec, 'a') - f.write("easyblock = 'ConfigureMake'") - f.close() + write_file(tweaked_toy_ec, "easyblock = 'ConfigureMake'", append=True) args = [ tweaked_toy_ec, @@ -2292,9 +2288,7 @@ def test_recursive_try(self): ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') copy_file(os.path.join(ecs_path, 't', 'toy', 'toy-0.0.eb'), tweaked_toy_ec) - f = open(tweaked_toy_ec, 'a') - f.write("dependencies = [('gzip', '1.4')]\n") # add fictious dependency - f.close() + write_file(tweaked_toy_ec, "dependencies = [('gzip', '1.4')]\n", append=True) # add fictious dependency sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'sources') args = [ @@ -2348,9 +2342,7 @@ def test_recursive_try(self): self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) # clear fictitious dependency - f = open(tweaked_toy_ec, 'a') - f.write("dependencies = []\n") - f.close() + write_file(tweaked_toy_ec, "dependencies = []\n", append=True) # no recursive try if --disable-map-toolchains is involved for extra_args in [['--try-software-version=1.2.3'], ['--software-version=1.2.3']]: @@ -2410,7 +2402,7 @@ def test_filter_deps(self): self.assertFalse(re.search('module: zlib', outtxt)) # clear log file - open(self.logfile, 'w').write('') + write_file(self.logfile, '') # filter deps (including a non-existing dep, i.e. zlib) args.extend(['--filter-deps', 'FFTW,ScaLAPACK,zlib']) @@ -2419,7 +2411,7 @@ def test_filter_deps(self): self.assertFalse(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt)) self.assertFalse(re.search('module: zlib', outtxt)) - open(self.logfile, 'w').write('') + write_file(self.logfile, '') # filter specific version of deps args[-1] = 'FFTW=3.2.3,zlib,ScaLAPACK=2.0.2' @@ -2428,7 +2420,7 @@ def test_filter_deps(self): self.assertFalse(re.search('module: ScaLAPACK', outtxt)) self.assertFalse(re.search('module: zlib', outtxt)) - open(self.logfile, 'w').write('') + write_file(self.logfile, '') args[-1] = 'zlib,FFTW=3.3.7,ScaLAPACK=2.0.1' outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) @@ -2436,7 +2428,7 @@ def test_filter_deps(self): self.assertTrue(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt)) self.assertFalse(re.search('module: zlib', outtxt)) - open(self.logfile, 'w').write('') + write_file(self.logfile, '') # filter deps with version range: only filter FFTW 3.x, ScaLAPACK 1.x args[-1] = 'zlib,ScaLAPACK=]1.0:2.0[,FFTW=[3.0:4.0[' @@ -2445,7 +2437,7 @@ def test_filter_deps(self): self.assertTrue(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt)) self.assertFalse(re.search('module: zlib', outtxt)) - open(self.logfile, 'w').write('') + write_file(self.logfile, '') # also test open ended ranges args[-1] = 'zlib,ScaLAPACK=[1.0:,FFTW=:4.0[' @@ -2454,7 +2446,7 @@ def test_filter_deps(self): self.assertFalse(re.search('module: ScaLAPACK', outtxt)) self.assertFalse(re.search('module: zlib', outtxt)) - open(self.logfile, 'w').write('') + write_file(self.logfile, '') args[-1] = 'zlib,ScaLAPACK=[2.1:,FFTW=:3.0[' outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) @@ -2469,7 +2461,7 @@ def test_filter_deps(self): self.assertFalse(re.search('module: ScaLAPACK', outtxt)) self.assertFalse(re.search('module: zlib', outtxt)) - open(self.logfile, 'w').write('') + write_file(self.logfile, '') # FFTW & ScaLAPACK versions are not included in range, so no filtering args[-1] = 'FFTW=]3.3.7:4.0],zlib,ScaLAPACK=[1.0:2.0.2[' @@ -2478,7 +2470,7 @@ def test_filter_deps(self): self.assertTrue(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt)) self.assertFalse(re.search('module: zlib', outtxt)) - open(self.logfile, 'w').write('') + write_file(self.logfile, '') # also test mix of ranges & specific versions args[-1] = 'FFTW=3.3.7,zlib,ScaLAPACK=[1.0:2.0.2[' @@ -2487,7 +2479,7 @@ def test_filter_deps(self): self.assertTrue(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt)) self.assertFalse(re.search('module: zlib', outtxt)) - open(self.logfile, 'w').write('') + write_file(self.logfile, '') args[-1] = 'FFTW=]3.3.7:4.0],zlib,ScaLAPACK=2.0.2' outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) self.assertTrue(re.search('module: FFTW/3.3.7-gompi', outtxt)) @@ -2496,7 +2488,7 @@ def test_filter_deps(self): # This easyconfig contains a dependency of CMake for which no easyconfig exists. It should still # succeed when called with --filter-deps=CMake=:2.8.10] - open(self.logfile, 'w').write('') + write_file(self.logfile, '') ec_file = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a-broken.eb') args[0] = ec_file args[-1] = 'FFTW=3.3.7,CMake=:2.8.10],zlib' @@ -2506,7 +2498,7 @@ def test_filter_deps(self): self.assertTrue(re.search(regexp, outtxt)) # The test below fails without PR 2983 - open(self.logfile, 'w').write('') + write_file(self.logfile, '') ec_file = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a-broken.eb') args[0] = ec_file args[-1] = 'FFTW=3.3.7,CMake=:2.8.10],zlib' @@ -2535,7 +2527,7 @@ def test_hide_deps(self): self.assertFalse(re.search('module: zlib', outtxt)) # clear log file - open(self.logfile, 'w').write('') + write_file(self.logfile, '') # hide deps (including a non-existing dep, i.e. zlib) args.append('--hide-deps=FFTW,ScaLAPACK,zlib') @@ -2583,9 +2575,7 @@ def toy(extra_args=None): software_path = os.path.join(self.test_installpath, 'software', 'toy', '0.0') test_report_path_pattern = os.path.join(software_path, 'easybuild', 'easybuild-toy-0.0*test_report.md') - f = open(glob.glob(test_report_path_pattern)[0], 'r') - test_report_txt = f.read() - f.close() + test_report_txt = read_file(glob.glob(test_report_path_pattern)[0]) return test_report_txt # define environment variables that should (not) show up in the test report diff --git a/test/framework/package.py b/test/framework/package.py index e2a97ad0de..5451fe64a8 100644 --- a/test/framework/package.py +++ b/test/framework/package.py @@ -49,9 +49,8 @@ import os, sys def verbose(msg): - fp = open('%(fpm_output_file)s', 'a') - fp.write(msg + '\\n') - fp.close() + with open('%(fpm_output_file)s', 'a') as fp: + fp.write(msg + '\\n') description, iteration, name, source, target, url, version, workdir = '', '', '', '', '', '', '', '' excludes = [] @@ -112,29 +111,26 @@ def verbose(msg): pkgfile = os.path.join(workdir, name + '-' + version + '.' + iteration + '.' + target) -fp = open(pkgfile, 'w') +with open(pkgfile, 'w') as fp: + fp.write('thisisan' + target + '\\n') + fp.write(' '.join(sys.argv[1:]) + '\\n') + fp.write("STARTCONTENTS of installdir " + installdir + ':\\n') -fp.write('thisisan' + target + '\\n') -fp.write(' '.join(sys.argv[1:]) + '\\n') -fp.write("STARTCONTENTS of installdir " + installdir + ':\\n') + find_cmd = 'find ' + installdir + ' ' + ''.join([" -not -path /" + x + ' ' for x in excludes]) + verbose("trying: " + find_cmd) + fp.write(find_cmd + '\\n') -find_cmd = 'find ' + installdir + ' ' + ''.join([" -not -path /" + x + ' ' for x in excludes]) -verbose("trying: " + find_cmd) -fp.write(find_cmd + '\\n') + fp.write('ENDCONTENTS\\n') -fp.write('ENDCONTENTS\\n') + fp.write("Contents of module file " + modulefile + ':') -fp.write("Contents of module file " + modulefile + ':') + fp.write('modulefile: ' + modulefile + '\\n') + #modtxt = open(modulefile).read() + #fp.write(modtxt + '\\n') -fp.write('modulefile: ' + modulefile + '\\n') -#modtxt = open(modulefile).read() -#fp.write(modtxt + '\\n') - -fp.write("I found excludes " + ' '.join(excludes) + '\\n') -fp.write("DESCRIPTION: " + description + '\\n') - -fp.close() + fp.write("I found excludes " + ' '.join(excludes) + '\\n') + fp.write("DESCRIPTION: " + description + '\\n') """ diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index 5b57a6f32d..f9a9f7a8c5 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -36,7 +36,7 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError from easybuild.tools.environment import setvar -from easybuild.tools.filetools import mkdir +from easybuild.tools.filetools import mkdir, write_file from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.run import run_cmd @@ -116,9 +116,7 @@ def install_step(self, name=None): # also install a dummy libtoy.a, to make the default sanity check happy libdir = os.path.join(self.installdir, 'lib') mkdir(libdir, parents=True) - f = open(os.path.join(libdir, 'lib%s.a' % name), 'w') - f.write(name.upper()) - f.close() + write_file(os.path.join(libdir, 'lib%s.a' % name), name.upper()) def run(self): """Install toy as extension.""" diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 9d741c7477..8d47366e5b 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -264,9 +264,8 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos logfile = self.logfile # clear log file if logfile: - f = open(logfile, 'w') - f.write('') - f.close() + with open(logfile, 'w') as f: + f.write('') env_before = copy.deepcopy(os.environ) From 37671fb9dc553b62ae41640c594da79147c6ecef Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 4 Feb 2021 16:45:01 +0100 Subject: [PATCH 068/864] Make sure the logging handler uses UTF-8 too --- easybuild/base/fancylogger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/base/fancylogger.py b/easybuild/base/fancylogger.py index e95fb0e89f..94c55e250e 100644 --- a/easybuild/base/fancylogger.py +++ b/easybuild/base/fancylogger.py @@ -580,6 +580,8 @@ def logToFile(filename, enable=True, filehandler=None, name=None, max_bytes=MAX_ 'maxBytes': max_bytes, 'backupCount': backup_count, } + if sys.version_info[0] >= 3: + handleropts['encoding'] = 'utf-8' # logging to a file is going to create the file later on, so let's try to be helpful and create the path if needed directory = os.path.dirname(filename) if not os.path.exists(directory): From 0bc7b80a5c0f17ea196fda2c930bf62df64a3128 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 8 Feb 2021 12:25:43 +0100 Subject: [PATCH 069/864] Add only 1 extra run for LC_ALL=C tests --- .github/workflows/unit_tests.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 87da44909c..f15da3dc53 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -9,6 +9,7 @@ jobs: python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] modules_tool: [Lmod-7.8.22, Lmod-8.2.9, modules-tcl-1.147, modules-3.2.10, modules-4.1.4] module_syntax: [Lua, Tcl] + lc_all: [""] # exclude some configuration for non-Lmod modules tool: # - don't test with Lua module syntax (only supported in Lmod) # - exclude Python 3.x versions other than 3.6, to limit test configurations @@ -51,6 +52,13 @@ jobs: python: 3.8 - modules_tool: Lmod-7.8.22 python: 3.9 + # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set + # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7) + include: + - python: 3.6 + modules_tool: Lmod-8.2.9 + module_syntax: Lua + lc_all: C fail-fast: false steps: - uses: actions/checkout@v2 @@ -129,6 +137,7 @@ jobs: EB_VERBOSE: 1 EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}} TEST_EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}} + LC_ALL: ${{matrix.lc_all}} run: | # run tests *outside* of checked out easybuild-framework directory, # to ensure we're testing installed version (see previous step) @@ -169,12 +178,6 @@ jobs: PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1) - # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set - # Hence run the tests again with LC_ALL=C - if [[ ${{matrix.python}} =~ "3." ]]; then - LC_ALL=C python -O -m test.framework.suite - fi - - name: test bootstrap script run: | # (re)initialize environment for modules tool From 039e9ce9d84addfedb809d140562e9ec5f36167e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 8 Feb 2021 13:38:37 +0100 Subject: [PATCH 070/864] Replace single letter variables and improve comments --- easybuild/base/optcomplete.py | 4 ++-- easybuild/scripts/fix_docs.py | 4 ++-- easybuild/scripts/mk_tmpl_easyblock_for.py | 4 ++-- easybuild/tools/configobj.py | 12 ++++++------ easybuild/tools/filetools.py | 7 ++++--- test/framework/utilities.py | 4 ++-- 6 files changed, 18 insertions(+), 17 deletions(-) diff --git a/easybuild/base/optcomplete.py b/easybuild/base/optcomplete.py index 0d80e2953e..263e4405cb 100644 --- a/easybuild/base/optcomplete.py +++ b/easybuild/base/optcomplete.py @@ -594,8 +594,8 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete if isinstance(debugfn, logging.Logger): debugfn.debug(txt) else: - with open(debugfn, 'a') as f: - f.write(txt) + with open(debugfn, 'a') as fh: + fh.write(txt) # Exit with error code (we do not let the caller continue on purpose, this # is a run for completions only.) diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py index dcf46cfff4..6bfa02c24a 100644 --- a/easybuild/scripts/fix_docs.py +++ b/easybuild/scripts/fix_docs.py @@ -55,10 +55,10 @@ # exclude self if os.path.basename(tmp) == os.path.basename(__file__): continue - with open(tmp) as f: + with open(tmp) as fh: temp = "tmp_file.py" with open(temp, 'w') as out: - for line in f: + for line in fh: if "@author" in line: out.write(re.sub(r"@author: (.*)", r":author: \1", line)) elif "@param" in line: diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py index ec4251153a..5cf2b36374 100644 --- a/easybuild/scripts/mk_tmpl_easyblock_for.py +++ b/easybuild/scripts/mk_tmpl_easyblock_for.py @@ -225,8 +225,8 @@ def make_module_extra(self): dirpath = os.path.dirname(easyblock_path) if not os.path.exists(dirpath): os.makedirs(dirpath) - with open(easyblock_path, "w") as f: - f.write(txt) + with open(easyblock_path, "w") as fh: + fh.write(txt) except (IOError, OSError) as err: sys.stderr.write("ERROR! Writing template easyblock for %s to %s failed: %s" % (name, easyblock_path, err)) sys.exit(1) diff --git a/easybuild/tools/configobj.py b/easybuild/tools/configobj.py index c42618739f..7931f2ca05 100644 --- a/easybuild/tools/configobj.py +++ b/easybuild/tools/configobj.py @@ -1213,8 +1213,8 @@ def _load(self, infile, configspec): if isinstance(infile, string_type): self.filename = infile if os.path.isfile(infile): - with open(infile, 'r') as h: - infile = h.read() or [] + with open(infile, 'r') as fh: + infile = fh.read() or [] elif self.file_error: # raise an error if the file doesn't exist raise IOError('Config file not found: "%s".' % self.filename) @@ -1223,8 +1223,8 @@ def _load(self, infile, configspec): if self.create_empty: # this is a good test that the filename specified # isn't impossible - like on a non-existent device - with open(infile, 'w') as h: - h.write('') + with open(infile, 'w') as fh: + fh.write('') infile = [] elif isinstance(infile, (list, tuple)): @@ -2050,8 +2050,8 @@ def write(self, outfile=None, section=None): if outfile is not None: outfile.write(output) else: - with open(self.filename, 'wb') as h: - h.write(output) + with open(self.filename, 'wb') as fh: + fh.write(output) def validate(self, validator, preserve_errors=False, copy=False, section=None): diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index ae0543697f..b7560c360a 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -189,7 +189,7 @@ def is_readable(path): def _open(path, mode): - """Open a file. If mode is not binary, then utf-8 encoding will be selected for Python3""" + """Open a file. If mode is not binary, then utf-8 encoding will be used for Python 3.x""" if sys.version_info[0] >= 3 and 'b' not in mode: return open(path, mode, encoding='utf-8') else: @@ -1018,8 +1018,8 @@ def calc_block_checksum(path, algorithm): _log.debug("Using blocksize %s for calculating the checksum" % blocksize) try: - with open(path, 'rb') as f: - for block in iter(lambda: f.read(blocksize), b''): + with open(path, 'rb') as fh: + for block in iter(lambda: fh.read(blocksize), b''): algorithm.update(block) except IOError as err: raise EasyBuildError("Failed to read %s: %s", path, err) @@ -2049,6 +2049,7 @@ def find_flexlm_license(custom_env_vars=None, lic_specs=None): if lic_files: for lic_file in lic_files: try: + # just try to open file for reading, no need to actually read it open(lic_file, 'r').close() valid_lic_specs.append(lic_file) except IOError as err: diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 8d47366e5b..57fafc3dbc 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -264,8 +264,8 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos logfile = self.logfile # clear log file if logfile: - with open(logfile, 'w') as f: - f.write('') + with open(logfile, 'w') as fh: + fh.write('') env_before = copy.deepcopy(os.environ) From 284167d8982a183b794e03de09032cf835807a21 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 8 Feb 2021 13:51:42 +0100 Subject: [PATCH 071/864] Avoid reading the file more often than required --- easybuild/tools/filetools.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index b7560c360a..c678800754 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1372,30 +1372,31 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb'): # it's possible this fails with UnicodeDecodeError when running EasyBuild with Python 3 try: with _open(path, 'r') as fp: - fp.read() + txt_utf8 = fp.read() except UnicodeDecodeError as err: _log.info("Encountered UnicodeDecodeError when opening %s in text mode: %s", path, err) path_backup = back_up_file(path) _log.info("Editing %s to strip out non-UTF-8 characters (backup at %s)", path, path_backup) txt = read_file(path, mode='rb') txt_utf8 = txt.decode(encoding='utf-8', errors='replace') + del txt write_file(path, txt_utf8) - backup_path = path + (backup or '.bak') - copy_file(path, backup_path) - with _open(backup_path, 'r') as in_file: - with _open(path, 'w') as out_file: - for line_id, line in enumerate(in_file): - for regex, subtxt in compiled_regex_subs: - match = regex.search(line) - if match: - origtxt = match.group(0) - _log.info("Replacing line %d in %s: '%s' -> '%s'", - (line_id + 1), path, origtxt, subtxt) + if backup: + copy_file(path, path + backup) + with _open(path, 'w') as out_file: + lines = txt_utf8.split('\n') + del txt_utf8 + for line_id, line in enumerate(lines): + for regex, subtxt in compiled_regex_subs: + match = regex.search(line) + if match: + origtxt = match.group(0) + _log.info("Replacing line %d in %s: '%s' -> '%s'", + (line_id + 1), path, origtxt, subtxt) line = regex.sub(subtxt, line) - out_file.write(line) - if not backup: - remove_file(backup_path) + lines[line_id] = line + out_file.write('\n'.join(lines)) except (IOError, OSError) as err: raise EasyBuildError("Failed to patch %s: %s", path, err) From 4039579b8f1eabd3f7fb967fb856221f22ff67c2 Mon Sep 17 00:00:00 2001 From: Bennet Fauber Date: Mon, 8 Feb 2021 15:25:27 -0500 Subject: [PATCH 072/864] Making symlinking of posix_c.so to posix.so conditional The missing lua posix.so symlink has been fixed in Ubuntu 2.04.1 so the creating of it breaks testing. I added a conditional that will only create it if the posix.so file does not exist as either a file or a symlink. --- .github/workflows/unit_tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 326810b96d..a9a568d0b9 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -69,7 +69,10 @@ jobs: # for modules tool sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 - sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so + # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists + if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then + sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so + fi # for GitPython, python-hglib sudo apt-get install git mercurial # dep for GC3Pie From a3d980945cc22e6294f6a2f271c4e751ef68f7a7 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Tue, 9 Feb 2021 09:37:36 +0000 Subject: [PATCH 073/864] Don't reuse variable name in the loop --- easybuild/tools/toolchain/compiler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index b8337b9b57..c0486ef03c 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -296,10 +296,10 @@ def _set_compiler_flags(self): self.variables.nextend(var, fflags) extra = 'extra_' + var.lower() if self.options.get(extra): - flags = self.options.option(extra) - if not flags or flags[0] != '-': - raise EasyBuildError("toolchainopts %s: '%s' must start with a '-'." % (extra, flags)) - self.variables.nappend_el(var, flags[1:]) + extraflags = self.options.option(extra) + if not extraflags or extraflags[0] != '-': + raise EasyBuildError("toolchainopts %s: '%s' must start with a '-'." % (extra, extraflags)) + self.variables.nappend_el(var, extraflags[1:]) def _set_optimal_architecture(self, default_optarch=None): """ From 2a06b3c46d620b1e46ebc4fba4ef7dedfa6aca8a Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Tue, 9 Feb 2021 10:21:21 +0000 Subject: [PATCH 074/864] Add a test --- test/framework/toolchain.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index f1a02ec9ca..24affe177b 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -631,6 +631,19 @@ def test_misc_flags_shared(self): self.assertTrue(tc.get_variable(var).endswith(' ' + value)) self.modtool.purge() + value = '--only-in-cxxflags' + flag_vars.remove('CXXFLAGS') + tc = self.get_toolchain('foss', version='2018a') + tc.set_options({'extra_cxxflags': value}) + tc.prepare() + self.assertTrue(tc.get_variable('CXXFLAGS').endswith(' ' + value)) + for var in flag_vars: + self.assertTrue(value not in tc.get_variable(var)) + # https://github.com/easybuilders/easybuild-framework/pull/3571 + # catch variable resued inside loop + self.assertTrue("-o -n -l -y" not in tc.get_variable(var)) + self.modtool.purge() + def test_misc_flags_unique(self): """Test whether unique compiler flags are set correctly.""" From e30995ac6dd761577eabf7355b6a29396c19911b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 8 Feb 2021 18:09:48 +0100 Subject: [PATCH 075/864] Create CUDA cache (for JIT compiled PTX code) in build dir instead of $HOME Add option to control the cache size and to disable it --- easybuild/framework/easyblock.py | 16 +++++++++ easybuild/tools/config.py | 1 + easybuild/tools/options.py | 3 ++ test/framework/easyblock.py | 51 ++++++++++++++++++++++++++++ test/framework/modules.py | 2 +- test/framework/modules/gcccuda/2018a | 26 ++++++++++++++ 6 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 test/framework/modules/gcccuda/2018a diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 776fedc96d..f3645ee78b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2150,6 +2150,22 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True): self.log.info("Loading extra modules: %s", extra_modules) self.modules_tool.load(extra_modules) + # Setup CUDA cache if required. If we don't do this, CUDA will use the $HOME for its cache files + if get_software_root('CUDA') or get_software_root('CUDAcore'): + cuda_cache_maxsize = build_option('cuda_cache_maxsize') + if cuda_cache_maxsize is None: + cuda_cache_maxsize = 1 * 1024 * 1024 * 1024 # 1 GB default value + if cuda_cache_maxsize == 0: + self.log.info('Disabling CUDA PTX cache as per request') + env.setvar('CUDA_CACHE_DISABLE', '1') + else: + cuda_cache_dir = os.path.join(self.builddir, 'eb-cuda-cache') + self.log.info('Enabling CUDA PTX cache of size %s MB at %s', + cuda_cache_maxsize / 1024 / 1024, cuda_cache_dir) + env.setvar('CUDA_CACHE_DISABLE', '0') + env.setvar('CUDA_CACHE_PATH', cuda_cache_dir) + env.setvar('CUDA_CACHE_MAXSIZE', str(cuda_cache_maxsize)) + # guess directory to start configure/build/install process in, and move there if start_dir: self.guess_start_dir() diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index b97186f3c5..1d686871e7 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -169,6 +169,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'container_image_name', 'container_template_recipe', 'container_tmpdir', + 'cuda_cache_maxsize', 'cuda_compute_capabilities', 'download_timeout', 'dump_test_report', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index a363cde325..8c7387ebf6 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -357,6 +357,9 @@ def override_options(self): 'consider-archived-easyconfigs': ("Also consider archived easyconfigs", None, 'store_true', False), 'containerize': ("Generate container recipe/image", None, 'store_true', False, 'C'), 'copy-ec': ("Copy specified easyconfig(s) to specified location", None, 'store_true', False), + 'cuda-cache-maxsize': ("Maximum size of the CUDA cache (in bytes) used for JIT compilation of PTX code. " + "Leave value empty to let EasyBuild choose a value or '0' to disable the cache", + int, 'store_or_None', None), 'cuda-compute-capabilities': ("List of CUDA compute capabilities to use when building GPU software; " "values should be specified as digits separated by a dot, " "for example: 3.5,5.0,7.2", 'strlist', 'extend', None), diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 399a269597..8b8921fcac 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1761,6 +1761,57 @@ def test_prepare_step_hmns(self): self.assertEqual(len(loaded_modules), 1) self.assertEqual(loaded_modules[0]['mod_name'], 'GCC/6.4.0-2.28') + def test_prepare_step_cuda_cache(self): + """Test handling cuda-cache-maxsize option.""" + + init_config(build_options={'cuda_cache_maxsize': None}) # Automatic mode + + test_ecs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + ec = process_easyconfig(toy_ec)[0] + eb = EasyBlock(ec['ec']) + eb.silent = True + eb.make_builddir() + + eb.prepare_step(start_dir=False) + logtxt = read_file(eb.logfile) + self.assertNotIn('Disabling CUDA PTX cache', logtxt) + self.assertNotIn('Enabling CUDA PTX cache', logtxt) + + # Now with CUDA + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ectxt = re.sub('^toolchain = .*', "toolchain = {'name': 'gcccuda', 'version': '2018a'}", + read_file(toy_ec), flags=re.M) + write_file(test_ec, test_ectxt) + ec = process_easyconfig(test_ec)[0] + eb = EasyBlock(ec['ec']) + eb.silent = True + eb.make_builddir() + + write_file(eb.logfile, '') + eb.prepare_step(start_dir=False) + logtxt = read_file(eb.logfile) + self.assertNotIn('Disabling CUDA PTX cache', logtxt) + self.assertIn('Enabling CUDA PTX cache', logtxt) + self.assertEqual(os.environ['CUDA_CACHE_DISABLE'], '0') + + init_config(build_options={'cuda_cache_maxsize': 0}) # Disable + write_file(eb.logfile, '') + eb.prepare_step(start_dir=False) + logtxt = read_file(eb.logfile) + self.assertIn('Disabling CUDA PTX cache', logtxt) + self.assertNotIn('Enabling CUDA PTX cache', logtxt) + self.assertEqual(os.environ['CUDA_CACHE_DISABLE'], '1') + + init_config(build_options={'cuda_cache_maxsize': 1234567890}) # Specified size + write_file(eb.logfile, '') + eb.prepare_step(start_dir=False) + logtxt = read_file(eb.logfile) + self.assertNotIn('Disabling CUDA PTX cache', logtxt) + self.assertIn('Enabling CUDA PTX cache', logtxt) + self.assertEqual(os.environ['CUDA_CACHE_DISABLE'], '0') + self.assertEqual(os.environ['CUDA_CACHE_MAXSIZE'], '1234567890') + def test_checksum_step(self): """Test checksum step""" testdir = os.path.abspath(os.path.dirname(__file__)) diff --git a/test/framework/modules.py b/test/framework/modules.py index f9b6ef6238..b1bea458ae 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -54,7 +54,7 @@ # number of modules included for testing purposes -TEST_MODULES_COUNT = 81 +TEST_MODULES_COUNT = 82 class ModulesTest(EnhancedTestCase): diff --git a/test/framework/modules/gcccuda/2018a b/test/framework/modules/gcccuda/2018a new file mode 100644 index 0000000000..f9779f1be5 --- /dev/null +++ b/test/framework/modules/gcccuda/2018a @@ -0,0 +1,26 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { GCC based compiler toolchain with CUDA support, and including + OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none) +} +} + +module-whatis {GNU Compiler Collection (GCC) based compiler toolchain, along with CUDA toolkit. - Homepage: (none)} + +set root /prefix/software/gcccuda/2018a + +conflict gcccuda + +if { ![is-loaded GCC/6.4.0-2.28] } { + module load GCC/6.4.0-2.28 +} + +if { ![is-loaded CUDA/9.1.85] } { + module load CUDA/9.1.85 +} + + +setenv EBROOTGCCCUDA "$root" +setenv EBVERSIONGCCCUDA "2018a" +setenv EBDEVELGCCCUDA "$root/easybuild/gcccuda-2018a-easybuild-devel" From 26b0ef014dec7f01c0dc8433065ca4b080d6dcb3 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 9 Feb 2021 12:18:29 +0100 Subject: [PATCH 076/864] Fix test_prefix_option not being run --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index cbdd57cc71..46fa9f70b2 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4430,7 +4430,7 @@ def test_modules_tool_vs_syntax_check(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) - def test_prefix(self): + def test_prefix_option(self): """Test which configuration settings are affected by --prefix.""" txt, _ = self._run_mock_eb(['--show-full-config', '--prefix=%s' % self.test_prefix], raise_error=True) From b0d56bb87cacc2989ee7a0551682db11d592a0d0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 9 Feb 2021 14:23:37 +0100 Subject: [PATCH 077/864] Rename _open to open_file --- easybuild/tools/filetools.py | 20 +++++++++++--------- test/framework/modulestool.py | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index c678800754..2ceaa2b3a1 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -188,8 +188,10 @@ def is_readable(path): raise EasyBuildError("Failed to check whether %s is readable: %s", path, err) -def _open(path, mode): - """Open a file. If mode is not binary, then utf-8 encoding will be used for Python 3.x""" +def open_file(path, mode): + """Open a (usually) text file. If mode is not binary, then utf-8 encoding will be used for Python 3.x""" + # This is required for text files in Python 3, especially until Python 3.7 which implements PEP 540. + # This PEP opens files in UTF-8 mode if the C locale is used, see https://www.python.org/dev/peps/pep-0540 if sys.version_info[0] >= 3 and 'b' not in mode: return open(path, mode, encoding='utf-8') else: @@ -200,7 +202,7 @@ def read_file(path, log_error=True, mode='r'): """Read contents of file at given path, in a robust way.""" txt = None try: - with _open(path, mode) as handle: + with open_file(path, mode) as handle: txt = handle.read() except IOError as err: if log_error: @@ -251,8 +253,8 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over # note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block try: mkdir(os.path.dirname(path), parents=True) - with _open(path, mode) as handle: - handle.write(data) + with open_file(path, mode) as fh: + fh.write(data) except IOError as err: raise EasyBuildError("Failed to write to %s: %s", path, err) @@ -1371,7 +1373,7 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb'): # make sure that file can be opened in text mode; # it's possible this fails with UnicodeDecodeError when running EasyBuild with Python 3 try: - with _open(path, 'r') as fp: + with open_file(path, 'r') as fp: txt_utf8 = fp.read() except UnicodeDecodeError as err: _log.info("Encountered UnicodeDecodeError when opening %s in text mode: %s", path, err) @@ -1384,7 +1386,7 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb'): if backup: copy_file(path, path + backup) - with _open(path, 'w') as out_file: + with open_file(path, 'w') as out_file: lines = txt_utf8.split('\n') del txt_utf8 for line_id, line in enumerate(lines): @@ -2051,7 +2053,7 @@ def find_flexlm_license(custom_env_vars=None, lic_specs=None): for lic_file in lic_files: try: # just try to open file for reading, no need to actually read it - open(lic_file, 'r').close() + open(lic_file, 'rb').close() valid_lic_specs.append(lic_file) except IOError as err: _log.warning("License file %s found, but failed to open it for reading: %s", lic_file, err) @@ -2385,7 +2387,7 @@ def install_fake_vsc(): fake_vsc_init_path = os.path.join(fake_vsc_path, 'vsc', '__init__.py') if not os.path.exists(os.path.dirname(fake_vsc_init_path)): os.makedirs(os.path.dirname(fake_vsc_init_path)) - with open(fake_vsc_init_path, 'w') as fp: + with open_file(fake_vsc_init_path, 'w') as fp: fp.write(fake_vsc_init) sys.path.insert(0, fake_vsc_path) diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index 4358134883..ec7a05830f 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -39,7 +39,7 @@ from easybuild.base import fancylogger from easybuild.tools import modules from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import which, write_file, read_file +from easybuild.tools.filetools import read_file, which, write_file from easybuild.tools.modules import Lmod from test.framework.utilities import init_config From 1d7ade4fcf2cd3b1c932a3b5f39a74898a5bff7a Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Wed, 10 Feb 2021 17:21:28 +0100 Subject: [PATCH 078/864] parse_http_header_fields_urlpat fixup: use sentinel value --- easybuild/tools/filetools.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 2363878ae1..390498dcc7 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -569,12 +569,15 @@ def derive_alt_pypi_url(url): return alt_pypi_url -def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_headers={}, maxdepth=3): +def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_headers=None, maxdepth=3): """ Recurse into [URLPAT::][HEADER: ]FILE|FIELD where FILE may be another such string or file containing lines matching the same format, and flatten the result as a dict e.g. {'^example.com': ['Authorization: Basic token', 'User-Agent: Special Agent']} """ + if urlpat_headers is None: + urlpat_headers = {} + # stop infinite recursion that might happen if a file.txt refers to itself if maxdepth < 0: _log.debug("Failed to parse_http_header_fields_urlpat (recursion limit)") @@ -619,7 +622,7 @@ def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_header if header is not None: # parent caller didn't want to forget about the header, reconstruct as recursion stops here. - argline = header.strip() + ': ' + argline + argline = header.strip() + ':' + argline if urlpat is not None: if urlpat in urlpat_headers.keys(): @@ -648,11 +651,11 @@ def download_file(filename, url, path, forced=False): # parse option HTTP header fields for URLs containing a pattern http_header_fields_urlpat = build_option('http_header_fields_urlpat') # compile a dict full of {urlpat: [header, list]} - urlpat_headers = dict() + urlpat_headers = None if http_header_fields_urlpat is not None: # there may be multiple options given, parse them all, while updating urlpat_headers for arg in http_header_fields_urlpat: - urlpat_headers = parse_http_header_fields_urlpat(arg, urlpat_headers) + urlpat_headers = parse_http_header_fields_urlpat(arg, urlpat_headers=urlpat_headers) # make sure directory exists basedir = os.path.dirname(path) From 3c21a166304cd9dcd4bb2ea46f47d9a640f6daba Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Fri, 12 Feb 2021 12:46:06 +0100 Subject: [PATCH 079/864] fixup! parse_http_header_fields_urlpat fixup: use sentinel value --- easybuild/tools/filetools.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 390498dcc7..41c623a4c0 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -671,12 +671,13 @@ def download_file(filename, url, path, forced=False): # permit additional or override headers via http_headers_fields_urlpat option # only append/override HTTP header fields that match current url - for urlpatkey, http_header_fields in urlpat_headers.items(): - if re.search(urlpatkey, url): - extraheaders = dict(hf.split(':', 1) for hf in http_header_fields) - for key, val in extraheaders.items(): - headers[key] = val - _log.debug('Custom HTTP header field set: %s (value omitted from log)' % (key)) + if urlpat_headers is not None: + for urlpatkey, http_header_fields in urlpat_headers.items(): + if re.search(urlpatkey, url): + extraheaders = dict(hf.split(':', 1) for hf in http_header_fields) + for key, val in extraheaders.items(): + headers[key] = val + _log.debug('Custom HTTP header field set: %s (value omitted from log)' % (key)) # for backward compatibility, and to avoid relying on 3rd party Python library 'requests' url_req = std_urllib.Request(url, headers=headers) From 5e8071697c2f12343939e38ba760e5e03b5b24d8 Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Fri, 12 Feb 2021 14:26:35 +0100 Subject: [PATCH 080/864] add test_parse_http_header_fields_urlpat --- easybuild/tools/filetools.py | 6 +-- test/framework/options.py | 71 +++++++++++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 41c623a4c0..11a8904dfa 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -580,12 +580,10 @@ def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_header # stop infinite recursion that might happen if a file.txt refers to itself if maxdepth < 0: - _log.debug("Failed to parse_http_header_fields_urlpat (recursion limit)") - return urlpat_headers + raise EasyBuildError("Failed to parse_http_header_fields_urlpat (recursion limit)") if not isinstance(arg, str): - _log.debug("Failed to parse_http_header_fields_urlpat (argument not a string)") - return urlpat_headers + raise EasyBuildError("Failed to parse_http_header_fields_urlpat (argument not a string)") # HTTP header fields are separated by CRLF but splitting on LF is more convenient for argline in arg.split('\n'): diff --git a/test/framework/options.py b/test/framework/options.py index ca5e7f0e25..7fae83e73c 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -52,7 +52,8 @@ from easybuild.tools.config import find_last_log, get_build_log_path, get_module_syntax, module_classes from easybuild.tools.environment import modify_env from easybuild.tools.filetools import change_dir, copy_dir, copy_file, download_file, is_patch_file, mkdir -from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file +from easybuild.tools.filetools import parse_http_header_fields_urlpat, read_file, remove_dir, remove_file +from easybuild.tools.filetools import which, write_file from easybuild.tools.github import GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO from easybuild.tools.github import URL_SEPARATOR, fetch_github_token from easybuild.tools.modules import Lmod @@ -2563,6 +2564,74 @@ def test_hide_toolchains(self): self.assertTrue(re.search(r'module: GCC/\.4\.9\.2', outtxt)) self.assertTrue(re.search(r'module: gzip/1\.6-GCC-4\.9\.2', outtxt)) + def test_parse_http_header_fields_urlpat(self): + """Test function parse_http_header_fields_urlpat""" + urlex = "example.com" + urlgnu = "gnu.org" + hdrauth = "Authorization" + valauth = "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==" + hdragent = "User-Agent" + valagent = "James/0.0.7 (MI6)" + hdrrefer = "Referer" + valrefer = "http://www.example.com/" + filesub1 = os.path.join(self.test_prefix, "testhttpheaders1.txt") + filesub2 = os.path.join(self.test_prefix, "testhttpheaders2.txt") + filesub3 = os.path.join(self.test_prefix, "testhttpheaders3.txt") + filesub4 = os.path.join(self.test_prefix, "testhttpheaders4.txt") + fileauth = os.path.join(self.test_prefix, "testhttpheadersauth.txt") + write_file(filesub4, filesub3) + write_file(filesub3, filesub2) + write_file(filesub2, filesub1) + write_file(filesub1, "\n".join([f"{urlgnu}::{hdrauth}:{valauth}"])) + write_file(filesub2, "\n".join([f"{urlex}::{filesub1}"])) + write_file(filesub3, "\n".join([f"{urlex}::{hdragent}:{filesub2}"])) + write_file(fileauth, "\n".join([f"{valauth}"])) + + # Case A: basic pattern + args = f"{urlgnu}::{hdragent}:{valagent}" + urlpat_headers = parse_http_header_fields_urlpat(args) + self.assertEqual({urlgnu: [f"{hdragent}:{valagent}"]}, urlpat_headers) + + # Case B: urlpat has another urlpat: retain deepest level + args = f"{urlgnu}::{urlgnu}::{urlex}::{hdragent}:{valagent}" + urlpat_headers = parse_http_header_fields_urlpat(args) + self.assertEqual({urlex: [f"{hdragent}:{valagent}"]}, urlpat_headers) + + # Case C: header value has a colon + args = f"{urlex}::{hdrrefer}:{valrefer}" + urlpat_headers = parse_http_header_fields_urlpat(args) + self.assertEqual({urlex: [f"{hdrrefer}:{valrefer}"]}, urlpat_headers) + + # Case D: recurse into files + args = filesub3 + urlpat_headers = parse_http_header_fields_urlpat(args) + self.assertEqual({urlgnu: [f"{hdrauth}:{valauth}"]}, urlpat_headers) + + # Case E: recurse into files as header + args = f"{urlex}::{filesub3}" + urlpat_headers = parse_http_header_fields_urlpat(args) + self.assertEqual({urlgnu: [f"{hdrauth}:{valauth}"]}, urlpat_headers) + + # Case F: recurse into files as value (header is replaced) + args = f"{urlex}::{hdrrefer}:{filesub3}" + urlpat_headers = parse_http_header_fields_urlpat(args) + self.assertEqual({urlgnu: [f"{hdrauth}:{valauth}"]}, urlpat_headers) + + # Case G: recurse into files as value (header is retained) + args = f"{urlgnu}::{hdrauth}:{fileauth}" + urlpat_headers = parse_http_header_fields_urlpat(args) + self.assertEqual({urlgnu: [f"{hdrauth}:{valauth}"]}, urlpat_headers) + + # Case H: recurse into files but hit limit + args = filesub4 + error_regex = r"Failed to parse_http_header_fields_urlpat \(recursion limit\)" + self.assertErrorRegex(EasyBuildError, error_regex, parse_http_header_fields_urlpat, args) + + # Case I: argument is not a string + args = list("foobar") + error_regex = r"Failed to parse_http_header_fields_urlpat \(argument not a string\)" + self.assertErrorRegex(EasyBuildError, error_regex, parse_http_header_fields_urlpat, args) + def test_http_header_fields_urlpat(self): """Test use of --http-header-fields-urlpat.""" test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') From 4410e04a338930b055cb3a57809230278cd7335b Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Fri, 12 Feb 2021 14:51:13 +0100 Subject: [PATCH 081/864] fixup! add test_parse_http_header_fields_urlpat --- test/framework/options.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 7fae83e73c..ba06b03fb1 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2582,45 +2582,45 @@ def test_parse_http_header_fields_urlpat(self): write_file(filesub4, filesub3) write_file(filesub3, filesub2) write_file(filesub2, filesub1) - write_file(filesub1, "\n".join([f"{urlgnu}::{hdrauth}:{valauth}"])) - write_file(filesub2, "\n".join([f"{urlex}::{filesub1}"])) - write_file(filesub3, "\n".join([f"{urlex}::{hdragent}:{filesub2}"])) - write_file(fileauth, "\n".join([f"{valauth}"])) + write_file(filesub1, "%s::%s:%s\n" % (urlgnu, hdrauth, valauth)) + write_file(filesub2, "%s::%s\n" % (urlex, filesub1)) + write_file(filesub3, "%s::%s:%s\n" % (urlex, hdragent, filesub2)) + write_file(fileauth, "%s\n" % (valauth)) # Case A: basic pattern - args = f"{urlgnu}::{hdragent}:{valagent}" + args = "%s::%s:%s" % (urlgnu, hdragent, valagent) urlpat_headers = parse_http_header_fields_urlpat(args) - self.assertEqual({urlgnu: [f"{hdragent}:{valagent}"]}, urlpat_headers) + self.assertEqual({urlgnu: ["%s:%s" % (hdragent, valagent)]}, urlpat_headers) # Case B: urlpat has another urlpat: retain deepest level - args = f"{urlgnu}::{urlgnu}::{urlex}::{hdragent}:{valagent}" + args = "%s::%s::%s::%s:%s" % (urlgnu, urlgnu, urlex, hdragent, valagent) urlpat_headers = parse_http_header_fields_urlpat(args) - self.assertEqual({urlex: [f"{hdragent}:{valagent}"]}, urlpat_headers) + self.assertEqual({urlex: ["%s:%s" % (hdragent, valagent)]}, urlpat_headers) # Case C: header value has a colon - args = f"{urlex}::{hdrrefer}:{valrefer}" + args = "%s::%s:%s" % (urlex, hdrrefer, valrefer) urlpat_headers = parse_http_header_fields_urlpat(args) - self.assertEqual({urlex: [f"{hdrrefer}:{valrefer}"]}, urlpat_headers) + self.assertEqual({urlex: ["%s:%s" % (hdrrefer, valrefer)]}, urlpat_headers) # Case D: recurse into files args = filesub3 urlpat_headers = parse_http_header_fields_urlpat(args) - self.assertEqual({urlgnu: [f"{hdrauth}:{valauth}"]}, urlpat_headers) + self.assertEqual({urlgnu: ["%s:%s" % (hdrauth, valauth)]}, urlpat_headers) # Case E: recurse into files as header - args = f"{urlex}::{filesub3}" + args = "%s::%s" % (urlex, filesub3) urlpat_headers = parse_http_header_fields_urlpat(args) - self.assertEqual({urlgnu: [f"{hdrauth}:{valauth}"]}, urlpat_headers) + self.assertEqual({urlgnu: ["%s:%s" % (hdrauth, valauth)]}, urlpat_headers) # Case F: recurse into files as value (header is replaced) - args = f"{urlex}::{hdrrefer}:{filesub3}" + args = "%s::%s:%s" % (urlex, hdrrefer, filesub3) urlpat_headers = parse_http_header_fields_urlpat(args) - self.assertEqual({urlgnu: [f"{hdrauth}:{valauth}"]}, urlpat_headers) + self.assertEqual({urlgnu: ["%s:%s" % (hdrauth, valauth)]}, urlpat_headers) # Case G: recurse into files as value (header is retained) - args = f"{urlgnu}::{hdrauth}:{fileauth}" + args = "%s::%s:%s" % (urlgnu, hdrauth, fileauth) urlpat_headers = parse_http_header_fields_urlpat(args) - self.assertEqual({urlgnu: [f"{hdrauth}:{valauth}"]}, urlpat_headers) + self.assertEqual({urlgnu: ["%s:%s" % (hdrauth, valauth)]}, urlpat_headers) # Case H: recurse into files but hit limit args = filesub4 From 57d93be66788d9eb5f4f1b3315e685099828f0d1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 13 Feb 2021 11:57:02 +0100 Subject: [PATCH 082/864] use http://sources/easybuild.io as fallback source URL --- easybuild/framework/easyblock.py | 8 +++++++- test/framework/easyblock.py | 33 +++++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 983478d255..5f2b5ec819 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -97,6 +97,8 @@ from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION +EASYBUILD_SOURCES_URL = 'http://sources.easybuild.io' + MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP] # string part of URL for Python packages on PyPI that indicates needs to be rewritten (see derive_alt_pypi_url) @@ -756,7 +758,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No break # no need to try other source paths - targetdir = os.path.join(srcpaths[0], self.name.lower()[0], self.name) + name_letter = self.name.lower()[0] + targetdir = os.path.join(srcpaths[0], name_letter, self.name) if foundfile: if self.dry_run: @@ -772,6 +775,9 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No source_urls = [] source_urls.extend(self.cfg['source_urls']) + # add sources.easybuild.io as fallback source URL + source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, self.name)) + mkdir(targetdir, parents=True) for url in source_urls: diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 399a269597..7e6911269b 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -46,7 +46,8 @@ from easybuild.tools import config from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import get_module_syntax -from easybuild.tools.filetools import change_dir, copy_dir, copy_file, mkdir, read_file, remove_file, write_file +from easybuild.tools.filetools import change_dir, copy_dir, copy_file, mkdir, read_file, remove_file +from easybuild.tools.filetools import verify_checksum, write_file from easybuild.tools.module_generator import module_generator from easybuild.tools.modules import reset_module_caches from easybuild.tools.utilities import time2str @@ -1396,6 +1397,36 @@ def test_obtain_file(self): shutil.rmtree(tmpdir) + def test_fallback_source_url(self): + """Check whether downloading from fallback source URL http://sources.easybuild.io works.""" + # cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/11951 + + init_config(args=["--sourcepath=%s" % self.test_prefix]) + + udunits_ec = os.path.join(self.test_prefix, 'UDUNITS.eb') + udunits_ec_txt = '\n'.join([ + "easyblock = 'ConfigureMake'", + "name = 'UDUNITS'", + "version = '2.2.26'", + "homepage = 'https://www.unidata.ucar.edu/software/udunits'", + "description = 'UDUNITS'", + "toolchain = {'name': 'GCC', 'version': '4.8.2'}", + "source_urls = ['https://broken.source.urls/nosuchdirectory']", + "sources = [SOURCELOWER_TAR_GZ]", + "checksums = ['368f4869c9c7d50d2920fa8c58654124e9ed0d8d2a8c714a9d7fdadc08c7356d']", + ]) + write_file(udunits_ec, udunits_ec_txt) + + ec = process_easyconfig(udunits_ec)[0] + eb = EasyBlock(ec['ec']) + + eb.fetch_step() + + expected_path = os.path.join(self.test_prefix, 'u', 'UDUNITS', 'udunits-2.2.26.tar.gz') + self.assertTrue(os.path.samefile(eb.src[0]['path'], expected_path)) + + self.assertTrue(verify_checksum(expected_path, eb.cfg['checksums'][0])) + def test_obtain_file_extension(self): """Test use of obtain_file method on an extension.""" From 2a155330f9ecdcf3697649637c219da701fbfcef Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 15 Feb 2021 11:18:30 +0100 Subject: [PATCH 083/864] Limit amount of attempts and test error conditions --- easybuild/tools/filetools.py | 7 ++----- test/framework/filetools.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index ef90af4c58..540d359e99 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2485,14 +2485,11 @@ def create_unused_dir(parent_folder, name): parent_folder = os.path.abspath(parent_folder) start_path = os.path.join(parent_folder, name) - number = None - while True: - if number is None: + for number in range(-1, 10000): # Start with no suffix and limit the number of attempts + if number < 0: path = start_path - number = 0 else: path = start_path + '_' + str(number) - number += 1 try: os.mkdir(path) break diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 6a7abf8ef6..7b34f6db61 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2889,6 +2889,7 @@ def test_locate_files(self): self.assertErrorRegex(EasyBuildError, error_pattern, ft.locate_files, ['2.txt'], []) def test_create_unused_dir(self): + """Test create_unused_dir function.""" path = ft.create_unused_dir(self.test_prefix, 'folder') self.assertEqual(path, os.path.join(self.test_prefix, 'folder')) self.assertTrue(os.path.exists(path)) @@ -2905,6 +2906,21 @@ def test_create_unused_dir(self): path = ft.create_unused_dir(self.test_prefix, 'folder2') self.assertEqual(path, os.path.join(self.test_prefix, 'folder2_%s' % i)) self.assertTrue(os.path.exists(path)) + # Fail cleanly if passed a readonly folder + readonly_dir = os.path.join(self.test_prefix, 'ro_folder') + os.mkdir(readonly_dir) + old_perms = os.lstat(readonly_dir)[stat.ST_MODE] + os.chmod(readonly_dir, stat.S_IREAD | stat.S_IEXEC) + try: + self.assertErrorRegex(EasyBuildError, 'Failed to create directory', + ft.create_unused_dir, readonly_dir, 'new_folder') + finally: + os.chmod(readonly_dir, old_perms) + # Ignore files same as folders. So first just create a file with no contents + open(os.path.join(self.test_prefix, 'file'), 'w').close() + path = ft.create_unused_dir(self.test_prefix, 'file') + self.assertEqual(path, os.path.join(self.test_prefix, 'file_0')) + self.assertTrue(os.path.exists(path)) def suite(): From 2f079a6eccd8ea0afc792abfad0471251e755bcf Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 15 Feb 2021 11:51:44 +0100 Subject: [PATCH 084/864] Change unit of cuda-cache-size to MiB and add cuda_cache_dir --- easybuild/framework/easyblock.py | 32 +++++++++++++++++++------------- easybuild/tools/config.py | 1 + easybuild/tools/options.py | 4 +++- test/framework/easyblock.py | 9 ++++++--- 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f3645ee78b..6ee0698904 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1027,6 +1027,24 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False): mkdir(dir_name, parents=True) + def setup_cuda_cache(self): + cuda_cache_maxsize = build_option('cuda_cache_maxsize') + if cuda_cache_maxsize is None: + cuda_cache_maxsize = 1 * 1024 # 1 GiB default value + else: + cuda_cache_maxsize = int(cuda_cache_maxsize) + if cuda_cache_maxsize == 0: + self.log.info('Disabling CUDA PTX cache as per request') + env.setvar('CUDA_CACHE_DISABLE', '1') + else: + cuda_cache_dir = build_option('cuda_cache_dir') + if not cuda_cache_dir: + cuda_cache_dir = os.path.join(self.builddir, 'eb-cuda-cache') + self.log.info('Enabling CUDA PTX cache of size %s MiB at %s', cuda_cache_maxsize, cuda_cache_dir) + env.setvar('CUDA_CACHE_DISABLE', '0') + env.setvar('CUDA_CACHE_PATH', cuda_cache_dir) + env.setvar('CUDA_CACHE_MAXSIZE', str(cuda_cache_maxsize * 1024 * 1024)) + # # MODULE UTILITY FUNCTIONS # @@ -2152,19 +2170,7 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True): # Setup CUDA cache if required. If we don't do this, CUDA will use the $HOME for its cache files if get_software_root('CUDA') or get_software_root('CUDAcore'): - cuda_cache_maxsize = build_option('cuda_cache_maxsize') - if cuda_cache_maxsize is None: - cuda_cache_maxsize = 1 * 1024 * 1024 * 1024 # 1 GB default value - if cuda_cache_maxsize == 0: - self.log.info('Disabling CUDA PTX cache as per request') - env.setvar('CUDA_CACHE_DISABLE', '1') - else: - cuda_cache_dir = os.path.join(self.builddir, 'eb-cuda-cache') - self.log.info('Enabling CUDA PTX cache of size %s MB at %s', - cuda_cache_maxsize / 1024 / 1024, cuda_cache_dir) - env.setvar('CUDA_CACHE_DISABLE', '0') - env.setvar('CUDA_CACHE_PATH', cuda_cache_dir) - env.setvar('CUDA_CACHE_MAXSIZE', str(cuda_cache_maxsize)) + self.setup_cuda_cache() # guess directory to start configure/build/install process in, and move there if start_dir: diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 1d686871e7..33efd1b99c 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -169,6 +169,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'container_image_name', 'container_template_recipe', 'container_tmpdir', + 'cuda_cache_dir', 'cuda_cache_maxsize', 'cuda_compute_capabilities', 'download_timeout', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 8c7387ebf6..69441614a6 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -357,7 +357,9 @@ def override_options(self): 'consider-archived-easyconfigs': ("Also consider archived easyconfigs", None, 'store_true', False), 'containerize': ("Generate container recipe/image", None, 'store_true', False, 'C'), 'copy-ec': ("Copy specified easyconfig(s) to specified location", None, 'store_true', False), - 'cuda-cache-maxsize': ("Maximum size of the CUDA cache (in bytes) used for JIT compilation of PTX code. " + 'cuda-cache-dir': ("Path to CUDA cache dir to use if enabled. Defaults to a path inside the build dir.", + str, 'store', None, {'metavar': "PATH"}), + 'cuda-cache-maxsize': ("Maximum size of the CUDA cache (in MiB) used for JIT compilation of PTX code. " "Leave value empty to let EasyBuild choose a value or '0' to disable the cache", int, 'store_or_None', None), 'cuda-compute-capabilities': ("List of CUDA compute capabilities to use when building GPU software; " diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 8b8921fcac..9257cb22af 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1762,7 +1762,7 @@ def test_prepare_step_hmns(self): self.assertEqual(loaded_modules[0]['mod_name'], 'GCC/6.4.0-2.28') def test_prepare_step_cuda_cache(self): - """Test handling cuda-cache-maxsize option.""" + """Test handling cuda-cache-* options.""" init_config(build_options={'cuda_cache_maxsize': None}) # Automatic mode @@ -1803,14 +1803,17 @@ def test_prepare_step_cuda_cache(self): self.assertNotIn('Enabling CUDA PTX cache', logtxt) self.assertEqual(os.environ['CUDA_CACHE_DISABLE'], '1') - init_config(build_options={'cuda_cache_maxsize': 1234567890}) # Specified size + # Specified size and location + cuda_cache_dir = os.path.join(self.test_prefix, 'custom-cuda-cache') + init_config(build_options={'cuda_cache_maxsize': 1234, 'cuda_cache_dir': cuda_cache_dir}) write_file(eb.logfile, '') eb.prepare_step(start_dir=False) logtxt = read_file(eb.logfile) self.assertNotIn('Disabling CUDA PTX cache', logtxt) self.assertIn('Enabling CUDA PTX cache', logtxt) self.assertEqual(os.environ['CUDA_CACHE_DISABLE'], '0') - self.assertEqual(os.environ['CUDA_CACHE_MAXSIZE'], '1234567890') + self.assertEqual(os.environ['CUDA_CACHE_MAXSIZE'], str(1234 * 1024 * 1024)) + self.assertEqual(os.environ['CUDA_CACHE_PATH'], cuda_cache_dir) def test_checksum_step(self): """Test checksum step""" From 8655e2da15e83c546cdc6275c2665e219d02d0b3 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Mon, 15 Feb 2021 13:02:51 +0100 Subject: [PATCH 085/864] Update easybuild/framework/easyblock.py Co-authored-by: Kenneth Hoste --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index bf7721378b..f4a0a2756a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1350,7 +1350,7 @@ def make_module_extend_modpath(self): self.log.debug("Including user module path extensions returned by naming scheme: %s", user_modpath_exts) for user_envvar in user_envvars: if not os.getenv(user_envvar): - raise EasyBuildError("Requested environment variable %s as an additional branch for user" + raise EasyBuildError("Requested environment variable $%s as an additional branch for user " "modules does not exist in current environment", user_envvar) txt += self.module_generator.use(user_modpath_exts, prefix=self.module_generator.getenv_cmd(user_envvar), guarded=True, From c0904bd55a02e2779198cd441bf097181b5d39d2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 15 Feb 2021 17:01:21 +0100 Subject: [PATCH 086/864] add toolchain definition for iibff toolchain --- easybuild/toolchains/iibff.py | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 easybuild/toolchains/iibff.py diff --git a/easybuild/toolchains/iibff.py b/easybuild/toolchains/iibff.py new file mode 100644 index 0000000000..71dde7534f --- /dev/null +++ b/easybuild/toolchains/iibff.py @@ -0,0 +1,41 @@ +## +# Copyright 2013-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for iibff compiler toolchain (includes Intel compilers + MPI, BLIS, libFLAME, ScaLAPACK and FFTW). + +:author: Kenneth Hoste (HPC-UGent) +""" + +from easybuild.toolchains.iimpi import Iimpi +from easybuild.toolchains.linalg.blis import Blis +from easybuild.toolchains.linalg.flame import Flame +from easybuild.toolchains.linalg.scalapack import ScaLAPACK +from easybuild.toolchains.fft.fftw import Fftw + + +class Iibff(Iimpi, Blis, Flame, ScaLAPACK, Fftw): + """Compiler toolchain with GCC, OpenMPI, BLIS, libFLAME, ScaLAPACK and FFTW.""" + NAME = 'iibff' + SUBTOOLCHAIN = Iimpi.NAME From 3b7cfc4b609965a929025c7ede41b9d47dd4759f Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Mon, 15 Feb 2021 22:27:02 +0100 Subject: [PATCH 087/864] cleanups for test_http_header_fields_urlpat --- easybuild/tools/filetools.py | 18 +++--- easybuild/tools/options.py | 8 ++- test/framework/options.py | 105 ++++++++++++----------------------- 3 files changed, 51 insertions(+), 80 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 11a8904dfa..043838f509 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -569,14 +569,18 @@ def derive_alt_pypi_url(url): return alt_pypi_url -def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_headers=None, maxdepth=3): +def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_headers_collection=None, maxdepth=3): """ - Recurse into [URLPAT::][HEADER: ]FILE|FIELD where FILE may be another such string or - file containing lines matching the same format, and flatten the result as a dict - e.g. {'^example.com': ['Authorization: Basic token', 'User-Agent: Special Agent']} + Recurse into multi-line string "[URLPAT::][HEADER:]FILE|FIELD" where FILE may be another such string or file + containing lines matching the same format, such as "^https://www.example.com::/path/to/headers.txt", and flatten + the result to dict e.g. {'^https://www.example.com': ['Authorization: Basic token', 'User-Agent: Special Agent']} """ - if urlpat_headers is None: + if urlpat_headers_collection is None: + # this function call is not a recursive call urlpat_headers = {} + else: + # copy existing header data to avoid modifying it + urlpat_headers = urlpat_headers_collection.copy() # stop infinite recursion that might happen if a file.txt refers to itself if maxdepth < 0: @@ -649,11 +653,11 @@ def download_file(filename, url, path, forced=False): # parse option HTTP header fields for URLs containing a pattern http_header_fields_urlpat = build_option('http_header_fields_urlpat') # compile a dict full of {urlpat: [header, list]} - urlpat_headers = None + urlpat_headers = dict() if http_header_fields_urlpat is not None: # there may be multiple options given, parse them all, while updating urlpat_headers for arg in http_header_fields_urlpat: - urlpat_headers = parse_http_header_fields_urlpat(arg, urlpat_headers=urlpat_headers) + urlpat_headers.update(parse_http_header_fields_urlpat(arg)) # make sure directory exists basedir = os.path.dirname(path) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 1419d29879..c238497658 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -399,9 +399,11 @@ def override_options(self): "(e.g. --hide-deps=zlib,ncurses)", 'strlist', 'extend', None), 'hide-toolchains': ("Comma separated list of toolchains that you want automatically hidden, " "(e.g. --hide-toolchains=GCCcore)", 'strlist', 'extend', None), - 'http-header-fields-urlpat': (("Set extra HTTP header fields (or file) for URL patterns;" - "(e.g. ^https://www.example.com::/path/to/headers.txt)"), - None, 'append', None, {'metavar': 'PAT::FIELD[,[PAT::]FIELDS..]'}), + 'http-header-fields-urlpat': (("Set extra HTTP header FIELDs when downloading files from URL PATterns. " + "To not log senstive values, specify a file containing newline separated " + "FIELDs. e.g. \"^https://www.example.com::/path/to/headers.txt\" or " + "\"client[A-z0-9]*.example.com': ['Authorization: Basic token']\"."), + None, 'append', None, {'metavar': '[URLPAT::][HEADER:]FILE|FIELD'}), 'ignore-checksums': ("Ignore failing checksum verification", None, 'store_true', False), 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), 'install-latest-eb-release': ("Install latest known version of easybuild", None, 'store_true', False), diff --git a/test/framework/options.py b/test/framework/options.py index ba06b03fb1..ecd7536b3d 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2675,104 +2675,69 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None): self.assertFalse(re.compile(thestring).search(stdout), "Pattern '%s' leaked into log (%s)" % (thestring, msg)) - # A: simple direct case (all is logged) + # A: simple direct case (all is logged because passed directly via EasyBuild configuration options) args = list(common_args) args.extend([ '--http-header-fields-urlpat=gnu.org::%s:%s' % (testdohdr, testdoval), '--http-header-fields-urlpat=nomatch.com::%s:%s' % (testdonthdr, testdontval), ]) # expect to find everything passed on cmdline - run_and_assert( - args, - 'case A', - [mentionhdr % (testdohdr), testdoval, testdonthdr, testdontval] - ) + expected = [mentionhdr % (testdohdr), testdoval, testdonthdr, testdontval] + run_and_assert(args, "case A", expected) # all subsequent tests share this argument list args = common_args args.append('--http-header-fields-urlpat=%s' % (testcmdfile)) # B: simple file case (secrets in file are not logged) - write_file( - testcmdfile, - '\n'.join( - [ - 'gnu.org::%s: %s' % (testdohdr, testdoval), - 'nomatch.com::%s: %s' % (testdonthdr, testdontval), - '', - ] - ), - ) + txt = '\n'.join([ + 'gnu.org::%s: %s' % (testdohdr, testdoval), + 'nomatch.com::%s: %s' % (testdonthdr, testdontval), + '', + ]) + write_file(testcmdfile, txt) # expect to find only the header key (not its value) and only for the appropriate url - run_and_assert( - args, - 'case B', - [mentionhdr % (testdohdr), mentionfile % (testcmdfile)], - [testdoval, testdonthdr, testdontval], - ) + expected = [mentionhdr % testdohdr, mentionfile % testcmdfile] + not_expected = [testdoval, testdonthdr, testdontval] + run_and_assert(args, "case B", expected, not_expected) # C: recursion one: header value is another file - write_file( - testcmdfile, - '\n'.join( - [ - 'gnu.org::%s: %s' % (testdohdr, testincfile), - 'nomatch.com::%s: %s' % (testdonthdr, testexcfile), - '', - ] - ), - ) + txt = '\n'.join([ + 'gnu.org::%s: %s' % (testdohdr, testincfile), + 'nomatch.com::%s: %s' % (testdonthdr, testexcfile), + '', + ]) + write_file(testcmdfile, txt) write_file(testincfile, '%s\n' % (testdoval)) write_file(testexcfile, '%s\n' % (testdontval)) # expect to find only the header key (not its value and not the filename) and only for the appropriate url - run_and_assert( - args, - 'case C', - [ - mentionhdr % (testdohdr), - mentionfile % (testcmdfile), - mentionfile % (testincfile), - mentionfile % (testexcfile), - ], - [testdoval, testdonthdr, testdontval], - ) + expected = [mentionhdr % (testdohdr), mentionfile % (testcmdfile), + mentionfile % (testincfile), mentionfile % (testexcfile)] + not_expected = [testdoval, testdonthdr, testdontval] + run_and_assert(args, "case C", expected, not_expected) # D: recursion two: header field+value is another file, write_file(testcmdfile, '\n'.join(['gnu.org::%s' % (testinchdrfile), 'nomatch.com::%s' % (testexchdrfile), ''])) write_file(testinchdrfile, '%s: %s\n' % (testdohdr, testdoval)) write_file(testexchdrfile, '%s: %s\n' % (testdonthdr, testdontval)) # expect to find only the header key (and the literal filename) and only for the appropriate url - run_and_assert( - args, - 'case D', - [ - mentionhdr % (testdohdr), - mentionfile % (testcmdfile), - mentionfile % (testinchdrfile), - mentionfile % (testexchdrfile), - ], - [testdoval, testdonthdr, testdontval], - ) + expected = [mentionhdr % (testdohdr), mentionfile % (testcmdfile), + mentionfile % (testinchdrfile), mentionfile % (testexchdrfile)] + not_expected = [testdoval, testdonthdr, testdontval] + run_and_assert(args, "case D", expected, not_expected) # E: recursion three: url pattern + header field + value in another file write_file(testcmdfile, '%s\n' % (testurlpatfile)) - write_file( - testurlpatfile, - '\n'.join( - [ - 'gnu.org::%s: %s' % (testdohdr, testdoval), - 'nomatch.com::%s: %s' % (testdonthdr, testdontval), - '', - ] - ), - ) + txt = '\n'.join([ + 'gnu.org::%s: %s' % (testdohdr, testdoval), + 'nomatch.com::%s: %s' % (testdonthdr, testdontval), + '', + ]) + write_file(testurlpatfile, txt) # expect to find only the header key (but not the literal filename) and only for the appropriate url - run_and_assert( - args, - 'case E', - [mentionhdr % (testdohdr), mentionfile % (testcmdfile), mentionfile % (testurlpatfile)], - [testdoval, testdonthdr, testdontval], - ) + expected = [mentionhdr % (testdohdr), mentionfile % (testcmdfile), mentionfile % (testurlpatfile)] + not_expected = [testdoval, testdonthdr, testdontval] + run_and_assert(args, "case E", expected, not_expected) def test_test_report_env_filter(self): """Test use of --test-report-env-filter.""" From 3a4f487a9bd8dfed937d562e2bccf183fc797763 Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Tue, 16 Feb 2021 08:33:49 +0100 Subject: [PATCH 088/864] fixup! cleanups for test_http_header_fields_urlpat --- easybuild/tools/options.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index c238497658..9a0716e47e 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -399,11 +399,11 @@ def override_options(self): "(e.g. --hide-deps=zlib,ncurses)", 'strlist', 'extend', None), 'hide-toolchains': ("Comma separated list of toolchains that you want automatically hidden, " "(e.g. --hide-toolchains=GCCcore)", 'strlist', 'extend', None), - 'http-header-fields-urlpat': (("Set extra HTTP header FIELDs when downloading files from URL PATterns. " - "To not log senstive values, specify a file containing newline separated " - "FIELDs. e.g. \"^https://www.example.com::/path/to/headers.txt\" or " - "\"client[A-z0-9]*.example.com': ['Authorization: Basic token']\"."), - None, 'append', None, {'metavar': '[URLPAT::][HEADER:]FILE|FIELD'}), + 'http-header-fields-urlpat': ("Set extra HTTP header FIELDs when downloading files from URL PATterns. " + "To not log senstive values, specify a file containing newline separated " + "FIELDs. e.g. \"^https://www.example.com::/path/to/headers.txt\" or " + "\"client[A-z0-9]*.example.com': ['Authorization: Basic token']\".", + None, 'append', None, {'metavar': '[URLPAT::][HEADER:]FILE|FIELD'}), 'ignore-checksums': ("Ignore failing checksum verification", None, 'store_true', False), 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), 'install-latest-eb-release': ("Install latest known version of easybuild", None, 'store_true', False), From a9476deb210e4420958c2e83b0d5ca539a0752e2 Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Tue, 16 Feb 2021 09:07:06 +0100 Subject: [PATCH 089/864] We usually use double quotes for sentences --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 043838f509..08a8097e2e 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -679,7 +679,7 @@ def download_file(filename, url, path, forced=False): extraheaders = dict(hf.split(':', 1) for hf in http_header_fields) for key, val in extraheaders.items(): headers[key] = val - _log.debug('Custom HTTP header field set: %s (value omitted from log)' % (key)) + _log.debug("Custom HTTP header field set: %s (value omitted from log)", key) # for backward compatibility, and to avoid relying on 3rd party Python library 'requests' url_req = std_urllib.Request(url, headers=headers) From 97c0374de8c98ab95bff87fd5b2bb726322ef411 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 16 Feb 2021 10:25:13 +0100 Subject: [PATCH 090/864] Address comments --- easybuild/framework/easyblock.py | 3 ++- easybuild/tools/options.py | 9 +++++---- test/framework/easyblock.py | 20 +++++++++++--------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e4e049a95c..7ab471b3f9 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -66,6 +66,7 @@ from easybuild.tools.build_details import get_build_stats from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs from easybuild.tools.build_log import print_error, print_msg, print_warning +from easybuild.tools.config import DEFAULT_ENVVAR_USERS_MODULES from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, package_path, source_paths @@ -1351,7 +1352,7 @@ def make_module_extend_modpath(self): # add user-specific module path; use statement will be guarded so no need to create the directories user_modpath = build_option('subdir_user_modules') if user_modpath: - user_envvars = build_option('envvars_user_modules') or ['HOME'] + user_envvars = build_option('envvars_user_modules') or [DEFAULT_ENVVAR_USERS_MODULES] user_modpath_exts = ActiveMNS().det_user_modpath_extensions(self.cfg) self.log.debug("Including user module path extensions returned by naming scheme: %s", user_modpath_exts) for user_envvar in user_envvars: diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 6d3d1c4b64..e46be243cc 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -60,7 +60,8 @@ from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError from easybuild.tools.build_log import init_logging, log_start, print_msg, print_warning, raise_easybuilderror from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE, DEFAULT_ALLOW_LOADED_MODULES -from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE +from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_ENVVAR_USERS_MODULES, DEFAULT_FORCE_DOWNLOAD +from easybuild.tools.config import DEFAULT_INDEX_MAX_AGE from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL @@ -491,7 +492,7 @@ def config_options(self): mk_full_default_path('containerpath')), 'envvars-user-modules': ("List of environment variables that hold the base paths for which user-specific " "modules will be installed relative to", 'strlist', 'store', - ['HOME']), + [DEFAULT_ENVVAR_USERS_MODULES]), 'external-modules-metadata': ("List of (glob patterns for) paths to files specifying metadata " "for external modules (INI format)", 'strlist', 'store', None), 'hooks': ("Location of Python module with hook implementations", 'str', 'store', None), @@ -550,8 +551,8 @@ def config_options(self): 'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']), 'subdir-software': ("Installpath subdir for software", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_software']), - 'subdir-user-modules': ("Base path of user-specific modules relative to --envvar-user-modules (which " - "defaults to $HOME)", None, 'store', None), + 'subdir-user-modules': ("Base path of user-specific modules relative to --envvar-user-modules", + None, 'store', None), 'suffix-modules-path': ("Suffix for module files install path", None, 'store', GENERAL_CLASS), # this one is sort of an exception, it's something jobscripts can set, # has no real meaning for regular eb usage diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 6eed8ef5fd..cc64fb29f5 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -303,9 +303,11 @@ def test_make_module_extend_modpath(self): self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) # Repeat this but using an alternate envvars (instead of $HOME) - list_of_envvars = ['TURKEY', 'HAM'] - os.environ['TURKEY'] = "1" - os.environ['HAM'] = "1" + list_of_envvars = ['SITE_INSTALLS', 'USER_INSTALLS'] + site_install_path = os.path.join('path','to','site','installs') + user_install_path = os.path.join('path','to','user','installs') + os.environ['SITE_INSTALLS'] = site_install_path + os.environ['USER_INSTALLS'] = user_install_path build_options = { 'envvars_user_modules': list_of_envvars, @@ -321,24 +323,24 @@ def test_make_module_extend_modpath(self): for envvar in list_of_envvars: if get_module_syntax() == 'Tcl': regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses] - home = r'\$::env\(%s\)' % envvar + module_envvar = r'\$::env\(%s\)' % envvar fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir regexs.extend([ # extension for user modules is guarded - r'if { \[ file isdirectory \[ file join %s \[ %s \] \] \] } {$' % (home, fj_usermodsdir), + r'if { \[ file isdirectory \[ file join %s \[ %s \] \] \] } {$' % (module_envvar, fj_usermodsdir), # no per-moduleclass extension for user modules - r'^\s+module use \[ file join %s \[ %s \] \]$' % (home, fj_usermodsdir), + r'^\s+module use \[ file join %s \[ %s \] \]$' % (module_envvar, fj_usermodsdir), ]) elif get_module_syntax() == 'Lua': regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c for c in modclasses] - home = r'os.getenv\("%s"\)' % envvar + module_envvar = r'os.getenv\("%s"\)' % envvar pj_usermodsdir = r'pathJoin\("%s", "funky", "Compiler/pi/3.14"\)' % usermodsdir regexs.extend([ # extension for user modules is guarded - r'if isDir\(pathJoin\(%s, %s\)\) then' % (home, pj_usermodsdir), + r'if isDir\(pathJoin\(%s, %s\)\) then' % (module_envvar, pj_usermodsdir), # no per-moduleclass extension for user modules - r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, %s\)\)' % (home, pj_usermodsdir), + r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, %s\)\)' % (module_envvar, pj_usermodsdir), ]) else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) From 595fb68c621ab22eac4fa32afb2ab2adf9bce960 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 16 Feb 2021 10:37:45 +0100 Subject: [PATCH 091/864] Appease hound --- test/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index cc64fb29f5..4bd879dc5f 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -304,8 +304,8 @@ def test_make_module_extend_modpath(self): # Repeat this but using an alternate envvars (instead of $HOME) list_of_envvars = ['SITE_INSTALLS', 'USER_INSTALLS'] - site_install_path = os.path.join('path','to','site','installs') - user_install_path = os.path.join('path','to','user','installs') + site_install_path = os.path.join('path', 'to', 'site', 'installs') + user_install_path = os.path.join('path', 'to', 'user', 'installs') os.environ['SITE_INSTALLS'] = site_install_path os.environ['USER_INSTALLS'] = user_install_path From 1047b36cf6715c3b676d1eb81c9219665d3cd136 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 16 Feb 2021 11:49:02 +0100 Subject: [PATCH 092/864] use https for sources.easybuild.io fallback URL --- easybuild/framework/easyblock.py | 4 ++-- test/framework/easyblock.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 5f2b5ec819..e730dfc1d2 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -97,7 +97,7 @@ from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION -EASYBUILD_SOURCES_URL = 'http://sources.easybuild.io' +EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io' MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP] @@ -775,7 +775,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No source_urls = [] source_urls.extend(self.cfg['source_urls']) - # add sources.easybuild.io as fallback source URL + # add https://sources.easybuild.io as fallback source URL source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, self.name)) mkdir(targetdir, parents=True) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 7e6911269b..a689883a92 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1398,7 +1398,7 @@ def test_obtain_file(self): shutil.rmtree(tmpdir) def test_fallback_source_url(self): - """Check whether downloading from fallback source URL http://sources.easybuild.io works.""" + """Check whether downloading from fallback source URL https://sources.easybuild.io works.""" # cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/11951 init_config(args=["--sourcepath=%s" % self.test_prefix]) From 086fc41df3b59d18f32cabfb8c461a8cc751f87b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 16 Feb 2021 12:47:58 +0100 Subject: [PATCH 093/864] Allow use of non-existent environment variables (defaulting to "" for those) --- easybuild/framework/easyblock.py | 4 ++-- easybuild/tools/module_generator.py | 4 ++-- test/framework/easyblock.py | 9 +++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 7ab471b3f9..294269c6be 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1357,8 +1357,8 @@ def make_module_extend_modpath(self): self.log.debug("Including user module path extensions returned by naming scheme: %s", user_modpath_exts) for user_envvar in user_envvars: if not os.getenv(user_envvar): - raise EasyBuildError("Requested environment variable $%s as an additional branch for user " - "modules does not exist in current environment", user_envvar) + self.log.warning("Requested environment variable $%s as an additional branch for user " + "modules does not exist in current environment", user_envvar) txt += self.module_generator.use(user_modpath_exts, prefix=self.module_generator.getenv_cmd(user_envvar), guarded=True, user_modpath=user_modpath) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index c031f14d83..dd685814b9 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -780,7 +780,7 @@ def getenv_cmd(self, envvar): """ Return module-syntax specific code to get value of specific environment variable. """ - return '$::env(%s)' % envvar + return '[if { [info exists ::env(%s)] } { concat $::env(%s) } ]' % (envvar, envvar) def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None, multi_dep_mods=None): """ @@ -1200,7 +1200,7 @@ def getenv_cmd(self, envvar): """ Return module-syntax specific code to get value of specific environment variable. """ - return 'os.getenv("%s")' % envvar + return 'os.getenv("%s") or ""' % envvar def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None, multi_dep_mods=None): """ diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 4bd879dc5f..075766f3a6 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -278,7 +278,7 @@ def test_make_module_extend_modpath(self): txt = eb.make_module_extend_modpath() if get_module_syntax() == 'Tcl': regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses] - home = r'\$::env\(HOME\)' + home = r'\[if \{ \[info exists ::env\(%s\)\] \} \{ concat \$::env\(%s\) \} \]' % ("HOME", "HOME") fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir regexs.extend([ # extension for user modules is guarded @@ -288,7 +288,7 @@ def test_make_module_extend_modpath(self): ]) elif get_module_syntax() == 'Lua': regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c for c in modclasses] - home = r'os.getenv\("HOME"\)' + home = r'os.getenv\("HOME"\) or ""' pj_usermodsdir = r'pathJoin\("%s", "funky", "Compiler/pi/3.14"\)' % usermodsdir regexs.extend([ # extension for user modules is guarded @@ -323,7 +323,8 @@ def test_make_module_extend_modpath(self): for envvar in list_of_envvars: if get_module_syntax() == 'Tcl': regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses] - module_envvar = r'\$::env\(%s\)' % envvar + module_envvar = r'\[if \{ \[info exists ::env\(%s\)\] \} \{ concat \$::env\(%s\) \} \]' % (envvar, + envvar) fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir regexs.extend([ # extension for user modules is guarded @@ -334,7 +335,7 @@ def test_make_module_extend_modpath(self): elif get_module_syntax() == 'Lua': regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c for c in modclasses] - module_envvar = r'os.getenv\("%s"\)' % envvar + module_envvar = r'os.getenv\("%s"\) or ""' % envvar pj_usermodsdir = r'pathJoin\("%s", "funky", "Compiler/pi/3.14"\)' % usermodsdir regexs.extend([ # extension for user modules is guarded From b57f1778fbece95f7185161a0250a365818c0031 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 16 Feb 2021 14:07:06 +0100 Subject: [PATCH 094/864] Add additional tests and keep old behaviour as much as possible --- easybuild/framework/easyblock.py | 4 +-- easybuild/tools/module_generator.py | 18 +++++++++---- test/framework/easyblock.py | 40 +++++++++++++++++++++++++++-- 3 files changed, 53 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 294269c6be..9b0c5f5143 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1360,8 +1360,8 @@ def make_module_extend_modpath(self): self.log.warning("Requested environment variable $%s as an additional branch for user " "modules does not exist in current environment", user_envvar) txt += self.module_generator.use(user_modpath_exts, - prefix=self.module_generator.getenv_cmd(user_envvar), guarded=True, - user_modpath=user_modpath) + prefix=self.module_generator.getenv_cmd(user_envvar, safe=True), + guarded=True, user_modpath=user_modpath) else: self.log.debug("Not including module path extensions, as specified.") return txt diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index dd685814b9..433422fca4 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -394,7 +394,7 @@ def get_description(self, conflict=True): """ raise NotImplementedError - def getenv_cmd(self, envvar): + def getenv_cmd(self, envvar, safe=False): """ Return module-syntax specific code to get value of specific environment variable. """ @@ -776,11 +776,15 @@ def get_description(self, conflict=True): return txt - def getenv_cmd(self, envvar): + def getenv_cmd(self, envvar, safe=False): """ Return module-syntax specific code to get value of specific environment variable. """ - return '[if { [info exists ::env(%s)] } { concat $::env(%s) } ]' % (envvar, envvar) + if safe: + cmd = '[if { [info exists ::env(%s)] } { concat $::env(%s) } ]' % (envvar, envvar) + else: + cmd = '$::env(%s)' % envvar + return cmd def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None, multi_dep_mods=None): """ @@ -1196,11 +1200,15 @@ def get_description(self, conflict=True): return txt - def getenv_cmd(self, envvar): + def getenv_cmd(self, envvar, safe=False): """ Return module-syntax specific code to get value of specific environment variable. """ - return 'os.getenv("%s") or ""' % envvar + if safe: + cmd = 'os.getenv("%s") or ""' % envvar + else: + cmd = 'os.getenv("%s")' % envvar + return cmd def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None, multi_dep_mods=None): """ diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 075766f3a6..5f0d5fa5be 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -304,8 +304,8 @@ def test_make_module_extend_modpath(self): # Repeat this but using an alternate envvars (instead of $HOME) list_of_envvars = ['SITE_INSTALLS', 'USER_INSTALLS'] - site_install_path = os.path.join('path', 'to', 'site', 'installs') - user_install_path = os.path.join('path', 'to', 'user', 'installs') + site_install_path = os.path.join(config.install_path(), 'site') + user_install_path = os.path.join(config.install_path(), 'user') os.environ['SITE_INSTALLS'] = site_install_path os.environ['USER_INSTALLS'] = user_install_path @@ -350,6 +350,42 @@ def test_make_module_extend_modpath(self): self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) os.unsetenv(envvar) + # Check behaviour when directories do and do not exist + site_modules = os.path.join(site_install_path, "my/own/modules", "funky", "Compiler/pi/3.14") + user_modules = os.path.join(user_install_path, "my/own/modules", "funky", "Compiler/pi/3.14") + # make a modules directory so that we can create our module files + temp_module_file_dir = os.path.join(site_install_path, "my/own/modules", "temp_module_files") + os.makedirs(temp_module_file_dir) + # write out a module file + if get_module_syntax() == 'Tcl': + module_file = os.path.join(temp_module_file_dir, "mytest") + module_txt = "#%Module\n" + txt + elif get_module_syntax() == 'Lua': + module_file = os.path.join(temp_module_file_dir, "mytest.lua") + module_txt = txt + handle = open(module_file, "w") + handle.write(module_txt) + handle.close() + # Set MODULEPATH and check the effect of `module load` + os.environ['MODULEPATH'] = temp_module_file_dir + + # Check MODULEPATH when neither directories exist + self.modtool.run_module('load', 'mytest') + self.assertFalse(site_modules in os.environ['MODULEPATH']) + self.assertFalse(user_modules in os.environ['MODULEPATH']) + self.modtool.run_module('unload', 'mytest') + # Now create the directory for site modules + os.makedirs(site_modules) + self.modtool.run_module('load', 'mytest') + self.assertTrue(os.environ['MODULEPATH'].startswith(site_modules)) + self.assertFalse(user_modules in os.environ['MODULEPATH']) + self.modtool.run_module('unload', 'mytest') + # Now create the directory for user modules + os.makedirs(user_modules) + self.modtool.run_module('load', 'mytest') + self.assertTrue(os.environ['MODULEPATH'].startswith(user_modules + ":" + site_modules)) + self.modtool.run_module('unload', 'mytest') + def test_make_module_req(self): """Testcase for make_module_req""" self.contents = '\n'.join([ From 57501e3b58cd75f5405e2201a35a9309adeacce7 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 17 Feb 2021 11:46:58 +0100 Subject: [PATCH 095/864] Replace "safe" kwarg with possibility to define default value when grabbing --- easybuild/framework/easyblock.py | 4 +++- easybuild/tools/module_generator.py | 15 ++++++++------- test/framework/easyblock.py | 14 +++++++++----- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 9b0c5f5143..0cef089064 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1359,8 +1359,10 @@ def make_module_extend_modpath(self): if not os.getenv(user_envvar): self.log.warning("Requested environment variable $%s as an additional branch for user " "modules does not exist in current environment", user_envvar) + default_value = user_envvar + "_NOT_DEFINED" txt += self.module_generator.use(user_modpath_exts, - prefix=self.module_generator.getenv_cmd(user_envvar, safe=True), + prefix=self.module_generator.getenv_cmd(user_envvar, + default=default_value), guarded=True, user_modpath=user_modpath) else: self.log.debug("Not including module path extensions, as specified.") diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 433422fca4..eeac9aadcf 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -394,7 +394,7 @@ def get_description(self, conflict=True): """ raise NotImplementedError - def getenv_cmd(self, envvar, safe=False): + def getenv_cmd(self, envvar, default=None): """ Return module-syntax specific code to get value of specific environment variable. """ @@ -776,12 +776,13 @@ def get_description(self, conflict=True): return txt - def getenv_cmd(self, envvar, safe=False): + def getenv_cmd(self, envvar, default=None): """ Return module-syntax specific code to get value of specific environment variable. """ - if safe: - cmd = '[if { [info exists ::env(%s)] } { concat $::env(%s) } ]' % (envvar, envvar) + if default is not None: + cmd = '[if { [info exists ::env(%s)] } { concat $::env(%s) } else { concat "%s" } ]' % (envvar, envvar, + default) else: cmd = '$::env(%s)' % envvar return cmd @@ -1200,12 +1201,12 @@ def get_description(self, conflict=True): return txt - def getenv_cmd(self, envvar, safe=False): + def getenv_cmd(self, envvar, default=None): """ Return module-syntax specific code to get value of specific environment variable. """ - if safe: - cmd = 'os.getenv("%s") or ""' % envvar + if default is not None: + cmd = 'os.getenv("%s") or "%s"' % (envvar, default) else: cmd = 'os.getenv("%s")' % envvar return cmd diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 5f0d5fa5be..d7d8b7dc33 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -278,7 +278,8 @@ def test_make_module_extend_modpath(self): txt = eb.make_module_extend_modpath() if get_module_syntax() == 'Tcl': regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses] - home = r'\[if \{ \[info exists ::env\(%s\)\] \} \{ concat \$::env\(%s\) \} \]' % ("HOME", "HOME") + home = r'\[if { \[info exists ::env\(%s\)\] } { concat \$::env\(%s\) } else { concat "%s" } \]'\ + % ("HOME", "HOME", "HOME_NOT_DEFINED") fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir regexs.extend([ # extension for user modules is guarded @@ -288,7 +289,7 @@ def test_make_module_extend_modpath(self): ]) elif get_module_syntax() == 'Lua': regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c for c in modclasses] - home = r'os.getenv\("HOME"\) or ""' + home = r'os.getenv\("HOME"\) or "HOME_NOT_DEFINED"' pj_usermodsdir = r'pathJoin\("%s", "funky", "Compiler/pi/3.14"\)' % usermodsdir regexs.extend([ # extension for user modules is guarded @@ -323,8 +324,9 @@ def test_make_module_extend_modpath(self): for envvar in list_of_envvars: if get_module_syntax() == 'Tcl': regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses] - module_envvar = r'\[if \{ \[info exists ::env\(%s\)\] \} \{ concat \$::env\(%s\) \} \]' % (envvar, - envvar) + module_envvar =\ + r'\[if \{ \[info exists ::env\(%s\)\] \} \{ concat \$::env\(%s\) \} else { concat "%s" } \]'\ + % (envvar, envvar, envvar + "_NOT_DEFINED") fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir regexs.extend([ # extension for user modules is guarded @@ -335,7 +337,7 @@ def test_make_module_extend_modpath(self): elif get_module_syntax() == 'Lua': regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c for c in modclasses] - module_envvar = r'os.getenv\("%s"\) or ""' % envvar + module_envvar = r'os.getenv\("%s"\) or "%s"' % (envvar, envvar + "_NOT_DEFINED") pj_usermodsdir = r'pathJoin\("%s", "funky", "Compiler/pi/3.14"\)' % usermodsdir regexs.extend([ # extension for user modules is guarded @@ -360,9 +362,11 @@ def test_make_module_extend_modpath(self): if get_module_syntax() == 'Tcl': module_file = os.path.join(temp_module_file_dir, "mytest") module_txt = "#%Module\n" + txt + print(module_txt) elif get_module_syntax() == 'Lua': module_file = os.path.join(temp_module_file_dir, "mytest.lua") module_txt = txt + print(module_txt) handle = open(module_file, "w") handle.write(module_txt) handle.close() From d1f29d46b2b4c5a036124f322e2f600ac3d0e10b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 17 Feb 2021 11:47:23 +0100 Subject: [PATCH 096/864] Replace "safe" kwarg with possibility to define default value when grabbing --- test/framework/easyblock.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index d7d8b7dc33..ce6fe56578 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -362,11 +362,9 @@ def test_make_module_extend_modpath(self): if get_module_syntax() == 'Tcl': module_file = os.path.join(temp_module_file_dir, "mytest") module_txt = "#%Module\n" + txt - print(module_txt) elif get_module_syntax() == 'Lua': module_file = os.path.join(temp_module_file_dir, "mytest.lua") module_txt = txt - print(module_txt) handle = open(module_file, "w") handle.write(module_txt) handle.close() From 2f894107198a44de9e674921c976b8784ebf0324 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 17 Feb 2021 12:08:28 +0100 Subject: [PATCH 097/864] Add additional test to make sure any matching path in the CWD is not picked up --- test/framework/easyblock.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index ce6fe56578..62feee215d 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -368,9 +368,21 @@ def test_make_module_extend_modpath(self): handle = open(module_file, "w") handle.write(module_txt) handle.close() + # Set MODULEPATH and check the effect of `module load` os.environ['MODULEPATH'] = temp_module_file_dir + # Let's switch to a dir where the paths we will use exist to make sure they can + # not be accidentally picked up is the variable is not defined but the paths exist + # relative to the current directory + cwd = os.getcwd() + os.makedirs(os.path.join(config.install_path(), "existing_dir", "my/own/modules", "funky", "Compiler/pi/3.14")) + change_dir(os.path.join(config.install_path(), "existing_dir")) + self.modtool.run_module('load', 'mytest') + self.assertFalse("existing_dir" in os.environ['MODULEPATH']) + self.modtool.run_module('unload', 'mytest') + change_dir(cwd) + # Check MODULEPATH when neither directories exist self.modtool.run_module('load', 'mytest') self.assertFalse(site_modules in os.environ['MODULEPATH']) From b67ce231fb36a8701ff819064f0cf44e0d70f358 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 17 Feb 2021 12:10:28 +0100 Subject: [PATCH 098/864] Appease hound --- easybuild/tools/module_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index eeac9aadcf..460c9b855f 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -782,7 +782,7 @@ def getenv_cmd(self, envvar, default=None): """ if default is not None: cmd = '[if { [info exists ::env(%s)] } { concat $::env(%s) } else { concat "%s" } ]' % (envvar, envvar, - default) + default) else: cmd = '$::env(%s)' % envvar return cmd From a36f81a81f5248f275ca342d56c6cc67e2927176 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 17 Feb 2021 12:24:35 +0100 Subject: [PATCH 099/864] Replace warning with debug message when for environment variables used to add to MODULEPATH --- easybuild/framework/easyblock.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 0cef089064..ced339d009 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1356,9 +1356,8 @@ def make_module_extend_modpath(self): user_modpath_exts = ActiveMNS().det_user_modpath_extensions(self.cfg) self.log.debug("Including user module path extensions returned by naming scheme: %s", user_modpath_exts) for user_envvar in user_envvars: - if not os.getenv(user_envvar): - self.log.warning("Requested environment variable $%s as an additional branch for user " - "modules does not exist in current environment", user_envvar) + self.log.debug("Requested environment variable $%s to host additional branch for modules", + user_envvar) default_value = user_envvar + "_NOT_DEFINED" txt += self.module_generator.use(user_modpath_exts, prefix=self.module_generator.getenv_cmd(user_envvar, From 34db2585095dd02f2f256b454f46bd0663f8dd90 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 17 Feb 2021 13:25:33 +0100 Subject: [PATCH 100/864] Be more careful with test case --- test/framework/easyblock.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 62feee215d..3811fbc1ed 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -263,7 +263,7 @@ def test_make_module_extend_modpath(self): # no $MODULEPATH extensions for default module naming scheme (EasyBuildMNS) self.assertEqual(eb.make_module_extend_modpath(), '') - usermodsdir = 'my/own/modules' + usermodsdir = 'my_own_modules' modclasses = ['compiler', 'tools'] os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS' build_options = { @@ -353,10 +353,10 @@ def test_make_module_extend_modpath(self): os.unsetenv(envvar) # Check behaviour when directories do and do not exist - site_modules = os.path.join(site_install_path, "my/own/modules", "funky", "Compiler/pi/3.14") - user_modules = os.path.join(user_install_path, "my/own/modules", "funky", "Compiler/pi/3.14") + site_modules = os.path.join(site_install_path, usermodsdir, "funky", "Compiler/pi/3.14") + user_modules = os.path.join(user_install_path, usermodsdir, "funky", "Compiler/pi/3.14") # make a modules directory so that we can create our module files - temp_module_file_dir = os.path.join(site_install_path, "my/own/modules", "temp_module_files") + temp_module_file_dir = os.path.join(site_install_path, usermodsdir, "temp_module_files") os.makedirs(temp_module_file_dir) # write out a module file if get_module_syntax() == 'Tcl': @@ -376,10 +376,10 @@ def test_make_module_extend_modpath(self): # not be accidentally picked up is the variable is not defined but the paths exist # relative to the current directory cwd = os.getcwd() - os.makedirs(os.path.join(config.install_path(), "existing_dir", "my/own/modules", "funky", "Compiler/pi/3.14")) + os.makedirs(os.path.join(config.install_path(), "existing_dir", usermodsdir, "funky", "Compiler/pi/3.14")) change_dir(os.path.join(config.install_path(), "existing_dir")) self.modtool.run_module('load', 'mytest') - self.assertFalse("existing_dir" in os.environ['MODULEPATH']) + self.assertFalse(os.path.join(usermodsdir, "funky") in os.environ['MODULEPATH']) self.modtool.run_module('unload', 'mytest') change_dir(cwd) From 7d63bf60d3b2b916a7561e167d960ad69f6eda04 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 17 Feb 2021 13:32:49 +0100 Subject: [PATCH 101/864] Make sure environment variables are not defined for initial module load test --- test/framework/easyblock.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 3811fbc1ed..27b2573494 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -307,8 +307,6 @@ def test_make_module_extend_modpath(self): list_of_envvars = ['SITE_INSTALLS', 'USER_INSTALLS'] site_install_path = os.path.join(config.install_path(), 'site') user_install_path = os.path.join(config.install_path(), 'user') - os.environ['SITE_INSTALLS'] = site_install_path - os.environ['USER_INSTALLS'] = user_install_path build_options = { 'envvars_user_modules': list_of_envvars, @@ -373,7 +371,7 @@ def test_make_module_extend_modpath(self): os.environ['MODULEPATH'] = temp_module_file_dir # Let's switch to a dir where the paths we will use exist to make sure they can - # not be accidentally picked up is the variable is not defined but the paths exist + # not be accidentally picked up if the variable is not defined but the paths exist # relative to the current directory cwd = os.getcwd() os.makedirs(os.path.join(config.install_path(), "existing_dir", usermodsdir, "funky", "Compiler/pi/3.14")) @@ -383,6 +381,10 @@ def test_make_module_extend_modpath(self): self.modtool.run_module('unload', 'mytest') change_dir(cwd) + # Now define our environment variables + os.environ['SITE_INSTALLS'] = site_install_path + os.environ['USER_INSTALLS'] = user_install_path + # Check MODULEPATH when neither directories exist self.modtool.run_module('load', 'mytest') self.assertFalse(site_modules in os.environ['MODULEPATH']) From 9e53ee624ecbf7fc8a7482493516fc493e889612 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 17 Feb 2021 13:39:40 +0100 Subject: [PATCH 102/864] Clean up a little --- test/framework/easyblock.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 27b2573494..a674813179 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -351,8 +351,9 @@ def test_make_module_extend_modpath(self): os.unsetenv(envvar) # Check behaviour when directories do and do not exist - site_modules = os.path.join(site_install_path, usermodsdir, "funky", "Compiler/pi/3.14") - user_modules = os.path.join(user_install_path, usermodsdir, "funky", "Compiler/pi/3.14") + usermodsdir_extension = os.path.join(usermodsdir, "funky", "Compiler/pi/3.14") + site_modules = os.path.join(site_install_path, usermodsdir_extension) + user_modules = os.path.join(user_install_path, usermodsdir_extension) # make a modules directory so that we can create our module files temp_module_file_dir = os.path.join(site_install_path, usermodsdir, "temp_module_files") os.makedirs(temp_module_file_dir) @@ -374,10 +375,10 @@ def test_make_module_extend_modpath(self): # not be accidentally picked up if the variable is not defined but the paths exist # relative to the current directory cwd = os.getcwd() - os.makedirs(os.path.join(config.install_path(), "existing_dir", usermodsdir, "funky", "Compiler/pi/3.14")) + os.makedirs(os.path.join(config.install_path(), "existing_dir", usermodsdir_extension)) change_dir(os.path.join(config.install_path(), "existing_dir")) self.modtool.run_module('load', 'mytest') - self.assertFalse(os.path.join(usermodsdir, "funky") in os.environ['MODULEPATH']) + self.assertFalse(usermodsdir_extension in os.environ['MODULEPATH']) self.modtool.run_module('unload', 'mytest') change_dir(cwd) From 7b99f065a1085654f224add6454997720f55f4c4 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 17 Feb 2021 18:10:49 +0100 Subject: [PATCH 103/864] Add cuda_cc_space_sep and cuda_cc_semc_sep templates The latter is required for e.g. PyTorch and TorchVision which need a CMake list, which is semicolon separated --- easybuild/framework/easyconfig/templates.py | 2 ++ test/framework/easyconfig.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 610aa3291e..29cc2bb1a3 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -316,6 +316,8 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) cuda_compute_capabilities = build_option('cuda_compute_capabilities') or config.get('cuda_compute_capabilities') if cuda_compute_capabilities: template_values['cuda_compute_capabilities'] = ','.join(cuda_compute_capabilities) + template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities) + template_values['cuda_cc_semc_sep'] = ';'.join(cuda_compute_capabilities) sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities] template_values['cuda_sm_comma_sep'] = ','.join(sm_values) template_values['cuda_sm_space_sep'] = ' '.join(sm_values) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index b75a0db7ed..d987678a8e 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4052,6 +4052,8 @@ def test_cuda_compute_capabilities(self): "toolchain = SYSTEM", "cuda_compute_capabilities = ['5.1', '7.0', '7.1']", "installopts = '%(cuda_compute_capabilities)s'", + "preinstallopts = '%(cuda_cc_space_sep)s'", + "postinstallcmds = '%(cuda_cc_semc_sep)s'", "configopts = '%(cuda_sm_comma_sep)s'", "preconfigopts = '%(cuda_sm_space_sep)s'", ]) @@ -4059,6 +4061,8 @@ def test_cuda_compute_capabilities(self): ec = EasyConfig(test_ec) self.assertEqual(ec['installopts'], '5.1,7.0,7.1') + self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1') + self.assertEqual(ec['postinstallcmds'], '5.1;7.0;7.1') self.assertEqual(ec['configopts'], 'sm_51,sm_70,sm_71') self.assertEqual(ec['preconfigopts'], 'sm_51 sm_70 sm_71') @@ -4066,6 +4070,8 @@ def test_cuda_compute_capabilities(self): init_config(build_options={'cuda_compute_capabilities': ['4.2', '6.3']}) ec = EasyConfig(test_ec) self.assertEqual(ec['installopts'], '4.2,6.3') + self.assertEqual(ec['preinstallopts'], '4.2 6.3') + self.assertEqual(ec['postinstallcmds'], '4.2;6.3') self.assertEqual(ec['configopts'], 'sm_42,sm_63') self.assertEqual(ec['preconfigopts'], 'sm_42 sm_63') From 2e2fb5f5c6a0f6001708cda2c63a8de053fe07e1 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 17 Feb 2021 18:12:31 +0100 Subject: [PATCH 104/864] Rename to cuda_cc_semicolon_sep --- easybuild/framework/easyconfig/templates.py | 2 +- test/framework/easyconfig.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 29cc2bb1a3..38619c647f 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -317,7 +317,7 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) if cuda_compute_capabilities: template_values['cuda_compute_capabilities'] = ','.join(cuda_compute_capabilities) template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities) - template_values['cuda_cc_semc_sep'] = ';'.join(cuda_compute_capabilities) + template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_compute_capabilities) sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities] template_values['cuda_sm_comma_sep'] = ','.join(sm_values) template_values['cuda_sm_space_sep'] = ' '.join(sm_values) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index d987678a8e..c15b9da5c0 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4053,7 +4053,7 @@ def test_cuda_compute_capabilities(self): "cuda_compute_capabilities = ['5.1', '7.0', '7.1']", "installopts = '%(cuda_compute_capabilities)s'", "preinstallopts = '%(cuda_cc_space_sep)s'", - "postinstallcmds = '%(cuda_cc_semc_sep)s'", + "postinstallcmds = '%(cuda_cc_semicolon_sep)s'", "configopts = '%(cuda_sm_comma_sep)s'", "preconfigopts = '%(cuda_sm_space_sep)s'", ]) From 5ec676a7832e8882c93ae0c9ef8dfeab3663716c Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 17 Feb 2021 18:51:44 +0100 Subject: [PATCH 105/864] Use postinstallcmds in test --- test/framework/easyconfig.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index c15b9da5c0..4b4a6a757f 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4053,7 +4053,7 @@ def test_cuda_compute_capabilities(self): "cuda_compute_capabilities = ['5.1', '7.0', '7.1']", "installopts = '%(cuda_compute_capabilities)s'", "preinstallopts = '%(cuda_cc_space_sep)s'", - "postinstallcmds = '%(cuda_cc_semicolon_sep)s'", + "prebuildopts = '%(cuda_cc_semicolon_sep)s'", "configopts = '%(cuda_sm_comma_sep)s'", "preconfigopts = '%(cuda_sm_space_sep)s'", ]) @@ -4062,7 +4062,7 @@ def test_cuda_compute_capabilities(self): ec = EasyConfig(test_ec) self.assertEqual(ec['installopts'], '5.1,7.0,7.1') self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1') - self.assertEqual(ec['postinstallcmds'], '5.1;7.0;7.1') + self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1') self.assertEqual(ec['configopts'], 'sm_51,sm_70,sm_71') self.assertEqual(ec['preconfigopts'], 'sm_51 sm_70 sm_71') @@ -4071,7 +4071,7 @@ def test_cuda_compute_capabilities(self): ec = EasyConfig(test_ec) self.assertEqual(ec['installopts'], '4.2,6.3') self.assertEqual(ec['preinstallopts'], '4.2 6.3') - self.assertEqual(ec['postinstallcmds'], '4.2;6.3') + self.assertEqual(ec['prebuildopts'], '4.2;6.3') self.assertEqual(ec['configopts'], 'sm_42,sm_63') self.assertEqual(ec['preconfigopts'], 'sm_42 sm_63') From a5aa48576ef17df8a45cc43ed10b20a2e2a0a6c2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 17 Feb 2021 20:42:59 +0100 Subject: [PATCH 106/864] fix tiny typo --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 9a0716e47e..ebffd26eae 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -400,7 +400,7 @@ def override_options(self): 'hide-toolchains': ("Comma separated list of toolchains that you want automatically hidden, " "(e.g. --hide-toolchains=GCCcore)", 'strlist', 'extend', None), 'http-header-fields-urlpat': ("Set extra HTTP header FIELDs when downloading files from URL PATterns. " - "To not log senstive values, specify a file containing newline separated " + "To not log sensitive values, specify a file containing newline separated " "FIELDs. e.g. \"^https://www.example.com::/path/to/headers.txt\" or " "\"client[A-z0-9]*.example.com': ['Authorization: Basic token']\".", None, 'append', None, {'metavar': '[URLPAT::][HEADER:]FILE|FIELD'}), From 7004eb19d62c1e3019eefe99c838c689bc06b8cb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 18 Feb 2021 10:30:23 +0100 Subject: [PATCH 107/864] use write_file function in Lmod.update --- easybuild/tools/modules.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 3ceefc44ac..5c6947b422 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -46,7 +46,7 @@ from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, LOADED_MODULES_ACTIONS from easybuild.tools.config import build_option, get_modules_tool, install_path from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env, setvar, unset_env_vars -from easybuild.tools.filetools import convert_name, mkdir, path_matches, read_file, which +from easybuild.tools.filetools import convert_name, mkdir, path_matches, read_file, which, write_file from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX from easybuild.tools.py2vs3 import subprocess_popen_text from easybuild.tools.run import run_cmd @@ -1403,16 +1403,12 @@ def update(self): # don't actually update local cache when testing, just return the cache contents return stdout else: - try: - cache_fp = os.path.join(self.USER_CACHE_DIR, 'moduleT.lua') - self.log.debug("Updating Lmod spider cache %s with output from '%s'" % (cache_fp, ' '.join(cmd))) - cache_dir = os.path.dirname(cache_fp) - if not os.path.exists(cache_dir): - mkdir(cache_dir, parents=True) - with open(cache_fp, 'w') as cache_file: - cache_file.write(stdout) - except (IOError, OSError) as err: - raise EasyBuildError("Failed to update Lmod spider cache %s: %s", cache_fp, err) + cache_fp = os.path.join(self.USER_CACHE_DIR, 'moduleT.lua') + self.log.debug("Updating Lmod spider cache %s with output from '%s'" % (cache_fp, ' '.join(cmd))) + cache_dir = os.path.dirname(cache_fp) + if not os.path.exists(cache_dir): + mkdir(cache_dir, parents=True) + write_file(cache_fp, stdout) def use(self, path, priority=None): """ From b8ca5301d419eb8ef037dfc00431d153b0c41d5f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 18 Feb 2021 11:47:05 +0100 Subject: [PATCH 108/864] cleanup in ModuleGenerator.getenv_cmd implementations + enhance tests to ensure (mainly Tcl) syntax is correct --- easybuild/tools/module_generator.py | 17 ++++++++++------- test/framework/module_generator.py | 25 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 460c9b855f..53a6c25916 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -780,11 +780,14 @@ def getenv_cmd(self, envvar, default=None): """ Return module-syntax specific code to get value of specific environment variable. """ - if default is not None: - cmd = '[if { [info exists ::env(%s)] } { concat $::env(%s) } else { concat "%s" } ]' % (envvar, envvar, - default) - else: + if default is None: cmd = '$::env(%s)' % envvar + else: + values = { + 'default': default, + 'envvar': '::env(%s)' % envvar, + } + cmd = '[if { [info exists %(envvar)s] } { concat $%(envvar)s } else { concat "%(default)s" } ]' % values return cmd def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None, multi_dep_mods=None): @@ -1205,10 +1208,10 @@ def getenv_cmd(self, envvar, default=None): """ Return module-syntax specific code to get value of specific environment variable. """ - if default is not None: - cmd = 'os.getenv("%s") or "%s"' % (envvar, default) - else: + if default is None: cmd = 'os.getenv("%s")' % envvar + else: + cmd = 'os.getenv("%s") or "%s"' % (envvar, default) return cmd def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None, multi_dep_mods=None): diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 4a8cea46f5..8a40240751 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -854,12 +854,37 @@ def test_env(self): def test_getenv_cmd(self): """Test getting value of environment variable.""" if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + # can't have $LMOD_QUIET set when testing with Tcl syntax, + # otherwise we won't get the output produced by the test module file... + if 'LMOD_QUIET' in os.environ: + del os.environ['LMOD_QUIET'] + self.assertEqual('$::env(HOSTNAME)', self.modgen.getenv_cmd('HOSTNAME')) self.assertEqual('$::env(HOME)', self.modgen.getenv_cmd('HOME')) + + expected = '[if { [info exists ::env(TEST)] } { concat $::env(TEST) } else { concat "foobar" } ]' + getenv_txt = self.modgen.getenv_cmd('TEST', default='foobar') + self.assertEqual(getenv_txt, expected) + + write_file(os.path.join(self.test_prefix, 'test', '1.2.3'), '#%%Module\nputs stderr %s' % getenv_txt) else: self.assertEqual('os.getenv("HOSTNAME")', self.modgen.getenv_cmd('HOSTNAME')) self.assertEqual('os.getenv("HOME")', self.modgen.getenv_cmd('HOME')) + expected = 'os.getenv("TEST") or "foobar"' + getenv_txt = self.modgen.getenv_cmd('TEST', default='foobar') + self.assertEqual(getenv_txt, expected) + + write_file(os.path.join(self.test_prefix, 'test', '1.2.3.lua'), "io.stderr:write(%s)" % getenv_txt) + + self.modtool.use(self.test_prefix) + out = self.modtool.run_module('load', 'test/1.2.3', return_stderr=True) + self.assertEqual(out.strip(), 'foobar') + + os.environ['TEST'] = 'test_value_that_is_not_foobar' + out = self.modtool.run_module('load', 'test/1.2.3', return_stderr=True) + self.assertEqual(out.strip(), 'test_value_that_is_not_foobar') + def test_alias(self): """Test setting of alias in modulefiles.""" if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: From d542e438677f98d95762c0b1e17ba34fb7cc163e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 18 Feb 2021 12:03:33 +0100 Subject: [PATCH 109/864] minor style cleanups --- easybuild/framework/easyblock.py | 5 ++-- test/framework/easyblock.py | 49 ++++++++++++++++++-------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index ced339d009..78b4cdcc1b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1359,9 +1359,8 @@ def make_module_extend_modpath(self): self.log.debug("Requested environment variable $%s to host additional branch for modules", user_envvar) default_value = user_envvar + "_NOT_DEFINED" - txt += self.module_generator.use(user_modpath_exts, - prefix=self.module_generator.getenv_cmd(user_envvar, - default=default_value), + getenv_txt = self.module_generator.getenv_cmd(user_envvar, default=default_value) + txt += self.module_generator.use(user_modpath_exts, prefix=getenv_txt, guarded=True, user_modpath=user_modpath) else: self.log.debug("Not including module path extensions, as specified.") diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index a674813179..7d308d287d 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -248,6 +248,9 @@ def test_fake_module_load(self): def test_make_module_extend_modpath(self): """Test for make_module_extend_modpath""" + + module_syntax = get_module_syntax() + self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "pi"', @@ -276,10 +279,10 @@ def test_make_module_extend_modpath(self): eb.installdir = config.install_path() txt = eb.make_module_extend_modpath() - if get_module_syntax() == 'Tcl': + if module_syntax == 'Tcl': regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses] - home = r'\[if { \[info exists ::env\(%s\)\] } { concat \$::env\(%s\) } else { concat "%s" } \]'\ - % ("HOME", "HOME", "HOME_NOT_DEFINED") + home = r'\[if { \[info exists ::env\(HOME\)\] } { concat \$::env\(HOME\) } ' + home += r'else { concat "HOME_NOT_DEFINED" } \]' fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir regexs.extend([ # extension for user modules is guarded @@ -287,7 +290,7 @@ def test_make_module_extend_modpath(self): # no per-moduleclass extension for user modules r'^\s+module use \[ file join %s \[ %s \] \]$' % (home, fj_usermodsdir), ]) - elif get_module_syntax() == 'Lua': + elif module_syntax == 'Lua': regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c for c in modclasses] home = r'os.getenv\("HOME"\) or "HOME_NOT_DEFINED"' pj_usermodsdir = r'pathJoin\("%s", "funky", "Compiler/pi/3.14"\)' % usermodsdir @@ -298,15 +301,14 @@ def test_make_module_extend_modpath(self): r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, %s\)\)' % (home, pj_usermodsdir), ]) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.assertTrue(False, "Unknown module syntax: %s" % module_syntax) + for regex in regexs: regex = re.compile(regex, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) # Repeat this but using an alternate envvars (instead of $HOME) list_of_envvars = ['SITE_INSTALLS', 'USER_INSTALLS'] - site_install_path = os.path.join(config.install_path(), 'site') - user_install_path = os.path.join(config.install_path(), 'user') build_options = { 'envvars_user_modules': list_of_envvars, @@ -320,11 +322,11 @@ def test_make_module_extend_modpath(self): txt = eb.make_module_extend_modpath() for envvar in list_of_envvars: - if get_module_syntax() == 'Tcl': + if module_syntax == 'Tcl': regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses] - module_envvar =\ - r'\[if \{ \[info exists ::env\(%s\)\] \} \{ concat \$::env\(%s\) \} else { concat "%s" } \]'\ - % (envvar, envvar, envvar + "_NOT_DEFINED") + module_envvar = r'\[if \{ \[info exists ::env\(%s\)\] \} ' % envvar + module_envvar += r'\{ concat \$::env\(%s\) \} ' % envvar + module_envvar += r'else { concat "%s" } \]' % (envvar + '_NOT_DEFINED') fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir regexs.extend([ # extension for user modules is guarded @@ -332,7 +334,7 @@ def test_make_module_extend_modpath(self): # no per-moduleclass extension for user modules r'^\s+module use \[ file join %s \[ %s \] \]$' % (module_envvar, fj_usermodsdir), ]) - elif get_module_syntax() == 'Lua': + elif module_syntax == 'Lua': regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c for c in modclasses] module_envvar = r'os.getenv\("%s"\) or "%s"' % (envvar, envvar + "_NOT_DEFINED") @@ -344,7 +346,8 @@ def test_make_module_extend_modpath(self): r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, %s\)\)' % (module_envvar, pj_usermodsdir), ]) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.assertTrue(False, "Unknown module syntax: %s" % module_syntax) + for regex in regexs: regex = re.compile(regex, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) @@ -352,21 +355,23 @@ def test_make_module_extend_modpath(self): # Check behaviour when directories do and do not exist usermodsdir_extension = os.path.join(usermodsdir, "funky", "Compiler/pi/3.14") + site_install_path = os.path.join(config.install_path(), 'site') site_modules = os.path.join(site_install_path, usermodsdir_extension) + user_install_path = os.path.join(config.install_path(), 'user') user_modules = os.path.join(user_install_path, usermodsdir_extension) + # make a modules directory so that we can create our module files temp_module_file_dir = os.path.join(site_install_path, usermodsdir, "temp_module_files") - os.makedirs(temp_module_file_dir) + mkdir(temp_module_file_dir, parents=True) + # write out a module file - if get_module_syntax() == 'Tcl': + if module_syntax == 'Tcl': module_file = os.path.join(temp_module_file_dir, "mytest") module_txt = "#%Module\n" + txt - elif get_module_syntax() == 'Lua': + elif module_syntax == 'Lua': module_file = os.path.join(temp_module_file_dir, "mytest.lua") module_txt = txt - handle = open(module_file, "w") - handle.write(module_txt) - handle.close() + write_file(module_file, module_txt) # Set MODULEPATH and check the effect of `module load` os.environ['MODULEPATH'] = temp_module_file_dir @@ -375,7 +380,7 @@ def test_make_module_extend_modpath(self): # not be accidentally picked up if the variable is not defined but the paths exist # relative to the current directory cwd = os.getcwd() - os.makedirs(os.path.join(config.install_path(), "existing_dir", usermodsdir_extension)) + mkdir(os.path.join(config.install_path(), "existing_dir", usermodsdir_extension), parents=True) change_dir(os.path.join(config.install_path(), "existing_dir")) self.modtool.run_module('load', 'mytest') self.assertFalse(usermodsdir_extension in os.environ['MODULEPATH']) @@ -392,13 +397,13 @@ def test_make_module_extend_modpath(self): self.assertFalse(user_modules in os.environ['MODULEPATH']) self.modtool.run_module('unload', 'mytest') # Now create the directory for site modules - os.makedirs(site_modules) + mkdir(site_modules, parents=True) self.modtool.run_module('load', 'mytest') self.assertTrue(os.environ['MODULEPATH'].startswith(site_modules)) self.assertFalse(user_modules in os.environ['MODULEPATH']) self.modtool.run_module('unload', 'mytest') # Now create the directory for user modules - os.makedirs(user_modules) + mkdir(user_modules, parents=True) self.modtool.run_module('load', 'mytest') self.assertTrue(os.environ['MODULEPATH'].startswith(user_modules + ":" + site_modules)) self.modtool.run_module('unload', 'mytest') From 0133222bc8051b2929f1efa1b967fced52c22231 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 18 Feb 2021 12:26:54 +0100 Subject: [PATCH 110/864] only load test module in Lua syntax to check getenv_cmd syntax when using Lmod as modules tool --- test/framework/module_generator.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 8a40240751..00ee2cd8ae 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -853,6 +853,9 @@ def test_env(self): def test_getenv_cmd(self): """Test getting value of environment variable.""" + + test_mod_file = os.path.join(self.test_prefix, 'test', '1.2.3') + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: # can't have $LMOD_QUIET set when testing with Tcl syntax, # otherwise we won't get the output produced by the test module file... @@ -866,7 +869,7 @@ def test_getenv_cmd(self): getenv_txt = self.modgen.getenv_cmd('TEST', default='foobar') self.assertEqual(getenv_txt, expected) - write_file(os.path.join(self.test_prefix, 'test', '1.2.3'), '#%%Module\nputs stderr %s' % getenv_txt) + write_file(test_mod_file, '#%%Module\nputs stderr %s' % getenv_txt) else: self.assertEqual('os.getenv("HOSTNAME")', self.modgen.getenv_cmd('HOSTNAME')) self.assertEqual('os.getenv("HOME")', self.modgen.getenv_cmd('HOME')) @@ -875,15 +878,18 @@ def test_getenv_cmd(self): getenv_txt = self.modgen.getenv_cmd('TEST', default='foobar') self.assertEqual(getenv_txt, expected) - write_file(os.path.join(self.test_prefix, 'test', '1.2.3.lua'), "io.stderr:write(%s)" % getenv_txt) + test_mod_file += '.lua' + write_file(test_mod_file, "io.stderr:write(%s)" % getenv_txt) - self.modtool.use(self.test_prefix) - out = self.modtool.run_module('load', 'test/1.2.3', return_stderr=True) - self.assertEqual(out.strip(), 'foobar') + # only test loading of test module in Lua syntax when using Lmod + if isinstance(self.modtool, Lmod) or not test_mod_file.endswith('.lua'): + self.modtool.use(self.test_prefix) + out = self.modtool.run_module('load', 'test/1.2.3', return_stderr=True) + self.assertEqual(out.strip(), 'foobar') - os.environ['TEST'] = 'test_value_that_is_not_foobar' - out = self.modtool.run_module('load', 'test/1.2.3', return_stderr=True) - self.assertEqual(out.strip(), 'test_value_that_is_not_foobar') + os.environ['TEST'] = 'test_value_that_is_not_foobar' + out = self.modtool.run_module('load', 'test/1.2.3', return_stderr=True) + self.assertEqual(out.strip(), 'test_value_that_is_not_foobar') def test_alias(self): """Test setting of alias in modulefiles.""" From e2e466e26331e37295ca53fc8e84394226ca12a1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 18 Feb 2021 15:46:10 +0100 Subject: [PATCH 111/864] symlink 'lib' to 'lib64' if it doesn't exist (fixes #3549) --- easybuild/framework/easyblock.py | 13 ++++++-- easybuild/tools/config.py | 1 + easybuild/tools/options.py | 2 ++ test/framework/toy_build.py | 55 +++++++++++++++++++++++++++++--- 4 files changed, 65 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 143f73a38b..5fb593aee0 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2410,18 +2410,27 @@ def post_install_step(self): run_cmd(cmd, simple=True, log_ok=True, log_all=True) self.fix_shebang() + + lib_dir = os.path.join(self.installdir, 'lib') + lib64_dir = os.path.join(self.installdir, 'lib64') + # GCC linker searches system /lib64 path before the $LIBRARY_PATH paths. # However for each in $LIBRARY_PATH (where is often /lib) it searches /../lib64 first. # So we create /lib64 as a symlink to /lib to make it prefer EB installed libraries. # See https://github.com/easybuilders/easybuild-easyconfigs/issues/5776 if build_option('lib64_lib_symlink'): - lib_dir = os.path.join(self.installdir, 'lib') - lib64_dir = os.path.join(self.installdir, 'lib64') if os.path.exists(lib_dir) and not os.path.exists(lib64_dir): # create *relative* 'lib64' symlink to 'lib'; # see https://github.com/easybuilders/easybuild-framework/issues/3564 symlink('lib', lib64_dir, use_abspath_source=False) + # symlink lib to lib64, which is helpful on OpenSUSE; + # see https://github.com/easybuilders/easybuild-framework/issues/3549 + if build_option('lib_lib64_symlink'): + if os.path.exists(lib64_dir) and not os.path.exists(lib_dir): + # create *relative* 'lib' symlink to 'lib64'; + symlink('lib64', lib_dir, use_abspath_source=False) + def sanity_check_step(self, *args, **kwargs): """ Do a sanity check on the installation diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index e36ff37533..1a7ff08c88 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -275,6 +275,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'cleanup_tmpdir', 'extended_dry_run_ignore_errors', 'fixed_installdir_naming_scheme', + 'lib_lib64_symlink', 'lib64_fallback_sanity_check', 'lib64_lib_symlink', 'mpi_tests', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index aa8294b074..2b44aaa2af 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -408,6 +408,8 @@ def override_options(self): 'ignore-checksums': ("Ignore failing checksum verification", None, 'store_true', False), 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), 'install-latest-eb-release': ("Install latest known version of easybuild", None, 'store_true', False), + 'lib-lib64-symlink': ("Automatically create symlinks for lib/ pointing to lib64/ if the former is missing", + None, 'store_true', True), 'lib64-fallback-sanity-check': ("Fallback in sanity check to lib64/ equivalent for missing libraries", None, 'store_true', True), 'lib64-lib-symlink': ("Automatically create symlinks for lib64/ pointing to lib/ if the former is missing", diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 5d02387f26..433ef89716 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3133,25 +3133,29 @@ def test_toy_build_unicode_description(self): self.test_toy_build(ec_file=test_ec, raise_error=True) - def test_test_toy_build_lib64_symlink(self): + def test_test_toy_build_lib64_lib_symlink(self): """Check whether lib64 symlink to lib subdirectory is created.""" # this is done to ensure that /lib64 is considered before /lib64 by GCC linker, # see https://github.com/easybuilders/easybuild-easyconfigs/issues/5776 - # by default, lib64 symlink is created + # by default, lib64 -> lib symlink is created (--lib64-lib-symlink is enabled by default) self.test_toy_build() toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') lib_path = os.path.join(toy_installdir, 'lib') lib64_path = os.path.join(toy_installdir, 'lib64') + # lib6 subdir exists, is not a symlink self.assertTrue(os.path.exists(lib_path)) - self.assertTrue(os.path.exists(lib64_path)) self.assertTrue(os.path.isdir(lib_path)) self.assertFalse(os.path.islink(lib_path)) + + # lib64 subdir is a symlink to lib subdir + self.assertTrue(os.path.exists(lib64_path)) self.assertTrue(os.path.islink(lib64_path)) self.assertTrue(os.path.samefile(lib_path, lib64_path)) - # Need relative path: https://github.com/easybuilders/easybuild-framework/issues/3564 + + # lib64 symlink should point to a relative path self.assertFalse(os.path.isabs(os.readlink(lib64_path))) # cleanup and try again with --disable-lib64-lib-symlink @@ -3164,6 +3168,49 @@ def test_test_toy_build_lib64_symlink(self): self.assertTrue(os.path.isdir(lib_path)) self.assertFalse(os.path.islink(lib_path)) + def test_test_toy_build_lib_lib64_symlink(self): + """Check whether lib64 symlink to lib subdirectory is created.""" + + test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + + test_ec_txt = read_file(toy_ec) + test_ec_txt += "\npostinstallcmds += ['mv %(installdir)s/lib %(installdir)s/lib64']" + + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + + # by default, lib -> lib64 symlink is created (--lib-lib64-symlink is enabled by default) + self.test_toy_build(ec_file=test_ec) + + toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') + lib_path = os.path.join(toy_installdir, 'lib') + lib64_path = os.path.join(toy_installdir, 'lib64') + + # lib64 subdir exists, is not a symlink + self.assertTrue(os.path.exists(lib64_path)) + self.assertTrue(os.path.isdir(lib64_path)) + self.assertFalse(os.path.islink(lib64_path)) + + # lib subdir is a symlink to lib64 subdir + self.assertTrue(os.path.exists(lib_path)) + self.assertTrue(os.path.isdir(lib_path)) + self.assertTrue(os.path.islink(lib_path)) + self.assertTrue(os.path.samefile(lib_path, lib64_path)) + + # lib symlink should point to a relative path + self.assertFalse(os.path.isabs(os.readlink(lib_path))) + + # cleanup and try again with --disable-lib-lib64-symlink + remove_dir(self.test_installpath) + self.test_toy_build(ec_file=test_ec, extra_args=['--disable-lib-lib64-symlink']) + + self.assertTrue(os.path.exists(lib64_path)) + self.assertFalse(os.path.exists(lib_path)) + self.assertFalse('lib' in os.listdir(toy_installdir)) + self.assertTrue(os.path.isdir(lib64_path)) + self.assertFalse(os.path.islink(lib64_path)) + def suite(): """ return all the tests in this file """ From 2e6c89f12d6937aec04fee97fc2cd56b3e8e7e5b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 18 Feb 2021 16:19:36 +0100 Subject: [PATCH 112/864] fix tiny typo in comment in tests --- test/framework/toy_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 433ef89716..653238f4ee 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3145,7 +3145,7 @@ def test_test_toy_build_lib64_lib_symlink(self): lib_path = os.path.join(toy_installdir, 'lib') lib64_path = os.path.join(toy_installdir, 'lib64') - # lib6 subdir exists, is not a symlink + # lib64 subdir exists, is not a symlink self.assertTrue(os.path.exists(lib_path)) self.assertTrue(os.path.isdir(lib_path)) self.assertFalse(os.path.islink(lib_path)) From cea9a110835bbfdf2e631ab63076b11a7a8b8361 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 18 Feb 2021 19:28:06 +0100 Subject: [PATCH 113/864] add support for intel-compiler compiler toolchain (>= 2021.x versions, oneAPI) --- .../toolchains/compiler/intel_compilers.py | 63 +++++++++++++++++++ easybuild/toolchains/intel-compilers.py | 41 ++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 easybuild/toolchains/compiler/intel_compilers.py create mode 100644 easybuild/toolchains/intel-compilers.py diff --git a/easybuild/toolchains/compiler/intel_compilers.py b/easybuild/toolchains/compiler/intel_compilers.py new file mode 100644 index 0000000000..98b4ea5b7f --- /dev/null +++ b/easybuild/toolchains/compiler/intel_compilers.py @@ -0,0 +1,63 @@ +## +# Copyright 2021-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for Intel compilers (icc, ifort) as toolchain compilers, version 2021.x and newer (oneAPI). + +:author: Kenneth Hoste (Ghent University) +""" +import os + +from easybuild.toolchains.compiler.inteliccifort import IntelIccIfort +from easybuild.tools.toolchain.compiler import Compiler + + +class IntelCompilers(IntelIccIfort): + """ + Compiler class for Intel oneAPI compilers + """ + + COMPILER_MODULE_NAME = ['intel-compilers'] + + def _set_compiler_vars(self): + """Intel compilers-specific adjustments after setting compiler variables.""" + + # skip IntelIccIfort._set_compiler_vars (no longer relevant for recent versions) + Compiler._set_compiler_vars(self) + + root = self.get_software_root(self.COMPILER_MODULE_NAME)[0] + + libpaths = [ + 'lib', + os.path.join('lib', 'x64'), + os.path.join('compiler', 'lib', 'intel64_lin'), + ] + + self.variables.append_subdirs("LDFLAGS", root, subdirs=libpaths) + + def set_variables(self): + """Set the variables.""" + + # skip IntelIccIfort.set_variables (no longer relevant for recent versions) + Compiler.set_variables(self) diff --git a/easybuild/toolchains/intel-compilers.py b/easybuild/toolchains/intel-compilers.py new file mode 100644 index 0000000000..21b118f291 --- /dev/null +++ b/easybuild/toolchains/intel-compilers.py @@ -0,0 +1,41 @@ +## +# Copyright 2021-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for Intel compilers toolchain (icc, ifort), v2021.x or newer (oneAPI). + +:author: Kenneth Hoste (Ghent University) +""" +from easybuild.toolchains.compiler.intel_compilers import IntelCompilers +from easybuild.toolchains.gcccore import GCCcore +from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME + + +class IntelCompilersToolchain(IntelCompilers): + """Compiler toolchain with Intel compilers (icc/ifort).""" + NAME = 'intel-compilers' + # use GCCcore as subtoolchain rather than GCC, since two 'real' compiler-only toolchains don't mix well, + # in particular in a hierarchical module naming scheme + SUBTOOLCHAIN = [GCCcore.NAME, SYSTEM_TOOLCHAIN_NAME] + OPTIONAL = False From 9d52dd1e8eee7e6d9eb0ffd99f451927e1d48f67 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 18 Feb 2021 20:15:35 +0100 Subject: [PATCH 114/864] fix Python module filename for intel-compilers toolchain --- easybuild/toolchains/{intel-compilers.py => intel_compilers.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easybuild/toolchains/{intel-compilers.py => intel_compilers.py} (100%) diff --git a/easybuild/toolchains/intel-compilers.py b/easybuild/toolchains/intel_compilers.py similarity index 100% rename from easybuild/toolchains/intel-compilers.py rename to easybuild/toolchains/intel_compilers.py From 01e3329442f61ca3ab660eb4e1710b9850bbe19f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Feb 2021 12:12:24 +0100 Subject: [PATCH 115/864] don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools, to avoid hitting GitHub rate limit --- .github/workflows/unit_tests.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index dc603b9866..e102ab36ed 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -100,12 +100,17 @@ jobs: # see https://github.com//easybuild-framework/settings/secrets GITHUB_TOKEN: ${{secrets.TEST_GITHUB_TOKEN}} run: | - if [ ! -z $GITHUB_TOKEN ]; then - if [ "x${{matrix.python}}" == 'x2.6' ]; + # don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools, + # to avoid hitting GitHub rate limit; + # tests that require a GitHub token are skipped automatically when no GitHub token is available + if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]]; then + if [ ! -z $GITHUB_TOKEN ]; then + if [ "x${{matrix.python}}" == 'x2.6' ]; then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())"; else SET_KEYRING="import keyrings; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; - fi; - python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; + fi; + python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; + fi fi - name: install modules tool From b5097d109fca0cba5c442a520ceb15bcc1646ab1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Feb 2021 14:15:25 +0100 Subject: [PATCH 116/864] fix failing GitHub tests due to trivial change in output of 'eb' command --- test/framework/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 4631f4fc76..9efd0ed240 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4184,7 +4184,7 @@ def test_sync_pr_with_develop(self): github_path = r"boegel/easybuild-easyconfigs\.git" pattern = '\n'.join([ - r"== temporary log file in case of crash .*", + r"== Temporary log file in case of crash .*", r"== Determined branch name corresponding to easybuilders/easybuild-easyconfigs PR #9150: develop", r"== fetching branch 'develop' from https://github\.com/%s\.\.\." % github_path, r"== pulling latest version of 'develop' branch from easybuilders/easybuild-easyconfigs\.\.\.", @@ -4215,7 +4215,7 @@ def test_sync_branch_with_develop(self): github_path = r"boegel/easybuild-easyconfigs\.git" pattern = '\n'.join([ - r"== temporary log file in case of crash .*", + r"== Temporary log file in case of crash .*", r"== fetching branch '%s' from https://github\.com/%s\.\.\." % (test_branch, github_path), r"== pulling latest version of 'develop' branch from easybuilders/easybuild-easyconfigs\.\.\.", r"== merging 'develop' branch into PR branch '%s'\.\.\." % test_branch, From 4add76f4effda28789889090f2b6ddd86e9ef9bb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Feb 2021 14:16:36 +0100 Subject: [PATCH 117/864] print message to make it clear whether GitHub token was installed or not --- .github/workflows/unit_tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index e102ab36ed..7375e80ed5 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -111,6 +111,9 @@ jobs: fi; python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; fi + echo "GitHub token installed!" + else + echo "Installation of GitHub token skipped!" fi - name: install modules tool From 3f84e06f671c62cab4368afd63be56199de01f53 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Sat, 20 Feb 2021 11:21:15 +0100 Subject: [PATCH 118/864] fix BLAS_LIB_MT for OpenBLAS --- easybuild/toolchains/linalg/openblas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/toolchains/linalg/openblas.py b/easybuild/toolchains/linalg/openblas.py index 7cfd042a94..4d57f51ba9 100644 --- a/easybuild/toolchains/linalg/openblas.py +++ b/easybuild/toolchains/linalg/openblas.py @@ -40,6 +40,7 @@ class OpenBLAS(LinAlg): """ BLAS_MODULE_NAME = ['OpenBLAS'] BLAS_LIB = ['openblas'] + BLAS_LIB_MT = ['openblas'] BLAS_FAMILY = TC_CONSTANT_OPENBLAS LAPACK_MODULE_NAME = ['OpenBLAS'] From f12e4d95abdf9a4dcdda8da6a9f568c3b27dea63 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Sat, 20 Feb 2021 13:26:02 +0100 Subject: [PATCH 119/864] fix test --- test/framework/toolchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 24affe177b..28013b468a 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1449,7 +1449,7 @@ def test_old_new_iccifort(self): libscalack_intel4 = "-lmkl_scalapack_lp64 -lmkl_blacs_intelmpi_lp64 -lmkl_intel_lp64 -lmkl_sequential " libscalack_intel4 += "-lmkl_core" - libblas_mt_fosscuda = "-lopenblas -lgfortran" + libblas_mt_fosscuda = "-lopenblas -lgfortran -lpthread" libscalack_fosscuda = "-lscalapack -lopenblas -lgfortran" libfft_mt_fosscuda = "-lfftw3_omp -lfftw3 -lpthread" From c8ed201285327104f56716c8c620a90f35f26364 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 20 Feb 2021 18:53:23 +0100 Subject: [PATCH 120/864] include %(mpi_cmd_prefix)s and %(cuda_*)s templates in output of --avail-easyconfig-templates (fixes #3585) --- easybuild/framework/easyconfig/templates.py | 25 ++++++++++- easybuild/tools/docs.py | 17 ++++++- test/framework/easyconfig.py | 1 + test/framework/options.py | 49 +++++++++++++++++++++ 4 files changed, 89 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 38619c647f..a1ef4d42b7 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -46,7 +46,6 @@ # derived from easyconfig, but not from ._config directly TEMPLATE_NAMES_EASYCONFIG = [ - ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"), ('module_name', "Module name"), ('nameletter', "First letter of software name"), ('toolchain_name', "Toolchain name"), @@ -86,6 +85,18 @@ ('Python', 'py'), ('R', 'r'), ] +# template values which are only generated dynamically +TEMPLATE_NAMES_DYNAMIC = [ + ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"), + ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"), + ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via " + "--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"), + ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"), + ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"), + ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"), + ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"), +] + # constant templates that can be used in easyconfigs TEMPLATE_CONSTANTS = [ # source url constants @@ -300,6 +311,10 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) except Exception: _log.warning("Failed to get .lower() for name %s value %s (type %s)", name, value, type(value)) + # keep track of names of defined templates until now, + # so we can check whether names of additional dynamic template values are all known + common_template_names = set(template_values.keys()) + # step 5. add additional conditional templates if toolchain is not None and hasattr(toolchain, 'mpi_cmd_prefix'): try: @@ -322,6 +337,14 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) template_values['cuda_sm_comma_sep'] = ','.join(sm_values) template_values['cuda_sm_space_sep'] = ' '.join(sm_values) + unknown_names = [] + for key in template_values: + dynamic_template_names = set(x for (x, _) in TEMPLATE_NAMES_DYNAMIC) + if not (key in common_template_names or key in dynamic_template_names): + unknown_names.append(key) + if unknown_names: + raise EasyBuildError("One or more template values found with unknown name: %s", ','.join(unknown_names)) + return template_values diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index a052e176e2..3036a66a92 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -46,9 +46,9 @@ from easybuild.framework.easyconfig.easyconfig import get_easyblock_class, process_easyconfig from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT from easybuild.framework.easyconfig.parser import EasyConfigParser -from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_CONFIG, TEMPLATE_NAMES_EASYCONFIG +from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_CONFIG, TEMPLATE_NAMES_DYNAMIC +from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, TEMPLATE_NAMES_EASYCONFIG from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_LOWER, TEMPLATE_NAMES_LOWER_TEMPLATE -from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, TEMPLATE_CONSTANTS from easybuild.framework.easyconfig.templates import TEMPLATE_SOFTWARE_VERSIONS, template_constant_dict from easybuild.framework.easyconfig.tools import avail_easyblocks from easybuild.framework.easyconfig.tweak import find_matching_easyconfigs @@ -344,6 +344,12 @@ def avail_easyconfig_templates_txt(): for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1])) + # some template values are only defined dynamically, + # see template_constant_dict function in easybuild.framework.easyconfigs.templates + doc.append('Template values which are defined dynamically') + for name in TEMPLATE_NAMES_DYNAMIC: + doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1])) + doc.append('Template constants that can be used in easyconfigs') for cst in TEMPLATE_CONSTANTS: doc.append('%s%s: %s (%s)' % (INDENT_4SPACES, cst[0], cst[2], cst[1])) @@ -395,6 +401,13 @@ def avail_easyconfig_templates_rst(): ] doc.extend(rst_title_and_table(title, table_titles, table_values)) + title = 'Template values which are defined dynamically' + table_values = [ + ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_DYNAMIC], + [name[1] for name in TEMPLATE_NAMES_DYNAMIC], + ] + doc.extend(rst_title_and_table(title, table_titles, table_values)) + title = 'Template constants that can be used in easyconfigs' titles = ['Constant', 'Template value', 'Template name'] table_values = [ diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 4b4a6a757f..d4011f124f 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1128,6 +1128,7 @@ def test_templating_doc(self): easyconfig.templates.TEMPLATE_NAMES_CONFIG, easyconfig.templates.TEMPLATE_NAMES_LOWER, easyconfig.templates.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, + easyconfig.templates.TEMPLATE_NAMES_DYNAMIC, easyconfig.templates.TEMPLATE_CONSTANTS, ] diff --git a/test/framework/options.py b/test/framework/options.py index 9efd0ed240..dedd242068 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -509,6 +509,55 @@ def run_test(fmt=None): for fmt in [None, 'txt', 'rst']: run_test(fmt=fmt) + def test_avail_easyconfig_templates(self): + """Test listing available easyconfig file templates.""" + + def run_test(fmt=None): + """Helper function to test --avail-easyconfig-templates.""" + + args = ['--avail-easyconfig-templates'] + if fmt is not None: + args.append('--output-format=%s' % fmt) + + self.mock_stderr(True) + self.mock_stdout(True) + self.eb_main(args, verbose=True, raise_error=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(stderr) + + if fmt == 'rst': + pattern_lines = [ + r'^``%\(version_major\)s``\s+Major version\s*$', + r'^``%\(cudaver\)s``\s+full version for CUDA\s*$', + r'^``%\(pyshortver\)s``\s+short version for Python \(.\)\s*$', + r'^\* ``%\(name\)s``$', + r'^``%\(namelower\)s``\s+lower case of value of name\s*$', + r'^``%\(arch\)s``\s+System architecture \(e.g. x86_64, aarch64, ppc64le, ...\)\s*$', + r'^``%\(cuda_cc_space_sep\)s``\s+Space-separated list of CUDA compute capabilities\s*$', + r'^``SOURCE_TAR_GZ``\s+Source \.tar\.gz bundle\s+``%\(name\)s-%\(version\)s.tar.gz``\s*$', + ] + else: + pattern_lines = [ + r'^\s+%\(version_major\)s: Major version$', + r'^\s+%\(cudaver\)s: full version for CUDA$', + r'^\s+%\(pyshortver\)s: short version for Python \(.\)$', + r'^\s+%\(name\)s$', + r'^\s+%\(namelower\)s: lower case of value of name$', + r'^\s+%\(arch\)s: System architecture \(e.g. x86_64, aarch64, ppc64le, ...\)$', + r'^\s+%\(cuda_cc_space_sep\)s: Space-separated list of CUDA compute capabilities$', + r'^\s+SOURCE_TAR_GZ: Source \.tar\.gz bundle \(%\(name\)s-%\(version\)s.tar.gz\)$', + ] + + for pattern_line in pattern_lines: + regex = re.compile(pattern_line, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should match in: %s" % (regex.pattern, stdout)) + + for fmt in [None, 'txt', 'rst']: + run_test(fmt=fmt) + def test_avail_easyconfig_params(self): """Test listing available easyconfig parameters.""" From 54cd0d436cadd3e708ebd03c84bc124842a795da Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 22 Feb 2021 09:41:36 +0100 Subject: [PATCH 121/864] update release notes + bump version for EasyBuild v4.3.3 --- RELEASE_NOTES | 33 +++++++++++++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 10cf597707..1d4cadbecb 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,6 +3,39 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. + +v4.3.3 (February 23rd 2021) +--------------------------- + +update/bugfix release + +- various enhancements, including: + - advise PR labels in --review-pr and add support for --add-pr-labels (#3177) + - add support for using customized HTTP headers in download_file (#3472, #3583) + - also take toolchain dependencies into account when defining template values (#3541, #3560) + - add support for --accept-eula configuration option + 'accept_eula' easyconfig parameter (#3535, #3536, #3546) + - detect 'SYSTEM' toolchain as special case in easystack files (#3543) + - enhance extract_cmd function to use 'cp -a' for shell scripts (.sh) (#3545) + - allow use of alternate envvar(s) to $HOME for user modules (#3558) + - use https://sources/easybuild.io as fallback source URL (#3572, #3576) + - add toolchain definition for iibff toolchain (#3574) + - add %(cuda_cc_space_sep)s and %(cuda_cc_semicolon_sep)s templates (#3578) + - add support for intel-compiler toolchain (>= 2021.x versions, oneAPI) (#3581, #3582) +- various bug fixes, including: + - add --init and --recursive options to 'git submodule update' command that is used when creating source tarball for specific commit (#3537) + - filter out duplicate paths in RPATH wrapper script (#3538) + - don't clean up imported modules after verifying imports of included Python modules (#3544) + - avoid no-op changes to $LD_* environment variables in ModulesTool (#3553) + - fix UTF-8 encoding errors when running EasyBuild with Python 3.0.x-3.6.x (#3565) + - create lib64 symlink as a relative symlink (#3566) + - don't reuse variable name in the loop to fix adding extra compiler flags via toolchainopts (#3571) + - symlink 'lib' to 'lib64' if it doesn't exist (#3580) + - include %(mpi_cmd_prefix)s and %(cuda_*)s templates in output of --avail-easyconfig-templates (#3586) +- other changes: + - rename EasyBlock._skip_step to EasyBlock.skip_step, to make it part of the public API (#3561) + - make symlinking of posix_c.so to posix.so in test suite configuration conditional (#3570) + + v4.3.2 (December 10th 2020) --------------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 125344e4d3..29a940011b 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.3.3.dev0') +VERSION = LooseVersion('4.3.3') UNKNOWN = 'UNKNOWN' From 21c81f4489525f89dab0c3598810f2d7c61db73c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 23 Feb 2021 11:59:20 +0100 Subject: [PATCH 122/864] use 'main' rather than 'master' branch in GitHub integration functionality --- easybuild/tools/github.py | 34 ++++++++++++++++++++++++++++------ test/framework/filetools.py | 2 +- test/framework/github.py | 21 ++++++++++++--------- test/framework/options.py | 6 +++--- test/framework/toy_build.py | 2 +- 5 files changed, 45 insertions(+), 20 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index f3ff4629e5..71f4718e5c 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -123,15 +123,21 @@ class Githubfs(object): """This class implements some higher level functionality on top of the Github api""" - def __init__(self, githubuser, reponame, branchname="master", username=None, password=None, token=None): + def __init__(self, githubuser, reponame, branchname=None, username=None, password=None, token=None): """Construct a new githubfs object :param githubuser: the github user's repo we want to use. :param reponame: The name of the repository we want to use. - :param branchname: Then name of the branch to use (defaults to master) + :param branchname: Then name of the branch to use (defaults to 'main' for easybuilders org, 'master' otherwise) :param username: (optional) your github username. :param password: (optional) your github password. :param token: (optional) a github api token. """ + if branchname is None: + if githubuser == GITHUB_EB_MAIN: + branchname = 'main' + else: + branchname = 'master' + if token is None: token = fetch_github_token(username) self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) @@ -218,7 +224,7 @@ def read(self, path, api=True): """Read the contents of a file and return it Or, if api=False it will download the file and return the location of the downloaded file""" # we don't need use the api for this, but can also use raw.github.com - # https://raw.github.com/easybuilders/easybuild/master/README.rst + # https://raw.github.com/easybuilders/easybuild/main/README.rst if not api: outfile = tempfile.mkstemp()[1] url = '/'.join([GITHUB_RAW, self.githubuser, self.reponame, self.branchname, path]) @@ -301,7 +307,7 @@ def github_api_put_request(request_f, github_user=None, token=None, **kwargs): return (status, data) -def fetch_latest_commit_sha(repo, account, branch='master', github_user=None, token=None): +def fetch_latest_commit_sha(repo, account, branch=None, github_user=None, token=None): """ Fetch latest SHA1 for a specified repository and branch. :param repo: GitHub repository @@ -311,6 +317,14 @@ def fetch_latest_commit_sha(repo, account, branch='master', github_user=None, to :param token: GitHub token to use :return: latest SHA1 """ + if branch is None: + # use 'main' as default branch for 'easybuilders' organisation, + # otherwise use 'master' + if account == GITHUB_EB_MAIN: + branch = 'main' + else: + branch = 'master' + status, data = github_api_get_request(lambda x: x.repos[account][repo].branches, github_user=github_user, token=token, per_page=GITHUB_MAX_PER_PAGE) if status != HTTP_STATUS_OK: @@ -332,7 +346,7 @@ def fetch_latest_commit_sha(repo, account, branch='master', github_user=None, to return res -def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_EB_MAIN, path=None, github_user=None): +def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, account=GITHUB_EB_MAIN, path=None, github_user=None): """ Download entire GitHub repo as a tar.gz archive, and extract it into specified path. :param repo: repo to download @@ -341,6 +355,14 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_ :param path: path to extract to :param github_user: name of GitHub user to use """ + if branch is None: + # use 'main' as default branch for 'easybuilders' organisation, + # otherwise use 'master' + if account == GITHUB_EB_MAIN: + branch = 'main' + else: + branch = 'master' + # make sure path exists, create it if necessary if path is None: path = tempfile.mkdtemp() @@ -1940,7 +1962,7 @@ def check_github(): branch_name = 'test_branch_%s' % ''.join(random.choice(ascii_letters) for _ in range(5)) try: git_repo = init_repo(git_working_dir, GITHUB_EASYCONFIGS_REPO, silent=not debug) - remote_name = setup_repo(git_repo, github_account, GITHUB_EASYCONFIGS_REPO, 'master', + remote_name = setup_repo(git_repo, github_account, GITHUB_EASYCONFIGS_REPO, 'main', silent=not debug, git_only=True) git_repo.create_head(branch_name) res = getattr(git_repo.remotes, remote_name).push(branch_name) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index b195ae58a5..c948477aae 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2394,7 +2394,7 @@ def test_get_source_tarball_from_git(self): git_config = { 'repo_name': 'testrepository', 'url': 'https://github.com/easybuilders', - 'tag': 'master', + 'tag': 'main', } target_dir = os.path.join(self.test_prefix, 'target') diff --git a/test/framework/github.py b/test/framework/github.py index f83088c4b2..8a11ef4dc2 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -60,7 +60,7 @@ GITHUB_USER = "easybuilders" GITHUB_REPO = "testrepository" # branch to test -GITHUB_BRANCH = 'master' +GITHUB_BRANCH = 'main' class GithubTest(EnhancedTestCase): @@ -71,12 +71,15 @@ class GithubTest(EnhancedTestCase): def setUp(self): """setup""" super(GithubTest, self).setUp() + self.github_token = gh.fetch_github_token(GITHUB_TEST_ACCOUNT) + if self.github_token is None: - self.ghfs = gh.Githubfs(GITHUB_USER, GITHUB_REPO, GITHUB_BRANCH, None, None, None) + username, token = None, None else: - self.ghfs = gh.Githubfs(GITHUB_USER, GITHUB_REPO, GITHUB_BRANCH, GITHUB_TEST_ACCOUNT, - None, self.github_token) + username, token = GITHUB_TEST_ACCOUNT, self.github_token + + self.ghfs = gh.Githubfs(GITHUB_USER, GITHUB_REPO, GITHUB_BRANCH, username, None, token) self.skip_github_tests = self.github_token is None and os.getenv('FORCE_EB_GITHUB_TESTS') is None @@ -452,7 +455,7 @@ def test_download_repo(self): # default: download tarball for master branch of easybuilders/easybuild-easyconfigs repo path = gh.download_repo(path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT) - repodir = os.path.join(self.test_prefix, 'easybuilders', 'easybuild-easyconfigs-master') + repodir = os.path.join(self.test_prefix, 'easybuilders', 'easybuild-easyconfigs-main') self.assertTrue(os.path.samefile(path, repodir)) self.assertTrue(os.path.exists(repodir)) shafile = os.path.join(repodir, 'latest-sha') @@ -634,7 +637,7 @@ def run_check(expected_result=False): pr_data = { 'base': { - 'ref': 'master', + 'ref': 'main', 'repo': { 'name': 'easybuild-easyconfigs', 'owner': {'login': 'easybuilders'}, @@ -652,7 +655,7 @@ def run_check(expected_result=False): expected_stdout = "Checking eligibility of easybuilders/easybuild-easyconfigs PR #1234 for merging...\n" # target branch for PR must be develop - expected_warning = "* targets develop branch: FAILED; found 'master' => not eligible for merging!\n" + expected_warning = "* targets develop branch: FAILED; found 'main' => not eligible for merging!\n" run_check() pr_data['base']['ref'] = 'develop' @@ -937,7 +940,7 @@ def test_push_branch_to_github(self): self.mock_stderr(True) self.mock_stdout(True) - gh.setup_repo(git_repo, GITHUB_USER, GITHUB_REPO, 'master') + gh.setup_repo(git_repo, GITHUB_USER, GITHUB_REPO, 'main') git_repo.create_head(branch, force=True) gh.push_branch_to_github(git_repo, GITHUB_USER, GITHUB_REPO, branch) stderr = self.get_stderr() @@ -949,7 +952,7 @@ def test_push_branch_to_github(self): github_path = '%s/%s.git' % (GITHUB_USER, GITHUB_REPO) pattern = r'^' + '\n'.join([ - r"== fetching branch 'master' from https://github.com/%s\.\.\." % github_path, + r"== fetching branch 'main' from https://github.com/%s\.\.\." % github_path, r"== pushing branch 'test123' to remote 'github_.*' \(git@github.com:%s\) \[DRY RUN\]" % github_path, ]) + r'$' regex = re.compile(pattern) diff --git a/test/framework/options.py b/test/framework/options.py index dedd242068..3a8539014c 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4103,7 +4103,7 @@ def test_new_update_pr(self): '--pr-branch-name=branch_name_for_new_pr_test', '--pr-commit-msg="this is a commit message. really!"', '--pr-descr="moar letters foar teh lettre box"', - '--pr-target-branch=master', + '--pr-target-branch=main', '--github-org=%s' % GITHUB_TEST_ORG, '--pr-target-account=boegel', # we need to be able to 'clone' from here (via https) '--pr-title=test-1-2-3', @@ -4111,9 +4111,9 @@ def test_new_update_pr(self): txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) regexs = [ - r"^== fetching branch 'master' from https://github.com/boegel/easybuild-easyconfigs.git...", + r"^== fetching branch 'main' from https://github.com/boegel/easybuild-easyconfigs.git...", r"^Opening pull request \[DRY RUN\]", - r"^\* target: boegel/easybuild-easyconfigs:master", + r"^\* target: boegel/easybuild-easyconfigs:main", r"^\* from: %s/easybuild-easyconfigs:branch_name_for_new_pr_test" % GITHUB_TEST_ORG, r"\(created using `eb --new-pr`\)", # description r"moar letters foar teh lettre box", # also description (see --pr-descr) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 653238f4ee..9809e5b999 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1341,7 +1341,7 @@ def test_toy_extension_sources_git_config(self): ' "git_config": {', ' "repo_name": "testrepository",', ' "url": "https://github.com/easybuilders",', - ' "tag": "master",', + ' "tag": "main",', ' },', ' },', ' }),', From 53e560ca85f9e0e0c5ef42baa9345869121df6c5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 23 Feb 2021 18:40:23 +0100 Subject: [PATCH 123/864] bump version to 4.3.4dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 29a940011b..d4822e22ad 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.3.3') +VERSION = LooseVersion('4.3.4.dev0') UNKNOWN = 'UNKNOWN' From 96bff223d2771eaff3398fcb511a5191fbe13e9a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 23 Feb 2021 19:49:40 +0100 Subject: [PATCH 124/864] fix typo in release notes --- RELEASE_NOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 1d4cadbecb..178f0fb4cb 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -17,7 +17,7 @@ update/bugfix release - detect 'SYSTEM' toolchain as special case in easystack files (#3543) - enhance extract_cmd function to use 'cp -a' for shell scripts (.sh) (#3545) - allow use of alternate envvar(s) to $HOME for user modules (#3558) - - use https://sources/easybuild.io as fallback source URL (#3572, #3576) + - use https://sources.easybuild.io as fallback source URL (#3572, #3576) - add toolchain definition for iibff toolchain (#3574) - add %(cuda_cc_space_sep)s and %(cuda_cc_semicolon_sep)s templates (#3578) - add support for intel-compiler toolchain (>= 2021.x versions, oneAPI) (#3581, #3582) From 99a95faf2df799b05ce7cbd982c570fc184fe8e8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 24 Feb 2021 09:23:56 +0100 Subject: [PATCH 125/864] add pick_default_branch function to clean up duplicate code in tools/github.py --- easybuild/tools/github.py | 35 ++++++++++++++++++----------------- test/framework/github.py | 7 +++++++ 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 71f4718e5c..7f6d7faefa 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -85,6 +85,8 @@ GITHUB_URL = 'https://github.com' GITHUB_API_URL = 'https://api.github.com' +GITHUB_BRANCH_MAIN = 'main' +GITHUB_BRANCH_MASTER = 'master' GITHUB_DIR_TYPE = u'dir' GITHUB_EB_MAIN = 'easybuilders' GITHUB_EASYBLOCKS_REPO = 'easybuild-easyblocks' @@ -120,6 +122,18 @@ } +def pick_default_branch(github_owner): + """Determine default name to use.""" + # use 'main' as default branch for 'easybuilders' organisation, + # otherwise use 'master' + if github_owner == GITHUB_EB_MAIN: + branch = GITHUB_BRANCH_MAIN + else: + branch = GITHUB_BRANCH_MASTER + + return branch + + class Githubfs(object): """This class implements some higher level functionality on top of the Github api""" @@ -133,10 +147,7 @@ def __init__(self, githubuser, reponame, branchname=None, username=None, passwor :param token: (optional) a github api token. """ if branchname is None: - if githubuser == GITHUB_EB_MAIN: - branchname = 'main' - else: - branchname = 'master' + branchname = pick_default_branch(githubuser) if token is None: token = fetch_github_token(username) @@ -318,12 +329,7 @@ def fetch_latest_commit_sha(repo, account, branch=None, github_user=None, token= :return: latest SHA1 """ if branch is None: - # use 'main' as default branch for 'easybuilders' organisation, - # otherwise use 'master' - if account == GITHUB_EB_MAIN: - branch = 'main' - else: - branch = 'master' + branch = pick_default_branch(account) status, data = github_api_get_request(lambda x: x.repos[account][repo].branches, github_user=github_user, token=token, per_page=GITHUB_MAX_PER_PAGE) @@ -356,12 +362,7 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, account=GITHUB_EB_M :param github_user: name of GitHub user to use """ if branch is None: - # use 'main' as default branch for 'easybuilders' organisation, - # otherwise use 'master' - if account == GITHUB_EB_MAIN: - branch = 'main' - else: - branch = 'master' + branch = pick_default_branch(account) # make sure path exists, create it if necessary if path is None: @@ -1962,7 +1963,7 @@ def check_github(): branch_name = 'test_branch_%s' % ''.join(random.choice(ascii_letters) for _ in range(5)) try: git_repo = init_repo(git_working_dir, GITHUB_EASYCONFIGS_REPO, silent=not debug) - remote_name = setup_repo(git_repo, github_account, GITHUB_EASYCONFIGS_REPO, 'main', + remote_name = setup_repo(git_repo, github_account, GITHUB_EASYCONFIGS_REPO, GITHUB_BRANCH_MAIN, silent=not debug, git_only=True) git_repo.create_head(branch_name) res = getattr(git_repo.remotes, remote_name).push(branch_name) diff --git a/test/framework/github.py b/test/framework/github.py index 8a11ef4dc2..2b9d7deb4d 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -43,6 +43,7 @@ from easybuild.tools.configobj import ConfigObj from easybuild.tools.filetools import read_file, write_file from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, VALID_CLOSE_PR_REASONS +from easybuild.tools.github import pick_default_branch from easybuild.tools.testing import post_pr_test_report, session_state from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters import easybuild.tools.github as gh @@ -83,6 +84,12 @@ def setUp(self): self.skip_github_tests = self.github_token is None and os.getenv('FORCE_EB_GITHUB_TESTS') is None + def test_pick_default_branch(self): + """Test pick_default_branch function.""" + + self.assertEqual(pick_default_branch('easybuilders'), 'main') + self.assertEqual(pick_default_branch('foobar'), 'master') + def test_walk(self): """test the gitubfs walk function""" if self.skip_github_tests: From f854941fdb9d1bf9d2c5b7ccdbab778ec869ee49 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 25 Feb 2021 09:03:52 +0100 Subject: [PATCH 126/864] use '--opt=val' for passing settings from config file to option parser to avoid error for values starting with '-' or '--' --- easybuild/base/generaloption.py | 3 +-- test/framework/config.py | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 79fa8a85f8..b74110cdbd 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -1376,8 +1376,7 @@ def parseconfigfiles(self): configfile_values[opt_dest] = newval else: configfile_cmdline_dest.append(opt_dest) - configfile_cmdline.append("--%s" % opt_name) - configfile_cmdline.append(val) + configfile_cmdline.append("--%s=%s" % (opt_name, val)) # reparse self.log.debug('parseconfigfiles: going to parse options through cmdline %s' % configfile_cmdline) diff --git a/test/framework/config.py b/test/framework/config.py index 0b17a4e876..08a0594825 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -244,6 +244,9 @@ def test_generaloption_config_file(self): cfgtxt = '\n'.join([ '[config]', 'installpath = %s' % testpath2, + # special case: configuration option to a value starting with '--' + '[override]', + 'optarch = --test', ]) write_file(config_file, cfgtxt) @@ -261,6 +264,8 @@ def test_generaloption_config_file(self): self.assertEqual(install_path(), installpath_software) # via cmdline arg self.assertEqual(install_path('mod'), os.path.join(testpath2, 'modules')) # via config file + self.assertEqual(options.optarch, '--test') # via config file + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory # to check whether easyconfigs install path is auto-included in robot path tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') From b7f483502d731c3822f5e348095df5e8a141407f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 25 Feb 2021 11:46:09 +0100 Subject: [PATCH 127/864] Add "Citing" section to module files --- easybuild/framework/easyconfig/default.py | 1 + easybuild/tools/module_generator.py | 6 ++++++ test/framework/toy_build.py | 8 +++++++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 99e3c8309f..317f862b8c 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -194,6 +194,7 @@ # MODULES documentation easyconfig parameters # (docurls is part of MANDATORY) + 'citing': [None, "Free-form text that describes how the software should be cited in publications", MODULES], 'docpaths': [None, "List of paths for documentation relative to installation directory", MODULES], 'examples': [None, "Free-form text with examples on using the software", MODULES], 'site_contacts': [None, "String/list of strings with site contacts for the software", MODULES], diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 53a6c25916..e7e0134695 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -559,6 +559,9 @@ def _generate_help_text(self): # Examples (optional) lines.extend(self._generate_section('Examples', self.app.cfg['examples'], strip=True)) + # Citing (optional) + lines.extend(self._generate_section('Citing', self.app.cfg['citing'], strip=True)) + # Additional information: homepage + (if available) doc paths/urls, upstream/site contact lines.extend(self._generate_section("More information", " - Homepage: %s" % self.app.cfg['homepage'])) @@ -570,6 +573,9 @@ def _generate_help_text(self): lines.extend([" - $%s/%s" % (root_envvar, path) for path in docpaths]) lines.extend([" - %s" % url for url in docurls]) + + + for contacts_type in ['upstream', 'site']: contacts = self.app.cfg['%s_contacts' % contacts_type] if contacts: diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9809e5b999..67c70593b9 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -270,6 +270,7 @@ def test_toy_tweaked(self): "modluafooter = 'io.stderr:write(\"oh hai!\")'", # ignored when module syntax is Tcl "usage = 'This toy is easy to use, 100%!'", "examples = 'No example available, 0% complete'", + "citing = 'If you use this package, please cite our paper https://ieeexplore.ieee.org/document/6495863'", "docpaths = ['share/doc/toy/readme.txt', 'share/doc/toy/html/index.html']", "docurls = ['https://easybuilders.github.io/easybuild/toy/docs.html']", "upstream_contacts = 'support@toy.org'", @@ -291,7 +292,6 @@ def test_toy_tweaked(self): if get_module_syntax() == 'Lua': toy_module += '.lua' toy_module_txt = read_file(toy_module) - if get_module_syntax() == 'Tcl': self.assertTrue(re.search(r'^setenv\s*FOO\s*"bar"$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root/foo/bar$', toy_module_txt, re.M)) @@ -300,6 +300,9 @@ def test_toy_tweaked(self): mod_load_msg = r'module-info mode load.*\n\s*puts stderr\s*.*%s$' % modloadmsg_regex_tcl self.assertTrue(re.search(mod_load_msg, toy_module_txt, re.M)) self.assertTrue(re.search(r'^puts stderr "oh hai!"$', toy_module_txt, re.M)) + self.assertTrue(re.search( + r'^If you use this package, please cite our paper https://ieeexplore.ieee.org/document/6495863$', + toy_module_txt, re.M)) elif get_module_syntax() == 'Lua': self.assertTrue(re.search(r'^setenv\("FOO", "bar"\)', toy_module_txt, re.M)) pattern = r'^prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)$' @@ -309,6 +312,9 @@ def test_toy_tweaked(self): mod_load_msg = r'^if mode\(\) == "load" then\n\s*io.stderr:write\(%s\)$' % modloadmsg_regex_lua regex = re.compile(mod_load_msg, re.M) self.assertTrue(regex.search(toy_module_txt), "Pattern '%s' found in: %s" % (regex.pattern, toy_module_txt)) + self.assertTrue(re.search( + r'^If you use this package, please cite our paper https://ieeexplore.ieee.org/document/6495863$', + toy_module_txt, re.M)) else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) From c71c13e4de6897f587c576a6787e1b26f4c45c96 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 25 Feb 2021 11:47:54 +0100 Subject: [PATCH 128/864] Clean up formatting --- easybuild/tools/module_generator.py | 3 --- test/framework/toy_build.py | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index e7e0134695..6cb2716a7c 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -573,9 +573,6 @@ def _generate_help_text(self): lines.extend([" - $%s/%s" % (root_envvar, path) for path in docpaths]) lines.extend([" - %s" % url for url in docurls]) - - - for contacts_type in ['upstream', 'site']: contacts = self.app.cfg['%s_contacts' % contacts_type] if contacts: diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 67c70593b9..7971391a4d 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -292,6 +292,7 @@ def test_toy_tweaked(self): if get_module_syntax() == 'Lua': toy_module += '.lua' toy_module_txt = read_file(toy_module) + if get_module_syntax() == 'Tcl': self.assertTrue(re.search(r'^setenv\s*FOO\s*"bar"$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root/foo/bar$', toy_module_txt, re.M)) From 78fecef3a507e9bc95d47a6fbafd6bc4ad622cbe Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 25 Feb 2021 12:34:17 +0100 Subject: [PATCH 129/864] Update module fulltxt test --- test/framework/toy_build.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 7971391a4d..186a7349aa 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -301,9 +301,6 @@ def test_toy_tweaked(self): mod_load_msg = r'module-info mode load.*\n\s*puts stderr\s*.*%s$' % modloadmsg_regex_tcl self.assertTrue(re.search(mod_load_msg, toy_module_txt, re.M)) self.assertTrue(re.search(r'^puts stderr "oh hai!"$', toy_module_txt, re.M)) - self.assertTrue(re.search( - r'^If you use this package, please cite our paper https://ieeexplore.ieee.org/document/6495863$', - toy_module_txt, re.M)) elif get_module_syntax() == 'Lua': self.assertTrue(re.search(r'^setenv\("FOO", "bar"\)', toy_module_txt, re.M)) pattern = r'^prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)$' @@ -313,9 +310,6 @@ def test_toy_tweaked(self): mod_load_msg = r'^if mode\(\) == "load" then\n\s*io.stderr:write\(%s\)$' % modloadmsg_regex_lua regex = re.compile(mod_load_msg, re.M) self.assertTrue(regex.search(toy_module_txt), "Pattern '%s' found in: %s" % (regex.pattern, toy_module_txt)) - self.assertTrue(re.search( - r'^If you use this package, please cite our paper https://ieeexplore.ieee.org/document/6495863$', - toy_module_txt, re.M)) else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) @@ -1393,6 +1387,11 @@ def test_toy_module_fulltxt(self): r'No example available, 0% complete', r'', r'', + r'Citing', + r'========', + r'If you use this package, please cite our paper https://ieeexplore.ieee.org/document/6495863', + r'', + r'', r'More information', r'================', r' - Homepage: https://easybuilders.github.io/easybuild', From 5b9c014f0636dec3624bf510d567bc8916f8042d Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 25 Feb 2021 13:02:07 +0100 Subject: [PATCH 130/864] Fix too much underlining --- test/framework/toy_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 186a7349aa..b146719db3 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1388,7 +1388,7 @@ def test_toy_module_fulltxt(self): r'', r'', r'Citing', - r'========', + r'======', r'If you use this package, please cite our paper https://ieeexplore.ieee.org/document/6495863', r'', r'', From a9352479051ed3d725f2f9eb84c768d4bbf050ec Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 25 Feb 2021 21:15:13 +0100 Subject: [PATCH 131/864] avoid raised exception when getting output from interactive command in run_cmd_qa (fixes #3593) --- easybuild/tools/run.py | 6 +++++- test/framework/run.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 67977156b0..9c77edf06a 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -107,7 +107,10 @@ def get_output_from_process(proc, read_size=None, asynchronous=False): """ if asynchronous: - output = asyncprocess.recv_some(proc) + # e=False is set to avoid raising an exception when command has completed; + # that's needed to ensure we get all output, + # see https://github.com/easybuilders/easybuild-framework/issues/3593 + output = asyncprocess.recv_some(proc, e=False) elif read_size: output = proc.stdout.read(read_size) else: @@ -411,6 +414,7 @@ def check_answers_list(answers): # - otherwise the stdout/stderr buffer gets filled and it all stops working try: out = get_output_from_process(proc, asynchronous=True) + if cmd_log: cmd_log.write(out) stdout_err += out diff --git a/test/framework/run.py b/test/framework/run.py index b6ceaf8127..8486119e24 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -351,9 +351,27 @@ def test_run_cmd_qa_buffering(self): cmd += 'echo "Pick a number: "; read number; echo "Picked number: $number"' (out, ec) = run_cmd_qa(cmd, {'Pick a number: ': '42'}, log_all=True, maxhits=5) + self.assertEqual(ec, 0) regex = re.compile("Picked number: 42$") self.assertTrue(regex.search(out), "Pattern '%s' found in: %s" % (regex.pattern, out)) + # also test with script run as interactive command that quickly exits with non-zero exit code; + # see https://github.com/easybuilders/easybuild-framework/issues/3593 + script_txt = '\n'.join([ + "#/bin/bash", + "echo 'Hello, I am about to exit'", + "echo 'ERROR: I failed' >&2", + "exit 1", + ]) + script = os.path.join(self.test_prefix, 'test.sh') + write_file(script, script_txt) + adjust_permissions(script, stat.S_IXUSR) + + out, ec = run_cmd_qa(script, {}, log_ok=False) + + self.assertEqual(ec, 1) + self.assertEqual(out, "Hello, I am about to exit\nERROR: I failed\n") + def test_run_cmd_qa_log_all(self): """Test run_cmd_qa with log_output enabled""" (out, ec) = run_cmd_qa("echo 'n: '; read n; seq 1 $n", {'n: ': '5'}, log_all=True) From 2bdf60285eccb8199bd4a701d79eef15cc500145 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 1 Mar 2021 09:30:20 +0100 Subject: [PATCH 132/864] add test to check behaviour for dependencies with False version --- test/framework/easyconfig.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index d4011f124f..02fc33f4b2 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -72,7 +72,7 @@ from easybuild.tools.options import parse_external_modules_metadata from easybuild.tools.py2vs3 import OrderedDict, reload from easybuild.tools.robot import resolve_dependencies -from easybuild.tools.systemtools import get_shared_lib_ext +from easybuild.tools.systemtools import get_cpu_architecture, get_shared_lib_ext from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.utilities import quote_str, quote_py_str from test.framework.utilities import find_full_path @@ -305,6 +305,38 @@ def test_dependency(self): self.assertErrorRegex(EasyBuildError, err_msg, eb._parse_dependency, (EXTERNAL_MODULE_MARKER,)) self.assertErrorRegex(EasyBuildError, err_msg, eb._parse_dependency, ('foo', '1.2.3', EXTERNAL_MODULE_MARKER)) + def test_false_dep_version(self): + """ + Test use False as dependency version via dict using 'arch=' keys, + which should result in filtering the dependency. + """ + + arch = get_cpu_architecture() + + self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.14"', + 'versionsuffix = "-test"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = {"name":"GCC", "version": "4.6.3"}', + 'dependencies = [' + ' ("first", "1.0"),', + ' ("seoond", {"arch=%s": False}),' % arch, + ']', + 'builddependencies = [', + ' ("first_build", {"arch=%s": False}),' % arch, + ' ("second_build", "2.0"),', + ']', + ]) + self.prep() + eb = EasyConfig(self.eb_file) + deps = eb.dependencies() + self.assertEqual(len(deps), 2) + self.assertEqual(deps[0]['name'], 'second_build') + self.assertEqual(deps[1]['name'], 'first') + def test_extra_options(self): """ extra_options should allow other variables to be stored """ init_config(build_options={'silent': True}) From 7ba5c4fa5159fb6d19e048d136369e68e59fddf2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 1 Mar 2021 09:40:08 +0100 Subject: [PATCH 133/864] add support for using fallback 'arch=*' key in dependency version specified as arch->version mapping --- easybuild/tools/systemtools.py | 23 ++++++++++++++++------- test/framework/systemtools.py | 11 +++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index d58cfc0abf..cc1e73637b 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -73,6 +73,8 @@ POWER = 'POWER' X86_64 = 'x86_64' +ARCH_KEY_PREFIX = 'arch=' + # Vendor constants AMD = 'AMD' APM = 'Applied Micro' @@ -921,18 +923,25 @@ def pick_dep_version(dep_version): result = None elif isinstance(dep_version, dict): - # figure out matches based on dict keys (after splitting on '=') - my_arch_key = 'arch=%s' % get_cpu_architecture() - arch_keys = [x for x in dep_version.keys() if x.startswith('arch=')] + arch_keys = [x for x in dep_version.keys() if x.startswith(ARCH_KEY_PREFIX)] other_keys = [x for x in dep_version.keys() if x not in arch_keys] if other_keys: raise EasyBuildError("Unexpected keys in version: %s. Only 'arch=' keys are supported", other_keys) if arch_keys: - if my_arch_key in dep_version: - result = dep_version[my_arch_key] - _log.info("Version selected from %s using key %s: %s", dep_version, my_arch_key, result) + host_arch_key = ARCH_KEY_PREFIX + get_cpu_architecture() + star_arch_key = ARCH_KEY_PREFIX + '*' + # check for specific 'arch=' key first + if host_arch_key in dep_version: + result = dep_version[host_arch_key] + _log.info("Version selected from %s using key %s: %s", dep_version, host_arch_key, result) + # fall back to 'arch=*' + elif star_arch_key in dep_version: + result = dep_version[star_arch_key] + _log.info("Version selected for %s using fallback key %s: %s", dep_version, star_arch_key, result) else: - raise EasyBuildError("No matches for version in %s (looking for %s)", dep_version, my_arch_key) + raise EasyBuildError("No matches for version in %s (looking for %s)", dep_version, host_arch_key) + else: + raise EasyBuildError("Found empty dict as version!") else: raise EasyBuildError("Unknown value type for version: %s", dep_version) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index eba1625a17..ca7ba4ba2d 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -916,6 +916,17 @@ def test_pick_dep_version(self): error_pattern = "Unknown value type for version" self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, ('1.2.3', '4.5.6')) + # check support for using 'arch=*' as fallback key + dep_ver_dict = { + 'arch=*': '1.2.3', + 'arch=foo': '1.2.3-foo', + 'arch=POWER': '1.2.3-ppc64le', + } + self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3-ppc64le') + + del dep_ver_dict['arch=POWER'] + self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3') + def test_check_os_dependency(self): """Test check_os_dependency.""" From 257df8e70e990ea06a6ffaa64a43a33d5b628dd6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 1 Mar 2021 09:48:40 +0100 Subject: [PATCH 134/864] extend test for dependencies with False version to cover use of 'arch=*' --- test/framework/easyconfig.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 02fc33f4b2..391a22249a 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -72,7 +72,7 @@ from easybuild.tools.options import parse_external_modules_metadata from easybuild.tools.py2vs3 import OrderedDict, reload from easybuild.tools.robot import resolve_dependencies -from easybuild.tools.systemtools import get_cpu_architecture, get_shared_lib_ext +from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_shared_lib_ext from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.utilities import quote_str, quote_py_str from test.framework.utilities import find_full_path @@ -104,6 +104,7 @@ class EasyConfigTest(EnhancedTestCase): def setUp(self): """Set up everything for running a unit test.""" super(EasyConfigTest, self).setUp() + self.orig_get_cpu_architecture = st.get_cpu_architecture self.cwd = os.getcwd() self.all_stops = [x[0] for x in EasyBlock.get_steps()] @@ -122,6 +123,7 @@ def prep(self): def tearDown(self): """ make sure to remove the temporary file """ + st.get_cpu_architecture = self.orig_get_cpu_architecture super(EasyConfigTest, self).tearDown() if os.path.exists(self.eb_file): os.remove(self.eb_file) @@ -310,6 +312,8 @@ def test_false_dep_version(self): Test use False as dependency version via dict using 'arch=' keys, which should result in filtering the dependency. """ + # silence warnings about missing easyconfigs for dependencies, we don't care + init_config(build_options={'silent': True}) arch = get_cpu_architecture() @@ -323,7 +327,7 @@ def test_false_dep_version(self): 'toolchain = {"name":"GCC", "version": "4.6.3"}', 'dependencies = [' ' ("first", "1.0"),', - ' ("seoond", {"arch=%s": False}),' % arch, + ' ("second", {"arch=%s": False}),' % arch, ']', 'builddependencies = [', ' ("first_build", {"arch=%s": False}),' % arch, @@ -337,6 +341,34 @@ def test_false_dep_version(self): self.assertEqual(deps[0]['name'], 'second_build') self.assertEqual(deps[1]['name'], 'first') + # more realistic example: only filter dep for POWER + self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.14"', + 'versionsuffix = "-test"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = {"name":"GCC", "version": "4.6.3"}', + 'dependencies = [' + ' ("not_on_power", {"arch=*": "1.2.3", "arch=POWER": False}),', + ']', + ]) + self.prep() + + # only non-POWER arch, dependency is retained + for arch in (AARCH64, X86_64): + eb = EasyConfig(self.eb_file) + deps = eb.dependencies() + self.assertEqual(len(deps), 1) + self.assertEqual(deps[0]['name'], 'not_on_power') + + # only power, dependency gets filtered + st.get_cpu_architecture = lambda: POWER + eb = EasyConfig(self.eb_file) + deps = eb.dependencies() + self.assertEqual(deps, []) + def test_extra_options(self): """ extra_options should allow other variables to be stored """ init_config(build_options={'silent': True}) From 5c1cfc642a169452e367919cdee12b704a569c76 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 1 Mar 2021 10:16:40 +0100 Subject: [PATCH 135/864] improve error message for pick_dep_version + check error handling in test for pick_dep_version --- easybuild/tools/systemtools.py | 6 ++++-- test/framework/systemtools.py | 7 +++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index cc1e73637b..d91d423b61 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -926,7 +926,8 @@ def pick_dep_version(dep_version): arch_keys = [x for x in dep_version.keys() if x.startswith(ARCH_KEY_PREFIX)] other_keys = [x for x in dep_version.keys() if x not in arch_keys] if other_keys: - raise EasyBuildError("Unexpected keys in version: %s. Only 'arch=' keys are supported", other_keys) + other_keys = ','.join(other_keys) + raise EasyBuildError("Unexpected keys in version: %s (only 'arch=' keys are supported)", other_keys) if arch_keys: host_arch_key = ARCH_KEY_PREFIX + get_cpu_architecture() star_arch_key = ARCH_KEY_PREFIX + '*' @@ -944,6 +945,7 @@ def pick_dep_version(dep_version): raise EasyBuildError("Found empty dict as version!") else: - raise EasyBuildError("Unknown value type for version: %s", dep_version) + typ = type(dep_version) + raise EasyBuildError("Unknown value type for version: %s (%s), should be string value", typ, dep_version) return result diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index ca7ba4ba2d..784fe18c11 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -927,6 +927,13 @@ def test_pick_dep_version(self): del dep_ver_dict['arch=POWER'] self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3') + # check how faulty input is handled + self.assertErrorRegex(EasyBuildError, "Found empty dict as version!", pick_dep_version, {}) + error_pattern = r"Unexpected keys in version: foo,bar \(only 'arch=' keys are supported\)" + self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, {'foo': '1.2', 'bar': '2.3'}) + error_pattern = r"Unknown value type for version: .* \(1.23\), should be string value" + self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, 1.23) + def test_check_os_dependency(self): """Test check_os_dependency.""" From a86e314981dcd29ffae28857be1e2f7b107bebb4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 1 Mar 2021 10:21:38 +0100 Subject: [PATCH 136/864] actually control arch we check with in test_false_dep_version --- test/framework/easyconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 391a22249a..c0697ed479 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -358,6 +358,7 @@ def test_false_dep_version(self): # only non-POWER arch, dependency is retained for arch in (AARCH64, X86_64): + st.get_cpu_architecture = lambda: arch eb = EasyConfig(self.eb_file) deps = eb.dependencies() self.assertEqual(len(deps), 1) From 37d90d90e56fc960f33bec75b86ed0df4dffa8a5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Mar 2021 16:13:30 +0100 Subject: [PATCH 137/864] ensure sorted keys in error message produced by pick_dep_version --- easybuild/tools/systemtools.py | 2 +- test/framework/systemtools.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index d91d423b61..6a3aa83861 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -926,7 +926,7 @@ def pick_dep_version(dep_version): arch_keys = [x for x in dep_version.keys() if x.startswith(ARCH_KEY_PREFIX)] other_keys = [x for x in dep_version.keys() if x not in arch_keys] if other_keys: - other_keys = ','.join(other_keys) + other_keys = ','.join(sorted(other_keys)) raise EasyBuildError("Unexpected keys in version: %s (only 'arch=' keys are supported)", other_keys) if arch_keys: host_arch_key = ARCH_KEY_PREFIX + get_cpu_architecture() diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 784fe18c11..3654882086 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -929,7 +929,7 @@ def test_pick_dep_version(self): # check how faulty input is handled self.assertErrorRegex(EasyBuildError, "Found empty dict as version!", pick_dep_version, {}) - error_pattern = r"Unexpected keys in version: foo,bar \(only 'arch=' keys are supported\)" + error_pattern = r"Unexpected keys in version: bar,foo \(only 'arch=' keys are supported\)" self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, {'foo': '1.2', 'bar': '2.3'}) error_pattern = r"Unknown value type for version: .* \(1.23\), should be string value" self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, 1.23) From 3c1eb7f9927e782b596fc68a7402121ee79fd3b7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Mar 2021 21:20:01 +0100 Subject: [PATCH 138/864] fix order of builddependencies vs dependencies in test easyconfig in test_false_dep_version --- test/framework/easyconfig.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index c0697ed479..d5d6458ef0 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -325,14 +325,14 @@ def test_false_dep_version(self): 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name":"GCC", "version": "4.6.3"}', - 'dependencies = [' - ' ("first", "1.0"),', - ' ("second", {"arch=%s": False}),' % arch, - ']', 'builddependencies = [', ' ("first_build", {"arch=%s": False}),' % arch, ' ("second_build", "2.0"),', ']', + 'dependencies = [' + ' ("first", "1.0"),', + ' ("second", {"arch=%s": False}),' % arch, + ']', ]) self.prep() eb = EasyConfig(self.eb_file) From 98f78294df313e33c689914b308673a08c55cde9 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 9 Mar 2021 11:22:25 +0800 Subject: [PATCH 139/864] also check for pending change requests in check_pr_eligible_to_merge --- easybuild/tools/github.py | 11 ++++++++++- test/framework/github.py | 8 +++++--- test/framework/options.py | 3 +++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 7f6d7faefa..362eca1cc7 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1136,11 +1136,20 @@ def not_eligible(msg): if not test_report_found: res = not_eligible(msg_tmpl % "(no test reports found)") - # check for approved review + # check for requested changes and approved review approved_review_by = [] + changes_requested_by = [] for review in pr_data['reviews']: if review['state'] == 'APPROVED': approved_review_by.append(review['user']['login']) + if review['state'] == 'CHANGES_REQUESTED': + changes_requested_by.append(review['user']['login']) + + msg_tmpl = "* no pending change requests: %s" + if changes_requested_by: + res = not_eligible(msg_tmpl % 'FAILED (changes requested by %s)' % ', '.join(changes_requested_by)) + else: + print_msg(msg_tmpl % 'OK', prefix=False) msg_tmpl = "* approved review: %s" if approved_review_by: diff --git a/test/framework/github.py b/test/framework/github.py index 2b9d7deb4d..e301dee0c7 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -654,7 +654,7 @@ def run_check(expected_result=False): 'issue_comments': [], 'milestone': None, 'number': '1234', - 'reviews': [], + 'reviews': [{'state': 'CHANGES_REQUESTED', 'user': {'login': 'boegel'}}], } test_result_warning_template = "* test suite passes: %s => not eligible for merging!" @@ -714,10 +714,12 @@ def run_check(expected_result=False): pr_data['issue_comments'].insert(2, {'body': 'lgtm'}) run_check() - pr_data['reviews'].append({'state': 'CHANGES_REQUESTED', 'user': {'login': 'boegel'}}) + expected_warning = "* no pending change requests: FAILED (changes requested by boegel)" + expected_warning += " => not eligible for merging!" run_check() - pr_data['reviews'].append({'state': 'APPROVED', 'user': {'login': 'boegel'}}) + pr_data['reviews'] = [{'state': 'APPROVED', 'user': {'login': 'boegel'}}] + expected_stdout += "* no pending change requests: OK\n" expected_stdout += "* approved review: OK (by boegel)\n" expected_warning = '' run_check() diff --git a/test/framework/options.py b/test/framework/options.py index 3a8539014c..48c92d6f1b 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4435,6 +4435,7 @@ def test_merge_pr(self): "Checking eligibility of easybuilders/easybuild-easyconfigs PR #4781 for merging...", "* test suite passes: OK", "* last test report is successful: OK", + "* no pending change requests: OK", "* milestone is set: OK (3.3.1)", ]) expected_stderr = '\n'.join([ @@ -4457,6 +4458,7 @@ def test_merge_pr(self): "* targets develop branch: OK", "* test suite passes: OK", "* last test report is successful: OK", + "* no pending change requests: OK", "* approved review: OK (by wpoely86)", "* milestone is set: OK (3.3.1)", '', @@ -4483,6 +4485,7 @@ def test_merge_pr(self): "Checking eligibility of easybuilders/easybuild-easyblocks PR #1206 for merging...", "* targets develop branch: OK", "* test suite passes: OK", + "* no pending change requests: OK", "* approved review: OK (by migueldiascosta)", "* milestone is set: OK (3.3.1)", '', From 0697c2c6421c990a50fc9f6cad626cd39373ff39 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 9 Mar 2021 15:09:17 +0800 Subject: [PATCH 140/864] also check mergeable_state in check_pr_eligible_to_merge --- easybuild/tools/github.py | 10 ++++++++++ test/framework/github.py | 12 +++++++++++- test/framework/options.py | 3 +++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 362eca1cc7..0d2097c41f 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1164,6 +1164,16 @@ def not_eligible(msg): else: res = not_eligible(msg_tmpl % 'no milestone found') + # check github mergeable state + msg_tmpl = "* mergeable state is clean: %s" + if pr_data['merged']: + print_msg(msg_tmpl % "PR is already merged", prefix=False) + elif pr_data['mergeable_state'] == GITHUB_MERGEABLE_STATE_CLEAN: + print_msg(msg_tmpl % "OK", prefix=False) + else: + reason = "FAILED (mergeable state is '%s')" % pr_data['mergeable_state'] + res = not_eligible(msg_tmpl % reason) + return res diff --git a/test/framework/github.py b/test/framework/github.py index e301dee0c7..a1b9b348ab 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -42,7 +42,8 @@ from easybuild.tools.config import build_option, module_classes from easybuild.tools.configobj import ConfigObj from easybuild.tools.filetools import read_file, write_file -from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, VALID_CLOSE_PR_REASONS +from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, GITHUB_MERGEABLE_STATE_CLEAN +from easybuild.tools.github import VALID_CLOSE_PR_REASONS from easybuild.tools.github import pick_default_branch from easybuild.tools.testing import post_pr_test_report, session_state from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters @@ -654,6 +655,8 @@ def run_check(expected_result=False): 'issue_comments': [], 'milestone': None, 'number': '1234', + 'merged': False, + 'mergeable_state': 'unknown', 'reviews': [{'state': 'CHANGES_REQUESTED', 'user': {'login': 'boegel'}}], } @@ -731,6 +734,13 @@ def run_check(expected_result=False): pr_data['milestone'] = {'title': '3.3.1'} expected_stdout += "* milestone is set: OK (3.3.1)\n" + # mergeable state must be clean + expected_warning = "* mergeable state is clean: FAILED (mergeable state is 'unknown')" + run_check() + + pr_data['mergeable_state'] = GITHUB_MERGEABLE_STATE_CLEAN + expected_stdout += "* mergeable state is clean: OK\n" + # all checks pass, PR is eligible for merging expected_warning = '' self.assertEqual(run_check(True), '') diff --git a/test/framework/options.py b/test/framework/options.py index 48c92d6f1b..36a4a06f95 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4437,6 +4437,7 @@ def test_merge_pr(self): "* last test report is successful: OK", "* no pending change requests: OK", "* milestone is set: OK (3.3.1)", + "* mergeable state is clean: PR is already merged", ]) expected_stderr = '\n'.join([ "* targets some_branch branch: FAILED; found 'develop' => not eligible for merging!", @@ -4461,6 +4462,7 @@ def test_merge_pr(self): "* no pending change requests: OK", "* approved review: OK (by wpoely86)", "* milestone is set: OK (3.3.1)", + "* mergeable state is clean: PR is already merged", '', "Review OK, merging pull request!", '', @@ -4488,6 +4490,7 @@ def test_merge_pr(self): "* no pending change requests: OK", "* approved review: OK (by migueldiascosta)", "* milestone is set: OK (3.3.1)", + "* mergeable state is clean: PR is already merged", '', "Review OK, merging pull request!", ]) From ea3873db02b53a0fe8764449f01ac880e2bad2cf Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 11 Mar 2021 16:33:10 +0800 Subject: [PATCH 141/864] add support for multiple PRs in --from-pr --- easybuild/framework/easyconfig/tools.py | 20 +++++++++----- easybuild/main.py | 5 ++-- easybuild/tools/filetools.py | 14 ++++++---- easybuild/tools/options.py | 13 ++++++--- easybuild/tools/testing.py | 33 +++++++++++++++-------- test/framework/options.py | 35 +++++++++++++++++++++++++ 6 files changed, 93 insertions(+), 27 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index e05af337ca..fc9c086ee4 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -321,7 +321,7 @@ def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_pr=False): # path where files touched in PR will be downloaded to pr_path = None if from_pr: - pr_path = os.path.join(tmpdir, "files_pr%s" % from_pr) + pr_path = os.path.join(tmpdir, "files_pr%s" % '_'.join(str(pr) for pr in from_pr)) return tweaked_ecs_paths, pr_path @@ -332,14 +332,20 @@ def det_easyconfig_paths(orig_paths): :param orig_paths: list of original easyconfig paths :return: list of paths to easyconfig files """ - from_pr = build_option('from_pr') + try: + from_pr_list = map(int, build_option('from_pr')) + except ValueError: + raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") + robot_path = build_option('robot_path') # list of specified easyconfig files ec_files = orig_paths[:] - if from_pr is not None: - pr_files = fetch_easyconfigs_from_pr(from_pr) + if from_pr_list is not None: + pr_files = [] + for pr in from_pr_list: + pr_files.extend(fetch_easyconfigs_from_pr(pr)) if ec_files: # replace paths for specified easyconfigs that are touched in PR @@ -746,8 +752,10 @@ def det_copy_ec_specs(orig_paths, from_pr): # to avoid potential trouble with already existing files in the working tmpdir # (note: we use a fixed subdirectory in the working tmpdir here rather than a unique random subdirectory, # to ensure that the caching for fetch_files_from_pr works across calls for the same PR) - tmpdir = os.path.join(tempfile.gettempdir(), 'fetch_files_from_pr_%s' % from_pr) - pr_paths = fetch_files_from_pr(pr=from_pr, path=tmpdir) + tmpdir = os.path.join(tempfile.gettempdir(), 'fetch_files_from_pr_%s' % '_'.join(str(pr) for pr in from_pr)) + pr_paths = [] + for pr in from_pr: + pr_paths.extend(fetch_files_from_pr(pr=pr, path=tmpdir)) # assume that files need to be copied to current working directory for now target_path = os.getcwd() diff --git a/easybuild/main.py b/easybuild/main.py index 3c7429ea78..af4b2d1ab0 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -207,7 +207,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): options, orig_paths = eb_go.options, eb_go.args global _log - (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, tweaked_ecs_paths) = cfg_settings + (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, + from_pr_list, tweaked_ecs_paths) = cfg_settings # load hook implementations (if any) hooks = load_hooks(options.hooks) @@ -318,7 +319,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if options.copy_ec: # figure out list of files to copy + target location (taking into account --from-pr) - orig_paths, target_path = det_copy_ec_specs(orig_paths, options.from_pr) + orig_paths, target_path = det_copy_ec_specs(orig_paths, from_pr_list) categorized_paths = categorize_files_by_type(orig_paths) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 2ac341a622..99c63ad2db 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -291,11 +291,15 @@ def symlink(source_path, symlink_path, use_abspath_source=True): if use_abspath_source: source_path = os.path.abspath(source_path) - try: - os.symlink(source_path, symlink_path) - _log.info("Symlinked %s to %s", source_path, symlink_path) - except OSError as err: - raise EasyBuildError("Symlinking %s to %s failed: %s", source_path, symlink_path, err) + if os.path.exists(symlink_path): + _log.info("Skipping symlinking %s to %s, link already exists", source_path, symlink_path) + # TODO: check if the symlink_path points to source_path + else: + try: + os.symlink(source_path, symlink_path) + _log.info("Symlinked %s to %s", source_path, symlink_path) + except OSError as err: + raise EasyBuildError("Symlinking %s to %s failed: %s", source_path, symlink_path, err) def remove_file(path): diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 605ed46ab9..18dcb82fd0 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -638,7 +638,7 @@ def github_options(self): 'check-style': ("Run a style check on the given easyconfigs", None, 'store_true', False), 'cleanup-easyconfigs': ("Clean up easyconfig files for pull request", None, 'store_true', True), 'dump-test-report': ("Dump test report to specified path", None, 'store_or_None', 'test_report.md'), - 'from-pr': ("Obtain easyconfigs from specified PR", int, 'store', None, {'metavar': 'PR#'}), + 'from-pr': ("Obtain easyconfigs from specified PR", 'strlist', 'store', [], {'metavar': 'PR#'}), 'git-working-dirs-path': ("Path to Git working directories for EasyBuild repositories", str, 'store', None), 'github-user': ("GitHub username", str, 'store', None), 'github-org': ("GitHub organization", str, 'store', None), @@ -1442,10 +1442,16 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) + # map --from-pr strlist to list of ints + try: + from_pr_list = map(int, eb_go.options.from_pr) + except ValueError: + raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") + # determine robot path # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs tweaked_ecs = try_to_generate and build_specs - tweaked_ecs_paths, pr_path = alt_easyconfig_paths(tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr) + tweaked_ecs_paths, pr_path = alt_easyconfig_paths(tmpdir, tweaked_ecs=tweaked_ecs, from_pr=from_pr_list) auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_path, auto_robot=auto_robot) log.debug("Full robot path: %s" % robot_path) @@ -1517,7 +1523,8 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): sys.path.remove(fake_vsc_path) sys.path.insert(0, new_fake_vsc_path) - return eb_go, (build_specs, log, logfile, robot_path, search_query, tmpdir, try_to_generate, tweaked_ecs_paths) + return eb_go, (build_specs, log, logfile, robot_path, search_query, tmpdir, try_to_generate, + from_pr_list, tweaked_ecs_paths) def process_software_build_specs(options): diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index cf93034570..1acd98d38d 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -138,7 +138,7 @@ def session_state(): } -def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_log=False): +def create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=None, gist_log=False): """Create test report for easyconfigs PR, in Markdown format.""" github_user = build_option('github_user') @@ -149,9 +149,10 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_l # create a gist with a full test report test_report = [] - if pr_nr is not None: + if pr_nrs is not None: + pr_urls = ["https://github.com/%s/%s/pull/%s" % (pr_target_account, pr_target_repo, pr_nr) for pr_nr in pr_nrs] test_report.extend([ - "Test report for https://github.com/%s/%s/pull/%s" % (pr_target_account, pr_target_repo, pr_nr), + "Test report for %s" % ', '.join(pr_urls), "", ]) test_report.extend([ @@ -182,8 +183,8 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_l logtxt = read_file(ec_res['log_file']) partial_log_txt = '\n'.join(logtxt.split('\n')[-500:]) descr = "(partial) EasyBuild log for failed build of %s" % ec['spec'] - if pr_nr is not None: - descr += " (PR #%s)" % pr_nr + if pr_nrs is not None: + descr += " (PR #%s)" % ', #'.join(pr_nrs) fn = '%s_partial.log' % os.path.basename(ec['spec'])[:-3] gist_url = create_gist(partial_log_txt, fn, descr=descr, github_user=github_user) test_log = "(partial log available at %s)" % gist_url @@ -317,19 +318,29 @@ def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state :param init_session_state: initial session state info to include in test report """ dump_path = build_option('dump_test_report') - pr_nr = build_option('from_pr') - eb_pr_nrs = build_option('include_easyblocks_from_pr') + + try: + pr_nrs = map(int, build_option('from_pr')) + except ValueError: + raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") + + try: + eb_pr_nrs = map(int, build_option('include_easyblocks_from_pr')) + except ValueError: + raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s.") + upload = build_option('upload_test_report') if upload: msg = msg + " (%d easyconfigs in total)" % orig_cnt - test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nr=pr_nr, gist_log=True) - if pr_nr: + test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=pr_nrs, gist_log=True) + if pr_nrs: # upload test report to gist and issue a comment in the PR to notify - txt = post_pr_test_report(pr_nr, GITHUB_EASYCONFIGS_REPO, test_report, msg, init_session_state, success) + for pr_nr in pr_nrs: + txt = post_pr_test_report(pr_nr, GITHUB_EASYCONFIGS_REPO, test_report, msg, init_session_state, success) elif eb_pr_nrs: # upload test report to gist and issue a comment in the easyblocks PR to notify - for eb_pr_nr in map(int, eb_pr_nrs): + for eb_pr_nr in eb_pr_nrs: txt = post_pr_test_report(eb_pr_nr, GITHUB_EASYBLOCKS_REPO, test_report, msg, init_session_state, success) else: diff --git a/test/framework/options.py b/test/framework/options.py index 3a8539014c..171cb430e0 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1731,6 +1731,41 @@ def test_from_pr(self): print("Ignoring URLError '%s' in test_from_pr" % err) shutil.rmtree(tmpdir) + # test with multiple prs + tmpdir = tempfile.mkdtemp() + args = [ + # PRs for ReFrame 3.4.1 and 3.5.0 + '--from-pr=12150,12366', + '--dry-run', + # an argument must be specified to --robot, since easybuild-easyconfigs may not be installed + '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '--unittest-file=%s' % self.logfile, + '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user + '--tmpdir=%s' % tmpdir, + ] + try: + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) + modules = [ + (tmpdir, 'ReFrame/3.4.1'), + (tmpdir, 'ReFrame/3.5.0'), + ] + for path_prefix, module in modules: + ec_fn = "%s.eb" % '-'.join(module.split('/')) + path = '.*%s' % os.path.dirname(path_prefix) + regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path, ec_fn, module), re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + + # make sure that *only* these modules are listed, no others + regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) + self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules)) + + pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr12150_12366') + regex = re.compile("Appended list of robot search paths with %s:" % pr_tmpdir, re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + except URLError as err: + print("Ignoring URLError '%s' in test_from_pr" % err) + shutil.rmtree(tmpdir) + def test_from_pr_token_log(self): """Check that --from-pr doesn't leak GitHub token in log.""" if self.github_token is None: From ca29ca9ddc5ec79b012c46345add1eaa04cbc9bd Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Fri, 12 Mar 2021 06:17:51 +0800 Subject: [PATCH 142/864] ignore undismissed 'changes requested' review if there is an 'approved' review by the same user --- easybuild/tools/github.py | 10 +++++++--- test/framework/github.py | 14 +++++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 0d2097c41f..e874642a74 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1136,14 +1136,18 @@ def not_eligible(msg): if not test_report_found: res = not_eligible(msg_tmpl % "(no test reports found)") - # check for requested changes and approved review + # check for approved review approved_review_by = [] - changes_requested_by = [] for review in pr_data['reviews']: if review['state'] == 'APPROVED': approved_review_by.append(review['user']['login']) + + # check for requested changes + changes_requested_by = [] + for review in pr_data['reviews']: if review['state'] == 'CHANGES_REQUESTED': - changes_requested_by.append(review['user']['login']) + if review['user']['login'] not in approved_review_by: + changes_requested_by.append(review['user']['login']) msg_tmpl = "* no pending change requests: %s" if changes_requested_by: diff --git a/test/framework/github.py b/test/framework/github.py index a1b9b348ab..9dadbfb25f 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -721,9 +721,17 @@ def run_check(expected_result=False): expected_warning += " => not eligible for merging!" run_check() - pr_data['reviews'] = [{'state': 'APPROVED', 'user': {'login': 'boegel'}}] - expected_stdout += "* no pending change requests: OK\n" - expected_stdout += "* approved review: OK (by boegel)\n" + # if PR is approved by a different user that requested changes and that request has not been dismissed, + # the PR is still not mergeable + pr_data['reviews'].append({'state': 'APPROVED', 'user': {'login': 'not_boegel'}}) + expected_stdout_saved = expected_stdout + expected_stdout += "* approved review: OK (by not_boegel)\n" + run_check() + + # if the user that requested changes approves the PR, it's mergeable + pr_data['reviews'].append({'state': 'APPROVED', 'user': {'login': 'boegel'}}) + expected_stdout = expected_stdout_saved + "* no pending change requests: OK\n" + expected_stdout += "* approved review: OK (by not_boegel, boegel)\n" expected_warning = '' run_check() From 9a7f0e4886feb95f76754eb60aca845dc02f3187 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Fri, 12 Mar 2021 14:04:06 +0800 Subject: [PATCH 143/864] fix tests after adding support for multiple PRs in --from-pr --- easybuild/framework/easyconfig/tools.py | 3 ++ test/framework/options.py | 1 + .../easyblocks/generic/pythonbundle.py | 40 +++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index fc9c086ee4..14913baff2 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -731,6 +731,9 @@ def avail_easyblocks(): def det_copy_ec_specs(orig_paths, from_pr): """Determine list of paths + target directory for --copy-ec.""" + if from_pr is not None and not isinstance(from_pr, list): + from_pr = [from_pr] + target_path, paths = None, [] # if only one argument is specified, use current directory as target directory diff --git a/test/framework/options.py b/test/framework/options.py index 1b363b3857..f4f290f340 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -821,6 +821,7 @@ def test_list_easyblocks(self): r'\| \| \|-- EB_toytoy', r'\| \|-- Toy_Extension', r'\|-- ModuleRC', + r'\|-- PythonBundle', r'\|-- Toolchain', r'Extension', r'\|-- ExtensionEasyBlock', diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py b/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py new file mode 100644 index 0000000000..7146b8ecd3 --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py @@ -0,0 +1,40 @@ +## +# Copyright 2009-2020 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Dummy easyblock for Makecp. + +@author: Miguel Dias Costa (National University of Singapore) +""" +from easybuild.framework.easyblock import EasyBlock + + +class PythonBundle(EasyBlock): + """Dummy support for bundle of modules.""" + + @staticmethod + def extra_options(extra_vars=None): + if extra_vars is None: + extra_vars = {} + return EasyBlock.extra_options(extra_vars) From 8cb4f405c4cf477307e04ec0d9d866a24b1d24ce Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Fri, 12 Mar 2021 16:23:42 +0800 Subject: [PATCH 144/864] filter duplicates in changes_requested_by --- easybuild/tools/github.py | 2 +- test/framework/github.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index e874642a74..63cdf81213 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1146,7 +1146,7 @@ def not_eligible(msg): changes_requested_by = [] for review in pr_data['reviews']: if review['state'] == 'CHANGES_REQUESTED': - if review['user']['login'] not in approved_review_by: + if review['user']['login'] not in approved_review_by + changes_requested_by: changes_requested_by.append(review['user']['login']) msg_tmpl = "* no pending change requests: %s" diff --git a/test/framework/github.py b/test/framework/github.py index 9dadbfb25f..deb1a706f7 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -657,7 +657,9 @@ def run_check(expected_result=False): 'number': '1234', 'merged': False, 'mergeable_state': 'unknown', - 'reviews': [{'state': 'CHANGES_REQUESTED', 'user': {'login': 'boegel'}}], + 'reviews': [{'state': 'CHANGES_REQUESTED', 'user': {'login': 'boegel'}}, + # to check that duplicates are filtered + {'state': 'CHANGES_REQUESTED', 'user': {'login': 'boegel'}}], } test_result_warning_template = "* test suite passes: %s => not eligible for merging!" From 987599cd9ac0f31ae17ecda03a767cde6df9298f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 12 Mar 2021 14:40:25 +0100 Subject: [PATCH 145/864] Sort output of eb --search in natural order (respecting numbers) --- easybuild/tools/filetools.py | 4 ++-- easybuild/tools/utilities.py | 7 ++++++ test/framework/filetools.py | 41 ++++++++++++++++++++++-------------- test/framework/options.py | 16 ++++++-------- test/framework/robot.py | 2 +- 5 files changed, 42 insertions(+), 28 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 2ac341a622..34cf1c131f 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -60,7 +60,7 @@ from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, GENERIC_EASYBLOCK_PKG, build_option, install_path from easybuild.tools.py2vs3 import HTMLParser, std_urllib, string_type -from easybuild.tools.utilities import nub, remove_unwanted_chars +from easybuild.tools.utilities import nub, remove_unwanted_chars, natural_keys try: import requests @@ -1021,7 +1021,7 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen else: path_hits.append(os.path.join(path, filepath)) - path_hits = sorted(path_hits) + path_hits = sorted(path_hits, key=natural_keys) if path_hits: common_prefix = det_common_path_prefix(path_hits) diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index e1b19b2985..15c501de8b 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -316,3 +316,10 @@ def time2str(delta): res = '%d %s %d min %d sec' % (hours, hours_str, mins, secs) return res + + +def natural_keys(key): + """Can be used as the sort key in list.sort(key=natural_keys) to sort in natural order (i.e. respecting numbers)""" + def try_to_int(key_part): + return int(key_part) if key_part.isdigit() else key_part + return [try_to_int(key_part) for key_part in re.split(r'(\d+)', key)] diff --git a/test/framework/filetools.py b/test/framework/filetools.py index c948477aae..7073e1b537 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2158,11 +2158,11 @@ def test_search_file(self): self.assertEqual(var_defs, []) self.assertEqual(len(hits), 5) self.assertTrue(all(os.path.exists(p) for p in hits)) - self.assertTrue(hits[0].endswith('/hwloc-1.11.8-GCC-4.6.4.eb')) - self.assertTrue(hits[1].endswith('/hwloc-1.11.8-GCC-6.4.0-2.28.eb')) - self.assertTrue(hits[2].endswith('/hwloc-1.11.8-GCC-7.3.0-2.30.eb')) - self.assertTrue(hits[3].endswith('/hwloc-1.6.2-GCC-4.9.3-2.26.eb')) - self.assertTrue(hits[4].endswith('/hwloc-1.8-gcccuda-2018a.eb')) + self.assertTrue(hits[0].endswith('/hwloc-1.6.2-GCC-4.9.3-2.26.eb')) + self.assertTrue(hits[1].endswith('/hwloc-1.8-gcccuda-2018a.eb')) + self.assertTrue(hits[2].endswith('/hwloc-1.11.8-GCC-4.6.4.eb')) + self.assertTrue(hits[3].endswith('/hwloc-1.11.8-GCC-6.4.0-2.28.eb')) + self.assertTrue(hits[4].endswith('/hwloc-1.11.8-GCC-7.3.0-2.30.eb')) # also test case-sensitive searching var_defs, hits_bis = ft.search_file([test_ecs], 'HWLOC', silent=True, case_sensitive=True) @@ -2176,9 +2176,12 @@ def test_search_file(self): # check filename-only mode var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, filename_only=True) self.assertEqual(var_defs, []) - self.assertEqual(hits, ['hwloc-1.11.8-GCC-4.6.4.eb', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb', - 'hwloc-1.11.8-GCC-7.3.0-2.30.eb', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb', - 'hwloc-1.8-gcccuda-2018a.eb']) + self.assertEqual(hits, ['hwloc-1.6.2-GCC-4.9.3-2.26.eb', + 'hwloc-1.8-gcccuda-2018a.eb', + 'hwloc-1.11.8-GCC-4.6.4.eb', + 'hwloc-1.11.8-GCC-6.4.0-2.28.eb', + 'hwloc-1.11.8-GCC-7.3.0-2.30.eb', + ]) # check specifying of ignored dirs var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, ignore_dirs=['hwloc']) @@ -2187,28 +2190,34 @@ def test_search_file(self): # check short mode var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, short=True) self.assertEqual(var_defs, [('CFGS1', os.path.join(test_ecs, 'h', 'hwloc'))]) - self.assertEqual(hits, ['$CFGS1/hwloc-1.11.8-GCC-4.6.4.eb', '$CFGS1/hwloc-1.11.8-GCC-6.4.0-2.28.eb', - '$CFGS1/hwloc-1.11.8-GCC-7.3.0-2.30.eb', '$CFGS1/hwloc-1.6.2-GCC-4.9.3-2.26.eb', - '$CFGS1/hwloc-1.8-gcccuda-2018a.eb']) + self.assertEqual(hits, ['$CFGS1/hwloc-1.6.2-GCC-4.9.3-2.26.eb', + '$CFGS1/hwloc-1.8-gcccuda-2018a.eb', + '$CFGS1/hwloc-1.11.8-GCC-4.6.4.eb', + '$CFGS1/hwloc-1.11.8-GCC-6.4.0-2.28.eb', + '$CFGS1/hwloc-1.11.8-GCC-7.3.0-2.30.eb' + ]) # check terse mode (implies 'silent', overrides 'short') var_defs, hits = ft.search_file([test_ecs], 'HWLOC', terse=True, short=True) self.assertEqual(var_defs, []) expected = [ + os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb'), + os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-7.3.0-2.30.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), ] self.assertEqual(hits, expected) # check combo of terse and filename-only var_defs, hits = ft.search_file([test_ecs], 'HWLOC', terse=True, filename_only=True) self.assertEqual(var_defs, []) - self.assertEqual(hits, ['hwloc-1.11.8-GCC-4.6.4.eb', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb', - 'hwloc-1.11.8-GCC-7.3.0-2.30.eb', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb', - 'hwloc-1.8-gcccuda-2018a.eb']) + self.assertEqual(hits, ['hwloc-1.6.2-GCC-4.9.3-2.26.eb', + 'hwloc-1.8-gcccuda-2018a.eb', + 'hwloc-1.11.8-GCC-4.6.4.eb', + 'hwloc-1.11.8-GCC-6.4.0-2.28.eb', + 'hwloc-1.11.8-GCC-7.3.0-2.30.eb', + ]) # patterns that include special characters + (or ++) shouldn't cause trouble # cfr. https://github.com/easybuilders/easybuild-framework/issues/2966 diff --git a/test/framework/options.py b/test/framework/options.py index 36a4a06f95..e046645514 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -988,7 +988,7 @@ def test_ignore_index(self): toy_ec = os.path.join(test_ecs_dir, 'test_ecs', 't', 'toy', 'toy-0.0.eb') copy_file(toy_ec, self.test_prefix) - toy_ec_list = ['toy-0.0.eb', 'toy-1.2.3.eb', 'toy-4.5.6.eb'] + toy_ec_list = ['toy-0.0.eb', 'toy-1.2.3.eb', 'toy-4.5.6.eb', 'toy-11.5.6.eb'] # install index that list more files than are actually available, # so we can check whether it's used @@ -998,15 +998,16 @@ def test_ignore_index(self): args = [ '--search=toy', '--robot-paths=%s' % self.test_prefix, + '--terse', ] self.mock_stdout(True) self.eb_main(args, testing=False, raise_error=True) stdout = self.get_stdout() self.mock_stdout(False) - for toy_ec_fn in toy_ec_list: - regex = re.compile(re.escape(os.path.join(self.test_prefix, toy_ec_fn)), re.M) - self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + # Also checks for ordering: 11.x comes last! + expected_output = '\n'.join(os.path.join(self.test_prefix, ec) for ec in toy_ec_list) + '\n' + self.assertEqual(stdout, expected_output) args.append('--ignore-index') self.mock_stdout(True) @@ -1014,11 +1015,8 @@ def test_ignore_index(self): stdout = self.get_stdout() self.mock_stdout(False) - regex = re.compile(re.escape(os.path.join(self.test_prefix, 'toy-0.0.eb')), re.M) - self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) - for toy_ec_fn in ['toy-1.2.3.eb', 'toy-4.5.6.eb']: - regex = re.compile(re.escape(os.path.join(self.test_prefix, toy_ec_fn)), re.M) - self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in: %s" % (regex.pattern, stdout)) + # This should be the only EC found + self.assertEqual(stdout, os.path.join(self.test_prefix, 'toy-0.0.eb') + '\n') def test_search_archived(self): "Test searching for archived easyconfigs" diff --git a/test/framework/robot.py b/test/framework/robot.py index 9c29cb25cd..0f44bf34e3 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1513,10 +1513,10 @@ def test_search_easyconfigs(self): paths = search_easyconfigs('8-gcc', consider_extra_paths=False, print_result=False) ref_paths = [ + os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-7.3.0-2.30.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), os.path.join(test_ecs, 'o', 'OpenBLAS', 'OpenBLAS-0.2.8-GCC-4.8.2-LAPACK-3.4.2.eb') ] self.assertEqual(paths, ref_paths) From 5f7697639226a6324527314a4a7dda35f801c6fe Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 12 Mar 2021 15:18:05 +0100 Subject: [PATCH 146/864] Add utilities_test.py --- test/framework/easyblock.py | 30 ----------- test/framework/utilities_test.py | 89 ++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 30 deletions(-) create mode 100644 test/framework/utilities_test.py diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index d72097a990..1a784dc521 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -34,7 +34,6 @@ import sys import tempfile from inspect import cleandoc -from datetime import datetime from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -50,7 +49,6 @@ from easybuild.tools.filetools import verify_checksum, write_file from easybuild.tools.module_generator import module_generator from easybuild.tools.modules import reset_module_caches -from easybuild.tools.utilities import time2str from easybuild.tools.version import get_git_revision, this_is_easybuild from easybuild.tools.py2vs3 import string_type @@ -2108,34 +2106,6 @@ def test_avail_easyblocks(self): self.assertEqual(hpl['class'], 'EB_HPL') self.assertTrue(hpl['loc'].endswith('sandbox/easybuild/easyblocks/h/hpl.py')) - def test_time2str(self): - """Test time2str function.""" - - start = datetime(2019, 7, 30, 5, 14, 23) - - test_cases = [ - (start, "0 sec"), - (datetime(2019, 7, 30, 5, 14, 37), "14 sec"), - (datetime(2019, 7, 30, 5, 15, 22), "59 sec"), - (datetime(2019, 7, 30, 5, 15, 23), "1 min 0 sec"), - (datetime(2019, 7, 30, 5, 16, 22), "1 min 59 sec"), - (datetime(2019, 7, 30, 5, 37, 26), "23 min 3 sec"), - (datetime(2019, 7, 30, 6, 14, 22), "59 min 59 sec"), - (datetime(2019, 7, 30, 6, 14, 23), "1 hour 0 min 0 sec"), - (datetime(2019, 7, 30, 6, 49, 14), "1 hour 34 min 51 sec"), - (datetime(2019, 7, 30, 7, 14, 23), "2 hours 0 min 0 sec"), - (datetime(2019, 7, 30, 8, 35, 59), "3 hours 21 min 36 sec"), - (datetime(2019, 7, 30, 16, 29, 24), "11 hours 15 min 1 sec"), - (datetime(2019, 7, 31, 5, 14, 22), "23 hours 59 min 59 sec"), - (datetime(2019, 7, 31, 5, 14, 23), "24 hours 0 min 0 sec"), - (datetime(2019, 8, 5, 20, 39, 44), "159 hours 25 min 21 sec"), - ] - for end, expected in test_cases: - self.assertEqual(time2str(end - start), expected) - - error_pattern = "Incorrect value type provided to time2str, should be datetime.timedelta: <.* 'int'>" - self.assertErrorRegex(EasyBuildError, error_pattern, time2str, 123) - def test_sanity_check_paths_verification(self): """Test verification of sanity_check_paths w.r.t. keys & values.""" diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py new file mode 100644 index 0000000000..689a2ab40c --- /dev/null +++ b/test/framework/utilities_test.py @@ -0,0 +1,89 @@ +## +# Copyright 2012-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Unit tests for utilities.py + +@author: Jens Timmerman (Ghent University) +@author: Kenneth Hoste (Ghent University) +@author: Alexander Grund (TU Dresden) +""" +import os +import sys +import tempfile +from datetime import datetime +from unittest import TextTestRunner + +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.utilities import time2str + + +class UtilitiesTest(EnhancedTestCase): + """Class for utilities testcases """ + + def setUp(self): + """ setup """ + super(UtilitiesTest, self).setUp() + + self.test_tmp_logdir = tempfile.mkdtemp() + os.environ['EASYBUILD_TMP_LOGDIR'] = self.test_tmp_logdir + + def test_time2str(self): + """Test time2str function.""" + + start = datetime(2019, 7, 30, 5, 14, 23) + + test_cases = [ + (start, "0 sec"), + (datetime(2019, 7, 30, 5, 14, 37), "14 sec"), + (datetime(2019, 7, 30, 5, 15, 22), "59 sec"), + (datetime(2019, 7, 30, 5, 15, 23), "1 min 0 sec"), + (datetime(2019, 7, 30, 5, 16, 22), "1 min 59 sec"), + (datetime(2019, 7, 30, 5, 37, 26), "23 min 3 sec"), + (datetime(2019, 7, 30, 6, 14, 22), "59 min 59 sec"), + (datetime(2019, 7, 30, 6, 14, 23), "1 hour 0 min 0 sec"), + (datetime(2019, 7, 30, 6, 49, 14), "1 hour 34 min 51 sec"), + (datetime(2019, 7, 30, 7, 14, 23), "2 hours 0 min 0 sec"), + (datetime(2019, 7, 30, 8, 35, 59), "3 hours 21 min 36 sec"), + (datetime(2019, 7, 30, 16, 29, 24), "11 hours 15 min 1 sec"), + (datetime(2019, 7, 31, 5, 14, 22), "23 hours 59 min 59 sec"), + (datetime(2019, 7, 31, 5, 14, 23), "24 hours 0 min 0 sec"), + (datetime(2019, 8, 5, 20, 39, 44), "159 hours 25 min 21 sec"), + ] + for end, expected in test_cases: + self.assertEqual(time2str(end - start), expected) + + error_pattern = "Incorrect value type provided to time2str, should be datetime.timedelta: <.* 'int'>" + self.assertErrorRegex(EasyBuildError, error_pattern, time2str, 123) + + +def suite(): + """ return all the tests in this file """ + return TestLoaderFiltered().loadTestsFromTestCase(UtilitiesTest, sys.argv[1:]) + + +if __name__ == '__main__': + res = TextTestRunner(verbosity=1).run(suite()) + sys.exit(len(res.failures)) From 9b23d0c61f00c45d08d6608eed58cccf8f0e4eb8 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 12 Mar 2021 15:25:53 +0100 Subject: [PATCH 147/864] Add test for natural_keys --- test/framework/utilities_test.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index 689a2ab40c..efe9fb7fd3 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -30,6 +30,7 @@ @author: Alexander Grund (TU Dresden) """ import os +import random import sys import tempfile from datetime import datetime @@ -37,7 +38,7 @@ from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.utilities import time2str +from easybuild.tools.utilities import time2str, natural_keys class UtilitiesTest(EnhancedTestCase): @@ -78,6 +79,25 @@ def test_time2str(self): error_pattern = "Incorrect value type provided to time2str, should be datetime.timedelta: <.* 'int'>" self.assertErrorRegex(EasyBuildError, error_pattern, time2str, 123) + def test_natural_keys(self): + """Test the natural_keys function""" + sorted_items = [ + 'ACoolSw-1.0', + 'ACoolSw-2.1', + 'ACoolSw-11.0', + 'ACoolSw-23.0', + 'ACoolSw-30.0', + 'ACoolSw-30.1', + 'BigNumber-1234567890', + 'BigNumber-1234567891', + 'NoNumbers', + 'VeryLastEntry-10' + ] + shuffled_items = sorted_items[:] + random.shuffle(shuffled_items) + shuffled_items.sort(key=natural_keys) + self.assertEqual(shuffled_items, sorted_items) + def suite(): """ return all the tests in this file """ From e8d859d38971335f48013d7ee8cca839e8728fc8 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 12 Mar 2021 16:58:54 +0100 Subject: [PATCH 148/864] Don't load index when it is ignored --- easybuild/tools/filetools.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 34cf1c131f..eca11a0f85 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1000,8 +1000,11 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen if not terse: print_msg("Searching (case-insensitive) for '%s' in %s " % (query.pattern, path), log=_log, silent=silent) - path_index = load_index(path, ignore_dirs=ignore_dirs) - if path_index is None or build_option('ignore_index'): + if build_option('ignore_index'): + path_index = None + else: + path_index = load_index(path, ignore_dirs=ignore_dirs) + if path_index is None: if os.path.exists(path): _log.info("No index found for %s, creating one...", path) path_index = create_index(path, ignore_dirs=ignore_dirs) From e1f744e96bedf661307a700fc418fcd567dd8463 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 12 Mar 2021 17:01:30 +0100 Subject: [PATCH 149/864] Avoid additional work handling the common prefix of paths --- easybuild/tools/filetools.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index eca11a0f85..3c79e57dd6 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1027,12 +1027,14 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen path_hits = sorted(path_hits, key=natural_keys) if path_hits: - common_prefix = det_common_path_prefix(path_hits) - if not terse and short and common_prefix is not None and len(common_prefix) > len(var) * 2: - var_defs.append((var, common_prefix)) - hits.extend([os.path.join('$%s' % var, fn[len(common_prefix) + 1:]) for fn in path_hits]) - else: - hits.extend(path_hits) + if not terse and short: + common_prefix = det_common_path_prefix(path_hits) + if common_prefix is not None and len(common_prefix) > len(var) * 2: + var_defs.append((var, common_prefix)) + var_spec = '$' + var + # Replace the common prefix by var_spec + path_hits = (var_spec + fn[len(common_prefix):] for fn in path_hits) + hits.extend(path_hits) return var_defs, hits From 724b8a9e438d0a302c44b3823d0a4398a90d7522 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 9 Mar 2021 17:40:58 +0100 Subject: [PATCH 150/864] add tests for 'eb' command --- .github/workflows/eb_command.yml | 88 ++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 .github/workflows/eb_command.yml diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml new file mode 100644 index 0000000000..fcda2b9c55 --- /dev/null +++ b/.github/workflows/eb_command.yml @@ -0,0 +1,88 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Tests for the 'eb' command +on: [push, pull_request] +jobs: + test-eb: + runs-on: ubuntu-18.04 + strategy: + matrix: + python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] + fail-fast: false + steps: + - uses: actions/checkout@v2 + + - name: set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{matrix.python}} + architecture: x64 + + - name: install OS & Python packages + run: | + # check Python version + python -V + # update to latest pip, check version + pip install --upgrade pip + pip --version + # install packages required for modules tool + sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev + # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 + # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists + if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then + sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so + fi + + - name: install modules tool + run: | + # avoid downloading modules tool sources into easybuild-framework dir + cd $HOME + export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh + # install Lmod + source $INSTALL_DEP Lmod-8.4.26 $HOME + # changes in environment are not passed to other steps, so need to create files... + echo $MOD_INIT > mod_init + echo $PATH > path + if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi + + - name: install EasyBuild framework + run: | + # install from source distribution tarball, to test release as published on PyPI + python setup.py sdist + ls dist + export PREFIX=/tmp/$USER/$GITHUB_SHA + pip install --prefix $PREFIX dist/easybuild-framework*tar.gz + + - name: run tests for 'eb' command + env: + EB_VERBOSE: 1 + run: | + # run tests *outside* of checked out easybuild-framework directory, + # to ensure we're testing installed version (see previous step) + cd $HOME + # initialize environment for modules tool + if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi + source $(cat $HOME/mod_init); type module + # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that); + # also pick up changes to $PATH set by sourcing $MOD_INIT + export PREFIX=/tmp/$USER/$GITHUB_SHA + export PATH=$PREFIX/bin:$(cat $HOME/path) + export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH + # run --version, capture (verbose) output + eb --version | tee eb_version.out 2>&1 + # determine active Python version + pymajver=$(python -c 'import sys; print(sys.version_info[0])') + pymajminver=$(python -c 'import sys; print(".".join(str(x) for x in sys.version_info[:2]))') + # check patterns in verbose output + for pattern in "^>> Considering .python.\.\.\." "^>> .python. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> Selected Python command: python \(.*/bin/python\)" "^This is EasyBuild 4\.[0-9.]\+"; do + echo "Looking for pattern \"${pattern}\" in eb_version.out..." + grep "$pattern" eb_version.out + done + # also check when specifying Python command via $EB_PYTHON + for eb_python in "python${pymajver}" "python${pymajminver}"; do + export EB_PYTHON="${eb_python}" + eb --version | tee eb_version.out 2>&1 + for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 4\.[0-9.]\+"; do + echo "Looking for pattern \"${pattern}\" in eb_version.out..." + grep "$pattern" eb_version.out + done + done From b9af38caf4df8517a6a4c70dbfa3cf3339994f4c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 14 Mar 2021 16:49:22 +0100 Subject: [PATCH 151/864] enhance 'eb' command to ensure that easybuild.main can be imported before settling on python* command to use --- .github/workflows/eb_command.yml | 4 ++-- eb | 24 +++++++++++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml index fcda2b9c55..d0b72e3079 100644 --- a/.github/workflows/eb_command.yml +++ b/.github/workflows/eb_command.yml @@ -73,7 +73,7 @@ jobs: pymajver=$(python -c 'import sys; print(sys.version_info[0])') pymajminver=$(python -c 'import sys; print(".".join(str(x) for x in sys.version_info[:2]))') # check patterns in verbose output - for pattern in "^>> Considering .python.\.\.\." "^>> .python. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> Selected Python command: python \(.*/bin/python\)" "^This is EasyBuild 4\.[0-9.]\+"; do + for pattern in "^>> Considering .python.\.\.\." "^>> .python. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python' is able to import 'easybuild.main', so retaining it" "^>> Selected Python command: python \(.*/bin/python\)" "^This is EasyBuild 4\.[0-9.]\+"; do echo "Looking for pattern \"${pattern}\" in eb_version.out..." grep "$pattern" eb_version.out done @@ -81,7 +81,7 @@ jobs: for eb_python in "python${pymajver}" "python${pymajminver}"; do export EB_PYTHON="${eb_python}" eb --version | tee eb_version.out 2>&1 - for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 4\.[0-9.]\+"; do + for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> '${eb_python}' is able to import 'easybuild.main', so retaining it" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 4\.[0-9.]\+"; do echo "Looking for pattern \"${pattern}\" in eb_version.out..." grep "$pattern" eb_version.out done diff --git a/eb b/eb index 8968892dbe..592b873432 100755 --- a/eb +++ b/eb @@ -37,6 +37,7 @@ REQ_MIN_PY2VER=6 REQ_MIN_PY3VER=5 +EASYBUILD_MAIN='easybuild.main' function verbose() { if [ ! -z ${EB_VERBOSE} ]; then echo ">> $1"; fi @@ -52,6 +53,8 @@ for python_cmd in ${EB_PYTHON} ${EB_INSTALLPYTHON} 'python' 'python3' 'python2'; verbose "Considering '$python_cmd'..." + # check whether python* command being considered is available + # (using 'command -v', since 'which' implies an extra dependency) command -v $python_cmd &> /dev/null if [ $? -eq 0 ]; then @@ -63,10 +66,25 @@ for python_cmd in ${EB_PYTHON} ${EB_INSTALLPYTHON} 'python' 'python3' 'python2'; if [ $pyver_maj -eq 2 ] && [ $pyver_min -ge $REQ_MIN_PY2VER ]; then verbose "'$python_cmd' version: $pyver, which matches Python 2 version requirement (>= 2.$REQ_MIN_PY2VER)" PYTHON=$python_cmd - break elif [ $pyver_maj -eq 3 ] && [ $pyver_min -ge $REQ_MIN_PY3VER ]; then verbose "'$python_cmd' version: $pyver, which matches Python 3 version requirement (>= 3.$REQ_MIN_PY3VER)" PYTHON=$python_cmd + fi + + if [ ! -z $PYTHON ]; then + # check whether easybuild.main is available for selected python command + $PYTHON -c "import $EASYBUILD_MAIN" 2> /dev/null + if [ $? -eq 0 ]; then + verbose "'$python_cmd' is able to import '$EASYBUILD_MAIN', so retaining it" + else + # if easybuild.main is not available, don't use this python command, keep searching... + verbose "'$python_cmd' is NOT able to import '$EASYBUILD_MAIN', so NOT retaining it" + unset PYTHON + fi + fi + + # break out of for loop if we've found a valid python command + if [ ! -z $PYTHON ]; then break fi else @@ -97,5 +115,5 @@ fi export EB_SCRIPT_PATH=$0 -verbose "$PYTHON -m easybuild.main `echo \"$@\"`" -$PYTHON -m easybuild.main "$@" +verbose "$PYTHON -m $EASYBUILD_MAIN `echo \"$@\"`" +$PYTHON -m $EASYBUILD_MAIN "$@" From 8092e69828fd2340cbdf82d76f94a5e283efdbb7 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 16 Mar 2021 14:53:48 +0800 Subject: [PATCH 152/864] use list comprehension instead of map when parsing from_pr strlist options --- easybuild/framework/easyconfig/tools.py | 2 +- easybuild/tools/options.py | 4 ++-- easybuild/tools/testing.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 14913baff2..20aa2e9430 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -333,7 +333,7 @@ def det_easyconfig_paths(orig_paths): :return: list of paths to easyconfig files """ try: - from_pr_list = map(int, build_option('from_pr')) + from_pr_list = [int(pr_nr) for pr_nr in build_option('from_pr')] except ValueError: raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 18dcb82fd0..2d496cbf49 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1444,7 +1444,7 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): # map --from-pr strlist to list of ints try: - from_pr_list = map(int, eb_go.options.from_pr) + from_pr_list = [int(pr_nr) for pr_nr in eb_go.options.from_pr] except ValueError: raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") @@ -1480,7 +1480,7 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): # done here instead of in _postprocess_include because github integration requires build_options to be initialized if eb_go.options.include_easyblocks_from_pr: try: - easyblock_prs = map(int, eb_go.options.include_easyblocks_from_pr) + easyblock_prs = [int(pr_nr) for pr_nr in eb_go.options.include_easyblocks_from_pr] except ValueError: raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s.") diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 1acd98d38d..7bf38b4c61 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -282,7 +282,7 @@ def post_pr_test_report(pr_nr, repo_type, test_report, msg, init_session_state, if build_option('include_easyblocks_from_pr'): if repo_type == GITHUB_EASYCONFIGS_REPO: - easyblocks_pr_nrs = map(int, build_option('include_easyblocks_from_pr')) + easyblocks_pr_nrs = [int(pr_nr) for pr_nr in build_option('include_easyblocks_from_pr')] comment_lines.append("Using easyblocks from PR(s) %s" % ", ".join(["https://github.com/%s/%s/pull/%s" % (pr_target_account, GITHUB_EASYBLOCKS_REPO, easyblocks_pr_nr) @@ -320,12 +320,12 @@ def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state dump_path = build_option('dump_test_report') try: - pr_nrs = map(int, build_option('from_pr')) + pr_nrs = [int(pr_nr) for pr_nr in build_option('from_pr')] except ValueError: raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") try: - eb_pr_nrs = map(int, build_option('include_easyblocks_from_pr')) + eb_pr_nrs = [int(pr_nr) for pr_nr in build_option('include_easyblocks_from_pr')] except ValueError: raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s.") From dbf9d283eb6973b9709695c2ac9335d440b2fa87 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 16 Mar 2021 14:22:13 +0100 Subject: [PATCH 153/864] include utilities_test subsuite in overall test suite (+ trivial code style fixes) --- easybuild/tools/filetools.py | 4 ++-- test/framework/suite.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 3c79e57dd6..cd8119f58d 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -60,7 +60,7 @@ from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, GENERIC_EASYBLOCK_PKG, build_option, install_path from easybuild.tools.py2vs3 import HTMLParser, std_urllib, string_type -from easybuild.tools.utilities import nub, remove_unwanted_chars, natural_keys +from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars try: import requests @@ -1032,7 +1032,7 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen if common_prefix is not None and len(common_prefix) > len(var) * 2: var_defs.append((var, common_prefix)) var_spec = '$' + var - # Replace the common prefix by var_spec + # Replace the common prefix by var_spec path_hits = (var_spec + fn[len(common_prefix):] for fn in path_hits) hits.extend(path_hits) diff --git a/test/framework/suite.py b/test/framework/suite.py index 466401aa4e..41c13d188f 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -77,6 +77,7 @@ import test.framework.toy_build as t import test.framework.type_checking as et import test.framework.tweak as tw +import test.framework.utilities_test as u import test.framework.variables as v import test.framework.yeb as y @@ -118,7 +119,7 @@ # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, lic, f_c, - tw, p, i, pkg, d, env, et, y, st, h, ct, lib] + tw, p, i, pkg, d, env, et, y, st, h, ct, lib, u] SUITE = unittest.TestSuite([x.suite() for x in tests]) res = unittest.TextTestRunner().run(SUITE) From 4929b7b3031b40c4a3389e7dc0fbe55e731aa406 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 16 Mar 2021 19:25:39 +0000 Subject: [PATCH 154/864] added configuration option to define the env command to use for shebangs --- easybuild/framework/easyblock.py | 2 +- easybuild/tools/config.py | 4 +++ easybuild/tools/options.py | 1 + test/framework/toy_build.py | 43 +++++++++++++++++++++++++++++++- 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 5fb593aee0..5b0b780029 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2367,7 +2367,7 @@ def fix_shebang(self): if isinstance(fix_shebang_for, string_type): fix_shebang_for = [fix_shebang_for] - shebang = '#!/usr/bin/env %s' % lang + shebang = '#!%s %s' % (build_option('env_for_shebang'), lang) for glob_pattern in fix_shebang_for: paths = glob.glob(os.path.join(self.installdir, glob_pattern)) self.log.info("Fixing '%s' shebang to '%s' for files that match '%s': %s", diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 1a7ff08c88..f49aefa4ef 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -79,6 +79,7 @@ DEFAULT_CONT_TYPE = CONT_TYPE_SINGULARITY DEFAULT_BRANCH = 'develop' +DEFAULT_ENV_FOR_SHEBANG = '/usr/bin/env' DEFAULT_ENVVAR_USERS_MODULES = 'HOME' DEFAULT_INDEX_MAX_AGE = 7 * 24 * 60 * 60 # 1 week (in seconds) DEFAULT_JOB_BACKEND = 'GC3Pie' @@ -295,6 +296,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_BRANCH: [ 'pr_target_branch', ], + DEFAULT_ENV_FOR_SHEBANG: [ + 'env_for_shebang', + ], DEFAULT_INDEX_MAX_AGE: [ 'index_max_age', ], diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 605ed46ab9..76df62643a 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -375,6 +375,7 @@ def override_options(self): None, 'store', None, 'e', {'metavar': 'CLASS'}), 'enforce-checksums': ("Enforce availability of checksums for all sources/patches, so they can be verified", None, 'store_true', False), + 'env-for-shebang': ("Define the env command to use when fixing shebangs", None, 'store', '/usr/bin/env'), 'experimental': ("Allow experimental code (with behaviour that can be changed/removed at any given time).", None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9809e5b999..a9bfac92c6 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -40,7 +40,7 @@ import sys import tempfile from distutils.version import LooseVersion -from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from test.framework.package import mock_fpm from unittest import TextTestRunner @@ -2854,6 +2854,47 @@ def test_fix_shebang(self): self.assertTrue(bash_shebang_regex.match(bashbin_txt), "Pattern '%s' found in %s: %s" % (bash_shebang_regex.pattern, bashbin_path, bashbin_txt)) + # now test with a custom env command + init_config(build_options={'env_for_shebang': "/usr/bin/env -S"}) + + self.test_toy_build(ec_file=test_ec, raise_error=True) + + toy_bindir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin') + + # bin/toy and bin/toy2 should *not* be patched, since they're binary files + toy_txt = read_file(os.path.join(toy_bindir, 'toy'), mode='rb') + for fn in ['toy.perl', 'toy.python']: + fn_txt = read_file(os.path.join(toy_bindir, fn), mode='rb') + # no shebang added + self.assertFalse(fn_txt.startswith(b"#!/")) + # exact same file as original binary (untouched) + self.assertEqual(toy_txt, fn_txt) + + # no re.M, this should match at start of file! + py_shebang_regex = re.compile(r'^#!/usr/bin/env -S python\n# test$') + for pybin in ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py']: + pybin_path = os.path.join(toy_bindir, pybin) + pybin_txt = read_file(pybin_path) + self.assertTrue(py_shebang_regex.match(pybin_txt), + "Pattern '%s' found in %s: %s" % (py_shebang_regex.pattern, pybin_path, pybin_txt)) + + # no re.M, this should match at start of file! + perl_shebang_regex = re.compile(r'^#!/usr/bin/env -S perl\n# test$') + for perlbin in ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl', 't7.pl']: + perlbin_path = os.path.join(toy_bindir, perlbin) + perlbin_txt = read_file(perlbin_path) + self.assertTrue(perl_shebang_regex.match(perlbin_txt), + "Pattern '%s' found in %s: %s" % (perl_shebang_regex.pattern, perlbin_path, perlbin_txt)) + + # There are 2 bash files which shouldn't be influenced by fix_shebang + bash_shebang_regex = re.compile(r'^#!/usr/bin/env bash\n# test$') + for bashbin in ['b1.sh', 'b2.sh']: + bashbin_path = os.path.join(toy_bindir, bashbin) + bashbin_txt = read_file(bashbin_path) + self.assertTrue(bash_shebang_regex.match(bashbin_txt), + "Pattern '%s' found in %s: %s" % (bash_shebang_regex.pattern, bashbin_path, bashbin_txt)) + + def test_toy_system_toolchain_alias(self): """Test use of 'system' toolchain alias.""" toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') From 00f8e7921f7f49d68604d25715d1c188190c49da Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 16 Mar 2021 19:30:28 +0000 Subject: [PATCH 155/864] removed extra blank line --- test/framework/toy_build.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index a9bfac92c6..0cf73cae29 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2894,7 +2894,6 @@ def test_fix_shebang(self): self.assertTrue(bash_shebang_regex.match(bashbin_txt), "Pattern '%s' found in %s: %s" % (bash_shebang_regex.pattern, bashbin_path, bashbin_txt)) - def test_toy_system_toolchain_alias(self): """Test use of 'system' toolchain alias.""" toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') From ccde372d16c81df7af38c4ab78b460e7ab112c9a Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 16 Mar 2021 19:50:29 +0000 Subject: [PATCH 156/864] reuse DEFAULT_ENV_FOR_SHEBANG --- easybuild/tools/options.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 76df62643a..860f97fb48 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -60,8 +60,8 @@ from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError from easybuild.tools.build_log import init_logging, log_start, print_msg, print_warning, raise_easybuilderror from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE, DEFAULT_ALLOW_LOADED_MODULES -from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_ENVVAR_USERS_MODULES, DEFAULT_FORCE_DOWNLOAD -from easybuild.tools.config import DEFAULT_INDEX_MAX_AGE +from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_ENV_FOR_SHEBANG, DEFAULT_ENVVAR_USERS_MODULES +from easybuild.tools.config import DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL @@ -375,7 +375,8 @@ def override_options(self): None, 'store', None, 'e', {'metavar': 'CLASS'}), 'enforce-checksums': ("Enforce availability of checksums for all sources/patches, so they can be verified", None, 'store_true', False), - 'env-for-shebang': ("Define the env command to use when fixing shebangs", None, 'store', '/usr/bin/env'), + 'env-for-shebang': ("Define the env command to use when fixing shebangs", None, 'store', + DEFAULT_ENV_FOR_SHEBANG), 'experimental': ("Allow experimental code (with behaviour that can be changed/removed at any given time).", None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", From 4a498d97a183b0b41c48ee00437110cb883ea820 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 16 Mar 2021 21:51:35 +0100 Subject: [PATCH 157/864] pass custom env command in test_fix_shebang via --env-for-shebang command line option --- test/framework/toy_build.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 0cf73cae29..296ab8b05b 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -40,7 +40,7 @@ import sys import tempfile from distutils.version import LooseVersion -from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from test.framework.package import mock_fpm from unittest import TextTestRunner @@ -2855,9 +2855,8 @@ def test_fix_shebang(self): "Pattern '%s' found in %s: %s" % (bash_shebang_regex.pattern, bashbin_path, bashbin_txt)) # now test with a custom env command - init_config(build_options={'env_for_shebang': "/usr/bin/env -S"}) - - self.test_toy_build(ec_file=test_ec, raise_error=True) + extra_args = ['--env-for-shebang=/usr/bin/env -S'] + self.test_toy_build(ec_file=test_ec, extra_args=extra_args, raise_error=True) toy_bindir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin') From 16d19bc24eaf270fa4dc6bc65d37fccc382ed047 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 17 Mar 2021 10:01:58 +0100 Subject: [PATCH 158/864] Add option to write file from file-like object and use in download_file Fixes downloads of large files where the size exceeds the maximum integer size Fixes #3455 --- easybuild/tools/filetools.py | 13 +++++++++---- test/framework/filetools.py | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index cd8119f58d..4f9a736793 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -217,7 +217,7 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over overwrites current file contents without backup by default! :param path: location of file - :param data: contents to write to file + :param data: contents to write to file. Can be a file-like object of binary data :param append: append to existing file rather than overwrite :param forced: force actually writing file in (extended) dry run mode :param backup: back up existing file before overwriting or modifying it @@ -246,15 +246,20 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over # cfr. https://docs.python.org/3/library/functions.html#open mode = 'a' if append else 'w' + is_file_like = hasattr(data, 'read') + # special care must be taken with binary data in Python 3 - if sys.version_info[0] >= 3 and isinstance(data, bytes): + if sys.version_info[0] >= 3 and (isinstance(data, bytes) or is_file_like): mode += 'b' # note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block try: mkdir(os.path.dirname(path), parents=True) with open_file(path, mode) as fh: - fh.write(data) + if is_file_like: + shutil.copyfileobj(data, fh) + else: + fh.write(data) except IOError as err: raise EasyBuildError("Failed to write to %s: %s", path, err) @@ -710,7 +715,7 @@ def download_file(filename, url, path, forced=False): url_fd = response.raw url_fd.decode_content = True _log.debug('response code for given url %s: %s' % (url, status_code)) - write_file(path, url_fd.read(), forced=forced, backup=True) + write_file(path, url_fd, forced=forced, backup=True) _log.info("Downloaded file %s from url %s to %s" % (filename, url, path)) downloaded = True url_fd.close() diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 7073e1b537..1506420a24 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -705,6 +705,26 @@ def test_read_write_file(self): # test use of 'mode' in read_file self.assertEqual(ft.read_file(foo, mode='rb'), b'bar') + def test_write_file_like(self): + """Test writing from a file handle directly""" + # Write a text file + fp = os.path.join(self.test_prefix, 'test.txt') + fp_out = os.path.join(self.test_prefix, 'test_out.txt') + ft.write_file(fp, b'Hyphen: \xe2\x80\x93\nEuro sign: \xe2\x82\xac\na with dots: \xc3\xa4') + + with ft.open_file(fp, 'rb') as fh: + ft.write_file(fp_out, fh) + self.assertEqual(ft.read_file(fp_out), ft.read_file(fp)) + + # Write a binary file + fp = os.path.join(self.test_prefix, 'test.bin') + fp_out = os.path.join(self.test_prefix, 'test_out.bin') + ft.write_file(fp, b'\x00\x01'+os.urandom(42)+b'\x02\x03') + + with ft.open_file(fp, 'rb') as fh: + ft.write_file(fp_out, fh) + self.assertEqual(ft.read_file(fp_out, mode='rb'), ft.read_file(fp, mode='rb')) + def test_is_binary(self): """Test is_binary function.""" From d32e203d160e96252a6d29ccfaea57bd762c8dcd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 17 Mar 2021 14:29:49 +0100 Subject: [PATCH 159/864] make sure we copy the entire file when a file-like object is passed to write_file + add comments w.r.t. passing file handle in download_file --- easybuild/tools/filetools.py | 12 +++++++++--- test/framework/filetools.py | 10 ++++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 4f9a736793..ed9b822a55 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -246,17 +246,19 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over # cfr. https://docs.python.org/3/library/functions.html#open mode = 'a' if append else 'w' - is_file_like = hasattr(data, 'read') + data_is_file_obj = all(hasattr(data, x) for x in ('read', 'seek', 'tell', 'write')) # special care must be taken with binary data in Python 3 - if sys.version_info[0] >= 3 and (isinstance(data, bytes) or is_file_like): + if sys.version_info[0] >= 3 and (isinstance(data, bytes) or data_is_file_obj): mode += 'b' # note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block try: mkdir(os.path.dirname(path), parents=True) with open_file(path, mode) as fh: - if is_file_like: + if data_is_file_obj: + # if a file-like object was provided, use copyfileobj (which reads the file in chunks) + data.seek(0) shutil.copyfileobj(data, fh) else: fh.write(data) @@ -715,6 +717,10 @@ def download_file(filename, url, path, forced=False): url_fd = response.raw url_fd.decode_content = True _log.debug('response code for given url %s: %s' % (url, status_code)) + # note: we pass the file object to write_file rather than reading the file first, + # to ensure the data is read in chunks (which prevents problems in Python 3.9+); + # cfr. https://github.com/easybuilders/easybuild-framework/issues/3455 + # and https://bugs.python.org/issue42853 write_file(path, url_fd, forced=forced, backup=True) _log.info("Downloaded file %s from url %s to %s" % (filename, url, path)) downloaded = True diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 1506420a24..55e5a56f42 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -705,8 +705,8 @@ def test_read_write_file(self): # test use of 'mode' in read_file self.assertEqual(ft.read_file(foo, mode='rb'), b'bar') - def test_write_file_like(self): - """Test writing from a file handle directly""" + def test_write_file_obj(self): + """Test writing from a file-like object directly""" # Write a text file fp = os.path.join(self.test_prefix, 'test.txt') fp_out = os.path.join(self.test_prefix, 'test_out.txt') @@ -716,6 +716,12 @@ def test_write_file_like(self): ft.write_file(fp_out, fh) self.assertEqual(ft.read_file(fp_out), ft.read_file(fp)) + # works fine even if same data was already read through the provided file handle + with ft.open_file(fp, 'rb') as fh: + fh.read(4) + ft.write_file(fp_out, fh) + self.assertEqual(ft.read_file(fp_out), ft.read_file(fp)) + # Write a binary file fp = os.path.join(self.test_prefix, 'test.bin') fp_out = os.path.join(self.test_prefix, 'test_out.bin') From 401cdd3b0d9f90a09e7f1a047aa555f2d2a59e16 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 17 Mar 2021 15:20:31 +0100 Subject: [PATCH 160/864] let caller decide whether or not to copy whole file + determine whether file-like object is called just based on 'read' method --- easybuild/tools/filetools.py | 3 +-- test/framework/filetools.py | 6 ------ 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index ed9b822a55..3eb66e782c 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -246,7 +246,7 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over # cfr. https://docs.python.org/3/library/functions.html#open mode = 'a' if append else 'w' - data_is_file_obj = all(hasattr(data, x) for x in ('read', 'seek', 'tell', 'write')) + data_is_file_obj = hasattr(data, 'read') # special care must be taken with binary data in Python 3 if sys.version_info[0] >= 3 and (isinstance(data, bytes) or data_is_file_obj): @@ -258,7 +258,6 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over with open_file(path, mode) as fh: if data_is_file_obj: # if a file-like object was provided, use copyfileobj (which reads the file in chunks) - data.seek(0) shutil.copyfileobj(data, fh) else: fh.write(data) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 55e5a56f42..977bac89a4 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -716,12 +716,6 @@ def test_write_file_obj(self): ft.write_file(fp_out, fh) self.assertEqual(ft.read_file(fp_out), ft.read_file(fp)) - # works fine even if same data was already read through the provided file handle - with ft.open_file(fp, 'rb') as fh: - fh.read(4) - ft.write_file(fp_out, fh) - self.assertEqual(ft.read_file(fp_out), ft.read_file(fp)) - # Write a binary file fp = os.path.join(self.test_prefix, 'test.bin') fp_out = os.path.join(self.test_prefix, 'test_out.bin') From 824f37d1cf78f4d8938bbc12da0f72044b3fd7b2 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 18 Mar 2021 11:25:33 +0100 Subject: [PATCH 161/864] Refactor the CI script to use inclusion instead of exclusion This provides more clarity on what exactly is run --- .github/workflows/unit_tests.yml | 65 ++++++++++++++------------------ 1 file changed, 28 insertions(+), 37 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 7375e80ed5..3115acdb73 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -6,13 +6,11 @@ jobs: runs-on: ubuntu-18.04 strategy: matrix: - python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] + python: [2.7, 3.6] modules_tool: [Lmod-7.8.22, Lmod-8.2.9, modules-tcl-1.147, modules-3.2.10, modules-4.1.4] module_syntax: [Lua, Tcl] lc_all: [""] - # exclude some configuration for non-Lmod modules tool: - # - don't test with Lua module syntax (only supported in Lmod) - # - exclude Python 3.x versions other than 3.6, to limit test configurations + # don't test with Lua module syntax (only supported in Lmod) exclude: - modules_tool: modules-tcl-1.147 module_syntax: Lua @@ -20,41 +18,34 @@ jobs: module_syntax: Lua - modules_tool: modules-4.1.4 module_syntax: Lua - - modules_tool: modules-tcl-1.147 - python: 3.5 - - modules_tool: modules-tcl-1.147 - python: 3.7 - - modules_tool: modules-tcl-1.147 - python: 3.8 - - modules_tool: modules-tcl-1.147 - python: 3.9 - - modules_tool: modules-3.2.10 - python: 3.5 - - modules_tool: modules-3.2.10 - python: 3.7 - - modules_tool: modules-3.2.10 - python: 3.8 - - modules_tool: modules-3.2.10 - python: 3.9 - - modules_tool: modules-4.1.4 - python: 3.5 - - modules_tool: modules-4.1.4 - python: 3.7 - - modules_tool: modules-4.1.4 - python: 3.8 - - modules_tool: modules-4.1.4 - python: 3.9 - - modules_tool: Lmod-7.8.22 - python: 3.5 - - modules_tool: Lmod-7.8.22 - python: 3.7 - - modules_tool: Lmod-7.8.22 - python: 3.8 - - modules_tool: Lmod-7.8.22 - python: 3.9 + include: + # Test different python versions with latest Lmod + - python: 3.5 + modules_tool: Lmod-8.2.9 + module_syntax: Lua + - python: 3.5 + modules_tool: Lmod-8.2.9 + module_syntax: Tcl + - python: 3.7 + modules_tool: Lmod-8.2.9 + module_syntax: Lua + - python: 3.7 + modules_tool: Lmod-8.2.9 + module_syntax: Tcl + - python: 3.8 + modules_tool: Lmod-8.2.9 + module_syntax: Lua + - python: 3.8 + modules_tool: Lmod-8.2.9 + module_syntax: Tcl + - python: 3.9 + modules_tool: Lmod-8.2.9 + module_syntax: Lua + - python: 3.9 + modules_tool: Lmod-8.2.9 + module_syntax: Tcl # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7) - include: - python: 3.6 modules_tool: Lmod-8.2.9 module_syntax: Lua From af0b3b65a962de4342eeb64984375afaeedeaf09 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 18 Mar 2021 17:16:03 +0100 Subject: [PATCH 162/864] bump to latest Lmod in test suite configurations --- .github/workflows/unit_tests.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 3115acdb73..44132d1e8a 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -7,7 +7,7 @@ jobs: strategy: matrix: python: [2.7, 3.6] - modules_tool: [Lmod-7.8.22, Lmod-8.2.9, modules-tcl-1.147, modules-3.2.10, modules-4.1.4] + modules_tool: [Lmod-7.8.22, Lmod-8.4.27, modules-tcl-1.147, modules-3.2.10, modules-4.1.4] module_syntax: [Lua, Tcl] lc_all: [""] # don't test with Lua module syntax (only supported in Lmod) @@ -19,35 +19,35 @@ jobs: - modules_tool: modules-4.1.4 module_syntax: Lua include: - # Test different python versions with latest Lmod + # Test different Python versions with Lmod 8.x (with both Lua and Tcl module syntax) - python: 3.5 - modules_tool: Lmod-8.2.9 + modules_tool: Lmod-8.4.27 module_syntax: Lua - python: 3.5 - modules_tool: Lmod-8.2.9 + modules_tool: Lmod-8.4.27 module_syntax: Tcl - python: 3.7 - modules_tool: Lmod-8.2.9 + modules_tool: Lmod-8.4.27 module_syntax: Lua - python: 3.7 - modules_tool: Lmod-8.2.9 + modules_tool: Lmod-8.4.27 module_syntax: Tcl - python: 3.8 - modules_tool: Lmod-8.2.9 + modules_tool: Lmod-8.4.27 module_syntax: Lua - python: 3.8 - modules_tool: Lmod-8.2.9 + modules_tool: Lmod-8.4.27 module_syntax: Tcl - python: 3.9 - modules_tool: Lmod-8.2.9 + modules_tool: Lmod-8.4.27 module_syntax: Lua - python: 3.9 - modules_tool: Lmod-8.2.9 + modules_tool: Lmod-8.4.27 module_syntax: Tcl - # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set - # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7) + # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set + # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7) - python: 3.6 - modules_tool: Lmod-8.2.9 + modules_tool: Lmod-8.4.27 module_syntax: Lua lc_all: C fail-fast: false From 1b2383169353333d1629c68557e3d321a39d1274 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 19 Mar 2021 12:25:45 +0100 Subject: [PATCH 163/864] Avoid repetition of module tool versions --- .github/workflows/unit_tests.yml | 42 ++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 44132d1e8a..7b774d0b6f 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -2,52 +2,68 @@ name: EasyBuild framework unit tests on: [push, pull_request] jobs: + setup: + runs-on: ubuntu-latest + outputs: + lmod7: Lmod-7.8.22 + lmod8: Lmod-8.4.27 + modulesTcl: modules-tcl-1.147 + modules3: modules-3.2.10 + modules4: modules-4.1.4 + steps: + - run: "true" build: + needs: setup runs-on: ubuntu-18.04 strategy: matrix: python: [2.7, 3.6] - modules_tool: [Lmod-7.8.22, Lmod-8.4.27, modules-tcl-1.147, modules-3.2.10, modules-4.1.4] + modules_tool: + - ${{needs.setup.outputs.lmod7}} + - ${{needs.setup.outputs.lmod8}} + - ${{needs.setup.outputs.modulesTcl}} + - ${{needs.setup.outputs.modules3}} + - ${{needs.setup.outputs.modules4}} module_syntax: [Lua, Tcl] lc_all: [""] # don't test with Lua module syntax (only supported in Lmod) exclude: - - modules_tool: modules-tcl-1.147 + - modules_tool: ${{needs.setup.outputs.modulesTcl}} module_syntax: Lua - - modules_tool: modules-3.2.10 + - modules_tool: ${{needs.setup.outputs.modules3}} module_syntax: Lua - - modules_tool: modules-4.1.4 + - modules_tool: ${{needs.setup.outputs.modules4}} module_syntax: Lua include: # Test different Python versions with Lmod 8.x (with both Lua and Tcl module syntax) - python: 3.5 - modules_tool: Lmod-8.4.27 + modules_tool: ${{needs.setup.outputs.lmod8}} module_syntax: Lua - python: 3.5 - modules_tool: Lmod-8.4.27 + modules_tool: ${{needs.setup.outputs.lmod8}} module_syntax: Tcl - python: 3.7 - modules_tool: Lmod-8.4.27 + modules_tool: ${{needs.setup.outputs.lmod8}} module_syntax: Lua - python: 3.7 - modules_tool: Lmod-8.4.27 + modules_tool: ${{needs.setup.outputs.lmod8}} module_syntax: Tcl - python: 3.8 - modules_tool: Lmod-8.4.27 + modules_tool: ${{needs.setup.outputs.lmod8}} module_syntax: Lua - python: 3.8 - modules_tool: Lmod-8.4.27 + modules_tool: ${{needs.setup.outputs.lmod8}} module_syntax: Tcl - python: 3.9 - modules_tool: Lmod-8.4.27 + modules_tool: ${{needs.setup.outputs.lmod8}} module_syntax: Lua - python: 3.9 - modules_tool: Lmod-8.4.27 + modules_tool: ${{needs.setup.outputs.lmod8}} module_syntax: Tcl # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7) - python: 3.6 - modules_tool: Lmod-8.4.27 + modules_tool: ${{needs.setup.outputs.lmod8}} module_syntax: Lua lc_all: C fail-fast: false From 3fef34ce89a88bb9dfc4b131bbb47aa9fd0c67d6 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 19 Mar 2021 13:28:11 +0100 Subject: [PATCH 164/864] Make sure that path to eb is always found by tests --- test/framework/utilities.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 57fafc3dbc..39f98384ff 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -50,7 +50,7 @@ from easybuild.tools.config import GENERAL_CLASS, Singleton, module_classes from easybuild.tools.configobj import ConfigObj from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import copy_dir, mkdir, read_file +from easybuild.tools.filetools import copy_dir, mkdir, read_file, which from easybuild.tools.modules import curr_module_paths, modules_tool, reset_module_caches from easybuild.tools.options import CONFIG_ENV_VAR_PREFIX, EasyBuildOptions, set_tmpdir from easybuild.tools.py2vs3 import reload @@ -124,6 +124,12 @@ def setUp(self): # make sure that the tests only pick up easyconfigs provided with the tests os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(testdir, 'easyconfigs', 'test_ecs') + # make sure that the EasyBuild installation is still known even if we purge an EB module + if os.getenv('EB_SCRIPT_PATH') is None: + eb_path = which('eb') + if eb_path is not None: + os.environ['EB_SCRIPT_PATH'] = eb_path + # make sure no deprecated behaviour is being triggered (unless intended by the test) self.orig_current_version = eb_build_log.CURRENT_VERSION self.disallow_deprecated_behaviour() From 8ee9b9218732b94a2988daf509a21704a97647ca Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 19 Mar 2021 14:29:29 +0100 Subject: [PATCH 165/864] Tweak test that actually checks EB_SCRIPT_PATH --- test/framework/easyconfig.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index d5d6458ef0..c43487ca25 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3374,6 +3374,11 @@ def test_get_paths_for(self): self.mock_stderr(False) self.assertTrue(os.path.samefile(test_ecs, res[0])) + # Can't have EB_SCRIPT_PATH set (for some of) these tests + env_eb_script_path = os.getenv('EB_SCRIPT_PATH') + if env_eb_script_path: + del os.environ['EB_SCRIPT_PATH'] + # easyconfigs location can also be derived from location of 'eb' write_file(os.path.join(self.test_prefix, 'bin', 'eb'), "#!/bin/bash; echo 'This is a fake eb'") adjust_permissions(os.path.join(self.test_prefix, 'bin', 'eb'), stat.S_IXUSR) @@ -3390,6 +3395,10 @@ def test_get_paths_for(self): res = get_paths_for(subdir='easyconfigs', robot_path=None) self.assertTrue(os.path.samefile(test_ecs, res[-1])) + # Restore EB_SCRIPT_PATH value if set originally + if env_eb_script_path: + os.environ['EB_SCRIPT_PATH'] = env_eb_script_path + # also locations in sys.path are considered os.environ['PATH'] = orig_path sys.path.insert(0, self.test_prefix) @@ -3440,6 +3449,10 @@ def test_get_paths_for(self): self.assertTrue(os.path.exists(res[0])) self.assertTrue(os.path.samefile(res[0], os.path.join(someprefix, 'easybuild', 'easyconfigs'))) + # Finally restore EB_SCRIPT_PATH value if set + if env_eb_script_path: + os.environ['EB_SCRIPT_PATH'] = env_eb_script_path + def test_is_generic_easyblock(self): """Test for is_generic_easyblock function.""" From d68a4951a611197c30545d9a7804c48318e59c3f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 19 Mar 2021 14:30:52 +0100 Subject: [PATCH 166/864] Improve comment --- test/framework/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index c43487ca25..777f9ffdb7 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3395,7 +3395,7 @@ def test_get_paths_for(self): res = get_paths_for(subdir='easyconfigs', robot_path=None) self.assertTrue(os.path.samefile(test_ecs, res[-1])) - # Restore EB_SCRIPT_PATH value if set originally + # Restore (temporarily) EB_SCRIPT_PATH value if set originally if env_eb_script_path: os.environ['EB_SCRIPT_PATH'] = env_eb_script_path From 8999ed1fd0b33a592aa3f38e9a217eda7b63b54e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 22 Mar 2021 09:49:04 +0100 Subject: [PATCH 167/864] Add/Improve comments --- .github/workflows/unit_tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 7b774d0b6f..1f31dd395a 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -19,6 +19,8 @@ jobs: matrix: python: [2.7, 3.6] modules_tool: + # use variables defined by 'setup' job above, see also + # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context - ${{needs.setup.outputs.lmod7}} - ${{needs.setup.outputs.lmod8}} - ${{needs.setup.outputs.modulesTcl}} @@ -35,7 +37,7 @@ jobs: - modules_tool: ${{needs.setup.outputs.modules4}} module_syntax: Lua include: - # Test different Python versions with Lmod 8.x (with both Lua and Tcl module syntax) + # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax) - python: 3.5 modules_tool: ${{needs.setup.outputs.lmod8}} module_syntax: Lua From c3674a8510e047e1b502cdaab99d0a4f90953d1d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 24 Mar 2021 12:36:57 +0100 Subject: [PATCH 168/864] Add templates for architecture independent Python wheels Wheels with the -none-any suffix are basically archives of source code which can often be installed with pip where using the tar.gz would fail for reasons such as missing versions --- easybuild/framework/easyconfig/templates.py | 13 +++++++++ test/framework/easyconfig.py | 32 +++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index a1ef4d42b7..6b3ba8763d 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -155,6 +155,19 @@ ('SOURCE_%s' % suffix, '%(name)s-%(version)s.' + ext, "Source .%s bundle" % ext), ('SOURCELOWER_%s' % suffix, '%(namelower)s-%(version)s.' + ext, "Source .%s bundle with lowercase name" % ext), ] +for pyver in ('py2.py3', 'py2', 'py3'): + if pyver == 'py2.py3': + desc = 'Python 2 & Python 3' + name_infix = '' + else: + desc = 'Python ' + pyver[-1] + name_infix = pyver.upper() + '_' + TEMPLATE_CONSTANTS += [ + ('SOURCE_%sWHL' % name_infix, '%%(name)s-%%(version)s-%s-none-any.whl' % pyver, + 'Generic (non-compiled) %s wheel package' % desc), + ('SOURCELOWER_%sWHL' % name_infix, '%%(namelower)s-%%(version)s-%s-none-any.whl' % pyver, + 'Generic (non-compiled) %s wheel package with lowercase name' % desc), + ] # TODO derived config templates # versionmajor, versionminor, versionmajorminor (eg '.'.join(version.split('.')[:2])) ) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 777f9ffdb7..f1a3b62efc 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -37,6 +37,7 @@ import stat import sys import tempfile +import textwrap from distutils.version import LooseVersion from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -1183,6 +1184,37 @@ def test_java_wrapper_templating(self): self.assertEqual(eb['modloadmsg'], "Java: 11, 11, 11") + def test_python_whl_templating(self): + """test templating for Python wheels""" + + self.contents = textwrap.dedent(""" + easyblock = "ConfigureMake" + name = "Pi" + version = "3.14" + homepage = "https://example.com" + description = "test easyconfig" + toolchain = {"name":"GCC", "version": "4.6.3"} + sources = [ + SOURCE_WHL, + SOURCELOWER_WHL, + SOURCE_PY2_WHL, + SOURCELOWER_PY2_WHL, + SOURCE_PY3_WHL, + SOURCELOWER_PY3_WHL, + ] + """) + self.prep() + ec = EasyConfig(self.eb_file) + + sources = ec['sources'] + + self.assertEqual(sources[0], 'Pi-3.14-py2.py3-none-any.whl') + self.assertEqual(sources[1], 'pi-3.14-py2.py3-none-any.whl') + self.assertEqual(sources[2], 'Pi-3.14-py2-none-any.whl') + self.assertEqual(sources[3], 'pi-3.14-py2-none-any.whl') + self.assertEqual(sources[4], 'Pi-3.14-py3-none-any.whl') + self.assertEqual(sources[5], 'pi-3.14-py3-none-any.whl') + def test_templating_doc(self): """test templating documentation""" doc = avail_easyconfig_templates() From c26e30298dd7f02ca4a4d152b8f9922dae53bb60 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 29 Mar 2021 11:15:07 +0200 Subject: [PATCH 169/864] Show the easyblocks repo and PR number when uploading a test report against that Also enhance and fix the clean_gists script to not confuse easyconfigs and easyblocks repo which leads to deleting gists to keep --- easybuild/scripts/clean_gists.py | 90 ++++++++++++++++++++++---------- easybuild/tools/testing.py | 19 +++++-- 2 files changed, 77 insertions(+), 32 deletions(-) diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py index b5761713be..cc64eb8e9c 100644 --- a/easybuild/scripts/clean_gists.py +++ b/easybuild/scripts/clean_gists.py @@ -32,9 +32,10 @@ from easybuild.base.generaloption import simple_option from easybuild.base.rest import RestClient from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.github import GITHUB_API_URL, HTTP_STATUS_OK, GITHUB_EASYCONFIGS_REPO +from easybuild.tools.github import GITHUB_API_URL, HTTP_STATUS_OK, GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO from easybuild.tools.github import GITHUB_EB_MAIN, fetch_github_token from easybuild.tools.options import EasyBuildOptions +from easybuild.tools.py2vs3 import HTTPError HTTP_DELETE_OK = 204 @@ -49,6 +50,7 @@ def main(): 'closed-pr': ('Delete all gists from closed pull-requests', None, 'store_true', True, 'p'), 'all': ('Delete all gists from Easybuild ', None, 'store_true', False, 'a'), 'orphans': ('Delete all gists without a pull-request', None, 'store_true', False, 'o'), + 'dry-run': ("Only show which gists will be deleted but don't actually delete them", None, 'store_true', False), } go = simple_option(options) @@ -88,7 +90,8 @@ def main(): break log.info("Found %s gists", len(all_gists)) - regex = re.compile(r"(EasyBuild test report|EasyBuild log for failed build).*?(?:PR #(?P[0-9]+))?\)?$") + re_eb_gist = re.compile(r"(EasyBuild test report|EasyBuild log for failed build)(.*?)$") + re_pr_nr = re.compile(r"(EB )?PR #([0-9]+)") pr_cache = {} num_deleted = 0 @@ -96,43 +99,74 @@ def main(): for gist in all_gists: if not gist["description"]: continue - re_pr_num = regex.search(gist["description"]) - delete_gist = False - - if re_pr_num: - log.debug("Found a Easybuild gist (id=%s)", gist["id"]) - pr_num = re_pr_num.group("PR") - if go.options.all: - delete_gist = True - elif pr_num and go.options.closed_pr: - log.debug("Found Easybuild test report for PR #%s", pr_num) - - if pr_num not in pr_cache: - status, pr = gh.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr_num].get() + + gist_match = re_eb_gist.search(gist["description"]) + + if not gist_match: + log.debug("Found a non-Easybuild gist (id=%s)", gist["id"]) + continue + + log.debug("Found an Easybuild gist (id=%s)", gist["id"]) + + pr_data = gist_match.group(2) + + pr_nrs_matches = re_pr_nr.findall(pr_data) + + if go.options.all: + delete_gist = True + elif not pr_nrs_matches: + log.info("Found Easybuild test report without PR (id=%s).", gist["id"]) + delete_gist = go.options.orphans + elif go.options.closed_pr: + # All PRs must be closed + delete_gist = True + for pr_nr_match in pr_nrs_matches: + eb_str, pr_num = pr_nr_match + if eb_str or GITHUB_EASYBLOCKS_REPO in pr_data: + repo = GITHUB_EASYBLOCKS_REPO + else: + repo = GITHUB_EASYCONFIGS_REPO + + cache_key = "%s-%s" % (repo, pr_num) + + if cache_key not in pr_cache: + try: + status, pr = gh.repos[GITHUB_EB_MAIN][repo].pulls[pr_num].get() + except HTTPError as e: + status, pr = e.code, e.msg if status != HTTP_STATUS_OK: raise EasyBuildError("Failed to get pull-request #%s: error code %s, message = %s", pr_num, status, pr) - pr_cache[pr_num] = pr["state"] - - if pr_cache[pr_num] == "closed": - log.debug("Found report from closed PR #%s (id=%s)", pr_num, gist["id"]) - delete_gist = True - - elif not pr_num and go.options.orphans: - log.debug("Found Easybuild test report without PR (id=%s)", gist["id"]) - delete_gist = True + pr_cache[cache_key] = pr["state"] + + if pr_cache[cache_key] == "closed": + log.info("Found report from closed %s PR #%s (id=%s)", repo, pr_num, gist["id"]) + elif delete_gist: + if len(pr_nrs_matches) > 1: + log.info("Found at least 1 PR, that is not closed yet: %s/%s (id=%s)", + repo, pr_num, gist["id"]) + delete_gist = False + else: + delete_gist = True if delete_gist: + if go.options.dry_run: + log.info("DRY-RUN: Delete gist with id=%s", gist["id"]) + num_deleted += 1 + continue status, del_gist = gh.gists[gist["id"]].delete() if status != HTTP_DELETE_OK: - raise EasyBuildError("Unable to remove gist (id=%s): error code %s, message = %s", - gist["id"], status, del_gist) + log.warning("Unable to remove gist (id=%s): error code %s, message = %s", + gist["id"], status, del_gist) else: - log.info("Delete gist with id=%s", gist["id"]) + log.info("Deleted gist with id=%s", gist["id"]) num_deleted += 1 - log.info("Deleted %s gists", num_deleted) + if go.options.dry_run: + log.info("DRY-RUN: Would delete %s gists", num_deleted) + else: + log.info("Deleted %s gists", num_deleted) if __name__ == '__main__': diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index cf93034570..8fa9101d19 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -138,22 +138,30 @@ def session_state(): } -def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_log=False): +def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_log=False, eb_pr_nrs=None): """Create test report for easyconfigs PR, in Markdown format.""" github_user = build_option('github_user') pr_target_account = build_option('pr_target_account') - pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO + pr_target_repo = build_option('pr_target_repo') end_time = gmtime() # create a gist with a full test report test_report = [] if pr_nr is not None: + repo = pr_target_repo or GITHUB_EASYCONFIGS_REPO test_report.extend([ - "Test report for https://github.com/%s/%s/pull/%s" % (pr_target_account, pr_target_repo, pr_nr), + "Test report for https://github.com/%s/%s/pull/%s" % (pr_target_account, repo, pr_nr), "", ]) + if eb_pr_nrs: + repo = pr_target_repo or GITHUB_EASYBLOCKS_REPO + test_report.extend([ + "Test report for https://github.com/%s/%s/pull/%s" % (pr_target_account, repo, nr) + for nr in eb_pr_nrs + ]) + test_report.append("") test_report.extend([ "#### Test result", "%s" % msg, @@ -184,6 +192,8 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_l descr = "(partial) EasyBuild log for failed build of %s" % ec['spec'] if pr_nr is not None: descr += " (PR #%s)" % pr_nr + if eb_pr_nrs: + descr += "".join(" (EB PR #%s)" % nr for nr in eb_pr_nrs) fn = '%s_partial.log' % os.path.basename(ec['spec'])[:-3] gist_url = create_gist(partial_log_txt, fn, descr=descr, github_user=github_user) test_log = "(partial log available at %s)" % gist_url @@ -323,7 +333,8 @@ def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state if upload: msg = msg + " (%d easyconfigs in total)" % orig_cnt - test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nr=pr_nr, gist_log=True) + test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nr=pr_nr, gist_log=True, + eb_pr_nrs=eb_pr_nrs) if pr_nr: # upload test report to gist and issue a comment in the PR to notify txt = post_pr_test_report(pr_nr, GITHUB_EASYCONFIGS_REPO, test_report, msg, init_session_state, success) From f5f9c12e653b49ab604b5db4351b8176b455423e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 29 Mar 2021 11:27:45 +0200 Subject: [PATCH 170/864] Handle errors on delete and reduce default verbosity --- easybuild/scripts/clean_gists.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py index cc64eb8e9c..2cd3a0ee3f 100644 --- a/easybuild/scripts/clean_gists.py +++ b/easybuild/scripts/clean_gists.py @@ -35,7 +35,7 @@ from easybuild.tools.github import GITHUB_API_URL, HTTP_STATUS_OK, GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO from easybuild.tools.github import GITHUB_EB_MAIN, fetch_github_token from easybuild.tools.options import EasyBuildOptions -from easybuild.tools.py2vs3 import HTTPError +from easybuild.tools.py2vs3 import HTTPError, URLError HTTP_DELETE_OK = 204 @@ -115,7 +115,7 @@ def main(): if go.options.all: delete_gist = True elif not pr_nrs_matches: - log.info("Found Easybuild test report without PR (id=%s).", gist["id"]) + log.debug("Found Easybuild test report without PR (id=%s).", gist["id"]) delete_gist = go.options.orphans elif go.options.closed_pr: # All PRs must be closed @@ -140,11 +140,11 @@ def main(): pr_cache[cache_key] = pr["state"] if pr_cache[cache_key] == "closed": - log.info("Found report from closed %s PR #%s (id=%s)", repo, pr_num, gist["id"]) + log.debug("Found report from closed %s PR #%s (id=%s)", repo, pr_num, gist["id"]) elif delete_gist: if len(pr_nrs_matches) > 1: - log.info("Found at least 1 PR, that is not closed yet: %s/%s (id=%s)", - repo, pr_num, gist["id"]) + log.debug("Found at least 1 PR, that is not closed yet: %s/%s (id=%s)", + repo, pr_num, gist["id"]) delete_gist = False else: delete_gist = True @@ -154,7 +154,12 @@ def main(): log.info("DRY-RUN: Delete gist with id=%s", gist["id"]) num_deleted += 1 continue - status, del_gist = gh.gists[gist["id"]].delete() + try: + status, del_gist = gh.gists[gist["id"]].delete() + except HTTPError as e: + status, del_gist = e.code, e.msg + except URLError as e: + status, del_gist = None, e.reason if status != HTTP_DELETE_OK: log.warning("Unable to remove gist (id=%s): error code %s, message = %s", From 4aac5763ee9d005adc5300be17f5a23c4755b5dc Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 29 Mar 2021 11:55:45 +0200 Subject: [PATCH 171/864] Don't overwrite default log level --- easybuild/scripts/clean_gists.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py index 2cd3a0ee3f..a290de3f42 100644 --- a/easybuild/scripts/clean_gists.py +++ b/easybuild/scripts/clean_gists.py @@ -60,6 +60,7 @@ def main(): raise EasyBuildError("Please tell me what to do?") if go.options.github_user is None: + EasyBuildOptions.DEFAULT_LOGLEVEL = None # Don't overwrite log level eb_go = EasyBuildOptions(envvar_prefix='EASYBUILD', go_args=[]) username = eb_go.options.github_user log.debug("Fetch github username from easybuild, found: %s", username) From 40ff9dd3cc4da26154aa20b0c36e2c05a3275ff2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 5 Apr 2021 14:51:23 +0200 Subject: [PATCH 172/864] fix typo in docstring of goblf toolchain definition --- easybuild/toolchains/goblf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/goblf.py b/easybuild/toolchains/goblf.py index 75c782b862..3bf2d50e81 100644 --- a/easybuild/toolchains/goblf.py +++ b/easybuild/toolchains/goblf.py @@ -23,7 +23,7 @@ # along with EasyBuild. If not, see . ## """ -EasyBuild support for foss compiler toolchain (includes GCC, OpenMPI, BLIS, LAPACK, ScaLAPACK and FFTW). +EasyBuild support for goblf compiler toolchain (includes GCC, OpenMPI, BLIS, LAPACK, ScaLAPACK and FFTW). :author: Kenneth Hoste (Ghent University) :author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) From 56dcefa3919dfd63af57d0003ebf5586183e529a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 5 Apr 2021 15:03:59 +0200 Subject: [PATCH 173/864] use develop branch when testing push access in --check-github (fixes #3590) --- easybuild/tools/github.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 63cdf81213..54d7ac2e00 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1457,7 +1457,7 @@ def post_pr_labels(pr, labels): return True -def add_pr_labels(pr, branch='develop'): +def add_pr_labels(pr, branch=GITHUB_DEVELOP_BRANCH): """ Try to determine and add labels to PR. :param pr: pull request number in easybuild-easyconfigs repo @@ -1986,7 +1986,7 @@ def check_github(): branch_name = 'test_branch_%s' % ''.join(random.choice(ascii_letters) for _ in range(5)) try: git_repo = init_repo(git_working_dir, GITHUB_EASYCONFIGS_REPO, silent=not debug) - remote_name = setup_repo(git_repo, github_account, GITHUB_EASYCONFIGS_REPO, GITHUB_BRANCH_MAIN, + remote_name = setup_repo(git_repo, github_account, GITHUB_EASYCONFIGS_REPO, GITHUB_DEVELOP_BRANCH, silent=not debug, git_only=True) git_repo.create_head(branch_name) res = getattr(git_repo.remotes, remote_name).push(branch_name) @@ -2168,7 +2168,8 @@ def validate_github_token(token, github_user): # try and determine sha of latest commit in easybuilders/easybuild-easyconfigs repo through authenticated access sha = None try: - sha = fetch_latest_commit_sha(GITHUB_EASYCONFIGS_REPO, GITHUB_EB_MAIN, github_user=github_user, token=token) + sha = fetch_latest_commit_sha(GITHUB_EASYCONFIGS_REPO, GITHUB_EB_MAIN, + branch=GITHUB_DEVELOP_BRANCH, github_user=github_user, token=token) except Exception as err: _log.warning("An exception occurred when trying to use token for authenticated GitHub access: %s", err) @@ -2185,7 +2186,8 @@ def find_easybuild_easyconfig(github_user=None): :param github_user: name of GitHub user to use when querying GitHub """ - dev_repo = download_repo(GITHUB_EASYCONFIGS_REPO, branch='develop', account=GITHUB_EB_MAIN, github_user=github_user) + dev_repo = download_repo(GITHUB_EASYCONFIGS_REPO, branch=GITHUB_DEVELOP_BRANCH, + account=GITHUB_EB_MAIN, github_user=github_user) eb_parent_path = os.path.join(dev_repo, 'easybuild', 'easyconfigs', 'e', 'EasyBuild') files = os.listdir(eb_parent_path) From ff05a370b23012f0f01e0198804dbb1d4fcbbeea Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 5 Apr 2021 16:16:08 +0200 Subject: [PATCH 174/864] deprecate --accept-eula, rename to --accept-eula-for --- easybuild/framework/easyblock.py | 4 ++-- easybuild/tools/config.py | 9 ++++++++- easybuild/tools/options.py | 4 +++- test/framework/options.py | 34 +++++++++++++++++++++++++++++--- 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 5b0b780029..6054b4ed80 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1655,7 +1655,7 @@ def check_accepted_eula(self, name=None, more_info=None): if name is None: name = self.name - accepted_eulas = build_option('accept_eula') or [] + accepted_eulas = build_option('accept_eula_for') or [] if self.cfg['accept_eula'] or name in accepted_eulas: self.log.info("EULA for %s is accepted", name) else: @@ -1667,7 +1667,7 @@ def check_accepted_eula(self, name=None, more_info=None): error_lines.extend([ "You should either:", - "- add --accept-eula=%(name)s to the 'eb' command;", + "- add --accept-eula-for=%(name)s to the 'eb' command;", "- update your EasyBuild configuration to always accept the EULA for %(name)s;", "- add 'accept_eula = True' to the easyconfig file you are using;", '', diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index f49aefa4ef..9fb49e348e 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -163,7 +163,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): # build options that have a perfectly matching command line option, listed by default value BUILD_OPTIONS_CMDLINE = { None: [ - 'accept_eula', + 'accept_eula_for', 'aggregate_regtest', 'backup_modules', 'container_config', @@ -504,6 +504,10 @@ def init_build_options(build_options=None, cmdline_options=None): _log.info("Auto-enabling ignoring of OS dependencies") cmdline_options.ignore_osdeps = True + if not cmdline_options.accept_eula_for and cmdline_options.accept_eula: + _log.deprecated("Use accept-eula-for configuration setting rather than accept-eula.", '5.0') + cmdline_options.accept_eula_for = cmdline_options.accept_eula + cmdline_build_option_names = [k for ks in BUILD_OPTIONS_CMDLINE.values() for k in ks] active_build_options.update(dict([(key, getattr(cmdline_options, key)) for key in cmdline_build_option_names])) # other options which can be derived but have no perfectly matching cmdline option @@ -537,6 +541,9 @@ def build_option(key, **kwargs): build_options = BuildOptions() if key in build_options: return build_options[key] + elif key == 'accept_eula': + _log.deprecated("Use accept_eula_for build option rather than accept_eula.", '5.0') + return build_options['accept_eula_for'] elif 'default' in kwargs: return kwargs['default'] else: diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 860f97fb48..31acffa334 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -334,7 +334,9 @@ def override_options(self): descr = ("Override options", "Override default EasyBuild behavior.") opts = OrderedDict({ - 'accept-eula': ("Accept EULA for specified software", 'strlist', 'store', []), + 'accept-eula': ("Accept EULA for specified software [DEPRECATED, use --accept-eula-for instead!]", + 'strlist', 'store', []), + 'accept-eula-for': ("Accept EULA for specified software", 'strlist', 'store', []), 'add-dummy-to-minimal-toolchains': ("Include dummy toolchain in minimal toolchain searches " "[DEPRECATED, use --add-system-to-minimal-toolchains instead!)", None, 'store_true', False), diff --git a/test/framework/options.py b/test/framework/options.py index e046645514..a8a9e29c7d 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5894,7 +5894,7 @@ def test_sysroot(self): self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, ['--show-config'], raise_error=True) def test_accept_eula_for(self): - """Test --accept-eula configuration option.""" + """Test --accept-eula-for configuration option.""" # use toy-0.0.eb easyconfig file that comes with the tests topdir = os.path.abspath(os.path.dirname(__file__)) @@ -5912,8 +5912,8 @@ def test_accept_eula_for(self): error_pattern = r"The End User License Argreement \(EULA\) for toy is currently not accepted!" self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True) - # installation proceeds if EasyBuild is configured to accept EULA for specified software via --accept-eula - self.eb_main(args + ['--accept-eula=foo,toy,bar'], do_build=True, raise_error=True) + # installation proceeds if EasyBuild is configured to accept EULA for specified software via --accept-eula-for + self.eb_main(args + ['--accept-eula-for=foo,toy,bar'], do_build=True, raise_error=True) toy_modfile = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') if get_module_syntax() == 'Lua': @@ -5924,14 +5924,42 @@ def test_accept_eula_for(self): self.assertFalse(os.path.exists(toy_modfile)) # also check use of $EASYBUILD_ACCEPT_EULA to accept EULA for specified software + os.environ['EASYBUILD_ACCEPT_EULA_FOR'] = 'toy' + self.eb_main(args, do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_modfile)) + + remove_dir(self.test_installpath) + self.assertFalse(os.path.exists(toy_modfile)) + + del os.environ['EASYBUILD_ACCEPT_EULA_FOR'] + + # also check deprecated --accept-eula configuration option + self.allow_deprecated_behaviour() + + self.mock_stderr(True) + self.eb_main(args + ['--accept-eula=foo,toy,bar'], do_build=True, raise_error=True) + stderr = self.get_stderr() + self.mock_stderr(False) + self.assertTrue("Use accept-eula-for configuration setting rather than accept-eula" in stderr) + + remove_dir(self.test_installpath) + self.assertFalse(os.path.exists(toy_modfile)) + + # also via $EASYBUILD_ACCEPT_EULA + self.mock_stderr(True) os.environ['EASYBUILD_ACCEPT_EULA'] = 'toy' self.eb_main(args, do_build=True, raise_error=True) + stderr = self.get_stderr() + self.mock_stderr(False) + self.assertTrue(os.path.exists(toy_modfile)) + self.assertTrue("Use accept-eula-for configuration setting rather than accept-eula" in stderr) remove_dir(self.test_installpath) self.assertFalse(os.path.exists(toy_modfile)) # also check accepting EULA via 'accept_eula = True' in easyconfig file + self.disallow_deprecated_behaviour() del os.environ['EASYBUILD_ACCEPT_EULA'] write_file(test_ec, test_ec_txt + '\naccept_eula = True') self.eb_main(args, do_build=True, raise_error=True) From c33acc2d6d02b89421098bcb852e6cdab245c3ac Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 5 Apr 2021 16:23:34 +0200 Subject: [PATCH 175/864] add support for using regular expression pattern to speicfy software names to accept EULA for --- easybuild/framework/easyblock.py | 2 +- test/framework/options.py | 29 +++++++++++++++-------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 6054b4ed80..dd23447611 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1656,7 +1656,7 @@ def check_accepted_eula(self, name=None, more_info=None): name = self.name accepted_eulas = build_option('accept_eula_for') or [] - if self.cfg['accept_eula'] or name in accepted_eulas: + if self.cfg['accept_eula'] or name in accepted_eulas or any(re.match(x, name) for x in accepted_eulas): self.log.info("EULA for %s is accepted", name) else: error_lines = [ diff --git a/test/framework/options.py b/test/framework/options.py index a8a9e29c7d..1cf0a11633 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5911,27 +5911,28 @@ def test_accept_eula_for(self): args = [test_ec, '--force'] error_pattern = r"The End User License Argreement \(EULA\) for toy is currently not accepted!" self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True) - - # installation proceeds if EasyBuild is configured to accept EULA for specified software via --accept-eula-for - self.eb_main(args + ['--accept-eula-for=foo,toy,bar'], do_build=True, raise_error=True) - toy_modfile = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') if get_module_syntax() == 'Lua': toy_modfile += '.lua' - self.assertTrue(os.path.exists(toy_modfile)) - remove_dir(self.test_installpath) - self.assertFalse(os.path.exists(toy_modfile)) + # installation proceeds if EasyBuild is configured to accept EULA for specified software via --accept-eula-for + for val in ('foo,toy,bar', '.*', 't.y'): + self.eb_main(args + ['--accept-eula-for=' + val], do_build=True, raise_error=True) - # also check use of $EASYBUILD_ACCEPT_EULA to accept EULA for specified software - os.environ['EASYBUILD_ACCEPT_EULA_FOR'] = 'toy' - self.eb_main(args, do_build=True, raise_error=True) - self.assertTrue(os.path.exists(toy_modfile)) + self.assertTrue(os.path.exists(toy_modfile)) - remove_dir(self.test_installpath) - self.assertFalse(os.path.exists(toy_modfile)) + remove_dir(self.test_installpath) + self.assertFalse(os.path.exists(toy_modfile)) + + # also check use of $EASYBUILD_ACCEPT_EULA to accept EULA for specified software + os.environ['EASYBUILD_ACCEPT_EULA_FOR'] = val + self.eb_main(args, do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_modfile)) + + remove_dir(self.test_installpath) + self.assertFalse(os.path.exists(toy_modfile)) - del os.environ['EASYBUILD_ACCEPT_EULA_FOR'] + del os.environ['EASYBUILD_ACCEPT_EULA_FOR'] # also check deprecated --accept-eula configuration option self.allow_deprecated_behaviour() From 4012ecba039c347dcf1d86d5c0f6f672f27f7035 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 5 Apr 2021 18:11:12 +0200 Subject: [PATCH 176/864] rename eb_pr_nrs to easyblock_pr_nrs --- easybuild/tools/testing.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 8fa9101d19..93f62babb1 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -138,7 +138,7 @@ def session_state(): } -def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_log=False, eb_pr_nrs=None): +def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_log=False, easyblock_pr_nrs=None): """Create test report for easyconfigs PR, in Markdown format.""" github_user = build_option('github_user') @@ -155,11 +155,11 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_l "Test report for https://github.com/%s/%s/pull/%s" % (pr_target_account, repo, pr_nr), "", ]) - if eb_pr_nrs: + if easyblock_pr_nrs: repo = pr_target_repo or GITHUB_EASYBLOCKS_REPO test_report.extend([ "Test report for https://github.com/%s/%s/pull/%s" % (pr_target_account, repo, nr) - for nr in eb_pr_nrs + for nr in easyblock_pr_nrs ]) test_report.append("") test_report.extend([ @@ -192,8 +192,8 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_l descr = "(partial) EasyBuild log for failed build of %s" % ec['spec'] if pr_nr is not None: descr += " (PR #%s)" % pr_nr - if eb_pr_nrs: - descr += "".join(" (EB PR #%s)" % nr for nr in eb_pr_nrs) + if easyblock_pr_nrs: + descr += "".join(" (easyblock PR #%s)" % nr for nr in easyblock_pr_nrs) fn = '%s_partial.log' % os.path.basename(ec['spec'])[:-3] gist_url = create_gist(partial_log_txt, fn, descr=descr, github_user=github_user) test_log = "(partial log available at %s)" % gist_url @@ -328,21 +328,21 @@ def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state """ dump_path = build_option('dump_test_report') pr_nr = build_option('from_pr') - eb_pr_nrs = build_option('include_easyblocks_from_pr') + easyblock_pr_nrs = build_option('include_easyblocks_from_pr') upload = build_option('upload_test_report') if upload: msg = msg + " (%d easyconfigs in total)" % orig_cnt test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nr=pr_nr, gist_log=True, - eb_pr_nrs=eb_pr_nrs) + easyblock_pr_nrs=easyblock_pr_nrs) if pr_nr: # upload test report to gist and issue a comment in the PR to notify txt = post_pr_test_report(pr_nr, GITHUB_EASYCONFIGS_REPO, test_report, msg, init_session_state, success) - elif eb_pr_nrs: + elif easyblock_pr_nrs: # upload test report to gist and issue a comment in the easyblocks PR to notify - for eb_pr_nr in map(int, eb_pr_nrs): - txt = post_pr_test_report(eb_pr_nr, GITHUB_EASYBLOCKS_REPO, test_report, msg, init_session_state, - success) + for easyblock_pr_nr in map(int, easyblock_pr_nrs): + txt = post_pr_test_report(easyblock_pr_nr, GITHUB_EASYBLOCKS_REPO, test_report, msg, + init_session_state, success) else: # only upload test report as a gist gist_url = upload_test_report_as_gist(test_report['full']) From 01bcca03b55d5e44740405aa7bdd853c2b5571ce Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 5 Apr 2021 18:42:47 +0200 Subject: [PATCH 177/864] rename setup_cuda_cache method to set_up_cuda_cache + minor tweaks to log messages --- easybuild/framework/easyblock.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 28add7d1ec..7b952dc2c7 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1034,20 +1034,23 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False): mkdir(dir_name, parents=True) - def setup_cuda_cache(self): + def set_up_cuda_cache(self): + """Set up CUDA PTX cache.""" + cuda_cache_maxsize = build_option('cuda_cache_maxsize') if cuda_cache_maxsize is None: cuda_cache_maxsize = 1 * 1024 # 1 GiB default value else: cuda_cache_maxsize = int(cuda_cache_maxsize) + if cuda_cache_maxsize == 0: - self.log.info('Disabling CUDA PTX cache as per request') + self.log.info("Disabling CUDA PTX cache since cache size was set to zero") env.setvar('CUDA_CACHE_DISABLE', '1') else: cuda_cache_dir = build_option('cuda_cache_dir') if not cuda_cache_dir: cuda_cache_dir = os.path.join(self.builddir, 'eb-cuda-cache') - self.log.info('Enabling CUDA PTX cache of size %s MiB at %s', cuda_cache_maxsize, cuda_cache_dir) + self.log.info("Enabling CUDA PTX cache of size %s MiB at %s", cuda_cache_maxsize, cuda_cache_dir) env.setvar('CUDA_CACHE_DISABLE', '0') env.setvar('CUDA_CACHE_PATH', cuda_cache_dir) env.setvar('CUDA_CACHE_MAXSIZE', str(cuda_cache_maxsize * 1024 * 1024)) @@ -2183,7 +2186,7 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True): # Setup CUDA cache if required. If we don't do this, CUDA will use the $HOME for its cache files if get_software_root('CUDA') or get_software_root('CUDAcore'): - self.setup_cuda_cache() + self.set_up_cuda_cache() # guess directory to start configure/build/install process in, and move there if start_dir: From 14b99ef96b6304a3f00c043e69a9b7d0979401ce Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 5 Apr 2021 19:08:28 +0200 Subject: [PATCH 178/864] flesh out duplicate code from mkdir and create_unused_dir functions into set_gid_sticky_bits --- easybuild/tools/filetools.py | 46 +++++++++++---------- test/framework/filetools.py | 78 ++++++++++++++++++++++++++++++++++-- 2 files changed, 98 insertions(+), 26 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 97e1501a38..a2e2709917 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1667,6 +1667,25 @@ def patch_perl_script_autoflush(path): write_file(path, newtxt) +def set_gid_sticky_bits(path, set_gid=None, sticky=None, recursive=False): + """Set GID/sticky bits on specified path.""" + if set_gid is None: + set_gid = build_option('set_gid_bit') + if sticky is None: + sticky = build_option('sticky_bit') + + bits = 0 + if set_gid: + bits |= stat.S_ISGID + if sticky: + bits |= stat.S_ISVTX + if bits: + try: + adjust_permissions(path, bits, add=True, relative=True, recursive=recursive, onlydirs=True) + except OSError as err: + raise EasyBuildError("Failed to set groud ID/sticky bit: %s", err) + + def mkdir(path, parents=False, set_gid=None, sticky=None): """ Create a directory @@ -1702,18 +1721,9 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): raise EasyBuildError("Failed to create directory %s: %s", path, err) # set group ID and sticky bits, if desired - bits = 0 - if set_gid: - bits |= stat.S_ISGID - if sticky: - bits |= stat.S_ISVTX - if bits: - try: - new_subdir = path[len(existing_parent_path):].lstrip(os.path.sep) - new_path = os.path.join(existing_parent_path, new_subdir.split(os.path.sep)[0]) - adjust_permissions(new_path, bits, add=True, relative=True, recursive=True, onlydirs=True) - except OSError as err: - raise EasyBuildError("Failed to set groud ID/sticky bit: %s", err) + new_subdir = path[len(existing_parent_path):].lstrip(os.path.sep) + new_path = os.path.join(existing_parent_path, new_subdir.split(os.path.sep)[0]) + set_gid_sticky_bits(new_path, set_gid, sticky, recursive=True) else: _log.debug("Not creating existing path %s" % path) @@ -2614,14 +2624,6 @@ def create_unused_dir(parent_folder, name): raise EasyBuildError("Failed to create directory %s: %s", path, err) # set group ID and sticky bits, if desired - bits = 0 - if build_option('set_gid_bit'): - bits |= stat.S_ISGID - if build_option('sticky_bit'): - bits |= stat.S_ISVTX - if bits: - try: - adjust_permissions(path, bits, add=True, relative=True, recursive=True, onlydirs=True) - except OSError as err: - raise EasyBuildError("Failed to set group ID/sticky bit: %s", err) + set_gid_sticky_bits(path, recursive=True) + return path diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 9ced5ac21a..fb9c2440e4 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2947,16 +2947,84 @@ def test_locate_files(self): error_pattern = r"One or more files not found: 2\.txt \(search paths: \)" self.assertErrorRegex(EasyBuildError, error_pattern, ft.locate_files, ['2.txt'], []) + def test_set_gid_sticky_bits(self): + """Test for set_gid_sticky_bits function.""" + test_dir = os.path.join(self.test_prefix, 'test_dir') + test_subdir = os.path.join(test_dir, 'subdir') + + ft.mkdir(test_subdir, parents=True) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, 0) + self.assertEqual(dir_perms & stat.S_ISVTX, 0) + dir_perms = os.lstat(test_subdir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, 0) + self.assertEqual(dir_perms & stat.S_ISVTX, 0) + + # by default, GID & sticky bits are not set + ft.set_gid_sticky_bits(test_dir) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, 0) + self.assertEqual(dir_perms & stat.S_ISVTX, 0) + + ft.set_gid_sticky_bits(test_dir, set_gid=True) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, 0) + ft.remove_dir(test_dir) + ft.mkdir(test_subdir, parents=True) + + ft.set_gid_sticky_bits(test_dir, sticky=True) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, 0) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + ft.remove_dir(test_dir) + ft.mkdir(test_subdir, parents=True) + + ft.set_gid_sticky_bits(test_dir, set_gid=True, sticky=True) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + # no recursion by default + dir_perms = os.lstat(test_subdir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, 0) + self.assertEqual(dir_perms & stat.S_ISVTX, 0) + + ft.remove_dir(test_dir) + ft.mkdir(test_subdir, parents=True) + + ft.set_gid_sticky_bits(test_dir, set_gid=True, sticky=True, recursive=True) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + dir_perms = os.lstat(test_subdir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + + ft.remove_dir(test_dir) + ft.mkdir(test_subdir, parents=True) + + # set_gid_sticky_bits honors relevant build options + init_config(build_options={'set_gid_bit': True, 'sticky_bit': True}) + ft.set_gid_sticky_bits(test_dir, recursive=True) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + dir_perms = os.lstat(test_subdir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + def test_create_unused_dir(self): """Test create_unused_dir function.""" path = ft.create_unused_dir(self.test_prefix, 'folder') self.assertEqual(path, os.path.join(self.test_prefix, 'folder')) self.assertTrue(os.path.exists(path)) + # Repeat with existing folder(s) should create new ones for i in range(10): path = ft.create_unused_dir(self.test_prefix, 'folder') self.assertEqual(path, os.path.join(self.test_prefix, 'folder_%s' % i)) self.assertTrue(os.path.exists(path)) + # Not influenced by similar folder path = ft.create_unused_dir(self.test_prefix, 'folder2') self.assertEqual(path, os.path.join(self.test_prefix, 'folder2')) @@ -2965,18 +3033,20 @@ def test_create_unused_dir(self): path = ft.create_unused_dir(self.test_prefix, 'folder2') self.assertEqual(path, os.path.join(self.test_prefix, 'folder2_%s' % i)) self.assertTrue(os.path.exists(path)) + # Fail cleanly if passed a readonly folder readonly_dir = os.path.join(self.test_prefix, 'ro_folder') - os.mkdir(readonly_dir) + ft.mkdir(readonly_dir) old_perms = os.lstat(readonly_dir)[stat.ST_MODE] - os.chmod(readonly_dir, stat.S_IREAD | stat.S_IEXEC) + ft.adjust_permissions(readonly_dir, stat.S_IREAD | stat.S_IEXEC, relative=False) try: self.assertErrorRegex(EasyBuildError, 'Failed to create directory', ft.create_unused_dir, readonly_dir, 'new_folder') finally: - os.chmod(readonly_dir, old_perms) + ft.adjust_permissions(readonly_dir, old_perms, relative=False) + # Ignore files same as folders. So first just create a file with no contents - open(os.path.join(self.test_prefix, 'file'), 'w').close() + ft.write_file(os.path.join(self.test_prefix, 'file'), '') path = ft.create_unused_dir(self.test_prefix, 'file') self.assertEqual(path, os.path.join(self.test_prefix, 'file_0')) self.assertTrue(os.path.exists(path)) From 52932185fdd5cbda7690978214369b8169bbef43 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 5 Apr 2021 19:25:48 +0200 Subject: [PATCH 179/864] extend test for 'module use', also test 'module unuse' --- test/framework/modules.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index e370b1a88f..1852b793a8 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1142,8 +1142,8 @@ def test_module_caches(self): self.assertEqual(mod.MODULE_AVAIL_CACHE, {}) self.assertEqual(mod.MODULE_SHOW_CACHE, {}) - def test_module_use(self): - """Test 'module use'.""" + def test_module_use_unuse(self): + """Test 'module use' and 'module unuse'.""" test_dir1 = os.path.join(self.test_prefix, 'one') test_dir2 = os.path.join(self.test_prefix, 'two') test_dir3 = os.path.join(self.test_prefix, 'three') @@ -1151,6 +1151,16 @@ def test_module_use(self): self.assertFalse(test_dir1 in os.environ.get('MODULEPATH', '')) self.modtool.use(test_dir1) self.assertTrue(os.environ.get('MODULEPATH', '').startswith('%s:' % test_dir1)) + self.modtool.use(test_dir2) + self.assertTrue(os.environ.get('MODULEPATH', '').startswith('%s:' % test_dir2)) + self.modtool.use(test_dir3) + self.assertTrue(os.environ.get('MODULEPATH', '').startswith('%s:' % test_dir3)) + self.modtool.unuse(test_dir3) + self.assertFalse(test_dir3 in os.environ.get('MODULEPATH', '')) + self.modtool.unuse(test_dir2) + self.assertFalse(test_dir2 in os.environ.get('MODULEPATH', '')) + self.modtool.unuse(test_dir1) + self.assertFalse(test_dir1 in os.environ.get('MODULEPATH', '')) # also test use with high priority self.modtool.use(test_dir2, priority=10000) @@ -1158,8 +1168,9 @@ def test_module_use(self): # check whether prepend with priority actually works (only for Lmod) if isinstance(self.modtool, Lmod): + self.modtool.use(test_dir1, priority=100) self.modtool.use(test_dir3) - self.assertTrue(os.environ['MODULEPATH'].startswith('%s:%s:' % (test_dir2, test_dir3))) + self.assertTrue(os.environ['MODULEPATH'].startswith('%s:%s:%s:' % (test_dir2, test_dir1, test_dir3))) def test_module_use_bash(self): """Test whether effect of 'module use' is preserved when a new bash session is started.""" From a45763f42be18f6b61960482b02ca4b52a8e56eb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 5 Apr 2021 19:36:46 +0200 Subject: [PATCH 180/864] also load module when testing use/unuse --- test/framework/modules.py | 40 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/framework/modules.py b/test/framework/modules.py index 1852b793a8..a6c2dcddcf 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1148,6 +1148,13 @@ def test_module_use_unuse(self): test_dir2 = os.path.join(self.test_prefix, 'two') test_dir3 = os.path.join(self.test_prefix, 'three') + for subdir in ('one', 'two', 'three'): + modtxt = '\n'.join([ + '#%Module', + "setenv TEST123 %s" % subdir, + ]) + write_file(os.path.join(self.test_prefix, subdir, 'test'), modtxt) + self.assertFalse(test_dir1 in os.environ.get('MODULEPATH', '')) self.modtool.use(test_dir1) self.assertTrue(os.environ.get('MODULEPATH', '').startswith('%s:' % test_dir1)) @@ -1155,10 +1162,26 @@ def test_module_use_unuse(self): self.assertTrue(os.environ.get('MODULEPATH', '').startswith('%s:' % test_dir2)) self.modtool.use(test_dir3) self.assertTrue(os.environ.get('MODULEPATH', '').startswith('%s:' % test_dir3)) + + # make sure the right test module is loaded + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'three') + self.modtool.unload(['test']) + self.modtool.unuse(test_dir3) self.assertFalse(test_dir3 in os.environ.get('MODULEPATH', '')) + + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'two') + self.modtool.unload(['test']) + self.modtool.unuse(test_dir2) self.assertFalse(test_dir2 in os.environ.get('MODULEPATH', '')) + + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'one') + self.modtool.unload(['test']) + self.modtool.unuse(test_dir1) self.assertFalse(test_dir1 in os.environ.get('MODULEPATH', '')) @@ -1166,11 +1189,28 @@ def test_module_use_unuse(self): self.modtool.use(test_dir2, priority=10000) self.assertTrue(os.environ['MODULEPATH'].startswith('%s:' % test_dir2)) + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'two') + self.modtool.unload(['test']) + # check whether prepend with priority actually works (only for Lmod) if isinstance(self.modtool, Lmod): self.modtool.use(test_dir1, priority=100) self.modtool.use(test_dir3) self.assertTrue(os.environ['MODULEPATH'].startswith('%s:%s:%s:' % (test_dir2, test_dir1, test_dir3))) + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'two') + self.modtool.unload(['test']) + + self.modtool.unuse(test_dir2) + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'one') + self.modtool.unload(['test']) + + self.modtool.unuse(test_dir1) + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'three') + self.modtool.unload(['test']) def test_module_use_bash(self): """Test whether effect of 'module use' is preserved when a new bash session is started.""" From 5509a090874fbbdcb5a7bd3f7c5271d35ff2d9d4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 6 Apr 2021 11:46:43 +0200 Subject: [PATCH 181/864] update validate_github_token function to accept GitHub token in new format --- easybuild/tools/github.py | 15 ++++++++++++--- test/framework/github.py | 5 +++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 54d7ac2e00..9c7d04d31c 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -2156,14 +2156,22 @@ def validate_github_token(token, github_user): * see if it conforms expectations (only [a-f]+[0-9] characters, length of 40) * see if it can be used for authenticated access """ - sha_regex = re.compile('^[0-9a-f]{40}') + # cfr. https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/ + token_regex = re.compile('^ghp_[a-zA-Z0-9]{36}$') + token_regex_old_format = re.compile('^[0-9a-f]{40}$') # token should be 40 characters long, and only contain characters in [0-9a-f] - sanity_check = bool(sha_regex.match(token)) + sanity_check = bool(token_regex.match(token)) if sanity_check: _log.info("Sanity check on token passed") else: - _log.warning("Sanity check on token failed; token doesn't match pattern '%s'", sha_regex.pattern) + _log.warning("Sanity check on token failed; token doesn't match pattern '%s'", token_regex.pattern) + sanity_check = bool(token_regex_old_format.match(token)) + if sanity_check: + _log.info("Sanity check on token (old format) passed") + else: + _log.warning("Sanity check on token failed; token doesn't match pattern '%s'", + token_regex_old_format.pattern) # try and determine sha of latest commit in easybuilders/easybuild-easyconfigs repo through authenticated access sha = None @@ -2173,6 +2181,7 @@ def validate_github_token(token, github_user): except Exception as err: _log.warning("An exception occurred when trying to use token for authenticated GitHub access: %s", err) + sha_regex = re.compile('^[0-9a-f]{40}$') token_test = bool(sha_regex.match(sha or '')) if token_test: _log.info("GitHub token can be used for authenticated GitHub access, validation passed") diff --git a/test/framework/github.py b/test/framework/github.py index deb1a706f7..dbae54fbf6 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -547,6 +547,11 @@ def test_validate_github_token(self): self.assertTrue(gh.validate_github_token(self.github_token, GITHUB_TEST_ACCOUNT)) + # if a token in the old format is available, test with that too + token_old_format = os.getenv('TEST_GITHUB_TOKEN_OLD_FORMAT') + if token_old_format: + self.assertTrue(gh.validate_github_token(token_old_format, GITHUB_TEST_ACCOUNT)) + def test_find_easybuild_easyconfig(self): """Test for find_easybuild_easyconfig function""" if self.skip_github_tests: From b5e154c95be5dfc709f6738a1fbca0fb10805a1c Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 6 Apr 2021 12:40:17 +0200 Subject: [PATCH 182/864] Avoid module call for unuse() for LMod and set $MODULEPATH directly Followup to #3557 No check for priorities is required as LMod correctly handles that Also adds a check for an empty path which is technically possible and handled by LMod like any other path --- easybuild/tools/modules.py | 15 +++++++++++++++ test/framework/modules.py | 18 +++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 14509e79d8..3e3f004dc5 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1443,6 +1443,21 @@ def use(self, path, priority=None): ('' if cur_mod_path is None else cur_mod_path, new_mod_path)) os.environ['MODULEPATH'] = new_mod_path + def unuse(self, path): + """Remove a module path""" + # We can simply remove the path from MODULEPATH to avoid the costly module call + cur_mod_path = os.environ.get('MODULEPATH') + if cur_mod_path is not None: + # Removing the last entry unsets the variable + if cur_mod_path == path: + self.log.debug('Changing MODULEPATH from %s to ' % cur_mod_path) + del os.environ['MODULEPATH'] + else: + new_mod_path = ':'.join(p for p in cur_mod_path.split(':') if p != path) + if new_mod_path != cur_mod_path: + self.log.debug('Changing MODULEPATH from %s to %s' % (cur_mod_path, new_mod_path)) + os.environ['MODULEPATH'] = new_mod_path + def prepend_module_path(self, path, set_mod_paths=True, priority=None): """ Prepend given module path to list of module paths, or bump it to 1st place. diff --git a/test/framework/modules.py b/test/framework/modules.py index c71b5f4360..33ef840e8e 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1157,11 +1157,23 @@ def test_module_use_unuse(self): self.assertFalse(test_dir1 in os.environ.get('MODULEPATH', '')) self.modtool.use(test_dir1) - self.assertTrue(os.environ.get('MODULEPATH', '').startswith('%s:' % test_dir1)) + self.assertTrue(os.environ['MODULEPATH'].startswith('%s:' % test_dir1)) self.modtool.use(test_dir2) - self.assertTrue(os.environ.get('MODULEPATH', '').startswith('%s:' % test_dir2)) + self.assertTrue(os.environ['MODULEPATH'].startswith('%s:' % test_dir2)) self.modtool.use(test_dir3) - self.assertTrue(os.environ.get('MODULEPATH', '').startswith('%s:' % test_dir3)) + self.assertTrue(os.environ['MODULEPATH'].startswith('%s:' % test_dir3)) + + # Using an empty path still works (technically) + old_module_path = os.environ['MODULEPATH'] + self.modtool.use('') + self.assertEqual(os.environ['MODULEPATH'], ':' + old_module_path) + self.modtool.unuse('') + self.assertEqual(os.environ['MODULEPATH'], old_module_path) + # Even works when the whole path is empty + os.environ['MODULEPATH'] = '' + self.modtool.unuse('') + self.assertFalse('MODULEPATH' in os.environ) + os.environ['MODULEPATH'] = old_module_path # Restore # make sure the right test module is loaded self.modtool.load(['test']) From 3262fc69b93387deb8f5f6ff08449ef27589b5a5 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 6 Apr 2021 12:45:04 +0200 Subject: [PATCH 183/864] Add test for adding to non-existing MODULEPATH and removing it --- test/framework/modules.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/framework/modules.py b/test/framework/modules.py index 33ef840e8e..a9111620b9 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1224,6 +1224,16 @@ def test_module_use_unuse(self): self.assertEqual(os.getenv('TEST123'), 'three') self.modtool.unload(['test']) + # Also test that load and unload a single path works when it is the only one + # Only for LMod as we have some shortcuts for avoiding the module call there + old_module_path = os.environ['MODULEPATH'] + del os.environ['MODULEPATH'] + self.modtool.use(test_dir1) + self.assertEqual(os.environ['MODULEPATH'], test_dir1) + self.modtool.unuse(test_dir1) + self.assertFalse('MODULEPATH' in os.environ) + os.environ['MODULEPATH'] = old_module_path # Restore + def test_module_use_bash(self): """Test whether effect of 'module use' is preserved when a new bash session is started.""" # this test is here as check for a nasty bug in how the modules tool is deployed From 21083d3b156e92ac69b8eddb20ab4525f22c3d18 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 6 Apr 2021 14:49:55 +0200 Subject: [PATCH 184/864] Test special cases for LMod only --- test/framework/modules.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index a9111620b9..a2f4633787 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1163,18 +1163,6 @@ def test_module_use_unuse(self): self.modtool.use(test_dir3) self.assertTrue(os.environ['MODULEPATH'].startswith('%s:' % test_dir3)) - # Using an empty path still works (technically) - old_module_path = os.environ['MODULEPATH'] - self.modtool.use('') - self.assertEqual(os.environ['MODULEPATH'], ':' + old_module_path) - self.modtool.unuse('') - self.assertEqual(os.environ['MODULEPATH'], old_module_path) - # Even works when the whole path is empty - os.environ['MODULEPATH'] = '' - self.modtool.unuse('') - self.assertFalse('MODULEPATH' in os.environ) - os.environ['MODULEPATH'] = old_module_path # Restore - # make sure the right test module is loaded self.modtool.load(['test']) self.assertEqual(os.getenv('TEST123'), 'three') @@ -1205,8 +1193,9 @@ def test_module_use_unuse(self): self.assertEqual(os.getenv('TEST123'), 'two') self.modtool.unload(['test']) - # check whether prepend with priority actually works (only for Lmod) + # Tests for Lmod only if isinstance(self.modtool, Lmod): + # check whether prepend with priority actually works (priority is specific to Lmod) self.modtool.use(test_dir1, priority=100) self.modtool.use(test_dir3) self.assertTrue(os.environ['MODULEPATH'].startswith('%s:%s:%s:' % (test_dir2, test_dir1, test_dir3))) @@ -1224,8 +1213,8 @@ def test_module_use_unuse(self): self.assertEqual(os.getenv('TEST123'), 'three') self.modtool.unload(['test']) - # Also test that load and unload a single path works when it is the only one - # Only for LMod as we have some shortcuts for avoiding the module call there + # Check load and unload for a single path when it is the only one + # Only for Lmod as we have some shortcuts for avoiding the module call there old_module_path = os.environ['MODULEPATH'] del os.environ['MODULEPATH'] self.modtool.use(test_dir1) @@ -1234,6 +1223,18 @@ def test_module_use_unuse(self): self.assertFalse('MODULEPATH' in os.environ) os.environ['MODULEPATH'] = old_module_path # Restore + # Using an empty path still works (technically) (Lmod only, ignored by Tcl) + old_module_path = os.environ['MODULEPATH'] + self.modtool.use('') + self.assertEqual(os.environ['MODULEPATH'], ':' + old_module_path) + self.modtool.unuse('') + self.assertEqual(os.environ['MODULEPATH'], old_module_path) + # Even works when the whole path is empty + os.environ['MODULEPATH'] = '' + self.modtool.unuse('') + self.assertFalse('MODULEPATH' in os.environ) + os.environ['MODULEPATH'] = old_module_path # Restore + def test_module_use_bash(self): """Test whether effect of 'module use' is preserved when a new bash session is started.""" # this test is here as check for a nasty bug in how the modules tool is deployed From 34a6c85b6c46d1cb08255d2d6fe51d5c6a5b0a2f Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 7 Apr 2021 10:52:51 +0800 Subject: [PATCH 185/864] prepare release notes for EasyBuild v4.3.4 + bump version to 4.3.4 --- RELEASE_NOTES | 37 +++++++++++++++++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 178f0fb4cb..89a605a683 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,42 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.3.4 (April 9th 2021) +----------------------- + +update/bugfix release + +- various enhancements, including: + - specifying False for a version removes the dep (#3506) + - add create_unused_dir function to create a directory which does not yet exist (#3551) + - create CUDA cache (for JIT compiled PTX code) in build dir instead of $HOME (#3569) + - add "Citing" section to module files (#3596) + - add support for using fallback 'arch=*' key in dependency version specified as arch->version mapping (#3600) + - also check for pending change requests and mergeable_state in check_pr_eligible_to_merge (#3604) + - ignore undismissed 'changes requested' review if there is an 'approved' review by the same user (#3607) + - filter duplicates in changes_requested_by (#3608) + - sort output of eb --search in natural order (respecting numbers) (#3609) + - enhance 'eb' command to ensure that easybuild.main can be imported before settling on python* command to use (#3610) + - added configuration option to define the env command to use for shebangs (#3613) + - add templates for architecture independent Python wheels (#3618) + - mention easyblocks PR in gist when uploading test report for it + fix clean_gists.py script (#3622) + - deprecate --accept-eula, rename to --accept-eula-for + also accept regular expression value (#3630) + - update validate_github_token function to accept GitHub token in new format (#3632) +- various bug fixes, including: + - fix $BLAS_LIB_MT for OpenBLAS, ensure -lpthread is included (#3584) + - use '--opt=val' for passing settings from config file to option parser to avoid error for values starting with '-' or '--' (#3594) + - avoid raised exception when getting output from interactive command in run_cmd_qa (#3599) + - add option to write file from file-like object and use in download_file (#3614) + - make sure that path to eb is always found by tests (#3617) + - fix typo in docstring of goblf toolchain definition (#3628) +- other changes: + - avoid running expensive 'module use' command when using Lmod as modules tool, update $MODULEPATH directly instead (#3557) + - add pick_default_branch function to clean up duplicate code in tools/github.py (#3592) + - refactor the CI configuration to use inclusion instead of exclusion (#3616) + - use develop branch when testing push access in --check-github (#3629) + - avoid module call for unuse() for Lmod and set $MODULEPATH directly (#3633) + + v4.3.3 (February 23rd 2021) --------------------------- @@ -34,6 +70,7 @@ update/bugfix release - other changes: - rename EasyBlock._skip_step to EasyBlock.skip_step, to make it part of the public API (#3561) - make symlinking of posix_c.so to posix.so in test suite configuration conditional (#3570) + - use 'main' rather than 'master' branch in GitHub integration functionality (#3589) v4.3.2 (December 10th 2020) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index d4822e22ad..74ab8d79d8 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.3.4.dev0') +VERSION = LooseVersion('4.3.4') UNKNOWN = 'UNKNOWN' From 2dc85b487b898bb95a509bbc7903665734cfb50d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 7 Apr 2021 21:46:57 +0200 Subject: [PATCH 186/864] tweak framework release notes for EasyBuild v4.3.4 --- RELEASE_NOTES | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 89a605a683..21d810f690 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -10,20 +10,20 @@ v4.3.4 (April 9th 2021) update/bugfix release - various enhancements, including: - - specifying False for a version removes the dep (#3506) + - add support for filtering dependencies by using False as version (#3506) - add create_unused_dir function to create a directory which does not yet exist (#3551) + - avoid running expensive 'module use' and 'module unuse' commands when using Lmod as modules tool, update $MODULEPATH directly instead (#3557, #3633) - create CUDA cache (for JIT compiled PTX code) in build dir instead of $HOME (#3569) - add "Citing" section to module files (#3596) - add support for using fallback 'arch=*' key in dependency version specified as arch->version mapping (#3600) - also check for pending change requests and mergeable_state in check_pr_eligible_to_merge (#3604) - - ignore undismissed 'changes requested' review if there is an 'approved' review by the same user (#3607) - - filter duplicates in changes_requested_by (#3608) - - sort output of eb --search in natural order (respecting numbers) (#3609) + - ignore undismissed 'changes requested' review if there is an 'approved' review by the same user (#3607, #3608) + - sort output of 'eb --search' in natural order (respecting numbers) (#3609) - enhance 'eb' command to ensure that easybuild.main can be imported before settling on python* command to use (#3610) - - added configuration option to define the env command to use for shebangs (#3613) + - add --env-for-shebang configuration option to define the env command to use for shebangs (#3613) - add templates for architecture independent Python wheels (#3618) - mention easyblocks PR in gist when uploading test report for it + fix clean_gists.py script (#3622) - - deprecate --accept-eula, rename to --accept-eula-for + also accept regular expression value (#3630) + - also accept regular expression value for --accept-eula-for (#3630) - update validate_github_token function to accept GitHub token in new format (#3632) - various bug fixes, including: - fix $BLAS_LIB_MT for OpenBLAS, ensure -lpthread is included (#3584) @@ -31,13 +31,11 @@ update/bugfix release - avoid raised exception when getting output from interactive command in run_cmd_qa (#3599) - add option to write file from file-like object and use in download_file (#3614) - make sure that path to eb is always found by tests (#3617) - - fix typo in docstring of goblf toolchain definition (#3628) - other changes: - - avoid running expensive 'module use' command when using Lmod as modules tool, update $MODULEPATH directly instead (#3557) - add pick_default_branch function to clean up duplicate code in tools/github.py (#3592) - refactor the CI configuration to use inclusion instead of exclusion (#3616) - use develop branch when testing push access in --check-github (#3629) - - avoid module call for unuse() for Lmod and set $MODULEPATH directly (#3633) + - deprecate --accept-eula, rename to --accept-eula-for (#3630) v4.3.3 (February 23rd 2021) From dfa852fc66a45b6d5fd9444d0bc312caa109d25c Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Fri, 9 Apr 2021 17:25:17 +0800 Subject: [PATCH 187/864] bump version to 4.3.5dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 74ab8d79d8..089f123624 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.3.4') +VERSION = LooseVersion('4.3.5.dev0') UNKNOWN = 'UNKNOWN' From 63e62b36187c3ee9c3595bbf376cf87de5954a68 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 9 Apr 2021 12:01:34 +0000 Subject: [PATCH 188/864] Bump cryptography from 3.2.1 to 3.3.2 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.2.1 to 3.3.2. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.2.1...3.3.2) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0defb13c7c..08d997317b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -55,5 +55,5 @@ archspec; python_version >= '2.7' # cryptography 3.0 deprecates Python 2.7 (but v3.2.1 still works with Python 2.7); # cryptography is not needed at all for Python 2.6 -cryptography==3.2.1; python_version == '2.7' +cryptography==3.3.2; python_version == '2.7' cryptography; python_version >= '3.5' From b8fa9b14e35d6ae073beb7c6ab41f8d72ba5e5ad Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 9 Apr 2021 15:36:19 +0200 Subject: [PATCH 189/864] Catch problems if no github user specified --- easybuild/tools/github.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 9c7d04d31c..34b08cf8be 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -688,8 +688,10 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa # git fetch # can't use --depth to only fetch a shallow copy, since pushing to another repo from a shallow copy doesn't work print_msg("fetching branch '%s' from %s..." % (branch_name, github_url), silent=silent) + res = None try: - res = origin.fetch() + if target_account != None: + res = origin.fetch() except GitCommandError as err: raise EasyBuildError("Failed to fetch branch '%s' from %s: %s", branch_name, github_url, err) if res: @@ -971,21 +973,24 @@ def push_branch_to_github(git_repo, target_account, target_repo, branch): if dry_run: print_msg(push_branch_msg + ' [DRY RUN]', log=_log) else: - print_msg(push_branch_msg, log=_log) - try: - res = remote.push(branch) - except GitCommandError as err: - raise EasyBuildError("Failed to push branch '%s' to GitHub (%s): %s", branch, github_url, err) + if target_account == None: + raise EasyBuildError("No valid GitHub username (--github-user) given, pushing branch will fail!") + else: + print_msg(push_branch_msg, log=_log) + try: + res = remote.push(branch) + except GitCommandError as err: + raise EasyBuildError("Failed to push branch '%s' to GitHub (%s): %s", branch, github_url, err) - if res: - if res[0].ERROR & res[0].flags: - raise EasyBuildError("Pushing branch '%s' to remote %s (%s) failed: %s", - branch, remote, github_url, res[0].summary) + if res: + if res[0].ERROR & res[0].flags: + raise EasyBuildError("Pushing branch '%s' to remote %s (%s) failed: %s", + branch, remote, github_url, res[0].summary) + else: + _log.debug("Pushed branch %s to remote %s (%s): %s", branch, remote, github_url, res[0].summary) else: - _log.debug("Pushed branch %s to remote %s (%s): %s", branch, remote, github_url, res[0].summary) - else: - raise EasyBuildError("Pushing branch '%s' to remote %s (%s) failed: empty result", - branch, remote, github_url) + raise EasyBuildError("Pushing branch '%s' to remote %s (%s) failed: empty result", + branch, remote, github_url) def is_patch_for(patch_name, ec): From 54fb85f90ee41bcc4ec47ef4b8e95f89631a6461 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 9 Apr 2021 15:43:57 +0200 Subject: [PATCH 190/864] Fix None comparisons --- easybuild/tools/github.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 34b08cf8be..e3d695abc4 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -690,7 +690,7 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa print_msg("fetching branch '%s' from %s..." % (branch_name, github_url), silent=silent) res = None try: - if target_account != None: + if target_account is not None: res = origin.fetch() except GitCommandError as err: raise EasyBuildError("Failed to fetch branch '%s' from %s: %s", branch_name, github_url, err) @@ -973,7 +973,7 @@ def push_branch_to_github(git_repo, target_account, target_repo, branch): if dry_run: print_msg(push_branch_msg + ' [DRY RUN]', log=_log) else: - if target_account == None: + if target_account is None: raise EasyBuildError("No valid GitHub username (--github-user) given, pushing branch will fail!") else: print_msg(push_branch_msg, log=_log) From c821a388b8e5e471bc40ccf04db6b1aac7a923cf Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 9 Apr 2021 18:55:04 +0200 Subject: [PATCH 191/864] Don't skip sanity check for --module-only --rebuild --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 77e0df72a5..bd6e4f3106 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3096,7 +3096,7 @@ def update_config_template_run_step(self): def skip_step(self, step, skippable): """Dedice whether or not to skip the specified step.""" module_only = build_option('module_only') - force = build_option('force') or build_option('rebuild') + force = build_option('force') skip = False # under --skip, sanity check is not skipped From 0339eb50c2c8fb9cf6b22c813925f8b299f87e1b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 9 Apr 2021 19:39:16 +0200 Subject: [PATCH 192/864] Fix test --- test/framework/toy_build.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index f3c0abda5a..64ae9de0f7 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1583,16 +1583,11 @@ def test_module_only(self): os.remove(toy_mod) - # --module-only --rebuild should be equivalent with --module-only --force - self.eb_main(args + ['--rebuild'], do_build=True, raise_error=True) - self.assertTrue(os.path.exists(toy_mod)) - - # make sure load statements for dependencies are included in additional module file generated with --module-only - modtxt = read_file(toy_mod) - self.assertTrue(re.search('load.*intel/2018a', modtxt), "load statement for intel/2018a found in module") - self.assertTrue(re.search('load.*GCC/6.4.0-2.28', modtxt), "load statement for GCC/6.4.0-2.28 found in module") - - os.remove(toy_mod) + # --module-only --rebuild should run sanity check + rebuild_args = args + ['--rebuild'] + err_msg = "Sanity check failed" + self.assertErrorRegex(EasyBuildError, err_msg, self.eb_main, rebuild_args, do_build=True, raise_error=True) + self.assertFalse(os.path.exists(toy_mod)) # installing another module under a different naming scheme and using Lua module syntax works fine From 38f0ce8a1c4dec83b87bf23ec427ab56e2e6141d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 10 Apr 2021 10:07:44 +0200 Subject: [PATCH 193/864] test bootstrap script in separate workflow, and limit test configurations a bit (fixes #3639) --- .github/workflows/bootstrap_script.yml | 120 +++++++++++++++++++++++++ .github/workflows/unit_tests.yml | 33 ------- 2 files changed, 120 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/bootstrap_script.yml diff --git a/.github/workflows/bootstrap_script.yml b/.github/workflows/bootstrap_script.yml new file mode 100644 index 0000000000..e98115cc79 --- /dev/null +++ b/.github/workflows/bootstrap_script.yml @@ -0,0 +1,120 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: test EasyBuild bootstrap script +on: [push, pull_request] +jobs: + setup: + runs-on: ubuntu-latest + outputs: + lmod7: Lmod-7.8.22 + lmod8: Lmod-8.4.27 + modulesTcl: modules-tcl-1.147 + modules3: modules-3.2.10 + modules4: modules-4.1.4 + steps: + - run: "true" + build: + needs: setup + runs-on: ubuntu-18.04 + strategy: + matrix: + python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] + modules_tool: + # use variables defined by 'setup' job above, see also + # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context + - ${{needs.setup.outputs.lmod7}} + - ${{needs.setup.outputs.lmod8}} + module_syntax: [Lua] + lc_all: [""] + include: + # also test with module tools other than Lmod (only Tcl syntax) + - modules_tool: ${{needs.setup.outputs.modulesTcl}} + module_syntax: Tcl + python: 2.7 + - modules_tool: ${{needs.setup.outputs.modulesTcl}} + module_syntax: Tcl + python: 3.6 + - modules_tool: ${{needs.setup.outputs.modules3}} + module_syntax: Tcl + python: 2.7 + - modules_tool: ${{needs.setup.outputs.modules3}} + module_syntax: Tcl + python: 3.6 + - modules_tool: ${{needs.setup.outputs.modules4}} + module_syntax: Tcl + python: 2.7 + - modules_tool: ${{needs.setup.outputs.modules4}} + module_syntax: Tcl + python: 3.6 + # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set + # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7) + - python: 3.6 + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Lua + lc_all: C + fail-fast: false + steps: + - uses: actions/checkout@v2 + + - name: set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{matrix.python}} + architecture: x64 + + - name: install OS & Python packages + run: | + # disable apt-get update, we don't really need it, + # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) + # sudo apt-get update + # for modules tool + sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev + # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 + # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists + if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then + sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so + fi + + - name: install modules tool + run: | + # avoid downloading modules tool sources into easybuild-framework dir + cd $HOME + export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh + # install Lmod + source $INSTALL_DEP ${{matrix.modules_tool}} $HOME + # changes in environment are not passed to other steps, so need to create files... + echo $MOD_INIT > mod_init + echo $PATH > path + if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi + + - name: test bootstrap script + run: | + # (re)initialize environment for modules tool + if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi + source $(cat $HOME/mod_init); type module + # also pick up changes to $PATH set by sourcing $HOME/mod_init + export PATH=$(cat $HOME/path) + + # define $EASYBUILD_MODULES_TOOL only for oldest module tools + # (for Lmod and EnvironmentModules 4.x the bootstrap script should correctly auto-detect the modules tool) + if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then + export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl + elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then + export EASYBUILD_MODULES_TOOL=EnvironmentModulesC + fi + + # version and SHA256 checksum are hardcoded below to avoid forgetting to update the version in the script along with contents + EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') + EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') + EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" + EB_BOOTSTRAP_EXPECTED="20210106.01 c2d93de0dd91123eb4f51cfc16d1f5efb80f1d238b3d6cd100994086887a1ae0" + test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) + + # test bootstrap script + export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap + python easybuild/scripts/bootstrap_eb.py $PREFIX + # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module + unset PYTHONPATH + # simple sanity check on bootstrapped EasyBuild module + module use $PREFIX/modules/all + module load EasyBuild + eb --version diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 1f31dd395a..ce18b787e6 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -197,36 +197,3 @@ jobs: # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1) - - - name: test bootstrap script - run: | - # (re)initialize environment for modules tool - if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi - source $(cat $HOME/mod_init); type module - # also pick up changes to $PATH set by sourcing $HOME/mod_init - export PATH=$(cat $HOME/path) - - # define $EASYBUILD_MODULES_TOOL only for oldest module tools - # (for Lmod and EnvironmentModules 4.x the bootstrap script should correctly auto-detect the modules tool) - if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then - export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl - elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then - export EASYBUILD_MODULES_TOOL=EnvironmentModulesC - fi - - # version and SHA256 checksum are hardcoded below to avoid forgetting to update the version in the script along with contents - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') - EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - EB_BOOTSTRAP_EXPECTED="20210106.01 c2d93de0dd91123eb4f51cfc16d1f5efb80f1d238b3d6cd100994086887a1ae0" - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) - - # test bootstrap script - export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap - python easybuild/scripts/bootstrap_eb.py $PREFIX - # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module - unset PYTHONPATH - # simple sanity check on bootstrapped EasyBuild module - module use $PREFIX/modules/all - module load EasyBuild - eb --version From 4bd96382e6aee961e07f9e8c0c5c02c02b7e7d5e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 10 Apr 2021 10:28:46 +0200 Subject: [PATCH 194/864] skip testing bootstrap script with Python 3.5, which often fails with 'Could not find suitable distribution' (cfr. #3639) --- .github/workflows/bootstrap_script.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bootstrap_script.yml b/.github/workflows/bootstrap_script.yml index e98115cc79..5a52b0d3a4 100644 --- a/.github/workflows/bootstrap_script.yml +++ b/.github/workflows/bootstrap_script.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-18.04 strategy: matrix: - python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] + python: [2.7, 3.6, 3.7, 3.8, 3.9] modules_tool: # use variables defined by 'setup' job above, see also # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context From 07c3576121f90fe1edd82943ba788dbc7273b292 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 10 Apr 2021 10:29:04 +0200 Subject: [PATCH 195/864] update setup.py to indicate compatibility with Python 3.8 and 3.9 --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 1fec1be8c9..2d5543aa37 100644 --- a/setup.py +++ b/setup.py @@ -114,6 +114,8 @@ def find_rel_test(): "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Topic :: Software Development :: Build Tools", ], platforms="Linux", From 5dd18902164b4feb14661b3e4b02ee2b416cf1fe Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 10 Apr 2021 11:38:09 +0200 Subject: [PATCH 196/864] make sure that --github-org or --github-user is specified in _easyconfigs_pr_common helper function + require that target_account is not None in setup_repo_from and push_branch_to_github --- easybuild/tools/github.py | 42 ++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index e3d695abc4..ecdec1a8a5 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -676,6 +676,9 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa """ _log.debug("Cloning from %s", github_url) + if target_account is None: + raise EasyBuildError("target_account not specified in setup_repo_from!") + # salt to use for names of remotes/branches that are created salt = ''.join(random.choice(ascii_letters) for _ in range(5)) @@ -690,10 +693,10 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa print_msg("fetching branch '%s' from %s..." % (branch_name, github_url), silent=silent) res = None try: - if target_account is not None: - res = origin.fetch() + res = origin.fetch() except GitCommandError as err: raise EasyBuildError("Failed to fetch branch '%s' from %s: %s", branch_name, github_url, err) + if res: if res[0].flags & res[0].ERROR: raise EasyBuildError("Fetching branch '%s' from remote %s failed: %s", branch_name, origin, res[0].note) @@ -809,6 +812,10 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ # if start branch is not specified, we're opening a new PR # account to use is determined by active EasyBuild configuration (--github-org or --github-user) target_account = build_option('github_org') or build_option('github_user') + + if target_account is None: + raise EasyBuildError("--github-org or --github-user must be specified!") + # if branch to start from is specified, we're updating an existing PR start_branch = build_option('pr_target_branch') else: @@ -961,6 +968,8 @@ def push_branch_to_github(git_repo, target_account, target_repo, branch): :param target_repo: repository name :param branch: name of branch to push """ + if target_account is None: + raise EasyBuildError("target_account not specified in push_branch_to_github!") # push to GitHub remote = create_remote(git_repo, target_account, target_repo) @@ -973,24 +982,21 @@ def push_branch_to_github(git_repo, target_account, target_repo, branch): if dry_run: print_msg(push_branch_msg + ' [DRY RUN]', log=_log) else: - if target_account is None: - raise EasyBuildError("No valid GitHub username (--github-user) given, pushing branch will fail!") - else: - print_msg(push_branch_msg, log=_log) - try: - res = remote.push(branch) - except GitCommandError as err: - raise EasyBuildError("Failed to push branch '%s' to GitHub (%s): %s", branch, github_url, err) + print_msg(push_branch_msg, log=_log) + try: + res = remote.push(branch) + except GitCommandError as err: + raise EasyBuildError("Failed to push branch '%s' to GitHub (%s): %s", branch, github_url, err) - if res: - if res[0].ERROR & res[0].flags: - raise EasyBuildError("Pushing branch '%s' to remote %s (%s) failed: %s", - branch, remote, github_url, res[0].summary) - else: - _log.debug("Pushed branch %s to remote %s (%s): %s", branch, remote, github_url, res[0].summary) + if res: + if res[0].ERROR & res[0].flags: + raise EasyBuildError("Pushing branch '%s' to remote %s (%s) failed: %s", + branch, remote, github_url, res[0].summary) else: - raise EasyBuildError("Pushing branch '%s' to remote %s (%s) failed: empty result", - branch, remote, github_url) + _log.debug("Pushed branch %s to remote %s (%s): %s", branch, remote, github_url, res[0].summary) + else: + raise EasyBuildError("Pushing branch '%s' to remote %s (%s) failed: empty result", + branch, remote, github_url) def is_patch_for(patch_name, ec): From b6b2725cd63598d50ba095c64662776ff77a7f38 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 10 Apr 2021 12:02:15 +0100 Subject: [PATCH 197/864] Correct cryptography comment --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 08d997317b..83bede56fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -53,7 +53,7 @@ requests archspec; python_version >= '2.7' -# cryptography 3.0 deprecates Python 2.7 (but v3.2.1 still works with Python 2.7); # cryptography is not needed at all for Python 2.6 +# cryptography 3.4.0 no longer supports Python 2.7 cryptography==3.3.2; python_version == '2.7' cryptography; python_version >= '3.5' From eb7b87f5f81d41a6750b3206036f03cfcdfe4669 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 12 Apr 2021 19:01:07 +0200 Subject: [PATCH 198/864] Re-enable write permissions when installing with read-only-installdir This allows e.g. adding additional packages (e.g. Python). Fixes #3033 --- easybuild/framework/easyblock.py | 7 ++++- test/framework/toy_build.py | 51 +++++++++++++++++++++----------- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 77e0df72a5..77f1572c72 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3401,6 +3401,11 @@ def build_and_install_one(ecdict, init_env): reprod_dir_root = os.path.dirname(app.logfile) reprod_dir = reproduce_build(app, reprod_dir_root) + if os.path.exists(app.installdir) and build_option('read_only_installdir') and ( + build_option('rebuild') or build_option('force')): + # re-enable write permissions so we can install additional modules + adjust_permissions(app.installdir, stat.S_IWUSR, add=True, recursive=True) + result = app.run_all_steps(run_test_cases=run_test_cases) if not dry_run: @@ -3435,7 +3440,7 @@ def build_and_install_one(ecdict, init_env): if build_option('read_only_installdir'): # temporarily re-enable write permissions for copying log/easyconfig to install dir if os.path.exists(new_log_dir): - adjust_permissions(new_log_dir, stat.S_IWUSR, add=True, recursive=False) + adjust_permissions(new_log_dir, stat.S_IWUSR, add=True, recursive=True) else: adjust_permissions(app.installdir, stat.S_IWUSR, add=True, recursive=False) mkdir(new_log_dir, parents=True) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index f3c0abda5a..72bb67b9f6 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -130,11 +130,11 @@ def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versio # make sure installation log file and easyconfig file are copied to install dir software_path = os.path.join(installpath, 'software', 'toy', full_version) install_log_path_pattern = os.path.join(software_path, 'easybuild', 'easybuild-toy-%s*.log' % version) - self.assertTrue(len(glob.glob(install_log_path_pattern)) == 1, "Found 1 file at %s" % install_log_path_pattern) + self.assertTrue(len(glob.glob(install_log_path_pattern)) >= 1, "Found 1 file at %s" % install_log_path_pattern) # make sure test report is available test_report_path_pattern = os.path.join(software_path, 'easybuild', 'easybuild-toy-%s*test_report.md' % version) - self.assertTrue(len(glob.glob(test_report_path_pattern)) == 1, "Found 1 file at %s" % test_report_path_pattern) + self.assertTrue(len(glob.glob(test_report_path_pattern)) >= 1, "Found 1 file at %s" % test_report_path_pattern) ec_file_path = os.path.join(software_path, 'easybuild', 'toy-%s.eb' % full_version) self.assertTrue(os.path.exists(ec_file_path)) @@ -144,7 +144,7 @@ def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versio def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True, fails=False, verbose=True, raise_error=False, test_report=None, versionsuffix='', testing=True, - raise_systemexit=False): + raise_systemexit=False, force=True): """Perform a toy build.""" if extra_args is None: extra_args = [] @@ -160,9 +160,10 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True '--installpath=%s' % self.test_installpath, '--debug', '--unittest-file=%s' % self.logfile, - '--force', '--robot=%s' % os.pathsep.join([self.test_buildpath, os.path.dirname(__file__)]), ] + if force: + args.append('--force') if tmpdir is not None: args.append('--tmpdir=%s' % tmpdir) if test_report is not None: @@ -574,13 +575,18 @@ def test_toy_permissions(self): def test_toy_permissions_installdir(self): """Test --read-only-installdir and --group-write-installdir.""" + # Avoid picking up the already prepared fake module + try: + del os.environ['MODULEPATH'] + except KeyError: + pass # set umask hard to verify default reliably orig_umask = os.umask(0o022) toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec_txt = read_file(toy_ec) # take away read permissions, to check whether they are correctly restored by EasyBuild after installation - test_ec_txt += "\npostinstallcmds = ['chmod -R og-r %(installdir)s']" + test_ec_txt += "\npostinstallcmds += ['chmod -R og-r %(installdir)s']" test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, test_ec_txt) @@ -600,19 +606,28 @@ def test_toy_permissions_installdir(self): shutil.rmtree(self.test_installpath) # check whether --read-only-installdir works as intended - self.test_toy_build(ec_file=test_ec, extra_args=['--read-only-installdir']) - installdir_perms = os.stat(toy_install_dir).st_mode & 0o777 - self.assertEqual(installdir_perms, 0o555, "%s has read-only permissions" % toy_install_dir) - installdir_perms = os.stat(os.path.dirname(toy_install_dir)).st_mode & 0o777 - self.assertEqual(installdir_perms, 0o755, "%s has default permissions" % os.path.dirname(toy_install_dir)) - - # also log file copied into install dir should be read-only (not just the 'easybuild/' subdir itself) - log_path = glob.glob(os.path.join(toy_install_dir, 'easybuild', '*log'))[0] - log_perms = os.stat(log_path).st_mode & 0o777 - self.assertEqual(log_perms, 0o444, "%s has read-only permissions" % log_path) - - toy_bin_perms = os.stat(toy_bin).st_mode & 0o777 - self.assertEqual(toy_bin_perms, 0o555, "%s has read-only permissions" % toy_bin_perms) + # Tested 5 times: + # 1. Non existing build -> Install and set read-only + # 2. Existing build with --rebuild -> Reinstall and set read-only + # 3. Existing build with --force -> Reinstall and set read-only + # 4-5: Same as 2-3 but with --skip + for extra_args in ([], ['--rebuild'], ['--force'], ['--skip', '--rebuild'], ['--skip', '--force']): + self.test_toy_build(ec_file=test_ec, extra_args=['--read-only-installdir'] + extra_args, force=False) + + installdir_perms = os.stat(os.path.dirname(toy_install_dir)).st_mode & 0o777 + self.assertEqual(installdir_perms, 0o755, "%s has default permissions" % os.path.dirname(toy_install_dir)) + + installdir_perms = os.stat(toy_install_dir).st_mode & 0o777 + self.assertEqual(installdir_perms, 0o555, "%s has read-only permissions" % toy_install_dir) + toy_bin_perms = os.stat(toy_bin).st_mode & 0o777 + self.assertEqual(toy_bin_perms, 0o555, "%s has read-only permissions" % toy_bin_perms) + toy_bin_perms = os.stat(os.path.join(toy_install_dir, 'README')).st_mode & 0o777 + self.assertEqual(toy_bin_perms, 0o444, "%s has read-only permissions" % toy_bin_perms) + + # also log file copied into install dir should be read-only (not just the 'easybuild/' subdir itself) + log_path = glob.glob(os.path.join(toy_install_dir, 'easybuild', '*log'))[0] + log_perms = os.stat(log_path).st_mode & 0o777 + self.assertEqual(log_perms, 0o444, "%s has read-only permissions" % log_path) adjust_permissions(toy_install_dir, stat.S_IWUSR, add=True) shutil.rmtree(self.test_installpath) From 321e67c561034dcd83a0a20daf8e2eca86aa58a2 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 12 Apr 2021 19:26:10 +0200 Subject: [PATCH 199/864] Don't assume PYTHONPATH is set in tests --- test/framework/options.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index f3a0a29fed..e7b1fd7dd0 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3405,9 +3405,12 @@ def mk_eb_test_cmd(self, args): """Construct test command for 'eb' with given options.""" # make sure that location to 'easybuild.main' is included in $PYTHONPATH - pythonpath = os.getenv('PYTHONPATH') easybuild_loc = os.path.dirname(os.path.dirname(easybuild.main.__file__)) - os.environ['PYTHONPATH'] = ':'.join([easybuild_loc, pythonpath]) + try: + pythonpath = easybuild_loc + ':' + os.environ['PYTHONPATH'] + except KeyError: + pythonpath = easybuild_loc + os.environ['PYTHONPATH'] = pythonpath return '; '.join([ "cd %s" % self.test_prefix, From 7b167e0544d2b092eaa924f33d5ca158bd673f40 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 13 Apr 2021 09:05:56 +0200 Subject: [PATCH 200/864] Ignore output like 'backup of existing module file' in the readonly-test --- test/framework/toy_build.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 72bb67b9f6..c1c4ee66ac 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -612,7 +612,9 @@ def test_toy_permissions_installdir(self): # 3. Existing build with --force -> Reinstall and set read-only # 4-5: Same as 2-3 but with --skip for extra_args in ([], ['--rebuild'], ['--force'], ['--skip', '--rebuild'], ['--skip', '--force']): + self.mock_stdout(True) self.test_toy_build(ec_file=test_ec, extra_args=['--read-only-installdir'] + extra_args, force=False) + self.mock_stdout(False) installdir_perms = os.stat(os.path.dirname(toy_install_dir)).st_mode & 0o777 self.assertEqual(installdir_perms, 0o755, "%s has default permissions" % os.path.dirname(toy_install_dir)) From fc9a27db9905faeb012967b7a82db987d2b43600 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 13 Apr 2021 13:33:10 +0200 Subject: [PATCH 201/864] Allow for overriding rpath with higher priority paths --- easybuild/framework/easyblock.py | 16 ++++++++++++++++ easybuild/tools/config.py | 3 ++- easybuild/tools/options.py | 2 ++ test/framework/toy_build.py | 28 +++++++++++++++++++++------- 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 77e0df72a5..5c1ced0ef7 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2144,6 +2144,22 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True): '$ORIGIN/../lib64', ] + if build_option('rpath_override_dirs') is not None: + # make sure we have a list + rpath_overrides = build_option('rpath_override_dirs') + if isinstance(rpath_overrides, string_type): + rpath_override_dirs = rpath_overrides.split(':') + _log.debug("Converted RPATH override directories ('%s') to a list of paths: %s" % (rpath_overrides, + rpath_override_dirs)) + for path in rpath_override_dirs: + if not os.path.isabs(path): + raise EasyBuildError( + "Path used in rpath_override_dirs is not an absolute path: %s", path) + else: + raise EasyBuildError("Value for rpath_override_dirs has invalid type (%s), should be string: %s", + type(rpath_overrides), rpath_overrides) + self.rpath_include_dirs.extend(rpath_override_dirs) + if self.iter_idx > 0: # reset toolchain for iterative runs before preparing it again self.toolchain.reset() diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index b491b0cde7..f86e47aef7 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -214,8 +214,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'pr_descr', 'pr_target_repo', 'pr_title', - 'rpath_filter', 'regtest_output_dir', + 'rpath_filter', + 'rpath_override_dirs', 'silence_deprecation_warnings', 'skip', 'stop', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 504b561eeb..eed416c36c 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -452,6 +452,8 @@ def override_options(self): None, 'store_true', False), 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', False), 'rpath-filter': ("List of regex patterns to use for filtering out RPATH paths", 'strlist', 'store', None), + 'rpath-override-dirs': ("Path(s) to be prepended when linking with RPATH (string, colon-separated)", + None, 'store', None), 'set-default-module': ("Set the generated module as default", None, 'store_true', False), 'set-gid-bit': ("Set group ID bit on newly created directories", None, 'store_true', False), 'silence-deprecation-warnings': ("Silence specified deprecation warnings", 'strlist', 'extend', None), diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index f3c0abda5a..fac78ca6ff 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2358,36 +2358,50 @@ def test_toy_rpath(self): top_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.insert(0, top_path) - def grab_gcc_rpath_wrapper_filter_arg(): - """Helper function to grab filter argument from last RPATH wrapper for 'gcc'.""" + def grab_gcc_rpath_wrapper_args(): + """Helper function to grab arguments from last RPATH wrapper for 'gcc'.""" rpath_wrappers_dir = glob.glob(os.path.join(os.getenv('TMPDIR'), '*', '*', 'rpath_wrappers'))[0] gcc_rpath_wrapper_txt = read_file(glob.glob(os.path.join(rpath_wrappers_dir, '*', 'gcc'))[0]) + # First get the filter argument rpath_args_regex = re.compile(r"^rpath_args_out=.*rpath_args.py \$CMD '([^ ]*)'.*", re.M) - res = rpath_args_regex.search(gcc_rpath_wrapper_txt) - self.assertTrue(res, "Pattern '%s' found in: %s" % (rpath_args_regex.pattern, gcc_rpath_wrapper_txt)) + res_filter = rpath_args_regex.search(gcc_rpath_wrapper_txt) + self.assertTrue(res_filter, "Pattern '%s' found in: %s" % (rpath_args_regex.pattern, gcc_rpath_wrapper_txt)) + + # Now get the include argument + rpath_args_regex = re.compile(r"^rpath_args_out=.*rpath_args.py \$CMD '.*' '([^ ]*)'.*", re.M) + res_include = rpath_args_regex.search(gcc_rpath_wrapper_txt) + self.assertTrue(res_include, "Pattern '%s' found in: %s" % (rpath_args_regex.pattern, gcc_rpath_wrapper_txt)) shutil.rmtree(rpath_wrappers_dir) - return res.group(1) + return {'filter_paths': res_filter.group(1), 'include_paths': res_include.group(1)} args = ['--rpath', '--experimental'] self.test_toy_build(extra_args=args, raise_error=True) # by default, /lib and /usr are included in RPATH filter, # together with temporary directory and build directory - rpath_filter_paths = grab_gcc_rpath_wrapper_filter_arg().split(',') + rpath_filter_paths = grab_gcc_rpath_wrapper_args()['filter_paths'].split(',') self.assertTrue('/lib.*' in rpath_filter_paths) self.assertTrue('/usr.*' in rpath_filter_paths) self.assertTrue(any(p.startswith(os.getenv('TMPDIR')) for p in rpath_filter_paths)) self.assertTrue(any(p.startswith(self.test_buildpath) for p in rpath_filter_paths)) + # Check that we can use --rpath-override-dirs + args = ['--rpath', '--experimental', '--rpath-override-dirs=/opt/eessi/2021.03/lib:/opt/eessi/lib'] + self.test_toy_build(extra_args=args, raise_error=True) + rpath_include_paths = grab_gcc_rpath_wrapper_args()['include_paths'].split(',') + # Make sure our directories appear in dirs to be included in the rpath (and in the right order) + self.assertEqual(rpath_include_paths[-2], '/opt/eessi/2021.03/lib') + self.assertEqual(rpath_include_paths[-1], '/opt/eessi/lib') + # also test use of --rpath-filter args.extend(['--rpath-filter=/test.*,/foo/bar.*', '--disable-cleanup-tmpdir']) self.test_toy_build(extra_args=args, raise_error=True) # check whether rpath filter was set correctly - rpath_filter_paths = grab_gcc_rpath_wrapper_filter_arg().split(',') + rpath_filter_paths = grab_gcc_rpath_wrapper_args()['filter_paths'].split(',') self.assertTrue('/test.*' in rpath_filter_paths) self.assertTrue('/foo/bar.*' in rpath_filter_paths) self.assertTrue(any(p.startswith(os.getenv('TMPDIR')) for p in rpath_filter_paths)) From fa607dd68bcceda971d05e0b8e62d67c00ea93b2 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 13 Apr 2021 13:37:42 +0200 Subject: [PATCH 202/864] Appease the hound --- test/framework/toy_build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index fac78ca6ff..4d08b07f7a 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2371,7 +2371,8 @@ def grab_gcc_rpath_wrapper_args(): # Now get the include argument rpath_args_regex = re.compile(r"^rpath_args_out=.*rpath_args.py \$CMD '.*' '([^ ]*)'.*", re.M) res_include = rpath_args_regex.search(gcc_rpath_wrapper_txt) - self.assertTrue(res_include, "Pattern '%s' found in: %s" % (rpath_args_regex.pattern, gcc_rpath_wrapper_txt)) + self.assertTrue(res_include, "Pattern '%s' found in: %s" % (rpath_args_regex.pattern, + gcc_rpath_wrapper_txt)) shutil.rmtree(rpath_wrappers_dir) From adfa4c880d5ce9292bb6875092fb36f1acdfee2a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 14 Apr 2021 16:09:02 +0200 Subject: [PATCH 203/864] Allow amending EC params which are not the default Overwriting parameters which are not in the EC and not one of a few standard ones is currently impossible. However it is useful when e.g. EasyBlocks introduce new params where the user wants to overwrite the default value without changing the EC manually. This change allows this by simply appending the new parameter when it is not in the default list. --- easybuild/framework/easyconfig/default.py | 5 +++++ easybuild/framework/easyconfig/tweak.py | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 317f862b8c..863fb275b0 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -230,3 +230,8 @@ def get_easyconfig_parameter_default(param): else: _log.debug("Returning default value for easyconfig parameter %s: %s" % (param, DEFAULT_CONFIG[param][0])) return DEFAULT_CONFIG[param][0] + + +def is_easyconfig_parameter_default_value(param, value): + """Return True if the parameters is one of the default ones and the value equals its default value""" + return param in DEFAULT_CONFIG and get_easyconfig_parameter_default(param) == value diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 7d15be21b0..d3dbee0fe3 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -45,7 +45,7 @@ from easybuild.base import fancylogger from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS -from easybuild.framework.easyconfig.default import get_easyconfig_parameter_default +from easybuild.framework.easyconfig.default import is_easyconfig_parameter_default_value from easybuild.framework.easyconfig.easyconfig import EasyConfig, create_paths, process_easyconfig from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION @@ -324,7 +324,7 @@ def __repr__(self): _log.debug("Overwriting %s with %s" % (key, fval)) ectxt = regexp.sub("%s = %s" % (res.group('key'), newval), ectxt) _log.info("Tweaked %s list to '%s'" % (key, newval)) - elif get_easyconfig_parameter_default(key) != val: + elif not is_easyconfig_parameter_default_value(key, val): additions.append("%s = %s" % (key, val)) tweaks.pop(key) @@ -350,7 +350,7 @@ def __repr__(self): if diff: ectxt = regexp.sub("%s = %s" % (res.group('key'), quote_str(val)), ectxt) _log.info("Tweaked '%s' to '%s'" % (key, quote_str(val))) - elif get_easyconfig_parameter_default(key) != val: + elif not is_easyconfig_parameter_default_value(key, val): additions.append("%s = %s" % (key, quote_str(val))) if additions: From befebc3c4af2c007e2663940ab55f8babbc6b06f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 14 Apr 2021 16:35:33 +0200 Subject: [PATCH 204/864] Handle amended True/False/None values --- easybuild/framework/easyconfig/tweak.py | 18 +++++++++++++++--- test/framework/easyconfig.py | 20 +++++++++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index d3dbee0fe3..1fa60a785f 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -330,7 +330,19 @@ def __repr__(self): tweaks.pop(key) # add parameters or replace existing ones + special_values = ('True', 'False', 'None') + quoted_special_values = ['"%s"' % val for val in special_values] + ["'%s'" % val for val in special_values] for (key, val) in tweaks.items(): + # if the value is True/False/None then take that + # if e.g. (literal) True is wanted, then it can be passed as "True"/'True' + if val in special_values: + str_val = val + val = eval(val) + elif val in quoted_special_values: + str_val = val + val = val.strip('"\'') + else: + str_val = quote_str(val) regexp = re.compile(r"^(?P\s*%s)\s*=\s*(?P.*)$" % key, re.M) _log.debug("Regexp pattern for replacing '%s': %s" % (key, regexp.pattern)) @@ -348,10 +360,10 @@ def __repr__(self): diff = res.group('val') != val if diff: - ectxt = regexp.sub("%s = %s" % (res.group('key'), quote_str(val)), ectxt) - _log.info("Tweaked '%s' to '%s'" % (key, quote_str(val))) + ectxt = regexp.sub("%s = %s" % (res.group('key'), str_val), ectxt) + _log.info("Tweaked '%s' to '%s'" % (key, str_val)) elif not is_easyconfig_parameter_default_value(key, val): - additions.append("%s = %s" % (key, quote_str(val))) + additions.append("%s = %s" % (key, str_val)) if additions: _log.info("Adding additional parameters to tweaked easyconfig file: %s" % additions) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index f1a3b62efc..49c6d220ed 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -631,6 +631,8 @@ def test_tweaking(self): 'version = "3.14"', 'toolchain = {"name": "GCC", "version": "4.6.3"}', 'patches = %s', + 'parallel = 1', + 'keepsymlinks = True', ]) % str(patches) self.prep() @@ -647,7 +649,17 @@ def test_tweaking(self): 'versionprefix': verpref, 'versionsuffix': versuff, 'toolchain_version': tcver, - 'patches': new_patches + 'patches': new_patches, + 'keepsymlinks': 'True', # Don't change this + # It should be possible to overwrite values with True/False/None as they often have special meaning + 'runtest': 'False', + 'hidden': 'True', + 'parallel': 'None', # Good example: parallel=None means "Auto detect" + # Adding new options (added only by easyblock) should also be possible + # and in case the string "True/False/None" is really wanted it is possible to quote it first + 'test_none': '"False"', + 'test_bool': '"True"', + 'test_123': '"None"', } tweak_one(self.eb_file, tweaked_fn, tweaks) @@ -657,6 +669,12 @@ def test_tweaking(self): self.assertEqual(eb['versionsuffix'], versuff) self.assertEqual(eb['toolchain']['version'], tcver) self.assertEqual(eb['patches'], new_patches) + self.assertTrue(eb['runtest'] is False) + self.assertTrue(eb['hidden'] is True) + self.assertTrue(eb['parallel'] is None) + self.assertEqual(eb['test_none'], 'False') + self.assertEqual(eb['test_bool'], 'True') + self.assertEqual(eb['test_123'], 'None') remove_file(tweaked_fn) From d45513a5e9e0952fa10d68a553ac4c940e688c8d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 14 Apr 2021 17:38:11 +0200 Subject: [PATCH 205/864] Remove test for tweak-adding an unknown EC parameter We already have this case covered by the variable naming enforcement, i.e. loading an EC with an unknown parameter not named local_* will already error out --- test/framework/easyconfig.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 49c6d220ed..a28db4db83 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -857,13 +857,6 @@ def test_obtain_easyconfig(self): self.assertEqual(ec['start_dir'], specs['start_dir']) remove_file(res[1]) - specs.update({ - 'foo': 'bar123' - }) - self.assertErrorRegex(EasyBuildError, "Unknown easyconfig parameter: foo", - obtain_ec_for, specs, [self.test_prefix], None) - del specs['foo'] - # should pick correct version, i.e. not newer than what's specified, if a choice needs to be made ver = '3.14' specs.update({'version': ver}) From 78dd6fdb6df39ac26cb2e312e9393aa77d4d97de Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 15 Apr 2021 09:33:53 +0200 Subject: [PATCH 206/864] Update HMNS for Intel OneAPI compilers --- easybuild/tools/module_naming_scheme/hierarchical_mns.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py index 33a2e32eb7..b934c3634d 100644 --- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py +++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py @@ -54,6 +54,7 @@ # required for use of iccifort toolchain 'icc,ifort': ('intel', '%(icc)s'), 'iccifort': ('intel', '%(iccifort)s'), + 'intel-compilers': ('intel', '%(intel-compilers)s'), # OneAPI compilers # required for use of ClangGCC toolchain 'Clang,GCC': ('Clang-GCC', '%(Clang)s-%(GCC)s'), # required for use of gcccuda toolchain, and for CUDA installed with GCC toolchain @@ -61,6 +62,7 @@ # required for use of iccifortcuda toolchain 'CUDA,icc,ifort': ('intel-CUDA', '%(icc)s-%(CUDA)s'), 'CUDA,iccifort': ('intel-CUDA', '%(iccifort)s-%(CUDA)s'), + 'CUDA,intel-compilers': ('intel-CUDA', '%(intel-compilers)s-%(CUDA)s'), # OneAPI compilers # required for CUDA installed with iccifort toolchain # need to use 'intel' here because 'iccifort' toolchain maps to 'intel' (see above) 'CUDA,intel': ('intel-CUDA', '%(intel)s-%(CUDA)s'), From b9a1535ae45e759c3afd0d2b66667bbaa0a852a9 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 15 Apr 2021 10:22:40 +0200 Subject: [PATCH 207/864] Update hierarchical_mns.py --- easybuild/tools/module_naming_scheme/hierarchical_mns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py index b934c3634d..673a31b6f5 100644 --- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py +++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py @@ -54,7 +54,8 @@ # required for use of iccifort toolchain 'icc,ifort': ('intel', '%(icc)s'), 'iccifort': ('intel', '%(iccifort)s'), - 'intel-compilers': ('intel', '%(intel-compilers)s'), # OneAPI compilers + # required for use of intel-compilers toolchain (OneAPI compilers) + 'intel-compilers': ('intel', '%(intel-compilers)s'), # required for use of ClangGCC toolchain 'Clang,GCC': ('Clang-GCC', '%(Clang)s-%(GCC)s'), # required for use of gcccuda toolchain, and for CUDA installed with GCC toolchain @@ -62,7 +63,6 @@ # required for use of iccifortcuda toolchain 'CUDA,icc,ifort': ('intel-CUDA', '%(icc)s-%(CUDA)s'), 'CUDA,iccifort': ('intel-CUDA', '%(iccifort)s-%(CUDA)s'), - 'CUDA,intel-compilers': ('intel-CUDA', '%(intel-compilers)s-%(CUDA)s'), # OneAPI compilers # required for CUDA installed with iccifort toolchain # need to use 'intel' here because 'iccifort' toolchain maps to 'intel' (see above) 'CUDA,intel': ('intel-CUDA', '%(intel)s-%(CUDA)s'), From d9fd770423989328f995d689e763708b2f7e6ba2 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 7 Apr 2021 17:07:13 +0200 Subject: [PATCH 208/864] Deprecate adding a non-existing path to $MODULEPATH --- easybuild/tools/modules.py | 17 +++++++++++++---- test/framework/modules.py | 18 ++++++------------ 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 3e3f004dc5..93d13aadd9 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -362,8 +362,12 @@ def use(self, path, priority=None): self.log.info("Ignoring specified priority '%s' when running 'module use %s' (Lmod-specific)", priority, path) - # make sure path exists before we add it - mkdir(path, parents=True) + if not path: + raise EasyBuildError("Cannot add empty path to $MODULEPATH") + if not os.path.exists(path): + self.log.deprecated("Path '%s' for module.use should exist" % path, '5.0') + # make sure path exists before we add it + mkdir(path, parents=True) self.run_module(['use', path]) def unuse(self, path): @@ -431,6 +435,7 @@ def check_module_path(self): eb_modpath = os.path.join(install_path(typ='modules'), build_option('suffix_modules_path')) # make sure EasyBuild module path is in 1st place + mkdir(eb_modpath, parents=True) self.prepend_module_path(eb_modpath) self.log.info("Prepended list of module paths with path used by EasyBuild: %s" % eb_modpath) @@ -1422,8 +1427,12 @@ def use(self, path, priority=None): :param path: path to add to $MODULEPATH :param priority: priority for this path in $MODULEPATH (Lmod-specific) """ - # make sure path exists before we add it - mkdir(path, parents=True) + if not path: + raise EasyBuildError("Cannot add empty path to $MODULEPATH") + if not os.path.exists(path): + self.log.deprecated("Path '%s' for module.use should exist" % path, '5.0') + # make sure path exists before we add it + mkdir(path, parents=True) if priority: self.run_module(['use', '--priority', str(priority), path]) diff --git a/test/framework/modules.py b/test/framework/modules.py index a2f4633787..21f36f10de 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1091,6 +1091,7 @@ def test_module_caches(self): # purposely extending $MODULEPATH with non-existing path, should be handled fine nonpath = os.path.join(self.test_prefix, 'nosuchfileordirectory') + mkdir(nonpath) self.modtool.use(nonpath) modulepaths = [p for p in os.environ.get('MODULEPATH', '').split(os.pathsep) if p] self.assertTrue(any([os.path.samefile(nonpath, mp) for mp in modulepaths])) @@ -1163,6 +1164,11 @@ def test_module_use_unuse(self): self.modtool.use(test_dir3) self.assertTrue(os.environ['MODULEPATH'].startswith('%s:' % test_dir3)) + # Adding an empty modulepath is not possible + modulepath = os.environ.get('MODULEPATH', '') + self.assertErrorRegex(EasyBuildError, "Cannot add empty path", self.modtool.use, '') + self.assertEqual(os.environ.get('MODULEPATH', ''), modulepath) + # make sure the right test module is loaded self.modtool.load(['test']) self.assertEqual(os.getenv('TEST123'), 'three') @@ -1223,18 +1229,6 @@ def test_module_use_unuse(self): self.assertFalse('MODULEPATH' in os.environ) os.environ['MODULEPATH'] = old_module_path # Restore - # Using an empty path still works (technically) (Lmod only, ignored by Tcl) - old_module_path = os.environ['MODULEPATH'] - self.modtool.use('') - self.assertEqual(os.environ['MODULEPATH'], ':' + old_module_path) - self.modtool.unuse('') - self.assertEqual(os.environ['MODULEPATH'], old_module_path) - # Even works when the whole path is empty - os.environ['MODULEPATH'] = '' - self.modtool.unuse('') - self.assertFalse('MODULEPATH' in os.environ) - os.environ['MODULEPATH'] = old_module_path # Restore - def test_module_use_bash(self): """Test whether effect of 'module use' is preserved when a new bash session is started.""" # this test is here as check for a nasty bug in how the modules tool is deployed From da22ce0023027d4162fc078a12659edaffae6a1b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Apr 2021 12:07:03 +0200 Subject: [PATCH 209/864] Add only existing module paths in tests --- test/framework/easyconfig.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index f1a3b62efc..21247f2ce1 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1655,7 +1655,6 @@ def test_external_dependencies(self): # by adding a couple of matching module files with some useful data in them # (use Tcl syntax, so it works with all varieties of module tools) mod_dir = os.path.join(self.test_prefix, 'modules') - self.modtool.use(mod_dir) pi_mod_txt = '\n'.join([ "#%Module", @@ -1680,6 +1679,8 @@ def test_external_dependencies(self): ]) write_file(os.path.join(mod_dir, 'foobar/2.3.4'), foobar_mod_txt) + self.modtool.use(mod_dir) + ec = EasyConfig(toy_ec) deps = ec.dependencies() From ededcdf45e5e36ecabf37bfdd94ab4b01a2fe91c Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Apr 2021 12:07:22 +0200 Subject: [PATCH 210/864] Add test for add_module_path and remove_module_path --- test/framework/modules.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/framework/modules.py b/test/framework/modules.py index 21f36f10de..10fc81ec20 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1229,6 +1229,38 @@ def test_module_use_unuse(self): self.assertFalse('MODULEPATH' in os.environ) os.environ['MODULEPATH'] = old_module_path # Restore + def test_add_and_remove_module_path(self): + """Test add_module_path and whether remove_module_path undoes changes of add_module_path""" + test_dir1 = tempfile.mkdtemp(suffix="_dir1") + test_dir2 = tempfile.mkdtemp(suffix="_dir2") + old_module_path = os.environ.get('MODULEPATH') + del os.environ['MODULEPATH'] + self.modtool.add_module_path(test_dir1) + self.assertEqual(os.environ['MODULEPATH'], test_dir1) + self.modtool.add_module_path(test_dir2) + self.assertEqual(os.environ['MODULEPATH'], test_dir2 + ':' + test_dir1) + # Adding the same path does not change the path + self.modtool.add_module_path(test_dir1) + self.assertEqual(os.environ['MODULEPATH'], test_dir2 + ':' + test_dir1) + self.modtool.add_module_path(test_dir2) + self.assertEqual(os.environ['MODULEPATH'], test_dir2 + ':' + test_dir1) + # Even when a (meaningless) slash is added + # This occurs when using an empty modules directory name + self.modtool.add_module_path(os.path.join(test_dir1, '')) + self.assertEqual(os.environ['MODULEPATH'], test_dir2 + ':' + test_dir1) + + # Similar tests for remove_module_path + self.modtool.remove_module_path(test_dir2) + self.assertEqual(os.environ['MODULEPATH'], test_dir1) + # Same again -> no-op + self.modtool.remove_module_path(test_dir2) + self.assertEqual(os.environ['MODULEPATH'], test_dir1) + # And with empty last part + self.modtool.remove_module_path(os.path.join(test_dir1, '')) + self.assertEqual(os.environ.get('MODULEPATH', ''), '') + + os.environ['MODULEPATH'] = old_module_path # Restore + def test_module_use_bash(self): """Test whether effect of 'module use' is preserved when a new bash session is started.""" # this test is here as check for a nasty bug in how the modules tool is deployed From 45993e9ef129d07244f3a0c5932901059e75f113 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Apr 2021 12:35:59 +0200 Subject: [PATCH 211/864] Strip trailing path separators when comparing against $MODULEPATH entries --- easybuild/tools/modules.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 93d13aadd9..696dd61451 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -381,6 +381,7 @@ def add_module_path(self, path, set_mod_paths=True): :param path: path to add to $MODULEPATH via 'use' :param set_mod_paths: (re)set self.mod_paths """ + path = path.rstrip(os.path.sep) if path not in curr_module_paths(): # add module path via 'module use' and make sure self.mod_paths is synced self.use(path) @@ -395,6 +396,7 @@ def remove_module_path(self, path, set_mod_paths=True): :param set_mod_paths: (re)set self.mod_paths """ # remove module path via 'module unuse' and make sure self.mod_paths is synced + path = path.rstrip(os.path.sep) if path in curr_module_paths(): self.unuse(path) @@ -1291,6 +1293,7 @@ def remove_module_path(self, path, set_mod_paths=True): # remove module path via 'module use' and make sure self.mod_paths is synced # modulecmd.tcl keeps track of how often a path was added via 'module use', # so we need to check to make sure it's really removed + path = path.rstrip(os.path.sep) while path in curr_module_paths(): self.unuse(path) if set_mod_paths: @@ -1442,6 +1445,7 @@ def use(self, path, priority=None): if os.environ.get('__LMOD_Priority_MODULEPATH'): self.run_module(['use', path]) else: + path = path.rstrip(os.path.sep) cur_mod_path = os.environ.get('MODULEPATH') if cur_mod_path is None: new_mod_path = path @@ -1462,6 +1466,7 @@ def unuse(self, path): self.log.debug('Changing MODULEPATH from %s to ' % cur_mod_path) del os.environ['MODULEPATH'] else: + path = path.rstrip(os.path.sep) new_mod_path = ':'.join(p for p in cur_mod_path.split(':') if p != path) if new_mod_path != cur_mod_path: self.log.debug('Changing MODULEPATH from %s to %s' % (cur_mod_path, new_mod_path)) From b357cd7c8f8982215e7c8f15fac468633042659f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Apr 2021 12:47:05 +0200 Subject: [PATCH 212/864] Don't add non-existing paths (provided by MNS) --- easybuild/framework/easyblock.py | 2 +- easybuild/tools/modules.py | 3 ++- easybuild/tools/toolchain/toolchain.py | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 77e0df72a5..274e60f874 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2155,7 +2155,7 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True): curr_modpaths = curr_module_paths() for init_modpath in init_modpaths: full_mod_path = os.path.join(self.installdir_mod, init_modpath) - if full_mod_path not in curr_modpaths: + if os.path.exists(full_mod_path) and full_mod_path not in curr_modpaths: self.modules_tool.prepend_module_path(full_mod_path) # prepare toolchain: load toolchain module and dependencies, set up build environment diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 696dd61451..b799a55dd4 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -672,7 +672,8 @@ def load(self, modules, mod_paths=None, purge=False, init_env=None, allow_reload # extend $MODULEPATH if needed for mod_path in mod_paths: full_mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'), mod_path) - self.prepend_module_path(full_mod_path) + if os.path.exists(full_mod_path): + self.prepend_module_path(full_mod_path) loaded_modules = self.loaded_modules() for mod in modules: diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index ca2f732824..985db8733e 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -634,7 +634,9 @@ def _load_toolchain_module(self, silent=False): if self.init_modpaths: mod_path_suffix = build_option('suffix_modules_path') for modpath in self.init_modpaths: - self.modules_tool.prepend_module_path(os.path.join(install_path('mod'), mod_path_suffix, modpath)) + modpath = os.path.join(install_path('mod'), mod_path_suffix, modpath) + if os.path.exists(modpath): + self.modules_tool.prepend_module_path(modpath) # load modules for all dependencies self.log.debug("Loading module for toolchain: %s", tc_mod) From 4384a983d740ac4d1eaec8fd486d9a3727b1e307 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Apr 2021 15:24:43 +0200 Subject: [PATCH 213/864] Normalize paths before adding/removing from $MODULEPATH --- easybuild/tools/filetools.py | 18 ++++++++++++++++++ easybuild/tools/modules.py | 27 ++++++++++++++++----------- test/framework/filetools.py | 15 +++++++++++++++ test/framework/modules.py | 21 +++++++++++++++++++++ 4 files changed, 70 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index a2e2709917..f6058e9e42 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -508,6 +508,24 @@ def det_common_path_prefix(paths): return None +def normalize_path(path): + """Normalize path removing empty and dot components. + + Similar to os.path.normpath but does not resolve '..' which may return a wrong path when symlinks are used + """ + # In POSIX 3 or more leading slashes are equivalent to 1 + if path.startswith(os.path.sep): + if path.startswith(os.path.sep * 2) and not path.startswith(os.path.sep * 3): + start_slashes = os.path.sep * 2 + else: + start_slashes = os.path.sep + else: + start_slashes = '' + + filtered_comps = (comp for comp in path.split(os.path.sep) if comp and comp != '.') + return start_slashes + os.path.sep.join(filtered_comps) + + def is_alt_pypi_url(url): """Determine whether specified URL is already an alternate PyPI URL, i.e. whether it contains a hash.""" # example: .../packages/5b/03/e135b19fadeb9b1ccb45eac9f60ca2dc3afe72d099f6bd84e03cb131f9bf/easybuild-2.7.0.tar.gz diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index b799a55dd4..3ce7655411 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -46,7 +46,7 @@ from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, LOADED_MODULES_ACTIONS from easybuild.tools.config import build_option, get_modules_tool, install_path from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env, setvar, unset_env_vars -from easybuild.tools.filetools import convert_name, mkdir, path_matches, read_file, which, write_file +from easybuild.tools.filetools import convert_name, mkdir, normalize_path, path_matches, read_file, which, write_file from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX from easybuild.tools.py2vs3 import subprocess_popen_text from easybuild.tools.run import run_cmd @@ -381,8 +381,8 @@ def add_module_path(self, path, set_mod_paths=True): :param path: path to add to $MODULEPATH via 'use' :param set_mod_paths: (re)set self.mod_paths """ - path = path.rstrip(os.path.sep) - if path not in curr_module_paths(): + path = normalize_path(path) + if path not in curr_module_paths(normalize=True): # add module path via 'module use' and make sure self.mod_paths is synced self.use(path) if set_mod_paths: @@ -396,8 +396,8 @@ def remove_module_path(self, path, set_mod_paths=True): :param set_mod_paths: (re)set self.mod_paths """ # remove module path via 'module unuse' and make sure self.mod_paths is synced - path = path.rstrip(os.path.sep) - if path in curr_module_paths(): + path = normalize_path(path) + if path in curr_module_paths(normalize=True): self.unuse(path) if set_mod_paths: @@ -1294,8 +1294,8 @@ def remove_module_path(self, path, set_mod_paths=True): # remove module path via 'module use' and make sure self.mod_paths is synced # modulecmd.tcl keeps track of how often a path was added via 'module use', # so we need to check to make sure it's really removed - path = path.rstrip(os.path.sep) - while path in curr_module_paths(): + path = normalize_path(path) + while path in curr_module_paths(normalize=True): self.unuse(path) if set_mod_paths: self.set_mod_paths() @@ -1446,7 +1446,7 @@ def use(self, path, priority=None): if os.environ.get('__LMOD_Priority_MODULEPATH'): self.run_module(['use', path]) else: - path = path.rstrip(os.path.sep) + path = normalize_path(path) cur_mod_path = os.environ.get('MODULEPATH') if cur_mod_path is None: new_mod_path = path @@ -1467,7 +1467,7 @@ def unuse(self, path): self.log.debug('Changing MODULEPATH from %s to ' % cur_mod_path) del os.environ['MODULEPATH'] else: - path = path.rstrip(os.path.sep) + path = normalize_path(path) new_mod_path = ':'.join(p for p in cur_mod_path.split(':') if p != path) if new_mod_path != cur_mod_path: self.log.debug('Changing MODULEPATH from %s to %s' % (cur_mod_path, new_mod_path)) @@ -1618,12 +1618,17 @@ def get_software_version(name): return version -def curr_module_paths(): +def curr_module_paths(normalize=False): """ Return a list of current module paths. + + :param normalize: Normalize the paths """ # avoid empty or nonexistent paths, which don't make any sense - return [p for p in os.environ.get('MODULEPATH', '').split(':') if p and os.path.exists(p)] + module_paths = (p for p in os.environ.get('MODULEPATH', '').split(':') if p and os.path.exists(p)) + if normalize: + module_paths = (normalize_path(p) for p in module_paths) + return list(module_paths) def mk_module_path(paths): diff --git a/test/framework/filetools.py b/test/framework/filetools.py index fb9c2440e4..7b176f2d83 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -351,6 +351,21 @@ def test_common_path_prefix(self): self.assertEqual(ft.det_common_path_prefix(['foo']), None) self.assertEqual(ft.det_common_path_prefix([]), None) + def test_normalize_path(self): + """Test normalize_path""" + self.assertEqual(ft.normalize_path(''), '') + self.assertEqual(ft.normalize_path('/'), '/') + self.assertEqual(ft.normalize_path('//'), '//') + self.assertEqual(ft.normalize_path('///'), '/') + self.assertEqual(ft.normalize_path('/foo/bar/baz'), '/foo/bar/baz') + self.assertEqual(ft.normalize_path('/foo//bar/././baz/'), '/foo/bar/baz') + self.assertEqual(ft.normalize_path('foo//bar/././baz/'), 'foo/bar/baz') + self.assertEqual(ft.normalize_path('//foo//bar/././baz/'), '//foo/bar/baz') + self.assertEqual(ft.normalize_path('///foo//bar/././baz/'), '/foo/bar/baz') + self.assertEqual(ft.normalize_path('////foo//bar/././baz/'), '/foo/bar/baz') + self.assertEqual(ft.normalize_path('/././foo//bar/././baz/'), '/foo/bar/baz') + self.assertEqual(ft.normalize_path('//././foo//bar/././baz/'), '//foo/bar/baz') + def test_download_file(self): """Test download_file function.""" fn = 'toy-0.0.tar.gz' diff --git a/test/framework/modules.py b/test/framework/modules.py index 10fc81ec20..d03a655d8c 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1259,6 +1259,27 @@ def test_add_and_remove_module_path(self): self.modtool.remove_module_path(os.path.join(test_dir1, '')) self.assertEqual(os.environ.get('MODULEPATH', ''), '') + # And with some more trickery + test_dir1_relative = os.path.join(test_dir1, '..', os.path.basename(test_dir1)) + test_dir2_dot = os.path.join(os.path.dirname(test_dir2), '.', os.path.basename(test_dir2)) + self.modtool.add_module_path(test_dir1_relative) + self.assertEqual(os.environ['MODULEPATH'], test_dir1_relative) + self.modtool.add_module_path(test_dir1) + self.assertEqual(os.environ['MODULEPATH'], test_dir1 + ':' + test_dir1_relative) + self.modtool.remove_module_path(test_dir1) + self.assertEqual(os.environ['MODULEPATH'], test_dir1_relative) + self.modtool.add_module_path(test_dir2_dot) + self.assertEqual(os.environ['MODULEPATH'], test_dir2 + ':' + test_dir1_relative) + self.modtool.remove_module_path(test_dir2_dot) + self.assertEqual(os.environ['MODULEPATH'], test_dir1_relative) + # Force adding such a dot path which can be removed with either variant + os.environ['MODULEPATH'] = test_dir2_dot + ':' + test_dir1_relative + self.modtool.remove_module_path(test_dir2_dot) + self.assertEqual(os.environ['MODULEPATH'], test_dir1_relative) + os.environ['MODULEPATH'] = test_dir2_dot + ':' + test_dir1_relative + self.modtool.remove_module_path(test_dir2) + self.assertEqual(os.environ['MODULEPATH'], test_dir1_relative) + os.environ['MODULEPATH'] = old_module_path # Restore def test_module_use_bash(self): From 006186fc57a2ff297576bc6d10c37b13ad81eb19 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Apr 2021 16:18:20 +0200 Subject: [PATCH 214/864] Unuse the actual (non-normalized) path from $MODULEPATH --- easybuild/tools/modules.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 3ce7655411..a93b16290b 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -397,8 +397,13 @@ def remove_module_path(self, path, set_mod_paths=True): """ # remove module path via 'module unuse' and make sure self.mod_paths is synced path = normalize_path(path) - if path in curr_module_paths(normalize=True): - self.unuse(path) + try: + # Unuse the path that is actually present in the environment + module_path = next(p for p in curr_module_paths() if normalize_path(p) == path) + except StopIteration: + pass + else: + self.unuse(module_path) if set_mod_paths: self.set_mod_paths() @@ -1295,8 +1300,13 @@ def remove_module_path(self, path, set_mod_paths=True): # modulecmd.tcl keeps track of how often a path was added via 'module use', # so we need to check to make sure it's really removed path = normalize_path(path) - while path in curr_module_paths(normalize=True): - self.unuse(path) + while True: + try: + # Unuse the path that is actually present in the environment + module_path = next(p for p in curr_module_paths() if normalize_path(p) == path) + except StopIteration: + break + self.unuse(module_path) if set_mod_paths: self.set_mod_paths() @@ -1451,7 +1461,7 @@ def use(self, path, priority=None): if cur_mod_path is None: new_mod_path = path else: - new_mod_path = [path] + [p for p in cur_mod_path.split(':') if p != path] + new_mod_path = [path] + [p for p in cur_mod_path.split(':') if normalize_path(p) != path] new_mod_path = ':'.join(new_mod_path) self.log.debug('Changing MODULEPATH from %s to %s' % ('' if cur_mod_path is None else cur_mod_path, new_mod_path)) @@ -1468,7 +1478,7 @@ def unuse(self, path): del os.environ['MODULEPATH'] else: path = normalize_path(path) - new_mod_path = ':'.join(p for p in cur_mod_path.split(':') if p != path) + new_mod_path = ':'.join(p for p in cur_mod_path.split(':') if normalize_path(p) != path) if new_mod_path != cur_mod_path: self.log.debug('Changing MODULEPATH from %s to %s' % (cur_mod_path, new_mod_path)) os.environ['MODULEPATH'] = new_mod_path From 65676df1b44fd3ec2902435c639383d2a2c6a09a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Apr 2021 17:06:33 +0200 Subject: [PATCH 215/864] Handle Environment-Modules 4.x behavior in tests --- test/framework/modules.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index d03a655d8c..679c636b1b 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1260,25 +1260,33 @@ def test_add_and_remove_module_path(self): self.assertEqual(os.environ.get('MODULEPATH', ''), '') # And with some more trickery + # Lmod seems to remove empty paths: /foo//bar/. -> /foo/bar + # Environment-Modules 4.x seems to resolve relative paths: /foo/../foo -> /foo + # Hence we can only check the real paths + def get_resolved_module_path(): + return ':'.join(os.path.realpath(p) for p in os.environ['MODULEPATH'].split(':')) + test_dir1_relative = os.path.join(test_dir1, '..', os.path.basename(test_dir1)) test_dir2_dot = os.path.join(os.path.dirname(test_dir2), '.', os.path.basename(test_dir2)) self.modtool.add_module_path(test_dir1_relative) - self.assertEqual(os.environ['MODULEPATH'], test_dir1_relative) + self.assertEqual(get_resolved_module_path(), test_dir1) + # Adding the same path, but in a different form may be possible, but may also be ignored, e.g. in EnvModules self.modtool.add_module_path(test_dir1) - self.assertEqual(os.environ['MODULEPATH'], test_dir1 + ':' + test_dir1_relative) - self.modtool.remove_module_path(test_dir1) - self.assertEqual(os.environ['MODULEPATH'], test_dir1_relative) + if get_resolved_module_path() != test_dir1: + self.assertEqual(get_resolved_module_path(), test_dir1 + ':' + test_dir1) + self.modtool.remove_module_path(test_dir1) + self.assertEqual(get_resolved_module_path(), test_dir1) self.modtool.add_module_path(test_dir2_dot) - self.assertEqual(os.environ['MODULEPATH'], test_dir2 + ':' + test_dir1_relative) + self.assertEqual(get_resolved_module_path(), test_dir2 + ':' + test_dir1) self.modtool.remove_module_path(test_dir2_dot) - self.assertEqual(os.environ['MODULEPATH'], test_dir1_relative) + self.assertEqual(get_resolved_module_path(), test_dir1) # Force adding such a dot path which can be removed with either variant os.environ['MODULEPATH'] = test_dir2_dot + ':' + test_dir1_relative self.modtool.remove_module_path(test_dir2_dot) - self.assertEqual(os.environ['MODULEPATH'], test_dir1_relative) + self.assertEqual(get_resolved_module_path(), test_dir1) os.environ['MODULEPATH'] = test_dir2_dot + ':' + test_dir1_relative self.modtool.remove_module_path(test_dir2) - self.assertEqual(os.environ['MODULEPATH'], test_dir1_relative) + self.assertEqual(get_resolved_module_path(), test_dir1) os.environ['MODULEPATH'] = old_module_path # Restore From 99a809c7fd09df94a7fd44cb1046d85c8360a61f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 29 Jan 2021 18:44:48 +0100 Subject: [PATCH 216/864] Close stdout handle in run Prior to this the handle was left open leading to a resource leak As we use this function quite often this can become serious --- easybuild/tools/run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 9c77edf06a..ec46116ce1 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -254,6 +254,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True # read remaining data (all of it) output = get_output_from_process(proc) + proc.stdout.close() if cmd_log: cmd_log.write(output) cmd_log.close() From 348654cacc225bf081d1cd8aa6aa0b54ef92c908 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 29 Jan 2021 19:03:10 +0100 Subject: [PATCH 217/864] Avoid inefficiencies in resolving software version templates Prior to this all dependencies were iterated over multiple times with quite expensive handling, e.g. a full copy of the dependencies, list concatenation, and many dict operations Changes: - To avoid the search in the TEMPLATE_SOFTWARE_VERSIONS it is converted to a dict with the key already lowercased (instead of for every dep) - The copy of the dependencies list is done only when required - Dependencies are iterated only once and instead a matching template is searched for which is a lot less work --- easybuild/framework/easyconfig/templates.py | 43 ++++++++++++--------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 6b3ba8763d..12bed87de3 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -31,7 +31,6 @@ :author: Fotis Georgatos (Uni.Lu, NTUA) :author: Kenneth Hoste (Ghent University) """ -import copy import re import platform @@ -242,10 +241,10 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) raise EasyBuildError("Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name) # step 2: define *ver and *shortver templates - for name, pref in TEMPLATE_SOFTWARE_VERSIONS: + if TEMPLATE_SOFTWARE_VERSIONS: - # copy to avoid changing original list below - deps = copy.copy(config.get('dependencies', [])) + name_to_prefix = dict((name.lower(), pref) for name, pref in TEMPLATE_SOFTWARE_VERSIONS) + deps = config.get('dependencies', []) # also consider build dependencies for *ver and *shortver templates; # we need to be a bit careful here, because for iterative installations @@ -262,15 +261,22 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # only consider build dependencies when we're actually in iterative mode! if 'builddependencies' in config.iterate_options: if config.iterating: - deps.extend(config.get('builddependencies', [])) + build_deps = config.get('builddependencies') + else: + build_deps = None else: - deps.extend(config.get('builddependencies', [])) - + build_deps = config.get('builddependencies') + if build_deps: + # Don't use += to avoid changing original list + deps = deps + build_deps # include all toolchain deps (e.g. CUDAcore component in fosscuda); # access Toolchain instance via _toolchain to avoid triggering initialization of the toolchain! - if config._toolchain is not None: - if config._toolchain.tcdeps is not None: + if config._toolchain is not None and config._toolchain.tcdeps: + # If we didn't create a new list above do it here + if build_deps: deps.extend(config._toolchain.tcdeps) + else: + deps = deps + config._toolchain.tcdeps for dep in deps: if isinstance(dep, dict): @@ -292,15 +298,16 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) else: raise EasyBuildError("Unexpected type for dependency: %s", dep) - if isinstance(dep_name, string_type) and dep_name.lower() == name.lower() and dep_version: - dep_version = pick_dep_version(dep_version) - template_values['%sver' % pref] = dep_version - dep_version_parts = dep_version.split('.') - template_values['%smajver' % pref] = dep_version_parts[0] - if len(dep_version_parts) > 1: - template_values['%sminver' % pref] = dep_version_parts[1] - template_values['%sshortver' % pref] = '.'.join(dep_version_parts[:2]) - break + if isinstance(dep_name, string_type) and dep_version: + pref = name_to_prefix.get(dep_name.lower()) + if pref: + dep_version = pick_dep_version(dep_version) + template_values['%sver' % pref] = dep_version + dep_version_parts = dep_version.split('.') + template_values['%smajver' % pref] = dep_version_parts[0] + if len(dep_version_parts) > 1: + template_values['%sminver' % pref] = dep_version_parts[1] + template_values['%sshortver' % pref] = '.'.join(dep_version_parts[:2]) # step 3: add remaining from config for name in TEMPLATE_NAMES_CONFIG: From 1387c38360a7c6dd82188a14a438aaa1eeb4e5f0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 29 Jan 2021 19:57:27 +0100 Subject: [PATCH 218/864] Add required param to get_software_version This avoids raising a traced and hence expensive exception although a more or less common case is fine with "no result" This yields a ~13% speedup in the parsing and sorting of all ECs in the EC unit tests (3m -> 2:30m) --- easybuild/tools/toolchain/mpi.py | 7 +++---- easybuild/tools/toolchain/toolchain.py | 15 ++++++++------- test/framework/toolchain.py | 20 ++++++++++++++++++++ 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/toolchain/mpi.py b/easybuild/tools/toolchain/mpi.py index 8121dbd5c8..ac9da42114 100644 --- a/easybuild/tools/toolchain/mpi.py +++ b/easybuild/tools/toolchain/mpi.py @@ -293,10 +293,9 @@ def mpi_cmd_for(self, cmd, nr_ranks): # for Intel MPI, try to determine impi version # this fails when it's done too early (before modules for toolchain/dependencies are loaded), # but it's safe to ignore this - try: - mpi_version = self.get_software_version(self.MPI_MODULE_NAME)[0] - except EasyBuildError as err: - self.log.debug("Ignoring error when trying to determine %s version: %s", self.MPI_MODULE_NAME, err) + mpi_version = self.get_software_version(self.MPI_MODULE_NAME, required=False)[0] + if not mpi_version: + self.log.debug("Ignoring error when trying to determine %s version", self.MPI_MODULE_NAME) # impi version is required to determine correct MPI command template, # so we have to return early if we couldn't determine the impi version... return None diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index ca2f732824..bcd8e0f0ce 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -403,20 +403,20 @@ def get_software_root(self, names): """Try to get the software root for all names""" return self._get_software_multiple(names, self._get_software_root) - def get_software_version(self, names): + def get_software_version(self, names, required=True): """Try to get the software version for all names""" - return self._get_software_multiple(names, self._get_software_version) + return self._get_software_multiple(names, self._get_software_version, required) - def _get_software_multiple(self, names, function): + def _get_software_multiple(self, names, function, required=True): """Execute function of each of names""" if isinstance(names, (str,)): names = [names] res = [] for name in names: - res.append(function(name)) + res.append(function(name, required=required)) return res - def _get_software_root(self, name): + def _get_software_root(self, name, required=True): """Try to get the software root for name""" root = get_software_root(name) if root is None: @@ -425,11 +425,12 @@ def _get_software_root(self, name): self.log.debug("get_software_root software root %s for %s was found in environment", root, name) return root - def _get_software_version(self, name): + def _get_software_version(self, name, required=True): """Try to get the software version for name""" version = get_software_version(name) if version is None: - raise EasyBuildError("get_software_version software version for %s was not found in environment", name) + if required: + raise EasyBuildError("get_software_version software version for %s was not found in environment", name) else: self.log.debug("get_software_version software version %s for %s was found in environment", version, name) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 28013b468a..69bc14350a 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1422,6 +1422,26 @@ def test_prepare_deps_external(self): self.assertEqual(modules.get_software_root('foobar'), '/foo/bar') self.assertEqual(modules.get_software_version('toy'), '1.2.3') + def test_get_software_version(self): + """Test that get_software_version works""" + os.environ['EBROOTTOY'] = '/foo/bar' + os.environ['EBVERSIONTOY'] = '1.2.3' + os.environ['EBROOTFOOBAR'] = '/foo/bar' + os.environ['EBVERSIONFOOBAR'] = '4.5' + tc = self.get_toolchain('GCC', version='6.4.0-2.28') + self.assertEqual(tc.get_software_version('toy'), ['1.2.3']) + self.assertEqual(tc.get_software_version(['toy']), ['1.2.3']) + self.assertEqual(tc.get_software_version(['toy', 'foobar']), ['1.2.3', '4.5']) + # Non existing modules raise an error + self.assertErrorRegex(EasyBuildError, 'non-existing was not found', + tc.get_software_version, 'non-existing') + self.assertErrorRegex(EasyBuildError, 'non-existing was not found', + tc.get_software_version, ['toy', 'non-existing', 'foobar']) + # Can use required=False to avoid + self.assertEqual(tc.get_software_version('non-existing', required=False), [None]) + self.assertEqual(tc.get_software_version(['toy', 'non-existing', 'foobar'], required=False), + ['1.2.3', None, '4.5']) + def test_old_new_iccifort(self): """Test whether preparing for old/new Intel compilers works correctly.""" self.setup_sandbox_for_intel_fftw(self.test_prefix, imklver='2018.1.163') From 2aaed1dc23f7b58a9c8a8df9425d0aa99f63b9f4 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 29 Jan 2021 20:38:47 +0100 Subject: [PATCH 219/864] Add missing required params --- easybuild/toolchains/linalg/libsci.py | 7 ++++--- easybuild/tools/toolchain/toolchain.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/easybuild/toolchains/linalg/libsci.py b/easybuild/toolchains/linalg/libsci.py index f31c3bf28c..ec3c0396e3 100644 --- a/easybuild/toolchains/linalg/libsci.py +++ b/easybuild/toolchains/linalg/libsci.py @@ -59,18 +59,19 @@ class LibSci(LinAlg): BLACS_MODULE_NAME = [] SCALAPACK_MODULE_NAME = [] - def _get_software_root(self, name): + def _get_software_root(self, name, required=True): """Get install prefix for specified software name; special treatment for Cray modules.""" if name == 'cray-libsci': # Cray-provided LibSci module env_var = 'CRAY_LIBSCI_PREFIX_DIR' root = os.getenv(env_var, None) if root is None: - raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var) + if required: + raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var) else: self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root) else: - root = super(LibSci, self)._get_software_root(name) + root = super(LibSci, self)._get_software_root(name, required=required) return root diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index bcd8e0f0ce..9542c96daa 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -420,7 +420,8 @@ def _get_software_root(self, name, required=True): """Try to get the software root for name""" root = get_software_root(name) if root is None: - raise EasyBuildError("get_software_root software root for %s was not found in environment", name) + if required: + raise EasyBuildError("get_software_root software root for %s was not found in environment", name) else: self.log.debug("get_software_root software root %s for %s was found in environment", root, name) return root From 3991aa7c106b2ccd86c0357e9c2ab78edf6972e1 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 1 Feb 2021 11:33:57 +0100 Subject: [PATCH 220/864] Fix environment dependency of tests Allow empty PYTHONPATH and non-English LANG/LC_ALL settings by explicitely handling those --- test/framework/easyblock.py | 3 +++ test/framework/modules.py | 1 + test/framework/options.py | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 35f617d939..dc250823a2 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1677,6 +1677,9 @@ def test_extensions_sanity_check(self): test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') toy_ec_fn = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + # Do this before loading the easyblock to check the non-translated output below + os.environ['LC_ALL'] = 'C' + # this import only works here, since EB_toy is a test easyblock from easybuild.easyblocks.toy import EB_toy diff --git a/test/framework/modules.py b/test/framework/modules.py index a2f4633787..d050b9fca4 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1036,6 +1036,7 @@ def test_modules_tool_stateless(self): load_err_msg = "Unable to locate a modulefile" # GCC/4.6.3 is *not* an available Core module + os.environ['LC_ALL'] = 'C' self.assertErrorRegex(EasyBuildError, load_err_msg, self.modtool.load, ['GCC/4.6.3']) # GCC/6.4.0-2.28 is one of the available Core modules diff --git a/test/framework/options.py b/test/framework/options.py index f3a0a29fed..b1485c60f7 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3406,8 +3406,9 @@ def mk_eb_test_cmd(self, args): # make sure that location to 'easybuild.main' is included in $PYTHONPATH pythonpath = os.getenv('PYTHONPATH') + pythonpath = [pythonpath] if pythonpath else [] easybuild_loc = os.path.dirname(os.path.dirname(easybuild.main.__file__)) - os.environ['PYTHONPATH'] = ':'.join([easybuild_loc, pythonpath]) + os.environ['PYTHONPATH'] = ':'.join([easybuild_loc] + pythonpath) return '; '.join([ "cd %s" % self.test_prefix, From 75bf03dc051f41f021d8e4e9fd906e9409699740 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 3 Feb 2021 15:53:42 +0100 Subject: [PATCH 221/864] Replace slow usage of gettext by faster one The Python docs suggest against the current usage. In profiles it has shown that this is very slow as for every translation request the translation file is searched and an exception thrown as none is found. The change caches the translation which would be (temporarily) created by _gettext (on every call) and reuses it. If none is found a fallback (NullTranslation) is automatically used. --- easybuild/base/generaloption.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index b74110cdbd..789c0ce152 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -40,13 +40,30 @@ from functools import reduce from optparse import Option, OptionGroup, OptionParser, OptionValueError, Values from optparse import SUPPRESS_HELP as nohelp # supported in optparse of python v2.4 -from optparse import gettext as _gettext # this is gettext.gettext normally from easybuild.base.fancylogger import getLogger, setroot, setLogLevel, getDetailsLogLevels from easybuild.base.optcomplete import autocomplete, CompleterOption from easybuild.tools.py2vs3 import StringIO, configparser, string_type from easybuild.tools.utilities import mk_rst_table, nub, shell_quote +try: + import gettext + eb_translation = None + + def get_translation(): + global eb_translation + if not eb_translation: + # Finding a translation is expensive, so do only once + domain = gettext.textdomain() + eb_translation = gettext.translation(domain, gettext.bindtextdomain(domain), fallback=True) + return eb_translation + + def _gettext(message): + return get_translation().gettext(message) +except ImportError: + def _gettext(message): + return message + HELP_OUTPUT_FORMATS = ['', 'rst', 'short', 'config'] From e20b78dbe0e12d4a052ab177705a62adc94b7357 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 17 Apr 2021 10:22:49 +0200 Subject: [PATCH 222/864] add support for --sanity-check-only --- easybuild/framework/easyblock.py | 38 ++++++++++++++------ easybuild/main.py | 5 ++- easybuild/tools/config.py | 1 + easybuild/tools/options.py | 2 ++ test/framework/options.py | 59 ++++++++++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 77e0df72a5..de4099962a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1502,13 +1502,14 @@ def make_module_req_guess(self): 'CMAKE_LIBRARY_PATH': ['lib64'], # lib and lib32 are searched through the above } - def load_module(self, mod_paths=None, purge=True, extra_modules=None): + def load_module(self, mod_paths=None, purge=True, extra_modules=None, verbose=True): """ Load module for this software package/version, after purging all currently loaded modules. :param mod_paths: list of (additional) module paths to take into account :param purge: boolean indicating whether or not to purge currently loaded modules first :param extra_modules: list of extra modules to load (these are loaded *before* loading the 'self' module) + :param verbose: print modules being loaded when trace mode is enabled """ # self.full_mod_name might not be set (e.g. during unit tests) if self.full_mod_name is not None: @@ -1530,6 +1531,9 @@ def load_module(self, mod_paths=None, purge=True, extra_modules=None): if self.mod_subdir and not self.toolchain.is_system_toolchain(): mods.insert(0, self.toolchain.det_short_module_name()) + if verbose: + trace_msg("loading modules: %s..." % ', '.join(mods)) + # pass initial environment, to use it for resetting the environment before loading the modules self.modules_tool.load(mods, mod_paths=all_mod_paths, purge=purge, init_env=self.initial_environ) @@ -1541,7 +1545,7 @@ def load_module(self, mod_paths=None, purge=True, extra_modules=None): else: self.log.warning("Not loading module, since self.full_mod_name is not set.") - def load_fake_module(self, purge=False, extra_modules=None): + def load_fake_module(self, purge=False, extra_modules=None, verbose=False): """ Create and load fake module. @@ -1556,7 +1560,7 @@ def load_fake_module(self, purge=False, extra_modules=None): # load fake module self.modules_tool.prepend_module_path(os.path.join(fake_mod_path, self.mod_subdir), priority=10000) - self.load_module(purge=purge, extra_modules=extra_modules) + self.load_module(purge=purge, extra_modules=extra_modules, verbose=verbose) return (fake_mod_path, env) @@ -2817,12 +2821,16 @@ def xs2str(xs): fake_mod_data = None + # skip loading of fake module when using --sanity-check-only, load real module instead + if build_option('sanity_check_only') and not extension: + self.load_module(extra_modules=extra_modules) + # only load fake module for non-extensions, and not during dry run - if not (extension or self.dry_run): + elif not (extension or self.dry_run): try: # unload all loaded modules before loading fake module # this ensures that loading of dependencies is tested, and avoids conflicts with build dependencies - fake_mod_data = self.load_fake_module(purge=True, extra_modules=extra_modules) + fake_mod_data = self.load_fake_module(purge=True, extra_modules=extra_modules, verbose=True) except EasyBuildError as err: self.sanity_check_fail_msgs.append("loading fake module failed: %s" % err) self.log.warning("Sanity check: %s" % self.sanity_check_fail_msgs[-1]) @@ -3095,16 +3103,19 @@ def update_config_template_run_step(self): def skip_step(self, step, skippable): """Dedice whether or not to skip the specified step.""" - module_only = build_option('module_only') - force = build_option('force') or build_option('rebuild') + skip = False + force = build_option('force') or build_option('rebuild') + module_only = build_option('module_only') + sanity_check_only = build_option('sanity_check_only') + skipsteps = self.cfg['skipsteps'] # under --skip, sanity check is not skipped cli_skip = self.skip and step != SANITYCHECK_STEP # skip step if specified as individual (skippable) step, or if --skip is used - if skippable and (cli_skip or step in self.cfg['skipsteps']): - self.log.info("Skipping %s step (skip: %s, skipsteps: %s)", step, self.skip, self.cfg['skipsteps']) + if skippable and (cli_skip or step in skipsteps): + self.log.info("Skipping %s step (skip: %s, skipsteps: %s)", step, self.skip, skipsteps) skip = True # skip step when only generating module file @@ -3119,9 +3130,14 @@ def skip_step(self, step, skippable): self.log.info("Skipping %s step because of forced module-only mode", step) skip = True + elif sanity_check_only and step != SANITYCHECK_STEP: + self.log.info("Skipping %s step because of sanity-check-only mode", step) + skip = True + else: - self.log.debug("Not skipping %s step (skippable: %s, skip: %s, skipsteps: %s, module_only: %s, force: %s", - step, skippable, self.skip, self.cfg['skipsteps'], module_only, force) + msg = "Not skipping %s step (skippable: %s, skip: %s, skipsteps: %s, module_only: %s, force: %s, " + msg += "sanity_check_only: %s" + self.log.debug(msg, step, skippable, self.skip, skipsteps, module_only, force, sanity_check_only) return skip diff --git a/easybuild/main.py b/easybuild/main.py index 3c7429ea78..0c53ea7b76 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -403,8 +403,11 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules + keep_available_modules = forced or dry_run_mode or options.extended_dry_run or pr_options + keep_available_modules = keep_available_modules or options.inject_checksums or options.sanity_check_only + # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping - if not (forced or dry_run_mode or options.extended_dry_run or pr_options or options.inject_checksums): + if not keep_available_modules: retained_ecs = skip_available(easyconfigs, modtool) if not testing: for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]: diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index b491b0cde7..3b5459662f 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -256,6 +256,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'rebuild', 'robot', 'rpath', + 'sanity_check_only', 'search_paths', 'sequential', 'set_gid_bit', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 504b561eeb..4c5612ccc6 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -452,6 +452,8 @@ def override_options(self): None, 'store_true', False), 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', False), 'rpath-filter': ("List of regex patterns to use for filtering out RPATH paths", 'strlist', 'store', None), + 'sanity-check-only': ("Only run sanity check (module is expected to be installed already", + None, 'store_true', False), 'set-default-module': ("Set the generated module as default", None, 'store_true', False), 'set-gid-bit': ("Set group ID bit on newly created directories", None, 'store_true', False), 'silence-deprecation-warnings': ("Silence specified deprecation warnings", 'strlist', 'extend', None), diff --git a/test/framework/options.py b/test/framework/options.py index f3a0a29fed..1ad1bf3fa4 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5736,6 +5736,65 @@ def test_tmp_logdir(self): logtxt = read_file(os.path.join(tmp_logdir, tmp_logs[0])) self.assertTrue("COMPLETED: Installation ended successfully" in logtxt) + def test_sanity_check_only(self): + """Test use of --sanity-check-only.""" + topdir = os.path.abspath(os.path.dirname(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + test_ec = os.path.join(self.test_prefix, 'test.ec') + test_ec_txt = read_file(toy_ec) + test_ec_txt += "\nsanity_check_commands = ['toy']" + write_file(test_ec, test_ec_txt) + + # sanity check fails if software was not installed yet + outtxt, error_thrown = self.eb_main([test_ec, '--sanity-check-only'], do_build=True, return_error=True) + self.assertTrue("Sanity check failed: no file found at \\'bin/yot\\' or \\'bin/toy\\'" in str(error_thrown)) + + # actually install, then try --sanity-check-only again; + # need to use --force to install toy because module already exists (but installation doesn't) + self.eb_main([test_ec, '--force'], do_build=True, raise_error=True) + + self.mock_stdout(True) + self.mock_stderr(True) + self.eb_main([test_ec, '--sanity-check-only', '--trace'], do_build=True, raise_error=True, testing=False) + stdout = self.get_stdout().strip() + stderr = self.get_stderr().strip() + self.mock_stdout(False) + self.mock_stderr(False) + + self.assertFalse(stderr) + skipped = [ + "fetching files", + "creating build dir, resetting environment", + "unpacking", + "patching", + "preparing", + "configuring", + "building", + "testing", + "installing", + "taking care of extensions", + "restore after iterating", + "postprocessing", + "cleaning up", + "creating module", + "permissions", + "packaging" + ] + for skip in skipped: + self.assertTrue("== %s [skipped]" % skip) + + self.assertTrue("== sanity checking..." in stdout) + self.assertTrue("COMPLETED: Installation ended successfully" in stdout) + msgs = [ + "file 'bin/yot' or 'bin/toy' found: OK", + "(non-empty) directory 'bin' found: OK", + "loading modules: toy/0.0...", + "result for command 'toy': OK", + ] + for msg in msgs: + self.assertTrue(" >> %s" % msg in stdout) + def test_fake_vsc_include(self): """Test whether fake 'vsc' namespace is triggered for modules included via --include-*.""" From d1cbe3a0bd5f10ae46e575f23abb7a5f2beb2004 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 17 Apr 2021 11:18:14 +0200 Subject: [PATCH 223/864] fix broken test test_toy_build_trace --- test/framework/toy_build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index f3c0abda5a..5d87fbbc64 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2494,6 +2494,7 @@ def test_toy_build_trace(self): r"== sanity checking\.\.\.", r" >> file 'bin/yot' or 'bin/toy' found: OK", r" >> \(non-empty\) directory 'bin' found: OK", + r" >> loading modules: toy/0.0\.\.\.", r" >> running command 'toy' \.\.\.", r" >> result for command 'toy': OK", ]) + r'$', From 93ca94bebb4370fa91f8afb5687427e7ba37e98c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 17 Apr 2021 17:50:02 +0200 Subject: [PATCH 224/864] add support for running commands asynchronously via run_cmd + complete_cmd functions --- easybuild/tools/run.py | 35 ++++++++++++++++++++++++---- test/framework/run.py | 53 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 77 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 9c77edf06a..4447f5bcb6 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -128,7 +128,7 @@ def get_output_from_process(proc, read_size=None, asynchronous=False): @run_cmd_cache def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None, - force_in_dry_run=False, verbose=True, shell=True, trace=True, stream_output=None): + force_in_dry_run=False, verbose=True, shell=True, trace=True, stream_output=None, asynchronous=False): """ Run specified command (in a subshell) :param cmd: command to run @@ -144,6 +144,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True :param shell: allow commands to not run in a shell (especially useful for cmd lists) :param trace: print command being executed as part of trace output :param stream_output: enable streaming command output to stdout + :param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True) """ cwd = os.getcwd() @@ -228,10 +229,35 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True stdin=subprocess.PIPE, close_fds=True, executable=exec_cmd) except OSError as err: raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err) + if inp: proc.stdin.write(inp.encode()) proc.stdin.close() + if asynchronous: + return (proc, cmd, cwd, start_time, cmd_log) + else: + return complete_cmd(proc, cmd, cwd, start_time, cmd_log, log_ok=log_ok, log_all=log_all, simple=simple, + regexp=regexp, stream_output=stream_output, trace=trace) + + +def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False, simple=False, + regexp=True, stream_output=None, trace=True, output=''): + """ + Complete running of command represented by passed subprocess.Popen instance. + + :param proc: subprocess.Popen instance representing running command + :param cmd: command being run + :param owd: original working directory + :param start_time: start time of command (datetime instance) + :param cmd_log: log file to print command output to + :param log_ok: only run output/exit code for failing commands (exit code non-zero) + :param log_all: always log command output and exit code + :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) + :param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) + :param stream_output: enable streaming command output to stdout + :param trace: print command being executed as part of trace output + """ # use small read size when streaming output, to make it stream more fluently # read size should not be too small though, to avoid too much overhead if stream_output: @@ -239,8 +265,9 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True else: read_size = 1024 * 8 + stdouterr = output + ec = proc.poll() - stdouterr = '' while ec is None: # need to read from time to time. # - otherwise the stdout/stderr buffer gets filled and it all stops working @@ -265,9 +292,9 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) try: - os.chdir(cwd) + os.chdir(owd) except OSError as err: - raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err) + raise EasyBuildError("Failed to return to %s after executing command: %s", owd, err) return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp) diff --git a/test/framework/run.py b/test/framework/run.py index 8486119e24..ea04d789f0 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -38,6 +38,7 @@ import subprocess import sys import tempfile +import time from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner from easybuild.base.fancylogger import setLogLevelDebug @@ -46,13 +47,8 @@ import easybuild.tools.utilities from easybuild.tools.build_log import EasyBuildError, init_logging, stop_logging from easybuild.tools.filetools import adjust_permissions, read_file, write_file -from easybuild.tools.run import ( - check_log_for_errors, - get_output_from_process, - run_cmd, - run_cmd_qa, - parse_log_for_error, -) +from easybuild.tools.run import check_log_for_errors, complete_cmd, get_output_from_process +from easybuild.tools.run import parse_log_for_error, run_cmd, run_cmd_qa from easybuild.tools.config import ERROR, IGNORE, WARN @@ -571,6 +567,49 @@ def test_run_cmd_stream(self): ]) self.assertEqual(stdout, expected) + def test_run_cmd_async(self): + """Test asynchronously running of a shell command via run_cmd + complete_cmd.""" + + os.environ['TEST'] = 'test123' + + cmd_info = run_cmd("sleep 2; echo $TEST", asynchronous=True) + proc = cmd_info[0] + + # change value of $TEST to check that command is completed with correct environment + os.environ['TEST'] = 'some_other_value' + + # initial poll should result in None, since it takes a while for the command to complete + ec = proc.poll() + self.assertEqual(ec, None) + + while ec is None: + time.sleep(1) + ec = proc.poll() + + out, ec = complete_cmd(*cmd_info, simple=False) + self.assertEqual(ec, 0) + self.assertEqual(out, 'test123\n') + + # also test with a command that produces a lot of output, + # since that tends to lock up things unless we frequently grab some output... + cmd = "echo start; for i in $(seq 1 50); do sleep 0.1; for j in $(seq 1000); do echo foo; done; done; echo done" + cmd_info = run_cmd(cmd, asynchronous=True) + proc = cmd_info[0] + + output = '' + ec = proc.poll() + self.assertEqual(ec, None) + + while ec is None: + time.sleep(1) + output += get_output_from_process(proc) + ec = proc.poll() + + out, ec = complete_cmd(*cmd_info, simple=False, output=output) + self.assertEqual(ec, 0) + self.assertTrue(out.startswith('start\n')) + self.assertTrue(out.endswith('\ndone\n')) + def test_check_log_for_errors(self): fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') os.close(fd) From af3cb7defd4d74160e8733fecf625950d934b494 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 20 Apr 2021 14:28:37 +0200 Subject: [PATCH 225/864] Improve logging when failing to load class from exts_classmap Posted in Slack: TypeError: __init__() takes 2 positional arguments but 3 were given This was not caught and "enhanced" by the information which class was tried to load. This change catches the base Exception class, which includes TypeError. It also logs the class found before instantiating it. --- easybuild/framework/easyblock.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 77e0df72a5..c4bdf459a8 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2269,49 +2269,51 @@ def extensions_step(self, fetch=False, install=True): # get class instances for all extensions self.ext_instances = [] for ext in self.exts: - self.log.debug("Creating class instance for extension %s...", ext['name']) + ext_name = ext['name'] + self.log.debug("Creating class instance for extension %s...", ext_name) cls, inst = None, None - class_name = encode_class_name(ext['name']) + class_name = encode_class_name(ext_name) mod_path = get_module_path(class_name, generic=False) # try instantiating extension-specific class try: # no error when importing class fails, in case we run into an existing easyblock # with a similar name (e.g., Perl Extension 'GO' vs 'Go' for which 'EB_Go' is available) - cls = get_easyblock_class(None, name=ext['name'], error_on_failed_import=False, + cls = get_easyblock_class(None, name=ext_name, error_on_failed_import=False, error_on_missing_easyblock=False) - self.log.debug("Obtained class %s for extension %s", cls, ext['name']) + self.log.debug("Obtained class %s for extension %s", cls, ext_name) if cls is not None: inst = cls(self, ext) except (ImportError, NameError) as err: - self.log.debug("Failed to use extension-specific class for extension %s: %s", ext['name'], err) + self.log.debug("Failed to use extension-specific class for extension %s: %s", ext_name, err) # alternative attempt: use class specified in class map (if any) - if inst is None and ext['name'] in exts_classmap: - - class_name = exts_classmap[ext['name']] + if inst is None and ext_name in exts_classmap: + class_name = exts_classmap[ext_name] mod_path = get_module_path(class_name) try: cls = get_class_for(mod_path, class_name) + self.log.debug("Obtained class %s for extension %s from exts_classmap", cls, ext_name) inst = cls(self, ext) - except (ImportError, NameError) as err: - raise EasyBuildError("Failed to load specified class %s for extension %s: %s", - class_name, ext['name'], err) + except Exception as err: + raise EasyBuildError("Failed to load specified class %s (from %s) specified via exts_classmap " + "for extension %s: %s", + class_name, mod_path, ext_name, err) # fallback attempt: use default class if inst is None: try: cls = get_class_for(default_class_modpath, default_class) - self.log.debug("Obtained class %s for installing extension %s", cls, ext['name']) + self.log.debug("Obtained class %s for installing extension %s", cls, ext_name) inst = cls(self, ext) self.log.debug("Installing extension %s with default class %s (from %s)", - ext['name'], default_class, default_class_modpath) + ext_name, default_class, default_class_modpath) except (ImportError, NameError) as err: raise EasyBuildError("Also failed to use default class %s from %s for extension %s: %s, giving up", - default_class, default_class_modpath, ext['name'], err) + default_class, default_class_modpath, ext_name, err) else: - self.log.debug("Installing extension %s with class %s (from %s)", ext['name'], class_name, mod_path) + self.log.debug("Installing extension %s with class %s (from %s)", ext_name, class_name, mod_path) self.ext_instances.append(inst) From 6b779256c85c89a3aa32b357c16efa1ff881f442 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 21 Apr 2021 08:17:29 +0200 Subject: [PATCH 226/864] add support for installing extensions in parallel --- easybuild/framework/easyblock.py | 196 +++++++++++++++++++++++++------ easybuild/framework/extension.py | 51 +++++++- 2 files changed, 210 insertions(+), 37 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 77e0df72a5..4b3777db34 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1620,6 +1620,166 @@ def skip_extensions(self): self.ext_instances = res + def install_extensions(self, install=True, parallel=False): + """ + Install extensions. + + :param install: actually install extensions, don't just prepare environment for installing + :param parallel: install extensions in parallel + + """ + self.log.debug("List of loaded modules: %s", self.modules_tool.list()) + + if parallel: + self.install_extensions_parallel(install=install) + else: + self.install_extensions_sequential(install=install) + + def install_extensions_sequential(self, install=True): + """ + Install extensions sequentially. + + :param install: actually install extensions, don't just prepare environment for installing + """ + self.log.info("Installing extensions sequentially...") + + exts_cnt = len(self.ext_instances) + for idx, ext in enumerate(self.ext_instances): + + self.log.debug("Starting extension %s" % ext.name) + + # always go back to original work dir to avoid running stuff from a dir that no longer exists + change_dir(self.orig_workdir) + + tup = (ext.name, ext.version or '', idx + 1, exts_cnt) + print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent, log=self.log) + + if self.dry_run: + tup = (ext.name, ext.version, ext.__class__.__name__) + msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup + self.dry_run_msg(msg) + + self.log.debug("List of loaded modules: %s", self.modules_tool.list()) + + # prepare toolchain build environment, but only when not doing a dry run + # since in that case the build environment is the same as for the parent + if self.dry_run: + self.dry_run_msg("defining build environment based on toolchain (options) and dependencies...") + else: + # don't reload modules for toolchain, there is no need since they will be loaded already; + # the (fake) module for the parent software gets loaded before installing extensions + ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, + rpath_filter_dirs=self.rpath_filter_dirs) + + # real work + if install: + ext.prerun() + txt = ext.run() + if txt: + self.module_extra_extensions += txt + ext.postrun() + + def install_extensions_parallel(self, install=True): + """ + Install extensions in parallel. + + :param install: actually install extensions, don't just prepare environment for installing + """ + self.log.info("Installing extensions in parallel...") + + running_exts = [] + installed_ext_names = [] + + all_ext_names = [x['name'] for x in self.exts_all] + self.log.debug("List of names of all extensions: %s", all_ext_names) + + # take into account that some extensions may be installed already + to_install_ext_names = [x.name for x in self.ext_instances] + installed_ext_names = [n for n in all_ext_names if n not in to_install_ext_names] + + exts_cnt = len(all_ext_names) + exts_queue = self.ext_instances[:] + + iter_id = 0 + while exts_queue or running_exts: + + iter_id += 1 + + # always go back to original work dir to avoid running stuff from a dir that no longer exists + change_dir(self.orig_workdir) + + # check for extension installations that have completed + if running_exts: + self.log.info("Checking for completed extension installations (%d running)...", len(running_exts)) + for ext in running_exts[:]: + if self.dry_run or ext.async_cmd_check(): + self.log.info("Installation of %s completed!", ext.name) + ext.postrun() + running_exts.remove(ext) + installed_ext_names.append(ext.name) + else: + self.log.debug("Installation of %s is still running...", ext.name) + + # print progress info every now and then + if iter_id % 1 == 0: + msg = "%d out of %d extensions installed (%d queued, %d running: %s)" + installed_cnt, queued_cnt, running_cnt = len(installed_ext_names), len(exts_queue), len(running_exts) + if running_cnt <= 3: + running_ext_names = ', '.join(x.name for x in running_exts) + else: + running_ext_names = ', '.join(x.name for x in running_exts[:3]) + ", ..." + print_msg(msg % (installed_cnt, exts_cnt, queued_cnt, running_cnt, running_ext_names), log=self.log) + + # try to start as many extension installations as we can, taking into account number of available cores, + # but only consider first 100 extensions still in the queue + max_iter = min(100, len(exts_queue)) + + for _ in range(max_iter): + + if not (exts_queue and len(running_exts) < self.cfg['parallel']): + break + + # check whether extension at top of the queue is ready to install + ext = exts_queue.pop(0) + + pending_deps = [x for x in ext.required_deps if x not in installed_ext_names] + + if self.dry_run: + tup = (ext.name, ext.version, ext.__class__.__name__) + msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup + self.dry_run_msg(msg) + running_exts.append(ext) + + # if some of the required dependencies are not installed yet, requeue this extension + elif pending_deps: + + # make sure all required dependencies are actually going to be installed, + # to avoid getting stuck in an infinite loop! + missing_deps = [x for x in ext.required_deps if x not in all_ext_names] + if missing_deps: + raise EasyBuildError("Missing required dependencies for %s are not going to be installed: %s", + ext.name, ', '.join(missing_deps)) + else: + self.log.info("Required dependencies missing for extension %s (%s), adding it back to queue...", + ext.name, ', '.join(pending_deps)) + # purposely adding extension back in the queue at Nth place rather than at the end, + # since we assume that the required dependencies will be installed soon... + exts_queue.insert(max_iter, ext) + + else: + tup = (ext.name, ext.version or '') + print_msg("starting installation of extension %s %s..." % tup, silent=self.silent, log=self.log) + + # don't reload modules for toolchain, there is no need since they will be loaded already; + # the (fake) module for the parent software gets loaded before installing extensions + ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, + rpath_filter_dirs=self.rpath_filter_dirs) + if install: + ext.prerun() + ext.run(asynchronous=True) + running_exts.append(ext) + self.log.debug("Started installation of extension %s in the background...", ext.name) + # # MISCELLANEOUS UTILITY FUNCTIONS # @@ -2318,41 +2478,7 @@ def extensions_step(self, fetch=False, install=True): if self.skip: self.skip_extensions() - exts_cnt = len(self.ext_instances) - for idx, ext in enumerate(self.ext_instances): - - self.log.debug("Starting extension %s" % ext.name) - - # always go back to original work dir to avoid running stuff from a dir that no longer exists - change_dir(self.orig_workdir) - - tup = (ext.name, ext.version or '', idx + 1, exts_cnt) - print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent) - - if self.dry_run: - tup = (ext.name, ext.version, cls.__name__) - msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup - self.dry_run_msg(msg) - - self.log.debug("List of loaded modules: %s", self.modules_tool.list()) - - # prepare toolchain build environment, but only when not doing a dry run - # since in that case the build environment is the same as for the parent - if self.dry_run: - self.dry_run_msg("defining build environment based on toolchain (options) and dependencies...") - else: - # don't reload modules for toolchain, there is no need since they will be loaded already; - # the (fake) module for the parent software gets loaded before installing extensions - ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, - rpath_filter_dirs=self.rpath_filter_dirs) - - # real work - if install: - ext.prerun() - txt = ext.run() - if txt: - self.module_extra_extensions += txt - ext.postrun() + self.install_extensions(install=install) # cleanup (unload fake module, remove fake module dir) if fake_mod_data: diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 569a3bb414..6b70afd966 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -40,7 +40,7 @@ from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.tools.build_log import EasyBuildError, raise_nosupport from easybuild.tools.filetools import change_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import complete_cmd, get_output_from_process, run_cmd from easybuild.tools.py2vs3 import string_type @@ -138,6 +138,12 @@ def __init__(self, mself, ext, extra_params=None): key, name, version, value) self.sanity_check_fail_msgs = [] + self.async_cmd_info = None + self.async_cmd_output = None + self.async_cmd_check_cnt = None + # initial read size should be relatively small, + # to avoid hanging for a long time until desired output is available in async_cmd_check + self.async_cmd_read_size = 1024 @property def name(self): @@ -159,7 +165,7 @@ def prerun(self): """ pass - def run(self): + def run(self, *args, **kwargs): """ Actual installation of a extension. """ @@ -171,6 +177,47 @@ def postrun(self): """ pass + def async_cmd_start(self, cmd, inp=None): + """ + Start installation asynchronously using specified command. + """ + self.async_cmd_output = '' + self.async_cmd_check_cnt = 0 + self.async_cmd_info = run_cmd(cmd, log_all=True, simple=False, inp=inp, regexp=False, asynchronous=True) + + def async_cmd_check(self): + """ + Check progress of installation command that was started asynchronously. + + :return: True if command completed, False otherwise + """ + if self.async_cmd_info is None: + raise EasyBuildError("No installation command running asynchronously for %s", self.name) + else: + self.log.debug("Checking on installation of extension %s...", self.name) + proc = self.async_cmd_info[0] + # use small read size, to avoid waiting for a long time until sufficient output is produced + self.async_cmd_output += get_output_from_process(proc, read_size=self.async_cmd_read_size) + ec = proc.poll() + if ec is None: + res = False + self.async_cmd_check_cnt += 1 + # increase read size after sufficient checks, + # to avoid that installation hangs due to output buffer filling up... + if self.async_cmd_check_cnt % 10 == 0 and self.async_cmd_read_size < (1024 ** 2): + self.async_cmd_read_size *= 2 + else: + self.log.debug("Completing installation of extension %s...", self.name) + self.async_cmd_output, _ = complete_cmd(*self.async_cmd_info, output=self.async_cmd_output) + res = True + + return res + + @property + def required_deps(self): + """Return list of required dependencies for this extension.""" + raise NotImplementedError("Don't know how to determine required dependencies for %s" % self.name) + @property def toolchain(self): """ From 1e808e7a18a9ba7ac94ef64585f73bbba506c83b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 26 Apr 2021 16:02:49 +0200 Subject: [PATCH 227/864] Allow module-only on existing installations (without write permissions) --- easybuild/framework/easyblock.py | 52 ++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index c4bdf459a8..6c90080d85 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2993,7 +2993,8 @@ def make_module_step(self, fake=False): mod_symlink_paths = ActiveMNS().det_module_symlink_paths(self.cfg) self.module_generator.create_symlinks(mod_symlink_paths, fake=fake) - if ActiveMNS().mns.det_make_devel_module() and not fake and build_option('generate_devel_module'): + generate_devel_module = ActiveMNS().mns.det_make_devel_module() and build_option('generate_devel_module') + if generate_devel_module and not fake and not build_option('module_only'): self.make_devel_module() else: self.log.info("Skipping devel module...") @@ -3449,14 +3450,17 @@ def build_and_install_one(ecdict, init_env): buildstats = get_build_stats(app, start_time, build_option('command_line')) _log.info("Build stats: %s" % buildstats) - # move the reproducability files to the final log directory - archive_reprod_dir = os.path.join(new_log_dir, REPROD) - if os.path.exists(archive_reprod_dir): - backup_dir = find_backup_name_candidate(archive_reprod_dir) - move_file(archive_reprod_dir, backup_dir) - _log.info("Existing reproducability directory %s backed up to %s", archive_reprod_dir, backup_dir) - move_file(reprod_dir, archive_reprod_dir) - _log.info("Wrote files for reproducability to %s", archive_reprod_dir) + if not build_option('module_only') or os.access(new_log_dir, os.W_OK): + # move the reproducability files to the final log directory + archive_reprod_dir = os.path.join(new_log_dir, REPROD) + if os.path.exists(archive_reprod_dir): + backup_dir = find_backup_name_candidate(archive_reprod_dir) + move_file(archive_reprod_dir, backup_dir) + _log.info("Existing reproducability directory %s backed up to %s", archive_reprod_dir, backup_dir) + move_file(reprod_dir, archive_reprod_dir) + _log.info("Wrote files for reproducability to %s", archive_reprod_dir) + else: + _log.debug("Using --module-only with non-writable log dir %s, cannot keep reprod dir", new_log_dir) try: # upload easyconfig (and patch files) to central repository @@ -3476,22 +3480,26 @@ def build_and_install_one(ecdict, init_env): # cleanup logs app.close_log() log_fn = os.path.basename(get_log_filename(app.name, app.version)) - application_log = os.path.join(new_log_dir, log_fn) - move_logs(app.logfile, application_log) + if not build_option('module_only') or os.access(new_log_dir, os.W_OK): + application_log = os.path.join(new_log_dir, log_fn) + move_logs(app.logfile, application_log) - newspec = os.path.join(new_log_dir, app.cfg.filename()) - copy_file(spec, newspec) - _log.debug("Copied easyconfig file %s to %s", spec, newspec) + newspec = os.path.join(new_log_dir, app.cfg.filename()) + copy_file(spec, newspec) + _log.debug("Copied easyconfig file %s to %s", spec, newspec) - # copy patches - for patch in app.patches: - target = os.path.join(new_log_dir, os.path.basename(patch['path'])) - copy_file(patch['path'], target) - _log.debug("Copied patch %s to %s", patch['path'], target) + # copy patches + for patch in app.patches: + target = os.path.join(new_log_dir, os.path.basename(patch['path'])) + copy_file(patch['path'], target) + _log.debug("Copied patch %s to %s", patch['path'], target) - if build_option('read_only_installdir'): - # take away user write permissions (again) - adjust_permissions(new_log_dir, stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False, recursive=True) + if build_option('read_only_installdir'): + # take away user write permissions (again) + adjust_permissions(new_log_dir, stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False, recursive=True) + else: + application_log = None + _log.debug("Using --module-only with non-writable log dir %s, cannot keep logs", new_log_dir) end_timestamp = datetime.now() From a9fe554f14c982b793e224487127a6571cad33c4 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 26 Apr 2021 16:50:36 +0200 Subject: [PATCH 228/864] Use try/except instead of checking write capability --- easybuild/framework/easyblock.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 6c90080d85..8a6bd4a220 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2993,9 +2993,14 @@ def make_module_step(self, fake=False): mod_symlink_paths = ActiveMNS().det_module_symlink_paths(self.cfg) self.module_generator.create_symlinks(mod_symlink_paths, fake=fake) - generate_devel_module = ActiveMNS().mns.det_make_devel_module() and build_option('generate_devel_module') - if generate_devel_module and not fake and not build_option('module_only'): - self.make_devel_module() + if ActiveMNS().mns.det_make_devel_module() and not fake and build_option('generate_devel_module'): + try: + self.make_devel_module() + except EasyBuildError as error: + if build_option('module_only'): + self.log.info("Using --module-only so can recover from error: %s", error) + else: + raise error else: self.log.info("Skipping devel module...") @@ -3450,7 +3455,7 @@ def build_and_install_one(ecdict, init_env): buildstats = get_build_stats(app, start_time, build_option('command_line')) _log.info("Build stats: %s" % buildstats) - if not build_option('module_only') or os.access(new_log_dir, os.W_OK): + try: # move the reproducability files to the final log directory archive_reprod_dir = os.path.join(new_log_dir, REPROD) if os.path.exists(archive_reprod_dir): @@ -3459,8 +3464,11 @@ def build_and_install_one(ecdict, init_env): _log.info("Existing reproducability directory %s backed up to %s", archive_reprod_dir, backup_dir) move_file(reprod_dir, archive_reprod_dir) _log.info("Wrote files for reproducability to %s", archive_reprod_dir) - else: - _log.debug("Using --module-only with non-writable log dir %s, cannot keep reprod dir", new_log_dir) + except EasyBuildError as error: + if build_option('module_only'): + _log.info("Using --module-only so can recover from error: %s", error) + else: + raise error try: # upload easyconfig (and patch files) to central repository @@ -3480,7 +3488,7 @@ def build_and_install_one(ecdict, init_env): # cleanup logs app.close_log() log_fn = os.path.basename(get_log_filename(app.name, app.version)) - if not build_option('module_only') or os.access(new_log_dir, os.W_OK): + try: application_log = os.path.join(new_log_dir, log_fn) move_logs(app.logfile, application_log) @@ -3497,9 +3505,12 @@ def build_and_install_one(ecdict, init_env): if build_option('read_only_installdir'): # take away user write permissions (again) adjust_permissions(new_log_dir, stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False, recursive=True) - else: - application_log = None - _log.debug("Using --module-only with non-writable log dir %s, cannot keep logs", new_log_dir) + except EasyBuildError as error: + if build_option('module_only'): + application_log = None + _log.debug("Using --module-only so can recover from error: %s", new_log_dir) + else: + raise error end_timestamp = datetime.now() From 0eabe0ab76a820dc78ba9479504b7637340ce355 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Mon, 26 Apr 2021 17:18:14 +0200 Subject: [PATCH 229/864] Update easyblock.py --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 8a6bd4a220..083e8e93bc 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3508,7 +3508,7 @@ def build_and_install_one(ecdict, init_env): except EasyBuildError as error: if build_option('module_only'): application_log = None - _log.debug("Using --module-only so can recover from error: %s", new_log_dir) + _log.debug("Using --module-only so can recover from error: %s", error) else: raise error From dea0a7e194a515e86e194f842801b0ba36791aec Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 27 Apr 2021 13:45:16 +0200 Subject: [PATCH 230/864] Correct help text for subdir-user-modules --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 504b561eeb..18f2184a54 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -567,7 +567,7 @@ def config_options(self): 'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']), 'subdir-software': ("Installpath subdir for software", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_software']), - 'subdir-user-modules': ("Base path of user-specific modules relative to --envvar-user-modules", + 'subdir-user-modules': ("Base path of user-specific modules relative to --envvars-user-modules", None, 'store', None), 'suffix-modules-path': ("Suffix for module files install path", None, 'store', GENERAL_CLASS), # this one is sort of an exception, it's something jobscripts can set, From f543cfe4cf1f3d04f291d16ea057d789e435c3ce Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 27 Apr 2021 14:27:23 +0200 Subject: [PATCH 231/864] Replace log_error parameter of which() by on_error This allows to additionally make the warning an error instead of only being able to ignore it. This is useful if the command must be found. --- easybuild/tools/filetools.py | 23 ++++++++++++++++++----- easybuild/tools/modules.py | 2 +- test/framework/filetools.py | 8 +++++++- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index a2e2709917..057f4513c2 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -58,7 +58,8 @@ from easybuild.tools import run # import build_log must stay, to use of EasyBuildLog from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning -from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, GENERIC_EASYBLOCK_PKG, build_option, install_path +from easybuild.tools.config import (DEFAULT_WAIT_ON_LOCK_INTERVAL, GENERIC_EASYBLOCK_PKG, build_option, install_path, + IGNORE, WARN, ERROR) from easybuild.tools.py2vs3 import HTMLParser, std_urllib, string_type from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars @@ -450,15 +451,23 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced return base_dir -def which(cmd, retain_all=False, check_perms=True, log_ok=True, log_error=True): +def which(cmd, retain_all=False, check_perms=True, log_ok=True, log_error=None, on_error=WARN): """ Return (first) path in $PATH for specified command, or None if command is not found :param retain_all: returns *all* locations to the specified command in $PATH, not just the first one :param check_perms: check whether candidate path has read/exec permissions before accepting it as a match :param log_ok: Log an info message where the command has been found (if any) - :param log_error: Log a warning message when command hasn't been found + :param on_error: What to do if the command was not found. Possible values: IGNORE, WARN, ERROR """ + if log_error is not None: + _log.deprecated("'log_error' named argument in which function has been replaced by 'on_error'", '5.0') + # If set, make sure on_error is at least WARN + if log_error and on_error == IGNORE: + on_error = WARN + if on_error not in (IGNORE, WARN, ERROR): + raise EasyBuildError("Invalid value for 'on_error': %s", on_error) + if retain_all: res = [] else: @@ -482,8 +491,12 @@ def which(cmd, retain_all=False, check_perms=True, log_ok=True, log_error=True): res = cmd_path break - if not res and log_error: - _log.warning("Could not find command '%s' (with permissions to read/execute it) in $PATH (%s)" % (cmd, paths)) + if not res and on_error != IGNORE: + msg = "Could not find command '%s' (with permissions to read/execute it) in $PATH (%s)" % (cmd, paths) + if on_error == WARN: + _log.warning(msg) + else: + raise EasyBuildError(msg) return res diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 3e3f004dc5..f98bd3e78a 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -181,7 +181,7 @@ def __init__(self, mod_paths=None, testing=False): self.set_mod_paths(mod_paths) if env_cmd_path: - cmd_path = which(self.cmd, log_ok=False, log_error=False) + cmd_path = which(self.cmd, log_ok=False, on_error=IGNORE) # only use command path in environment variable if command in not available in $PATH if cmd_path is None: self.cmd = env_cmd_path diff --git a/test/framework/filetools.py b/test/framework/filetools.py index fb9c2440e4..0eab3d83fa 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -45,6 +45,7 @@ import easybuild.tools.filetools as ft from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import IGNORE, ERROR from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import std_urllib @@ -213,8 +214,13 @@ def test_which(self): python = ft.which('python') self.assertTrue(python and os.path.exists(python) and os.path.isabs(python)) - path = ft.which('i_really_do_not_expect_a_command_with_a_name_like_this_to_be_available') + invalid_cmd = 'i_really_do_not_expect_a_command_with_a_name_like_this_to_be_available' + path = ft.which(invalid_cmd) self.assertTrue(path is None) + path = ft.which(invalid_cmd, on_error=IGNORE) + self.assertTrue(path is None) + error_msg = "Could not find command '%s'" % invalid_cmd + self.assertErrorRegex(EasyBuildError, error_msg, ft.which, invalid_cmd, on_error=ERROR) os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.environ['PATH']) # put a directory 'foo' in place (should be ignored by 'which') From 6c4f059aa4c6979c3b80daa6b74b2f7af0b75432 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 28 Apr 2021 08:20:49 +0200 Subject: [PATCH 232/864] add toolchain definition for gofbf (foss with FlexiBLAS rather than OpenBLAS) --- easybuild/toolchains/gofbf.py | 40 +++++++++++++++++++++ easybuild/toolchains/linalg/flexiblas.py | 46 ++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 easybuild/toolchains/gofbf.py create mode 100644 easybuild/toolchains/linalg/flexiblas.py diff --git a/easybuild/toolchains/gofbf.py b/easybuild/toolchains/gofbf.py new file mode 100644 index 0000000000..294f62cd03 --- /dev/null +++ b/easybuild/toolchains/gofbf.py @@ -0,0 +1,40 @@ +## +# Copyright 2021-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gofbf compiler toolchain (includes GCC, OpenMPI, FlexiBLAS, ScaLAPACK and FFTW) + +:author: Kenneth Hoste (Ghent University) +""" + +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.gompi import Gompi +from easybuild.toolchains.linalg.flexiblas import FlexiBLAS +from easybuild.toolchains.linalg.scalapack import ScaLAPACK + + +class Gofbf(Gompi, FlexiBLAS, ScaLAPACK, Fftw): + """Compiler toolchain with GCC, OpenMPI, FlexiBLAS, ScaLAPACK and FFTW.""" + NAME = 'gofbf' + SUBTOOLCHAIN = Gompi.NAME diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py new file mode 100644 index 0000000000..c8d7755f04 --- /dev/null +++ b/easybuild/toolchains/linalg/flexiblas.py @@ -0,0 +1,46 @@ +## +# Copyright 2021-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for FlexiBLAS as toolchain linear algebra library. + +:author: Kenneth Hoste (Ghent University) +""" +from easybuild.tools.toolchain.linalg import LinAlg + + +TC_CONSTANT_FLEXIBLAS = 'FlexiBLAS' + + +class FlexiBLAS(LinAlg): + """ + Trivial class, provides FlexiBLAS support. + """ + BLAS_MODULE_NAME = ['FlexiBLAS'] + BLAS_LIB = ['flexiblas'] + BLAS_FAMILY = TC_CONSTANT_FLEXIBLAS + + LAPACK_MODULE_NAME = ['FlexiBLAS'] + LAPACK_IS_BLAS = True + LAPACK_FAMILY = TC_CONSTANT_FLEXIBLAS From 0a26932e8625f39a4743021efb5511eb7641afc4 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 28 Apr 2021 08:50:38 +0200 Subject: [PATCH 233/864] Make which(on_error change backward compatible with log_error=False --- easybuild/tools/filetools.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 057f4513c2..aec01d7000 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -451,20 +451,26 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced return base_dir -def which(cmd, retain_all=False, check_perms=True, log_ok=True, log_error=None, on_error=WARN): +def which(cmd, retain_all=False, check_perms=True, log_ok=True, log_error=None, on_error=None): """ Return (first) path in $PATH for specified command, or None if command is not found :param retain_all: returns *all* locations to the specified command in $PATH, not just the first one :param check_perms: check whether candidate path has read/exec permissions before accepting it as a match :param log_ok: Log an info message where the command has been found (if any) - :param on_error: What to do if the command was not found. Possible values: IGNORE, WARN, ERROR + :param on_error: What to do if the command was not found, default: WARN. Possible values: IGNORE, WARN, ERROR """ if log_error is not None: _log.deprecated("'log_error' named argument in which function has been replaced by 'on_error'", '5.0') # If set, make sure on_error is at least WARN if log_error and on_error == IGNORE: on_error = WARN + elif not log_error and on_error is None: # If set to False, use IGNORE unless on_error is also set + on_error = IGNORE + # Set default + # TODO: After removal of log_error from the parameters, on_error=WARN can be used instead of this + if on_error is None: + on_error = WARN if on_error not in (IGNORE, WARN, ERROR): raise EasyBuildError("Invalid value for 'on_error': %s", on_error) From c4d780afeb932dbeb2939bc6f7c89d53600f9a69 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 28 Apr 2021 09:50:07 +0200 Subject: [PATCH 234/864] update iimpi toolchain to intel-compiler subtoolchain for oneAPI versions (>= iimpi 2020.12) --- easybuild/toolchains/iimpi.py | 47 +++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/easybuild/toolchains/iimpi.py b/easybuild/toolchains/iimpi.py index d36e0ba843..a7e374526a 100644 --- a/easybuild/toolchains/iimpi.py +++ b/easybuild/toolchains/iimpi.py @@ -32,32 +32,59 @@ import re from easybuild.toolchains.iccifort import IccIfort +from easybuild.toolchains.intel_compilers import IntelCompilers from easybuild.toolchains.mpi.intelmpi import IntelMPI -class Iimpi(IccIfort, IntelMPI): +class Iimpi(IntelCompilers, IccIfort, IntelMPI): """ Compiler toolchain with Intel compilers (icc/ifort), Intel MPI. """ NAME = 'iimpi' - SUBTOOLCHAIN = IccIfort.NAME + SUBTOOLCHAIN = None + + def __init__(self, *args, **kwargs): + """Constructor for Iimpi toolchain class.""" + + super(Iimpi, self).__init__(*args, **kwargs) + + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', self.version): + # need to transform a version like '2016a' with something that is safe to compare with '8.0', '2016.01' + # comparing subversions that include letters causes TypeErrors in Python 3 + # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) + # (good enough for this purpose) + self.iimpi_ver = LooseVersion(self.version.replace('a', '.01').replace('b', '.07')) + if self.iimpi_ver >= LooseVersion('2020.12'): + self.SUBTOOLCHAIN = IntelCompilers.NAME + self.COMPILER_MODULE_NAME = IntelCompilers.COMPILER_MODULE_NAME + else: + self.SUBTOOLCHAIN = IccIfort.NAME + self.COMPILER_MODULE_NAME = IccIfort.COMPILER_MODULE_NAME + else: + self.iimpi_ver = self.version def is_deprecated(self): """Return whether or not this toolchain is deprecated.""" - # need to transform a version like '2016a' with something that is safe to compare with '8.0', '2000', '2016.01' - # comparing subversions that include letters causes TypeErrors in Python 3 - # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) - version = self.version.replace('a', '.01').replace('b', '.07') deprecated = False + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion - if re.match('^[0-9]', version): - iimpi_ver = LooseVersion(version) + if re.match('^[0-9]', str(self.iimpi_ver)): # iimpi toolchains older than iimpi/2016.01 are deprecated # iimpi 8.1.5 is an exception, since it used in intel/2016a (which is not deprecated yet) - if iimpi_ver < LooseVersion('8.0'): + if self.iimpi_ver < LooseVersion('8.0'): deprecated = True - elif iimpi_ver > LooseVersion('2000') and iimpi_ver < LooseVersion('2016.01'): + elif self.iimpi_ver > LooseVersion('2000') and self.iimpi_ver < LooseVersion('2016.01'): deprecated = True return deprecated + + def is_dep_in_toolchain_module(self, *args, **kwargs): + """Check whether a specific software name is listed as a dependency in the module for this toolchain.""" + if re.match('^[0-9]', str(self.iimpi_ver)) and self.iimpi_ver < LooseVersion('2020.12'): + res = IccIfort.is_dep_in_toolchain_module(self, *args, **kwargs) + else: + res = super(Iimpi, self).is_dep_in_toolchain_module(*args, **kwargs) + + return res From a9143af19eab190d2235cab2bc9dcb520a09cc5c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 28 Apr 2021 10:04:30 +0200 Subject: [PATCH 235/864] fix check for Intel compilers in Intel FFT toolchain support --- easybuild/toolchains/fft/intelfftw.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py index 037acf823e..dca03a0476 100644 --- a/easybuild/toolchains/fft/intelfftw.py +++ b/easybuild/toolchains/fft/intelfftw.py @@ -57,15 +57,16 @@ def _set_fftw_variables(self): bitsuff = '_lp64' if self.options.get('i8', None): bitsuff = '_ilp64' - compsuff = '_intel' - if get_software_root('icc') is None: - if get_software_root('PGI'): - compsuff = '_pgi' - elif get_software_root('GCC'): - compsuff = '_gnu' - else: - error_msg = "Not using Intel compilers, PGI nor GCC, don't know compiler suffix for FFTW libraries." - raise EasyBuildError(error_msg) + + if get_software_root('icc') or get_software_root('intel-compilers'): + compsuff = '_intel' + elif get_software_root('PGI'): + compsuff = '_pgi' + elif get_software_root('GCC'): + compsuff = '_gnu' + else: + error_msg = "Not using Intel compilers, PGI nor GCC, don't know compiler suffix for FFTW libraries." + raise EasyBuildError(error_msg) interface_lib = "fftw3xc%s%s" % (compsuff, picsuff) fftw_libs = [interface_lib] From 0f7e69089db6c1933fe1b3915459d7fda416ce64 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 28 Apr 2021 10:09:25 +0200 Subject: [PATCH 236/864] fix location of mkl subdirectory for Intel MKL oneAPI versions >= 2021.x in toolchain support --- easybuild/toolchains/linalg/intelmkl.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index d115ca2a09..cefd110eac 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -28,6 +28,7 @@ :author: Stijn De Weirdt (Ghent University) :author: Kenneth Hoste (Ghent University) """ +import os from distutils.version import LooseVersion from easybuild.toolchains.compiler.gcc import TC_CONSTANT_GCC @@ -152,13 +153,18 @@ def _set_blas_variables(self): raise EasyBuildError("_set_blas_variables: 32-bit libraries not supported yet for IMKL v%s (> v10.3)", found_version) else: - self.BLAS_LIB_DIR = ['mkl/lib/intel64'] - if ver >= LooseVersion('10.3.4') and ver < LooseVersion('11.1'): - self.BLAS_LIB_DIR.append('compiler/lib/intel64') + if ver >= LooseVersion('2021'): + basedir = os.path.join('mkl', found_version) else: - self.BLAS_LIB_DIR.append('lib/intel64') + basedir = 'mkl' + + self.BLAS_LIB_DIR = [os.path.join(basedir, 'lib', 'intel64')] + if ver >= LooseVersion('10.3.4') and ver < LooseVersion('11.1'): + self.BLAS_LIB_DIR.append(os.path.join('compiler', 'lib', 'intel64')) + elif ver < LooseVersion('2021'): + self.BLAS_LIB_DIR.append(os.path.join('lib', 'intel64')) - self.BLAS_INCLUDE_DIR = ['mkl/include'] + self.BLAS_INCLUDE_DIR = [os.path.join(basedir, 'include')] super(IntelMKL, self)._set_blas_variables() From 035f26688218e145914c7c90adbf01c1fb09ddfe Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 28 Apr 2021 12:13:57 +0200 Subject: [PATCH 237/864] Add test for `--module-only` when installation directory is not writable --- test/framework/toy_build.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index f3c0abda5a..47c9781b7b 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -36,6 +36,7 @@ import re import shutil import signal +import subprocess import stat import sys import tempfile @@ -1625,8 +1626,25 @@ def test_module_only(self): modtxt = read_file(toy_core_mod) self.assertTrue(re.search('load.*intel/2018a', modtxt), "load statement for intel/2018a found in module") - os.remove(toy_mod) + # Test we can create a module even for an installation where we don't have write permissions os.remove(toy_core_mod) + subprocess.call(['chmod', '-R', '-w', prefix]) + self.assertFalse(os.path.exists(toy_core_mod)) + self.eb_main(args, do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_core_mod)) + # existing install is reused + modtxt2 = read_file(toy_core_mod) + self.assertTrue(re.search("set root %s" % prefix, modtxt2)) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 3) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) + + # make sure load statements for dependencies are included + modtxt = read_file(toy_core_mod) + self.assertTrue(re.search('load.*intel/2018a', modtxt), "load statement for intel/2018a found in module") + + os.remove(toy_core_mod) + os.remove(toy_mod) + # test installing (only) additional module in Lua syntax (if Lmod is available) lmod_abspath = os.environ.get('LMOD_CMD') or which('lmod') From ff8719a0ebcf304bedeb82405f0c65b9c7d9ba46 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 28 Apr 2021 12:27:55 +0200 Subject: [PATCH 238/864] Use adjust_permissions rather than subprocess --- test/framework/toy_build.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 47c9781b7b..364e9bab5f 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -36,7 +36,6 @@ import re import shutil import signal -import subprocess import stat import sys import tempfile @@ -1628,7 +1627,7 @@ def test_module_only(self): # Test we can create a module even for an installation where we don't have write permissions os.remove(toy_core_mod) - subprocess.call(['chmod', '-R', '-w', prefix]) + adjust_permissions(prefix, stat.S_IRUSR) self.assertFalse(os.path.exists(toy_core_mod)) self.eb_main(args, do_build=True, raise_error=True) self.assertTrue(os.path.exists(toy_core_mod)) From 93683dda8df11ff6327bc6dfd31c26c5d9860516 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 28 Apr 2021 12:36:25 +0200 Subject: [PATCH 239/864] Use existing adjust_permissions function --- test/framework/toy_build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 364e9bab5f..ec0b0cd599 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1627,7 +1627,8 @@ def test_module_only(self): # Test we can create a module even for an installation where we don't have write permissions os.remove(toy_core_mod) - adjust_permissions(prefix, stat.S_IRUSR) + # remove the write permissions on the installation + adjust_permissions(prefix, stat.S_IRUSR | stat.S_IXUSR, relative=False) self.assertFalse(os.path.exists(toy_core_mod)) self.eb_main(args, do_build=True, raise_error=True) self.assertTrue(os.path.exists(toy_core_mod)) From 05d2dc29ccac943c6363d3b82ece38aadc9494f8 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 28 Apr 2021 12:39:31 +0200 Subject: [PATCH 240/864] Fix linting issue --- test/framework/toy_build.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index ec0b0cd599..4e325843de 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1645,7 +1645,6 @@ def test_module_only(self): os.remove(toy_core_mod) os.remove(toy_mod) - # test installing (only) additional module in Lua syntax (if Lmod is available) lmod_abspath = os.environ.get('LMOD_CMD') or which('lmod') if lmod_abspath is not None: From 4c3cd395bd8ab92234d492f84fa4822b4cef44a3 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 28 Apr 2021 16:02:18 +0200 Subject: [PATCH 241/864] Move disable_templating into the EC class Allows easier use from outside as e.g. a few tests and easyblocks use it The free function now just redirects to the member function --- contrib/hooks/hpc2n_hooks.py | 44 +++++------ easybuild/framework/easyblock.py | 81 +++++++++----------- easybuild/framework/easyconfig/easyconfig.py | 42 ++++++---- test/framework/easyblock.py | 9 +-- test/framework/easyconfig.py | 16 ++-- 5 files changed, 92 insertions(+), 100 deletions(-) diff --git a/contrib/hooks/hpc2n_hooks.py b/contrib/hooks/hpc2n_hooks.py index 1e0bbd8fcd..602694e1a7 100644 --- a/contrib/hooks/hpc2n_hooks.py +++ b/contrib/hooks/hpc2n_hooks.py @@ -162,21 +162,19 @@ def pre_module_hook(self, *args, **kwargs): self.log.info("[pre-module hook] Set I_MPI_PMI_LIBRARY in impi module") # Must be done this way, updating self.cfg['modextravars'] # directly doesn't work due to templating. - en_templ = self.cfg.enable_templating - self.cfg.enable_templating = False - shlib_ext = get_shared_lib_ext() - pmix_root = get_software_root('PMIx') - if pmix_root: - mpi_type = 'pmix_v3' - self.cfg['modextravars'].update({ - 'I_MPI_PMI_LIBRARY': os.path.join(pmix_root, "lib", "libpmi." + shlib_ext) - }) - self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type}) - # Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off. - self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'}) - else: - self.cfg['modextravars'].update({'I_MPI_PMI_LIBRARY': "/lap/slurm/lib/libpmi.so"}) - self.cfg.enable_templating = en_templ + with self.cfg.disable_templating(): + shlib_ext = get_shared_lib_ext() + pmix_root = get_software_root('PMIx') + if pmix_root: + mpi_type = 'pmix_v3' + self.cfg['modextravars'].update({ + 'I_MPI_PMI_LIBRARY': os.path.join(pmix_root, "lib", "libpmi." + shlib_ext) + }) + self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type}) + # Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off. + self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'}) + else: + self.cfg['modextravars'].update({'I_MPI_PMI_LIBRARY': "/lap/slurm/lib/libpmi.so"}) if self.name == 'OpenBLAS': self.log.info("[pre-module hook] Set OMP_NUM_THREADS=1 in OpenBLAS module") @@ -197,18 +195,14 @@ def pre_module_hook(self, *args, **kwargs): self.log.info("[pre-module hook] Set SLURM_MPI_TYPE=%s in OpenMPI module" % mpi_type) # Must be done this way, updating self.cfg['modextravars'] # directly doesn't work due to templating. - en_templ = self.cfg.enable_templating - self.cfg.enable_templating = False - self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type}) - # Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off. - self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'}) - self.cfg.enable_templating = en_templ + with self.cfg.disable_templating(): + self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type}) + # Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off. + self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'}) if self.name == 'PMIx': # This is a, hopefully, temporary workaround for https://github.com/pmix/pmix/issues/1114 if LooseVersion(self.version) > LooseVersion('2') and LooseVersion(self.version) < LooseVersion('3'): self.log.info("[pre-module hook] Set PMIX_MCA_gds=^ds21 in PMIx module") - en_templ = self.cfg.enable_templating - self.cfg.enable_templating = False - self.cfg['modextravars'].update({'PMIX_MCA_gds': '^ds21'}) - self.cfg.enable_templating = en_templ + with self.cfg.disable_templating(): + self.cfg['modextravars'].update({'PMIX_MCA_gds': '^ds21'}) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index c4bdf459a8..d184e40ea3 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1704,59 +1704,50 @@ def handle_iterate_opts(self): self.iter_idx += 1 # disable templating in this function, since we're messing about with values in self.cfg - prev_enable_templating = self.cfg.enable_templating - self.cfg.enable_templating = False - - # start iterative mode (only need to do this once) - if self.iter_idx == 0: - self.cfg.start_iterating() - - # handle configure/build/install options that are specified as lists (+ perhaps builddependencies) - # set first element to be used, keep track of list in self.iter_opts - # only needs to be done during first iteration, since after that the options won't be lists anymore - if self.iter_idx == 0: - # keep track of list, supply first element as first option to handle - for opt in self.cfg.iterate_options: - self.iter_opts[opt] = self.cfg[opt] # copy - self.log.debug("Found list for %s: %s", opt, self.iter_opts[opt]) - - if self.iter_opts: - print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent) - self.log.info("Current iteration index: %s", self.iter_idx) - - # pop first element from all iterative easyconfig parameters as next value to use - for opt in self.iter_opts: - if len(self.iter_opts[opt]) > self.iter_idx: - self.cfg[opt] = self.iter_opts[opt][self.iter_idx] - else: - self.cfg[opt] = '' # empty list => empty option as next value - self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt]))) - - # re-generate template values, which may be affected by changed parameters we're iterating over - self.cfg.generate_template_values() + with self.cfg.disable_templating(): + + # start iterative mode (only need to do this once) + if self.iter_idx == 0: + self.cfg.start_iterating() + + # handle configure/build/install options that are specified as lists (+ perhaps builddependencies) + # set first element to be used, keep track of list in self.iter_opts + # only needs to be done during first iteration, since after that the options won't be lists anymore + if self.iter_idx == 0: + # keep track of list, supply first element as first option to handle + for opt in self.cfg.iterate_options: + self.iter_opts[opt] = self.cfg[opt] # copy + self.log.debug("Found list for %s: %s", opt, self.iter_opts[opt]) + + if self.iter_opts: + print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent) + self.log.info("Current iteration index: %s", self.iter_idx) + + # pop first element from all iterative easyconfig parameters as next value to use + for opt in self.iter_opts: + if len(self.iter_opts[opt]) > self.iter_idx: + self.cfg[opt] = self.iter_opts[opt][self.iter_idx] + else: + self.cfg[opt] = '' # empty list => empty option as next value + self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt]))) - # re-enable templating before self.cfg values are used - self.cfg.enable_templating = prev_enable_templating + # re-generate template values, which may be affected by changed parameters we're iterating over + self.cfg.generate_template_values() def post_iter_step(self): """Restore options that were iterated over""" # disable templating, since we're messing about with values in self.cfg - prev_enable_templating = self.cfg.enable_templating - self.cfg.enable_templating = False - - for opt in self.iter_opts: - self.cfg[opt] = self.iter_opts[opt] - - # also need to take into account extensions, since those were iterated over as well - for ext in self.ext_instances: - ext.cfg[opt] = self.iter_opts[opt] + with self.cfg.disable_templating(): + for opt in self.iter_opts: + self.cfg[opt] = self.iter_opts[opt] - self.log.debug("Restored value of '%s' that was iterated over: %s", opt, self.cfg[opt]) + # also need to take into account extensions, since those were iterated over as well + for ext in self.ext_instances: + ext.cfg[opt] = self.iter_opts[opt] - self.cfg.stop_iterating() + self.log.debug("Restored value of '%s' that was iterated over: %s", opt, self.cfg[opt]) - # re-enable templating before self.cfg values are used - self.cfg.enable_templating = prev_enable_templating + self.cfg.stop_iterating() def det_iter_cnt(self): """Determine iteration count based on configure/build/install options that may be lists.""" diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index c57f55c476..e9e86d7773 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -394,12 +394,9 @@ def disable_templating(ec): # Do what you want without templating # Templating set to previous value """ - old_enable_templating = ec.enable_templating - ec.enable_templating = False - try: - yield old_enable_templating - finally: - ec.enable_templating = old_enable_templating + _log.deprecated("disable_templating(ec) was replaced by ec.disable_templating()", '5.0') + with ec.disable_templating() as old_value: + yield old_value class EasyConfig(object): @@ -525,6 +522,22 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi self.software_license = None + @contextmanager + def disable_templating(self): + """Temporarily disable templating on the given EasyConfig + + Usage: + with ec.disable_templating(): + # Do what you want without templating + # Templating set to previous value + """ + old_enable_templating = self.enable_templating + self.enable_templating = False + try: + yield old_enable_templating + finally: + self.enable_templating = old_enable_templating + def filename(self): """Determine correct filename for this easyconfig file.""" @@ -634,7 +647,7 @@ def set_keys(self, params): """ # disable templating when setting easyconfig parameters # required to avoid problems with values that need more parsing to be done (e.g. dependencies) - with disable_templating(self): + with self.disable_templating(): for key in sorted(params.keys()): # validations are skipped, just set in the config if key in self._config.keys(): @@ -686,7 +699,7 @@ def parse(self): # templating is disabled when parse_hook is called to allow for easy updating of mutable easyconfig parameters # (see also comment in resolve_template) - with disable_templating(self): + with self.disable_templating(): # if any lists of dependency versions are specified over which we should iterate, # deal with them now, before calling parse hook, parsing of dependencies & iterative easyconfig parameters self.handle_multi_deps() @@ -1148,7 +1161,7 @@ def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=Fals :param backup: create backup of existing file before overwriting it """ # templated values should be dumped unresolved - with disable_templating(self): + with self.disable_templating(): # build dict of default values default_values = dict([(key, DEFAULT_CONFIG[key][0]) for key in DEFAULT_CONFIG]) default_values.update(dict([(key, self.extra_options[key][0]) for key in self.extra_options])) @@ -1639,7 +1652,7 @@ def _generate_template_values(self, ignore=None): # step 1-3 work with easyconfig.templates constants # disable templating with creating dict with template values to avoid looping back to here via __getitem__ - with disable_templating(self): + with self.disable_templating(): if self.template_values is None: # if no template values are set yet, initiate with a minimal set of template values; # this is important for easyconfig that use %(version_minor)s to define 'toolchain', @@ -1652,7 +1665,7 @@ def _generate_template_values(self, ignore=None): # get updated set of template values, now with toolchain instance # (which is used to define the %(mpi_cmd_prefix)s template) - with disable_templating(self): + with self.disable_templating(): template_values = template_constant_dict(self, ignore=ignore, toolchain=toolchain) # update the template_values dict @@ -1696,7 +1709,7 @@ def get_ref(self, key): # see also comments in resolve_template # temporarily disable templating - with disable_templating(self): + with self.disable_templating(): ref = self[key] return ref @@ -1931,9 +1944,8 @@ def resolve_template(value, tmpl_dict): # self['x'] is a get, will return a reference to a templated version of self._config['x'] # and the ['y] = z part will be against this new reference # you will need to do - # self.enable_templating = False - # self['x']['y'] = z - # self.enable_templating = True + # with self.disable_templating(): + # self['x']['y'] = z # or (direct but evil) # self._config['x']['y'] = z # it can not be intercepted with __setitem__ because the set is done at a deeper level diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 35f617d939..7832f80119 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1682,11 +1682,10 @@ def test_extensions_sanity_check(self): # purposely inject failing custom extension filter for last extension toy_ec = EasyConfig(toy_ec_fn) - toy_ec.enable_templating = False - exts_list = toy_ec['exts_list'] - exts_list[-1][2]['exts_filter'] = ("thisshouldfail", '') - toy_ec['exts_list'] = exts_list - toy_ec.enable_templating = True + with toy_ec.disable_templating(): + exts_list = toy_ec['exts_list'] + exts_list[-1][2]['exts_filter'] = ("thisshouldfail", '') + toy_ec['exts_list'] = exts_list eb = EB_toy(toy_ec) eb.silent = True diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index f1a3b62efc..5a83753fe9 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -662,10 +662,9 @@ def test_tweaking(self): eb = EasyConfig(self.eb_file) # eb['toolchain']['version'] = tcver does not work as expected with templating enabled - eb.enable_templating = False - eb['version'] = ver - eb['toolchain']['version'] = tcver - eb.enable_templating = True + with eb.disable_templating(): + eb['version'] = ver + eb['toolchain']['version'] = tcver eb.dump(self.eb_file) tweaks = { @@ -1070,12 +1069,9 @@ def test_templating(self): eb.validate() # temporarily disable templating, just so we can check later whether it's *still* disabled - eb.enable_templating = False - - eb.generate_template_values() - - self.assertFalse(eb.enable_templating) - eb.enable_templating = True + with eb.disable_templating(): + eb.generate_template_values() + self.assertFalse(eb.enable_templating) self.assertEqual(eb['description'], "test easyconfig PI") self.assertEqual(eb['sources'][0], 'PI-3.04.tar.gz') From 23745121c90a13119e38b896664f9a36f5105c56 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Sat, 1 May 2021 16:49:38 +0200 Subject: [PATCH 242/864] Enhance assertion messages Co-authored-by: Sam Moors --- test/framework/toy_build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index c1c4ee66ac..253e092750 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -130,11 +130,11 @@ def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versio # make sure installation log file and easyconfig file are copied to install dir software_path = os.path.join(installpath, 'software', 'toy', full_version) install_log_path_pattern = os.path.join(software_path, 'easybuild', 'easybuild-toy-%s*.log' % version) - self.assertTrue(len(glob.glob(install_log_path_pattern)) >= 1, "Found 1 file at %s" % install_log_path_pattern) + self.assertTrue(len(glob.glob(install_log_path_pattern)) >= 1, "Found at least 1 file at %s" % install_log_path_pattern) # make sure test report is available test_report_path_pattern = os.path.join(software_path, 'easybuild', 'easybuild-toy-%s*test_report.md' % version) - self.assertTrue(len(glob.glob(test_report_path_pattern)) >= 1, "Found 1 file at %s" % test_report_path_pattern) + self.assertTrue(len(glob.glob(test_report_path_pattern)) >= 1, "Found at least 1 file at %s" % test_report_path_pattern) ec_file_path = os.path.join(software_path, 'easybuild', 'toy-%s.eb' % full_version) self.assertTrue(os.path.exists(ec_file_path)) From ea1fa8429766f2257dfc3ba0c1b8ef3201bbf680 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 3 May 2021 08:08:55 +0200 Subject: [PATCH 243/864] Fix line length --- test/framework/toy_build.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 253e092750..e6268e3a71 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -130,11 +130,13 @@ def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versio # make sure installation log file and easyconfig file are copied to install dir software_path = os.path.join(installpath, 'software', 'toy', full_version) install_log_path_pattern = os.path.join(software_path, 'easybuild', 'easybuild-toy-%s*.log' % version) - self.assertTrue(len(glob.glob(install_log_path_pattern)) >= 1, "Found at least 1 file at %s" % install_log_path_pattern) + self.assertTrue(len(glob.glob(install_log_path_pattern)) >= 1, + "Found at least 1 file at %s" % install_log_path_pattern) # make sure test report is available test_report_path_pattern = os.path.join(software_path, 'easybuild', 'easybuild-toy-%s*test_report.md' % version) - self.assertTrue(len(glob.glob(test_report_path_pattern)) >= 1, "Found at least 1 file at %s" % test_report_path_pattern) + self.assertTrue(len(glob.glob(test_report_path_pattern)) >= 1, + "Found at least 1 file at %s" % test_report_path_pattern) ec_file_path = os.path.join(software_path, 'easybuild', 'toy-%s.eb' % full_version) self.assertTrue(os.path.exists(ec_file_path)) From b8972f171f5524dc7df45ecbafccf7808185e900 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 3 May 2021 08:10:37 +0200 Subject: [PATCH 244/864] Use explicit dict of True/False/None values Co-authored-by: Sam Moors --- easybuild/framework/easyconfig/tweak.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 1fa60a785f..f0258a2227 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -330,17 +330,17 @@ def __repr__(self): tweaks.pop(key) # add parameters or replace existing ones - special_values = ('True', 'False', 'None') - quoted_special_values = ['"%s"' % val for val in special_values] + ["'%s'" % val for val in special_values] + special_values = { + 'True': True, "'True'": 'True', '"True"': 'True', + 'False': False, "'False'": 'False', '"False"': 'False', + 'None': None, "'None'": 'None', '"None"': 'None', + } for (key, val) in tweaks.items(): # if the value is True/False/None then take that # if e.g. (literal) True is wanted, then it can be passed as "True"/'True' if val in special_values: str_val = val - val = eval(val) - elif val in quoted_special_values: - str_val = val - val = val.strip('"\'') + val = special_values[val] else: str_val = quote_str(val) From 2ed39d8bf0d1f032c9c615c10da4b2db84ed71ae Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 3 May 2021 08:26:56 +0200 Subject: [PATCH 245/864] Reformat dict and add string-check --- easybuild/framework/easyconfig/tweak.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index f0258a2227..137790a039 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -331,14 +331,20 @@ def __repr__(self): # add parameters or replace existing ones special_values = { - 'True': True, "'True'": 'True', '"True"': 'True', - 'False': False, "'False'": 'False', '"False"': 'False', - 'None': None, "'None'": 'None', '"None"': 'None', - } - for (key, val) in tweaks.items(): # if the value is True/False/None then take that + 'True': True, + 'False': False, + 'None': None, # if e.g. (literal) True is wanted, then it can be passed as "True"/'True' - if val in special_values: + "'True'": 'True', + '"True"': 'True', + "'False'": 'False', + '"False"': 'False', + "'None'": 'None', + '"None"': 'None', + } + for (key, val) in tweaks.items(): + if isinstance(val, string_type) and val in special_values: str_val = val val = special_values[val] else: From 633acbc64ddc6843ab0e42256190974798076936 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 4 May 2021 14:51:18 +0200 Subject: [PATCH 246/864] add support for using ARCH constant in easyconfig files --- easybuild/framework/easyconfig/constants.py | 1 + test/framework/easyconfig.py | 14 +++++++++----- test/framework/options.py | 4 ++++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyconfig/constants.py b/easybuild/framework/easyconfig/constants.py index 5ad1ccd904..4b5b4e4ffe 100644 --- a/easybuild/framework/easyconfig/constants.py +++ b/easybuild/framework/easyconfig/constants.py @@ -44,6 +44,7 @@ # constants that can be used in easyconfig EASYCONFIG_CONSTANTS = { + 'ARCH': (platform.uname()[4], "CPU architecture of current system (aarch64, x86_64, ppc64le, ...)"), 'EXTERNAL_MODULE': (EXTERNAL_MODULE_MARKER, "External module marker"), 'HOME': (os.path.expanduser('~'), "Home directory ($HOME)"), 'OS_TYPE': (get_os_type(), "System type (e.g. 'Linux' or 'Darwin')"), diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index f1a3b62efc..552cb5cd7a 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1021,8 +1021,8 @@ def trim_path(path): self.assertEqual(ec['name'], specs['name']) os.remove(res[1]) - def test_templating(self): - """ test easyconfig templating """ + def test_templating_constants(self): + """Test use of template values and constants in an easyconfig file.""" inp = { 'name': 'PI', # purposely using minor version that starts with a 0, to check for correct version_minor value @@ -1043,7 +1043,7 @@ def test_templating(self): 'sources = [SOURCE_TAR_GZ, (SOURCELOWER_TAR_BZ2, "%(cmd)s")]', 'sanity_check_paths = {', ' "files": ["bin/pi_%%(version_major)s_%%(version_minor)s", "lib/python%%(pyshortver)s/site-packages"],', - ' "dirs": ["libfoo.%%s" %% SHLIB_EXT, "lib/%%(arch)s"],', + ' "dirs": ["libfoo.%%s" %% SHLIB_EXT, "lib/%%(arch)s/" + SYS_PYTHON_VERSION, "include/" + ARCH],', '}', 'dependencies = [', ' ("CUDA", "10.1.105"),' @@ -1086,9 +1086,13 @@ def test_templating(self): self.assertEqual(eb['sanity_check_paths']['files'][0], 'bin/pi_3_04') self.assertEqual(eb['sanity_check_paths']['files'][1], 'lib/python2.7/site-packages') self.assertEqual(eb['sanity_check_paths']['dirs'][0], 'libfoo.%s' % get_shared_lib_ext()) - lib_arch_regex = re.compile('^lib/[a-z0-9_]+$') # should match lib/x86_64, lib/aarch64, lib/ppc64le, etc. + # should match lib/x86_64/2.7.18, lib/aarch64/3.8.6, lib/ppc64le/3.9.2, etc. + lib_arch_regex = re.compile(r'^lib/[a-z0-9_]+/[23]\.[0-9]+\.[0-9]+$') dirs1 = eb['sanity_check_paths']['dirs'][1] - self.assertTrue(lib_arch_regex.match(dirs1), "Pattern '%s' matches '%s'" % (lib_arch_regex.pattern, dirs1)) + self.assertTrue(lib_arch_regex.match(dirs1), "Pattern '%s' should match '%s'" % (lib_arch_regex.pattern, dirs1)) + inc_regex = re.compile('^include/(aarch64|ppc64le|x86_64)$') + dirs2 = eb['sanity_check_paths']['dirs'][2] + self.assertTrue(inc_regex.match(dirs2), "Pattern '%s' should match '%s'" % (inc_regex, dirs2)) self.assertEqual(eb['homepage'], "http://example.com/P/p/v3/") expected = ("CUDA: 10.1.105, 10, 1, 10.1; " "Java: 1.7.80, 1, 7, 1.7; " diff --git a/test/framework/options.py b/test/framework/options.py index f3a0a29fed..7e54888f5d 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -492,12 +492,16 @@ def run_test(fmt=None): if fmt == 'rst': pattern_lines = [ + r'^``ARCH``\s*``(aarch64|ppc64le|x86_64)``\s*CPU architecture .*', + r'^``EXTERNAL_MODULE``.*', r'^``HOME``.*', r'``OS_NAME``.*', r'``OS_PKG_IBVERBS_DEV``.*', ] else: pattern_lines = [ + r'^\s*ARCH: (aarch64|ppc64le|x86_64) \(CPU architecture .*', + r'^\s*EXTERNAL_MODULE:.*', r'^\s*HOME:.*', r'\s*OS_NAME: .*', r'\s*OS_PKG_IBVERBS_DEV: .*', From 7db56e49d4793459f6c438a356caf55ad4255eec Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 5 May 2021 13:31:32 +0200 Subject: [PATCH 247/864] add 'missing' \) in test_avail_easyconfig_constants to avoid confusion --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 7e54888f5d..556e483a07 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -500,7 +500,7 @@ def run_test(fmt=None): ] else: pattern_lines = [ - r'^\s*ARCH: (aarch64|ppc64le|x86_64) \(CPU architecture .*', + r'^\s*ARCH: (aarch64|ppc64le|x86_64) \(CPU architecture .*\)', r'^\s*EXTERNAL_MODULE:.*', r'^\s*HOME:.*', r'\s*OS_NAME: .*', From b454a3c9b266bbf37ab8a1e45983ccacf33452c0 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 7 May 2021 16:27:32 +0200 Subject: [PATCH 248/864] Additional tests --- easybuild/framework/easyblock.py | 2 ++ test/framework/toy_build.py | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index d2b1233f23..0f6db70361 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2149,6 +2149,8 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True): rpath_overrides = build_option('rpath_override_dirs') if isinstance(rpath_overrides, string_type): rpath_override_dirs = rpath_overrides.split(':') + # Filter out any empty values + rpath_override_dirs = list(filter(None, rpath_override_dirs)) _log.debug("Converted RPATH override directories ('%s') to a list of paths: %s" % (rpath_overrides, rpath_override_dirs)) for path in rpath_override_dirs: diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index f3b5e29877..844175e7d2 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2304,7 +2304,7 @@ def test_toy_build_enhanced_sanity_check(self): test_ec_txt = test_ec_txt + '\nenhance_sanity_check = False' write_file(test_ec, test_ec_txt) - error_pattern = " Missing mandatory key 'dirs' in sanity_check_paths." + error_pattern = r" Missing mandatory key 'dirs' in sanity_check_paths." self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec, extra_args=eb_args, raise_error=True, verbose=False) @@ -2428,6 +2428,20 @@ def grab_gcc_rpath_wrapper_args(): self.assertEqual(rpath_include_paths[-2], '/opt/eessi/2021.03/lib') self.assertEqual(rpath_include_paths[-1], '/opt/eessi/lib') + # Check that when we use --rpath-override-dirs empty values are filters + args = ['--rpath', '--experimental', '--rpath-override-dirs=/opt/eessi/2021.03/lib::/opt/eessi/lib'] + self.test_toy_build(extra_args=args, raise_error=True) + rpath_include_paths = grab_gcc_rpath_wrapper_args()['include_paths'].split(',') + # Make sure our directories appear in dirs to be included in the rpath (and in the right order) + self.assertEqual(rpath_include_paths[-2], '/opt/eessi/2021.03/lib') + self.assertEqual(rpath_include_paths[-1], '/opt/eessi/lib') + + # Check that when we use --rpath-override-dirs we can only provide absolute paths + args = ['--rpath', '--experimental', '--rpath-override-dirs=/opt/eessi/2021.03/lib:eessi/lib'] + error_pattern = r"Path used in rpath_override_dirs is not an absolute path" + self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, extra_args=args, raise_error=True, + verbose=False) + # also test use of --rpath-filter args.extend(['--rpath-filter=/test.*,/foo/bar.*', '--disable-cleanup-tmpdir']) self.test_toy_build(extra_args=args, raise_error=True) From 4b640612379e5b57b2f9793d301563a1c188c4f8 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 7 May 2021 16:36:41 +0200 Subject: [PATCH 249/864] Additional tests --- test/framework/toy_build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 844175e7d2..f228e51983 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2428,7 +2428,7 @@ def grab_gcc_rpath_wrapper_args(): self.assertEqual(rpath_include_paths[-2], '/opt/eessi/2021.03/lib') self.assertEqual(rpath_include_paths[-1], '/opt/eessi/lib') - # Check that when we use --rpath-override-dirs empty values are filters + # Check that when we use --rpath-override-dirs empty values are filtered args = ['--rpath', '--experimental', '--rpath-override-dirs=/opt/eessi/2021.03/lib::/opt/eessi/lib'] self.test_toy_build(extra_args=args, raise_error=True) rpath_include_paths = grab_gcc_rpath_wrapper_args()['include_paths'].split(',') @@ -2437,9 +2437,9 @@ def grab_gcc_rpath_wrapper_args(): self.assertEqual(rpath_include_paths[-1], '/opt/eessi/lib') # Check that when we use --rpath-override-dirs we can only provide absolute paths - args = ['--rpath', '--experimental', '--rpath-override-dirs=/opt/eessi/2021.03/lib:eessi/lib'] + eb_args = ['--rpath', '--experimental', '--rpath-override-dirs=/opt/eessi/2021.03/lib:eessi/lib'] error_pattern = r"Path used in rpath_override_dirs is not an absolute path" - self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, extra_args=args, raise_error=True, + self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, extra_args=eb_args, raise_error=True, verbose=False) # also test use of --rpath-filter From 12fd5f8c8882c6b06c121f16f70556808474f123 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 7 May 2021 17:22:46 +0200 Subject: [PATCH 250/864] Make the check more explicit --- test/framework/toy_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index f228e51983..aac512f50c 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2438,7 +2438,7 @@ def grab_gcc_rpath_wrapper_args(): # Check that when we use --rpath-override-dirs we can only provide absolute paths eb_args = ['--rpath', '--experimental', '--rpath-override-dirs=/opt/eessi/2021.03/lib:eessi/lib'] - error_pattern = r"Path used in rpath_override_dirs is not an absolute path" + error_pattern = r"Path used in rpath_override_dirs is not an absolute path: eessi/lib" self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, extra_args=eb_args, raise_error=True, verbose=False) From 9bca1027c8dd0f536817e27eebae957435df190f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 12 May 2021 08:41:33 +0200 Subject: [PATCH 251/864] honor keyboard interrupt in 'eb' command --- eb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/eb b/eb index 592b873432..53a345f08f 100755 --- a/eb +++ b/eb @@ -33,6 +33,13 @@ # @author: Pieter De Baets (Ghent University) # @author: Jens Timmerman (Ghent University) +keyboard_interrupt() { + echo "Keyboard interrupt!" + exit 1 +} + +trap keyboard_interrupt SIGINT + # Python 2.6+ or 3.5+ required REQ_MIN_PY2VER=6 REQ_MIN_PY3VER=5 From daa8648ea4fb7dd518dd71c2b2ad1f0cb540924a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 13 May 2021 09:45:59 +0200 Subject: [PATCH 252/864] Make rpath overrides override _everything_! --- easybuild/framework/easyblock.py | 24 +++++++++++++----------- test/framework/toy_build.py | 8 ++++---- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 0f6db70361..d4cbfef286 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2133,17 +2133,8 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True): if not self.build_in_installdir: self.rpath_filter_dirs.append(self.builddir) - # always include '/lib', '/lib64', $ORIGIN, $ORIGIN/../lib and $ORIGIN/../lib64 - # $ORIGIN will be resolved by the loader to be the full path to the executable or shared object - # see also https://linux.die.net/man/8/ld-linux; - self.rpath_include_dirs = [ - os.path.join(self.installdir, 'lib'), - os.path.join(self.installdir, 'lib64'), - '$ORIGIN', - '$ORIGIN/../lib', - '$ORIGIN/../lib64', - ] - + # If we have override directories for RPATH, insert them first. This means they override all other options + # (including the installation itself). if build_option('rpath_override_dirs') is not None: # make sure we have a list rpath_overrides = build_option('rpath_override_dirs') @@ -2162,6 +2153,17 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True): type(rpath_overrides), rpath_overrides) self.rpath_include_dirs.extend(rpath_override_dirs) + # always include '/lib', '/lib64', $ORIGIN, $ORIGIN/../lib and $ORIGIN/../lib64 + # $ORIGIN will be resolved by the loader to be the full path to the executable or shared object + # see also https://linux.die.net/man/8/ld-linux; + self.rpath_include_dirs.extend([ + os.path.join(self.installdir, 'lib'), + os.path.join(self.installdir, 'lib64'), + '$ORIGIN', + '$ORIGIN/../lib', + '$ORIGIN/../lib64', + ]) + if self.iter_idx > 0: # reset toolchain for iterative runs before preparing it again self.toolchain.reset() diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index f228e51983..0105cb8433 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2425,16 +2425,16 @@ def grab_gcc_rpath_wrapper_args(): self.test_toy_build(extra_args=args, raise_error=True) rpath_include_paths = grab_gcc_rpath_wrapper_args()['include_paths'].split(',') # Make sure our directories appear in dirs to be included in the rpath (and in the right order) - self.assertEqual(rpath_include_paths[-2], '/opt/eessi/2021.03/lib') - self.assertEqual(rpath_include_paths[-1], '/opt/eessi/lib') + self.assertEqual(rpath_include_paths[0], '/opt/eessi/2021.03/lib') + self.assertEqual(rpath_include_paths[1], '/opt/eessi/lib') # Check that when we use --rpath-override-dirs empty values are filtered args = ['--rpath', '--experimental', '--rpath-override-dirs=/opt/eessi/2021.03/lib::/opt/eessi/lib'] self.test_toy_build(extra_args=args, raise_error=True) rpath_include_paths = grab_gcc_rpath_wrapper_args()['include_paths'].split(',') # Make sure our directories appear in dirs to be included in the rpath (and in the right order) - self.assertEqual(rpath_include_paths[-2], '/opt/eessi/2021.03/lib') - self.assertEqual(rpath_include_paths[-1], '/opt/eessi/lib') + self.assertEqual(rpath_include_paths[0], '/opt/eessi/2021.03/lib') + self.assertEqual(rpath_include_paths[1], '/opt/eessi/lib') # Check that when we use --rpath-override-dirs we can only provide absolute paths eb_args = ['--rpath', '--experimental', '--rpath-override-dirs=/opt/eessi/2021.03/lib:eessi/lib'] From 2d8e175246ea2714e7c8ad4f8c72accde775c5c0 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 13 May 2021 10:36:19 +0200 Subject: [PATCH 253/864] Pin GitPython version for Python<3.6 --- requirements.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 83bede56fa..cdf387c9c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,9 +9,12 @@ keyrings.alt; python_version >= '2.7' gitdb==0.6.4; python_version < '2.7' gitdb; python_version >= '2.7' +# GitPython 3.1.15 deprecates Python 3.5 +GitPython==3.1.14; python_version < '3.6' # GitPython 2.1.9 no longer supports Python 2.6 GitPython==2.1.8; python_version < '2.7' -GitPython; python_version >= '2.7' +GitPython; python_version >= '3.6' + # pydot (dep for python-graph-dot) 1.2.0 and more recent doesn't work with Python 2.6 pydot==1.1.0; python_version < '2.7' From d77a49e98e18f0480da8fc7f8483130ea73d49b4 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 13 May 2021 11:43:29 +0200 Subject: [PATCH 254/864] Don't pin Python2 GitPython unnecessarily --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cdf387c9c2..04a2135141 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ gitdb==0.6.4; python_version < '2.7' gitdb; python_version >= '2.7' # GitPython 3.1.15 deprecates Python 3.5 -GitPython==3.1.14; python_version < '3.6' +GitPython==3.1.14; python_version >= '3', < '3.6' # GitPython 2.1.9 no longer supports Python 2.6 GitPython==2.1.8; python_version < '2.7' GitPython; python_version >= '3.6' From 5e8221f378cb1da75aab3c7d5f9a2bd315f58a02 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 13 May 2021 12:42:15 +0200 Subject: [PATCH 255/864] Keep Python 2.7 loose --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 04a2135141..e5d99f3e1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,10 +10,10 @@ gitdb==0.6.4; python_version < '2.7' gitdb; python_version >= '2.7' # GitPython 3.1.15 deprecates Python 3.5 -GitPython==3.1.14; python_version >= '3', < '3.6' +GitPython==3.1.14; python_version >= '3.0' and python_version < '3.6' # GitPython 2.1.9 no longer supports Python 2.6 GitPython==2.1.8; python_version < '2.7' -GitPython; python_version >= '3.6' +GitPython; python_version >= '3.6' or (python_version <= '3.0' and python_version >= '2.7') # pydot (dep for python-graph-dot) 1.2.0 and more recent doesn't work with Python 2.6 From 52c53eed06061ba9068ad19fd8fb05dbf7a51d64 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 13 May 2021 16:11:15 +0200 Subject: [PATCH 256/864] Don't run bootstrap script CI for Python 3.8/3.9 --- .github/workflows/bootstrap_script.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bootstrap_script.yml b/.github/workflows/bootstrap_script.yml index 5a52b0d3a4..e46f5cb41e 100644 --- a/.github/workflows/bootstrap_script.yml +++ b/.github/workflows/bootstrap_script.yml @@ -17,7 +17,8 @@ jobs: runs-on: ubuntu-18.04 strategy: matrix: - python: [2.7, 3.6, 3.7, 3.8, 3.9] + # Don't run for Python 3.8, 3.9 , people should just use `pip install easybuild` + python: [2.7, 3.6, 3.7] modules_tool: # use variables defined by 'setup' job above, see also # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context From 7bf596dc1d7fb7fddc807a948090ea64f599537b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 13 May 2021 16:59:42 +0200 Subject: [PATCH 257/864] explicitly initialize rpath_include_dirs with empty list in prepare_step --- easybuild/framework/easyblock.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 3c2c2df7d3..751d5280c1 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2124,8 +2124,10 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True): if not self.build_in_installdir: self.rpath_filter_dirs.append(self.builddir) - # If we have override directories for RPATH, insert them first. This means they override all other options - # (including the installation itself). + self.rpath_include_dirs = [] + + # If we have override directories for RPATH, insert them first. + # This means they override all other options (including the installation itself). if build_option('rpath_override_dirs') is not None: # make sure we have a list rpath_overrides = build_option('rpath_override_dirs') From d5960b4d4368c15658620c69b589a5e4d100b414 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 14 May 2021 15:03:13 +0200 Subject: [PATCH 258/864] make sure that build dependencies don't get added to 'dependencies' easyconfig parameter by template_constant_dict function --- test/framework/easyconfig.py | 40 ++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index affd3468b8..f4820f7ebf 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3081,26 +3081,29 @@ def test_template_constant_dict(self): " 'arch=fooarch': '1.8.0-foo',", " })", "]", + "builddependencies = [", + " ('CMake', '3.18.4'),", + "]", ]) test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, toy_ec_txt) - # only perform shallow/quick parse (as is done in list_software function) - ec = EasyConfigParser(filename=test_ec).get_config_dict() - expected = { + 'bitbucket_account': 'toy', + 'github_account': 'toy', 'javamajver': '1', 'javaminver': '8', 'javashortver': '1.8', 'javaver': '1.8.0_221', - 'module_name': None, + 'module_name': 'toy/0.01-deps', 'name': 'toy', 'namelower': 'toy', 'nameletter': 't', 'toolchain_name': 'system', 'toolchain_version': 'system', 'nameletterlower': 't', + 'parallel': None, 'pymajver': '3', 'pyminver': '7', 'pyshortver': '3.7', @@ -3109,9 +3112,38 @@ def test_template_constant_dict(self): 'version_major': '0', 'version_major_minor': '0.01', 'version_minor': '01', + 'versionprefix': '', 'versionsuffix': '-deps', } + + # proper EasyConfig instance + ec = EasyConfig(test_ec) + + # CMake should *not* be included, since it's a build-only dependency + dep_names = [x['name'] for x in ec['dependencies']] + self.assertFalse('CMake' in dep_names, "CMake should not be included in list of dependencies: %s" % dep_names) + res = template_constant_dict(ec) + dep_names = [x['name'] for x in ec['dependencies']] + self.assertFalse('CMake' in dep_names, "CMake should not be included in list of dependencies: %s" % dep_names) + + self.assertTrue('arch' in res) + arch = res.pop('arch') + self.assertTrue(arch_regex.match(arch), "'%s' matches with pattern '%s'" % (arch, arch_regex.pattern)) + + self.assertEqual(res, expected) + + # only perform shallow/quick parse (as is done in list_software function) + ec = EasyConfigParser(filename=test_ec).get_config_dict() + + expected['module_name'] = None + for key in ('bitbucket_account', 'github_account', 'parallel', 'versionprefix'): + del expected[key] + + dep_names = [x[0] for x in ec['dependencies']] + self.assertFalse('CMake' in dep_names, "CMake should not be included in list of dependencies: %s" % dep_names) res = template_constant_dict(ec) + dep_names = [x[0] for x in ec['dependencies']] + self.assertFalse('CMake' in dep_names, "CMake should not be included in list of dependencies: %s" % dep_names) self.assertTrue('arch' in res) arch = res.pop('arch') From 0a135401da48abdc36d6f7d31c85bf17d8873969 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 14 May 2021 15:08:14 +0200 Subject: [PATCH 259/864] pass down 'required' as named argument in Toolchain.get_software_version --- easybuild/tools/toolchain/toolchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 9542c96daa..22777b60e3 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -405,7 +405,7 @@ def get_software_root(self, names): def get_software_version(self, names, required=True): """Try to get the software version for all names""" - return self._get_software_multiple(names, self._get_software_version, required) + return self._get_software_multiple(names, self._get_software_version, required=required) def _get_software_multiple(self, names, function, required=True): """Execute function of each of names""" From 2a257f3949c2ea9e62aa08387928729eb66b991b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 14 May 2021 18:23:33 +0200 Subject: [PATCH 260/864] extend sanity check step to check whether specific libraries are (not) linked into installed binaries/libraries (WIP) --- easybuild/framework/easyblock.py | 67 +++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 751d5280c1..aff7dc02cb 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -92,7 +92,7 @@ from easybuild.tools.package.utilities import package from easybuild.tools.py2vs3 import extract_method_name, string_type from easybuild.tools.repository.repository import init_repository -from easybuild.tools.systemtools import det_parallelism, use_group +from easybuild.tools.systemtools import det_parallelism, get_shared_lib_ext, use_group from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, quote_str from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION @@ -2552,7 +2552,9 @@ def sanity_check_rpath(self, rpath_dirs=None): out, ec = run_cmd("file %s" % path, simple=False, trace=False) if ec: - fails.append("Failed to run 'file %s': %s" % (path, out)) + fail_msg = "Failed to run 'file %s': %s" % (path, out) + self.log.warning(fail_msg) + fails.append(fail_msg) # only run ldd/readelf on dynamically linked executables/libraries # example output: @@ -2594,6 +2596,62 @@ def sanity_check_rpath(self, rpath_dirs=None): return fails + def sanity_check_libs(self, subdirs=None): + """ + Check whether specific libraries are (not) linked into installed binaries/libraries. + """ + + fails = [] + + # libraries that *must* be linked in every installed binary/library + required_libs = [] + # libraries that can *not* be linked in any installed binary/library + banned_libs = ['openblas'] + + shlib_ext = get_shared_lib_ext() + required_lib_regexs = [(x, re.compile(r'/lib%s\.%s' % (x, shlib_ext))) for x in required_libs] + banned_lib_regexs = [(x, re.compile(r'/lib%s\.%s' % (x, shlib_ext))) for x in banned_libs] + + if subdirs is None: + subdirs = ['bin', 'lib', 'lib64'] + + for dirpath in [os.path.join(self.installdir, d) for d in subdirs]: + if os.path.exists(dirpath): + self.log.debug("Checking required/banned libraries in %s", dirpath) + + for path in [os.path.join(dirpath, x) for x in os.listdir(dirpath)]: + self.log.debug("Checking required/banned libraries for %s", path) + + out, ec = run_cmd("file %s" % path, simple=False, trace=False) + if ec: + fail_msg = "Failed to run 'file %s': %s" % (path, out) + self.log.warning(fail_msg) + fails.append(fail_msg) + + # only run ldd/readelf on dynamically linked executables/libraries + if "dynamically linked" in out: + # determine dynamically linked libraries for this file via 'ldd' + # example output line: + # libopenblas.so.0 => /software/OpenBLAS/0.3.15-GCC-10.3.0/lib/libopenblas.so.0 (0x00...0) + out, ec = run_cmd("ldd %s" % path, simple=False, trace=False) + if ec: + fail_msg = "Failed to run 'ldd %s': %s" % (path, out) + self.log.warning(fail_msg) + fails.append(fail_msg) + else: + for req_lib, regex in required_lib_regexs: + if not regex.search(out): + fail_msg = "Required library '%s' not found for %s" % (req_lib, path) + self.log.warning(fail_msg) + fails.append(fail_msg) + for banned_lib, regex in banned_lib_regexs: + if regex.search(out): + fail_msg = "Banned library '%s' found for %s" % (banned_lib, path) + self.log.warning(fail_msg) + fails.append(fail_msg) + + return fails + def _sanity_check_step_common(self, custom_paths, custom_commands): """ Determine sanity check paths and commands to use. @@ -2880,6 +2938,11 @@ def xs2str(xs): else: self.log.debug("Skiping RPATH sanity check") + lib_fails = self.sanity_check_libs() + if lib_fails: + self.log.warning("Check for required/banned libraries failed!") + self.sanity_check_fail_msgs.extend(lib_fails) + # pass or fail if self.sanity_check_fail_msgs: raise EasyBuildError("Sanity check failed: %s", '\n'.join(self.sanity_check_fail_msgs)) From b2a3d0b97de978097546dcf0b1478caef2fc07e9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 11:48:01 +0200 Subject: [PATCH 261/864] make sanity check for banned/required linked shared libraries configurable --- easybuild/framework/easyblock.py | 146 +++++++++++++++++----- easybuild/framework/easyconfig/default.py | 6 + easybuild/tools/config.py | 2 + easybuild/tools/options.py | 6 + easybuild/tools/toolchain/toolchain.py | 22 +++- 5 files changed, 147 insertions(+), 35 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index aff7dc02cb..0ef035d02d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -100,6 +100,8 @@ EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io' +DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64') + MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP] # string part of URL for Python packages on PyPI that indicates needs to be rewritten (see derive_alt_pypi_url) @@ -2538,10 +2540,15 @@ def sanity_check_rpath(self, rpath_dirs=None): readelf_rpath_regex = re.compile('(RPATH)', re.M) if rpath_dirs is None: - rpath_dirs = ['bin', 'lib', 'lib64'] - self.log.info("Using default subdirs for binaries/libraries to verify RPATH linking: %s", rpath_dirs) + rpath_dirs = self.cfg['bin_lib_subdirs'] or self.bin_lib_subdirs + + if not rpath_dirs: + rpath_dirs = DEFAULT_BIN_LIB_SUBDIRS + self.log.info("Using default subdirectories for binaries/libraries to verify RPATH linking: %s", + rpath_dirs) else: - self.log.info("Using specified subdirs for binaries/libraries to verify RPATH linking: %s", rpath_dirs) + self.log.info("Using specified subdirectories for binaries/libraries to verify RPATH linking: %s", + rpath_dirs) for dirpath in [os.path.join(self.installdir, d) for d in rpath_dirs]: if os.path.exists(dirpath): @@ -2596,61 +2603,134 @@ def sanity_check_rpath(self, rpath_dirs=None): return fails - def sanity_check_libs(self, subdirs=None): + @property + def bin_lib_subdirs(self): """ - Check whether specific libraries are (not) linked into installed binaries/libraries. + List of subdirectories for binaries and libraries for this software installation. + This is used during the sanity check to check RPATH linking and banned/required linked shared libraries. """ + return None - fails = [] + @property + def banned_linked_shared_libs(self): + """ + List of shared libraries which are not allowed to be linked in any installed binary/library. + Supported values are pure library names without 'lib' prefix or extension ('example'), + file names ('libexample.so'), and full paths ('/usr/lib64/libexample.so'). + """ + return [] + + @property + def required_linked_shared_libs(self): + """ + List of shared libraries which must be linked in all installed binaries/libraries. + Supported values are pure library names without 'lib' prefix or extension ('example'), + file names ('libexample.so'), and full paths ('/usr/lib64/libexample.so'). + """ + return [] + + def sanity_check_linked_shared_libs(self, subdirs=None): + """ + Check whether specific shared libraries are (not) linked into installed binaries/libraries. + """ + + res = [] + + # list of libraries that can *not* be linked in any installed binary/library + banned_libs = build_option('banned_linked_shared_libs') or [] + banned_libs.extend(self.toolchain.banned_linked_shared_libs) + banned_libs.extend(self.banned_linked_shared_libs) + banned_libs.extend(self.cfg['banned_linked_shared_libs']) + + # list of libraries that *must* be linked in every installed binary/library + required_libs = build_option('required_linked_shared_libs') or [] + required_libs.extend(self.toolchain.required_linked_shared_libs) + required_libs.extend(self.required_linked_shared_libs) + required_libs.extend(self.cfg['required_linked_shared_libs']) - # libraries that *must* be linked in every installed binary/library - required_libs = [] - # libraries that can *not* be linked in any installed binary/library - banned_libs = ['openblas'] + # early return if there are no banned/required libraries + if not (banned_libs + required_libs): + return [] shlib_ext = get_shared_lib_ext() - required_lib_regexs = [(x, re.compile(r'/lib%s\.%s' % (x, shlib_ext))) for x in required_libs] - banned_lib_regexs = [(x, re.compile(r'/lib%s\.%s' % (x, shlib_ext))) for x in banned_libs] + + # compose regular expressions for banned/required libraries + def regex_for_lib(lib): + """Compose regular expression for specified banned/required library.""" + # absolute path to library ('/usr/lib64/libexample.so') + if os.path.isabs(lib): + regex = re.compile(re.escape(lib)) + # full filename for library ('libexample.so') + elif lib.startswith('lib'): + regex = re.compile('/' + re.escape(lib)) + # pure library name, without 'lib' prefix or extension ('example') + else: + regex = re.compile(r'/lib%s\.%s' % (lib, shlib_ext)) + + return regex + + banned_lib_regexs = [(x, regex_for_lib(x)) for x in banned_libs] + required_lib_regexs = [(x, regex_for_lib(x)) for x in required_libs] if subdirs is None: - subdirs = ['bin', 'lib', 'lib64'] + subdirs = self.cfg['bin_lib_subdirs'] or self.bin_lib_subdirs + + if not subdirs: + subdirs = DEFAULT_BIN_LIB_SUBDIRS + self.log.info("Using default subdirectories to check for banned/required linked shared libraries: %s", + subdirs) + else: + self.log.info("Using specified subdirectories to check for banned/required linked shared libraries: %s", + subdirs) for dirpath in [os.path.join(self.installdir, d) for d in subdirs]: if os.path.exists(dirpath): - self.log.debug("Checking required/banned libraries in %s", dirpath) + self.log.debug("Checking banned/required linked shared libraries in %s", dirpath) for path in [os.path.join(dirpath, x) for x in os.listdir(dirpath)]: - self.log.debug("Checking required/banned libraries for %s", path) + self.log.debug("Checking banned/required linked shared libraries for %s", path) + + fail_msgs = [] + dyn_linked = False out, ec = run_cmd("file %s" % path, simple=False, trace=False) if ec: - fail_msg = "Failed to run 'file %s': %s" % (path, out) - self.log.warning(fail_msg) - fails.append(fail_msg) + fail_msgs.append("Failed to run 'file %s': %s" % (path, out)) # only run ldd/readelf on dynamically linked executables/libraries if "dynamically linked" in out: + dyn_linked = True # determine dynamically linked libraries for this file via 'ldd' # example output line: # libopenblas.so.0 => /software/OpenBLAS/0.3.15-GCC-10.3.0/lib/libopenblas.so.0 (0x00...0) out, ec = run_cmd("ldd %s" % path, simple=False, trace=False) if ec: - fail_msg = "Failed to run 'ldd %s': %s" % (path, out) - self.log.warning(fail_msg) - fails.append(fail_msg) + fail_msgs.append("Failed to run 'ldd %s': %s" % (path, out)) else: + missing_required_libs = [] for req_lib, regex in required_lib_regexs: if not regex.search(out): - fail_msg = "Required library '%s' not found for %s" % (req_lib, path) - self.log.warning(fail_msg) - fails.append(fail_msg) + missing_required_libs.append(req_lib) + if missing_required_libs: + fail_msg = "Required linked shared libraries not found in %s: %s" + fail_msgs.append(fail_msg % (path, ', '.join(missing_required_libs))) + + found_banned_libs = [] for banned_lib, regex in banned_lib_regexs: if regex.search(out): - fail_msg = "Banned library '%s' found for %s" % (banned_lib, path) - self.log.warning(fail_msg) - fails.append(fail_msg) + found_banned_libs.append(banned_lib) + if found_banned_libs: + fail_msg = "Banned linked shared libraries found in %s: %s" + fail_msgs.append(fail_msg % (path, ', '.join(found_banned_libs))) + + if fail_msgs: + res.extend(fail_msgs) + for fail_msg in fail_msgs: + self.log.warning(fail_msg) + elif dyn_linked: + self.log.debug("Check for banned/required linked shared libraries passed for %s", path) - return fails + return res def _sanity_check_step_common(self, custom_paths, custom_commands): """ @@ -2791,6 +2871,8 @@ def _sanity_check_step_dry_run(self, custom_paths=None, custom_commands=None, ** else: self.dry_run_msg(" (none)") + self.sanity_check_linked_shared_libs() + if self.toolchain.use_rpath: self.sanity_check_rpath() else: @@ -2938,10 +3020,10 @@ def xs2str(xs): else: self.log.debug("Skiping RPATH sanity check") - lib_fails = self.sanity_check_libs() - if lib_fails: - self.log.warning("Check for required/banned libraries failed!") - self.sanity_check_fail_msgs.extend(lib_fails) + linked_shared_lib_fails = self.sanity_check_linked_shared_libs() + if linked_shared_lib_fails: + self.log.warning("Check for required/banned linked shared libraries failed!") + self.sanity_check_fail_msgs.extend(linked_shared_lib_fails) # pass or fail if self.sanity_check_fail_msgs: diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 863fb275b0..414b363d59 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -82,6 +82,8 @@ 'toolchainopts': [None, 'Extra options for compilers', TOOLCHAIN], # BUILD easyconfig parameters + 'banned_linked_shared_libs': [[], "List of shared libraries (names, file names, or paths) which are not allowed " + "to be linked in any installed binary/library", BUILD], 'bitbucket_account': ['%(namelower)s', "Bitbucket account name to be used to resolve template values in source" " URLs", BUILD], 'buildopts': ['', 'Extra options passed to make step (default already has -j X)', BUILD], @@ -110,8 +112,12 @@ 'preinstallopts': ['', 'Extra prefix options for installation.', BUILD], 'pretestopts': ['', 'Extra prefix options for test.', BUILD], 'postinstallcmds': [[], 'Commands to run after the install step.', BUILD], + 'required_linked_shared_libs': [[], "List of shared libraries (names, file names, or paths) which must be " + "linked in all installed binaries/libraries", BUILD], 'runtest': [None, ('Indicates if a test should be run after make; should specify argument ' 'after make (for e.g.,"test" for make test)'), BUILD], + 'bin_lib_subdirs': [[], "List of subdirectories for binaries and libraries, which is used during sanity check " + "to check RPATH linking and banned/required libraries", BUILD], 'sanity_check_commands': [[], ("format: [(name, options)] e.g. [('gzip','-h')]. " "Using a non-tuple is equivalent to (name, '-h')"), BUILD], 'sanity_check_paths': [{}, ("List of files and directories to check " diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index f86e47aef7..1e7da68ced 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -217,6 +217,8 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'regtest_output_dir', 'rpath_filter', 'rpath_override_dirs', + 'banned_linked_shared_libs', + 'required_linked_shared_libs', 'silence_deprecation_warnings', 'skip', 'stop', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 5d3426facd..f046a0c9fe 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -350,6 +350,9 @@ def override_options(self): None, 'store_true', False), 'backup-modules': ("Back up an existing module file, if any. Only works when using --module-only", None, 'store_true', None), # default None to allow auto-enabling if not disabled + 'banned-linked-shared-libs': ("Comma-separated list of shared libraries (names, file names, or paths) " + "which are not allowed to be linked in any installed binary/library", + 'strlist', 'extend', None), 'check-ebroot-env-vars': ("Action to take when defined $EBROOT* environment variables are found " "for which there is no matching loaded module; " "supported values: %s" % ', '.join(EBROOT_ENV_VAR_ACTIONS), None, 'store', WARN), @@ -450,6 +453,9 @@ def override_options(self): 'remove-ghost-install-dirs': ("Remove ghost installation directories when --force or --rebuild is used, " "rather than just warning about them", None, 'store_true', False), + 'required-linked-shared-libs': ("Comma-separated list of shared libraries (names, file names, or paths) " + "which must be linked in all installed binaries/libraries", + 'strlist', 'extend', None), 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', False), 'rpath-filter': ("List of regex patterns to use for filtering out RPATH paths", 'strlist', 'store', None), 'rpath-override-dirs': ("Path(s) to be prepended when linking with RPATH (string, colon-separated)", diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index ca2f732824..73d3380c81 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1130,17 +1130,33 @@ def comp_family(self): raise NotImplementedError def blas_family(self): - "Return type of BLAS library used in this toolchain, or 'None' if BLAS is not supported." + """Return type of BLAS library used in this toolchain, or 'None' if BLAS is not supported.""" return None def lapack_family(self): - "Return type of LAPACK library used in this toolchain, or 'None' if LAPACK is not supported." + """Return type of LAPACK library used in this toolchain, or 'None' if LAPACK is not supported.""" return None def mpi_family(self): - "Return type of MPI library used in this toolchain, or 'None' if MPI is not supported." + """Return type of MPI library used in this toolchain, or 'None' if MPI is not supported.""" return None + @property + def banned_linked_shared_libs(self): + """ + List of shared libraries (names, file names, paths) which are + not allowed to be linked in any installed binary/library. + """ + return [] + + @property + def required_linked_shared_libs(self): + """ + List of shared libraries (names, file names, paths) which + must be linked in all installed binaries/libraries. + """ + return [] + def cleanup(self): """Clean up after using this toolchain""" pass From c74f1904feb4e170d3cab33513e7e13304db4234 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 13:46:33 +0200 Subject: [PATCH 262/864] more logging in sanity_check_linked_shared_libs --- easybuild/framework/easyblock.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 0ef035d02d..775227eddd 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2528,6 +2528,8 @@ def _sanity_check_step_multi_deps(self, *args, **kwargs): def sanity_check_rpath(self, rpath_dirs=None): """Sanity check binaries/libraries w.r.t. RPATH linking.""" + self.log.info("Checking RPATH linkage for binaries/libraries...") + fails = [] # hard reset $LD_LIBRARY_PATH before running RPATH sanity check @@ -2633,6 +2635,7 @@ def sanity_check_linked_shared_libs(self, subdirs=None): """ Check whether specific shared libraries are (not) linked into installed binaries/libraries. """ + self.log.info("Checking for banned/required linked shared libraries...") res = [] @@ -2650,7 +2653,13 @@ def sanity_check_linked_shared_libs(self, subdirs=None): # early return if there are no banned/required libraries if not (banned_libs + required_libs): + self.log.info("No banned/required libraries specified") return [] + else: + if banned_libs: + self.log.info("Banned libraries to check for: %s", ', '.join(banned_libs)) + if required_libs: + self.log.info("Required libraries to check for: %s", ', '.join(banned_libs)) shlib_ext = get_shared_lib_ext() @@ -2670,17 +2679,24 @@ def regex_for_lib(lib): return regex banned_lib_regexs = [(x, regex_for_lib(x)) for x in banned_libs] + if banned_lib_regexs: + self.log.debug("Regular expressions to check for banned libraries: %s", + '\n'.join("'%s'" % regex.pattern for (_, regex) in banned_lib_regexs)) + required_lib_regexs = [(x, regex_for_lib(x)) for x in required_libs] + if required_lib_regexs: + self.log.debug("Regular expressions to check for required libraries: %s", + '\n'.join("'%s'" % regex.pattern for (_, regex) in required_lib_regexs)) if subdirs is None: subdirs = self.cfg['bin_lib_subdirs'] or self.bin_lib_subdirs - if not subdirs: - subdirs = DEFAULT_BIN_LIB_SUBDIRS - self.log.info("Using default subdirectories to check for banned/required linked shared libraries: %s", + if subdirs: + self.log.info("Using specified subdirectories to check for banned/required linked shared libraries: %s", subdirs) else: - self.log.info("Using specified subdirectories to check for banned/required linked shared libraries: %s", + subdirs = DEFAULT_BIN_LIB_SUBDIRS + self.log.info("Using default subdirectories to check for banned/required linked shared libraries: %s", subdirs) for dirpath in [os.path.join(self.installdir, d) for d in subdirs]: From ce8f28700551bcfcd588175e120ad736dde5f51e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 16:15:18 +0200 Subject: [PATCH 263/864] implement check_linked_libs function in systemtools module --- easybuild/tools/systemtools.py | 63 ++++++++++++++++++++++++++++++++++ test/framework/systemtools.py | 63 ++++++++++++++++++++++++++++++++-- 2 files changed, 124 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 6a3aa83861..6399f1a1b9 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -772,6 +772,69 @@ def get_glibc_version(): return glibc_ver +def check_linked_libs(path, patterns=None, anti_patterns=None): + """ + Check for (lack of) patterns in linked libraries for binary/library at specified path. + Uses 'ldd' on Linux and 'otool -L' on macOS to determine linked libraries. + + Returns True or False for dynamically linked binaries and libraries to indicate + whether all patterns match and antipatterns don't match. + + Returns None if given path is not a dynamically linked binary or library. + """ + if patterns is None: + regexs = [] + else: + regexs = [re.compile(p) if isinstance(p, string_type) else p for p in patterns] + + if anti_patterns is None: + anti_regexs = [] + else: + anti_regexs = [re.compile(p) if isinstance(p, string_type) else p for p in anti_patterns] + + # resolve symbolic links (unless they're broken) + if os.path.islink(path) and os.path.exists(path): + path = os.path.realpath(path) + + file_cmd_out, _ = run_cmd("file %s" % path, simple=False, trace=False) + + os_type = get_os_type() + + # check whether specified path is a dynamically linked binary or a shared library + if os_type == LINUX: + # example output for dynamically linked binaries: + # /usr/bin/ls: ELF 64-bit LSB executable, x86-64, ..., dynamically linked (uses shared libs), ... + # example output for shared libraries: + # /lib64/libc-2.17.so: ELF 64-bit LSB shared object, x86-64, ..., dynamically linked (uses shared libs), ... + if "dynamically linked" in file_cmd_out: + linked_libs_out, _ = run_cmd("ldd %s" % path, simple=False, trace=False) + else: + return None + + elif os_type == DARWIN: + # example output for dynamically linked binaries: + # /bin/ls: Mach-O 64-bit executable x86_64 + # example output for shared libraries: + # /usr/lib/libz.dylib: Mach-O 64-bit dynamically linked shared library x86_64 + bin_lib_regex = re.compile('(Mach-O .* executable)|(dynamically linked)', re.M) + if bin_lib_regex.search(file_cmd_out): + linked_libs_out, _ = run_cmd("otool -L %s" % path, simple=False, trace=False) + else: + return None + else: + raise EasyBuildError("Unknown OS type: %s", os_type) + + res = True + for regex in regexs: + if not regex.search(linked_libs_out): + res = False + for anti_regex in anti_regexs: + if anti_regex.search(linked_libs_out): + res = False + + return res + + def get_system_info(): """Return a dictionary with system information.""" python_version = '; '.join(sys.version.split('\n')) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 3654882086..e345d23c5d 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -38,14 +38,14 @@ import easybuild.tools.systemtools as st from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import adjust_permissions, read_file, which, write_file +from easybuild.tools.filetools import adjust_permissions, read_file, symlink, which, write_file from easybuild.tools.py2vs3 import string_type from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import CPU_ARCHITECTURES, AARCH32, AARCH64, POWER, X86_64 from easybuild.tools.systemtools import CPU_FAMILIES, POWER_LE, DARWIN, LINUX, UNKNOWN from easybuild.tools.systemtools import CPU_VENDORS, AMD, APM, ARM, CAVIUM, IBM, INTEL from easybuild.tools.systemtools import MAX_FREQ_FP, PROC_CPUINFO_FP, PROC_MEMINFO_FP -from easybuild.tools.systemtools import check_os_dependency, check_python_version, pick_dep_version +from easybuild.tools.systemtools import check_linked_libs, check_os_dependency, check_python_version, pick_dep_version from easybuild.tools.systemtools import det_parallelism, get_avail_core_count, get_cpu_arch_name, get_cpu_architecture from easybuild.tools.systemtools import get_cpu_family, get_cpu_features, get_cpu_model, get_cpu_speed, get_cpu_vendor from easybuild.tools.systemtools import get_gcc_version, get_glibc_version, get_os_type, get_os_name, get_os_version @@ -977,6 +977,65 @@ def test_check_os_dependency(self): write_file(bash_profile, 'export LD_LIBRARY_PATH=%s' % self.test_prefix) self.assertTrue(check_os_dependency('bar')) + def test_check_linked_libs(self): + """Test for check_linked_libs function.""" + + txt_path = os.path.join(self.test_prefix, 'test.txt') + write_file(txt_path, "some text") + + broken_symlink_path = os.path.join(self.test_prefix, 'broken_symlink') + symlink('/doesnotexist', broken_symlink_path, use_abspath_source=False) + + # result is always None for anything other than dynamically linked binaries or shared libraries + self.assertEqual(check_linked_libs(self.test_prefix), None) + self.assertEqual(check_linked_libs(txt_path), None) + self.assertEqual(check_linked_libs(broken_symlink_path), None) + + bin_ls_path = which('ls') + + os_type = get_os_type() + if os_type == LINUX: + out, _ = run_cmd("ldd %s" % bin_ls_path) + elif os_type == DARWIN: + out, _ = run_cmd("otool -L %s" % bin_ls_path) + else: + raise EasyBuildError("Unknown OS type: %s" % os_type) + + shlib_ext = get_shared_lib_ext() + lib_path_regex = re.compile(r'(?P[^\s]*/lib[^ ]+\.%s[^ ]*)' % shlib_ext, re.M) + lib_path = lib_path_regex.search(out).group(1) + + test_pattern_named_args = [ + # if no patterns are specified, result is always True + {}, + {'patterns': ['/lib', shlib_ext]}, + {'anti_patterns': ['this_pattern_should_not_match']}, + {'patterns': ['/lib', shlib_ext], 'anti_patterns': ['weirdstuff']}, + ] + for pattern_named_args in test_pattern_named_args: + # result is always None for anything other than dynamically linked binaries or shared libraries + self.assertEqual(check_linked_libs(self.test_prefix, **pattern_named_args), None) + self.assertEqual(check_linked_libs(txt_path, **pattern_named_args), None) + self.assertEqual(check_linked_libs(broken_symlink_path, **pattern_named_args), None) + for path in (bin_ls_path, lib_path): + error_msg = "Check on linked libs should pass for %s with %s" % (path, pattern_named_args) + self.assertTrue(check_linked_libs(path, **pattern_named_args), error_msg) + + # also test with input that should result in failing check + test_pattern_named_args = [ + {'patterns': ['this_pattern_will_not_match']}, + {'anti_patterns': ['/lib']}, + {'patterns': ['weirdstuff'], 'anti_patterns': ['/lib', shlib_ext]}, + ] + for pattern_named_args in test_pattern_named_args: + # result is always None for anything other than dynamically linked binaries or shared libraries + self.assertEqual(check_linked_libs(self.test_prefix, **pattern_named_args), None) + self.assertEqual(check_linked_libs(txt_path, **pattern_named_args), None) + self.assertEqual(check_linked_libs(broken_symlink_path, **pattern_named_args), None) + for path in (bin_ls_path, lib_path): + error_msg = "Check on linked libs should fail for %s with %s" % (path, pattern_named_args) + self.assertFalse(check_linked_libs(path, **pattern_named_args), error_msg) + def suite(): """ returns all the testcases in this module """ From 982ea596c6dadc026c280ac8cb17465806f52cc0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 17:17:21 +0200 Subject: [PATCH 264/864] log warnings for failing pattern checks in check_linked_libs function --- easybuild/tools/systemtools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 6399f1a1b9..e7a4c72e00 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -827,9 +827,11 @@ def check_linked_libs(path, patterns=None, anti_patterns=None): res = True for regex in regexs: if not regex.search(linked_libs_out): + _log.warning("Required pattern '%s' not found in linked libraries output for %s", regex.pattern, path) res = False for anti_regex in anti_regexs: if anti_regex.search(linked_libs_out): + _log.warning("Non-allowed pattern '%s' found in linked libraries output for %s", anti_regex.pattern, path) res = False return res From d23a8087556a55a00c3c7b91459cc6700cfb0915 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 18:30:48 +0200 Subject: [PATCH 265/864] rename check_linked_libs to check_linked_shared_libs + tweak logged warnings + rename named arguments to avoid confusion --- easybuild/tools/systemtools.py | 46 +++++++++++++++++++++------------- test/framework/systemtools.py | 40 ++++++++++++++--------------- 2 files changed, 48 insertions(+), 38 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index e7a4c72e00..20d2412b8f 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -772,25 +772,25 @@ def get_glibc_version(): return glibc_ver -def check_linked_libs(path, patterns=None, anti_patterns=None): +def check_linked_shared_libs(path, required_patterns=None, banned_patterns=None): """ - Check for (lack of) patterns in linked libraries for binary/library at specified path. - Uses 'ldd' on Linux and 'otool -L' on macOS to determine linked libraries. + Check for (lack of) patterns in linked shared libraries for binary/library at specified path. + Uses 'ldd' on Linux and 'otool -L' on macOS to determine linked shared libraries. - Returns True or False for dynamically linked binaries and libraries to indicate + Returns True or False for dynamically linked binaries and shared libraries to indicate whether all patterns match and antipatterns don't match. Returns None if given path is not a dynamically linked binary or library. """ - if patterns is None: - regexs = [] + if required_patterns is None: + required_regexs = [] else: - regexs = [re.compile(p) if isinstance(p, string_type) else p for p in patterns] + required_regexs = [re.compile(p) if isinstance(p, string_type) else p for p in required_patterns] - if anti_patterns is None: - anti_regexs = [] + if banned_patterns is None: + banned_regexs = [] else: - anti_regexs = [re.compile(p) if isinstance(p, string_type) else p for p in anti_patterns] + banned_regexs = [re.compile(p) if isinstance(p, string_type) else p for p in banned_patterns] # resolve symbolic links (unless they're broken) if os.path.islink(path) and os.path.exists(path): @@ -824,17 +824,27 @@ def check_linked_libs(path, patterns=None, anti_patterns=None): else: raise EasyBuildError("Unknown OS type: %s", os_type) - res = True - for regex in regexs: + found_banned_patterns = [] + missing_required_patterns = [] + for regex in required_regexs: if not regex.search(linked_libs_out): _log.warning("Required pattern '%s' not found in linked libraries output for %s", regex.pattern, path) - res = False - for anti_regex in anti_regexs: - if anti_regex.search(linked_libs_out): - _log.warning("Non-allowed pattern '%s' found in linked libraries output for %s", anti_regex.pattern, path) - res = False + missing_required_patterns.append(regex.pattern) - return res + for regex in banned_regexs: + if regex.search(linked_libs_out): + _log.warning("Banned pattern '%s' found in linked libraries output for %s", regex.pattern, path) + found_banned_patterns.append(regex.pattern) + + if missing_required_patterns: + patterns = ', '.join("'%s'" % p for p in missing_required_patterns) + _log.warning("Required patterns not found in linked libraries output for %s: %s", path, patterns) + + if found_banned_patterns: + patterns = ', '.join("'%s'" % p for p in found_banned_patterns) + _log.warning("Banned patterns found in linked libraries output for %s: %s", path, patterns) + + return not (found_banned_patterns or missing_required_patterns) def get_system_info(): diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index e345d23c5d..f5829c3d8e 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -45,7 +45,7 @@ from easybuild.tools.systemtools import CPU_FAMILIES, POWER_LE, DARWIN, LINUX, UNKNOWN from easybuild.tools.systemtools import CPU_VENDORS, AMD, APM, ARM, CAVIUM, IBM, INTEL from easybuild.tools.systemtools import MAX_FREQ_FP, PROC_CPUINFO_FP, PROC_MEMINFO_FP -from easybuild.tools.systemtools import check_linked_libs, check_os_dependency, check_python_version, pick_dep_version +from easybuild.tools.systemtools import check_linked_shared_libs, check_os_dependency, check_python_version, pick_dep_version from easybuild.tools.systemtools import det_parallelism, get_avail_core_count, get_cpu_arch_name, get_cpu_architecture from easybuild.tools.systemtools import get_cpu_family, get_cpu_features, get_cpu_model, get_cpu_speed, get_cpu_vendor from easybuild.tools.systemtools import get_gcc_version, get_glibc_version, get_os_type, get_os_name, get_os_version @@ -977,8 +977,8 @@ def test_check_os_dependency(self): write_file(bash_profile, 'export LD_LIBRARY_PATH=%s' % self.test_prefix) self.assertTrue(check_os_dependency('bar')) - def test_check_linked_libs(self): - """Test for check_linked_libs function.""" + def test_check_linked_shared_libs(self): + """Test for check_linked_shared_libs function.""" txt_path = os.path.join(self.test_prefix, 'test.txt') write_file(txt_path, "some text") @@ -987,9 +987,9 @@ def test_check_linked_libs(self): symlink('/doesnotexist', broken_symlink_path, use_abspath_source=False) # result is always None for anything other than dynamically linked binaries or shared libraries - self.assertEqual(check_linked_libs(self.test_prefix), None) - self.assertEqual(check_linked_libs(txt_path), None) - self.assertEqual(check_linked_libs(broken_symlink_path), None) + self.assertEqual(check_linked_shared_libs(self.test_prefix), None) + self.assertEqual(check_linked_shared_libs(txt_path), None) + self.assertEqual(check_linked_shared_libs(broken_symlink_path), None) bin_ls_path = which('ls') @@ -1008,33 +1008,33 @@ def test_check_linked_libs(self): test_pattern_named_args = [ # if no patterns are specified, result is always True {}, - {'patterns': ['/lib', shlib_ext]}, - {'anti_patterns': ['this_pattern_should_not_match']}, - {'patterns': ['/lib', shlib_ext], 'anti_patterns': ['weirdstuff']}, + {'required_patterns': ['/lib', shlib_ext]}, + {'banned_patterns': ['this_pattern_should_not_match']}, + {'required_patterns': ['/lib', shlib_ext], 'banned_patterns': ['weirdstuff']}, ] for pattern_named_args in test_pattern_named_args: # result is always None for anything other than dynamically linked binaries or shared libraries - self.assertEqual(check_linked_libs(self.test_prefix, **pattern_named_args), None) - self.assertEqual(check_linked_libs(txt_path, **pattern_named_args), None) - self.assertEqual(check_linked_libs(broken_symlink_path, **pattern_named_args), None) + self.assertEqual(check_linked_shared_libs(self.test_prefix, **pattern_named_args), None) + self.assertEqual(check_linked_shared_libs(txt_path, **pattern_named_args), None) + self.assertEqual(check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None) for path in (bin_ls_path, lib_path): error_msg = "Check on linked libs should pass for %s with %s" % (path, pattern_named_args) - self.assertTrue(check_linked_libs(path, **pattern_named_args), error_msg) + self.assertTrue(check_linked_shared_libs(path, **pattern_named_args), error_msg) # also test with input that should result in failing check test_pattern_named_args = [ - {'patterns': ['this_pattern_will_not_match']}, - {'anti_patterns': ['/lib']}, - {'patterns': ['weirdstuff'], 'anti_patterns': ['/lib', shlib_ext]}, + {'required_patterns': ['this_pattern_will_not_match']}, + {'banned_patterns': ['/lib']}, + {'required_patterns': ['weirdstuff'], 'banned_patterns': ['/lib', shlib_ext]}, ] for pattern_named_args in test_pattern_named_args: # result is always None for anything other than dynamically linked binaries or shared libraries - self.assertEqual(check_linked_libs(self.test_prefix, **pattern_named_args), None) - self.assertEqual(check_linked_libs(txt_path, **pattern_named_args), None) - self.assertEqual(check_linked_libs(broken_symlink_path, **pattern_named_args), None) + self.assertEqual(check_linked_shared_libs(self.test_prefix, **pattern_named_args), None) + self.assertEqual(check_linked_shared_libs(txt_path, **pattern_named_args), None) + self.assertEqual(check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None) for path in (bin_ls_path, lib_path): error_msg = "Check on linked libs should fail for %s with %s" % (path, pattern_named_args) - self.assertFalse(check_linked_libs(path, **pattern_named_args), error_msg) + self.assertFalse(check_linked_shared_libs(path, **pattern_named_args), error_msg) def suite(): From 9f96840c955cb70825b95296da17d2311746c7ed Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 18:44:05 +0200 Subject: [PATCH 266/864] leverage check_linked_shared_libs in EasyBlock.sanity_check_linked_shared_libs + add test to verify it works as intended --- easybuild/framework/easyblock.py | 69 +++++----------- .../test_ecs/l/libtoy/libtoy-0.0.eb | 17 ++++ .../sandbox/easybuild/easyblocks/l/libtoy.py | 62 ++++++++++++++ .../sources/l/libtoy/libtoy-0.0.tar.gz | Bin 0 -> 1782 bytes test/framework/toy_build.py | 76 +++++++++++++++++- 5 files changed, 173 insertions(+), 51 deletions(-) create mode 100644 test/framework/easyconfigs/test_ecs/l/libtoy/libtoy-0.0.eb create mode 100644 test/framework/sandbox/easybuild/easyblocks/l/libtoy.py create mode 100644 test/framework/sandbox/sources/l/libtoy/libtoy-0.0.tar.gz diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 775227eddd..6c0b365e94 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -92,7 +92,7 @@ from easybuild.tools.package.utilities import package from easybuild.tools.py2vs3 import extract_method_name, string_type from easybuild.tools.repository.repository import init_repository -from easybuild.tools.systemtools import det_parallelism, get_shared_lib_ext, use_group +from easybuild.tools.systemtools import check_linked_shared_libs, det_parallelism, get_shared_lib_ext, use_group from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, quote_str from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION @@ -2637,8 +2637,6 @@ def sanity_check_linked_shared_libs(self, subdirs=None): """ self.log.info("Checking for banned/required linked shared libraries...") - res = [] - # list of libraries that can *not* be linked in any installed binary/library banned_libs = build_option('banned_linked_shared_libs') or [] banned_libs.extend(self.toolchain.banned_linked_shared_libs) @@ -2671,22 +2669,22 @@ def regex_for_lib(lib): regex = re.compile(re.escape(lib)) # full filename for library ('libexample.so') elif lib.startswith('lib'): - regex = re.compile('/' + re.escape(lib)) + regex = re.compile(r'(/|\s)' + re.escape(lib)) # pure library name, without 'lib' prefix or extension ('example') else: - regex = re.compile(r'/lib%s\.%s' % (lib, shlib_ext)) + regex = re.compile(r'(/|\s)lib%s\.%s' % (lib, shlib_ext)) return regex - banned_lib_regexs = [(x, regex_for_lib(x)) for x in banned_libs] + banned_lib_regexs = [regex_for_lib(x) for x in banned_libs] if banned_lib_regexs: self.log.debug("Regular expressions to check for banned libraries: %s", - '\n'.join("'%s'" % regex.pattern for (_, regex) in banned_lib_regexs)) + '\n'.join("'%s'" % regex.pattern for regex in banned_lib_regexs)) - required_lib_regexs = [(x, regex_for_lib(x)) for x in required_libs] + required_lib_regexs = [regex_for_lib(x) for x in required_libs] if required_lib_regexs: self.log.debug("Regular expressions to check for required libraries: %s", - '\n'.join("'%s'" % regex.pattern for (_, regex) in required_lib_regexs)) + '\n'.join("'%s'" % regex.pattern for regex in required_lib_regexs)) if subdirs is None: subdirs = self.cfg['bin_lib_subdirs'] or self.bin_lib_subdirs @@ -2699,6 +2697,8 @@ def regex_for_lib(lib): self.log.info("Using default subdirectories to check for banned/required linked shared libraries: %s", subdirs) + failed_paths = [] + for dirpath in [os.path.join(self.installdir, d) for d in subdirs]: if os.path.exists(dirpath): self.log.debug("Checking banned/required linked shared libraries in %s", dirpath) @@ -2706,47 +2706,18 @@ def regex_for_lib(lib): for path in [os.path.join(dirpath, x) for x in os.listdir(dirpath)]: self.log.debug("Checking banned/required linked shared libraries for %s", path) - fail_msgs = [] - dyn_linked = False - - out, ec = run_cmd("file %s" % path, simple=False, trace=False) - if ec: - fail_msgs.append("Failed to run 'file %s': %s" % (path, out)) - - # only run ldd/readelf on dynamically linked executables/libraries - if "dynamically linked" in out: - dyn_linked = True - # determine dynamically linked libraries for this file via 'ldd' - # example output line: - # libopenblas.so.0 => /software/OpenBLAS/0.3.15-GCC-10.3.0/lib/libopenblas.so.0 (0x00...0) - out, ec = run_cmd("ldd %s" % path, simple=False, trace=False) - if ec: - fail_msgs.append("Failed to run 'ldd %s': %s" % (path, out)) - else: - missing_required_libs = [] - for req_lib, regex in required_lib_regexs: - if not regex.search(out): - missing_required_libs.append(req_lib) - if missing_required_libs: - fail_msg = "Required linked shared libraries not found in %s: %s" - fail_msgs.append(fail_msg % (path, ', '.join(missing_required_libs))) - - found_banned_libs = [] - for banned_lib, regex in banned_lib_regexs: - if regex.search(out): - found_banned_libs.append(banned_lib) - if found_banned_libs: - fail_msg = "Banned linked shared libraries found in %s: %s" - fail_msgs.append(fail_msg % (path, ', '.join(found_banned_libs))) - - if fail_msgs: - res.extend(fail_msgs) - for fail_msg in fail_msgs: - self.log.warning(fail_msg) - elif dyn_linked: + libs_check = check_linked_shared_libs(path, banned_patterns=banned_lib_regexs, + required_patterns=required_lib_regexs) + if libs_check: self.log.debug("Check for banned/required linked shared libraries passed for %s", path) + else: + failed_paths.append(path) + + fail_msg = None + if failed_paths: + fail_msg = "Check for banned/required shared libraries failed for %s" % ', '.join(failed_paths) - return res + return fail_msg def _sanity_check_step_common(self, custom_paths, custom_commands): """ @@ -3039,7 +3010,7 @@ def xs2str(xs): linked_shared_lib_fails = self.sanity_check_linked_shared_libs() if linked_shared_lib_fails: self.log.warning("Check for required/banned linked shared libraries failed!") - self.sanity_check_fail_msgs.extend(linked_shared_lib_fails) + self.sanity_check_fail_msgs.append(linked_shared_lib_fails) # pass or fail if self.sanity_check_fail_msgs: diff --git a/test/framework/easyconfigs/test_ecs/l/libtoy/libtoy-0.0.eb b/test/framework/easyconfigs/test_ecs/l/libtoy/libtoy-0.0.eb new file mode 100644 index 0000000000..189b0cd5a9 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/l/libtoy/libtoy-0.0.eb @@ -0,0 +1,17 @@ +name = 'libtoy' +version = '0.0' + +homepage = 'https://easybuild.io' +description = "Toy C library." + +toolchain = SYSTEM + +sources = [SOURCE_TAR_GZ] +checksums = ['00331af4fd154e809beb0e3295e088b92def909da73d7edb34c1e369fa6d13cb'] + +sanity_check_paths = { + 'files': ['bin/toy', 'lib/libtoy.%s' % SHLIB_EXT], + 'dirs': [], +} + +moduleclass = 'lib' diff --git a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py new file mode 100644 index 0000000000..8c62e61bfa --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py @@ -0,0 +1,62 @@ +## +# Copyright 2021-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing libtoy, implemented as an easyblock + +@author: Kenneth Hoste (Ghent University) +""" +import os + +from easybuild.framework.easyblock import EasyBlock +from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import get_shared_lib_ext + +SHLIB_EXT = get_shared_lib_ext() + + +class EB_libtoy(EasyBlock): + """Support for building/installing libtoy.""" + + @property + def banned_linked_shared_libs(self): + default = '/thiswillnotbethere,libtoytoytoy.%s,toytoytoy' % SHLIB_EXT + return os.getenv('EB_LIBTOY_BANNED_SHARED_LIBS', default).split(',') + + @property + def required_linked_shared_libs(self): + default = '/libtoy,toy,libtoy.%s' % SHLIB_EXT + return os.getenv('EB_LIBTOY_REQUIRED_SHARED_LIBS', default).split(',') + + def configure_step(self, name=None): + """No configuration for libtoy.""" + pass + + def build_step(self, name=None, buildopts=None): + """Build libtoy.""" + run_cmd('make') + + def install_step(self, name=None): + """Install libtoy.""" + run_cmd('make install PREFIX="%s"' % self.installdir) diff --git a/test/framework/sandbox/sources/l/libtoy/libtoy-0.0.tar.gz b/test/framework/sandbox/sources/l/libtoy/libtoy-0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..3e77a23c0344bcf96db4ea9f8539884efe06d87a GIT binary patch literal 1782 zcmVXT?)Gv^_x7%Pdu>AuR7=Hzg*+7>^wkIPp$`f^_)w@Qg5Zk};){JzD=5@A z5i!o}%-zmz?=EfrLF(@V!|ZQ%<~P6D$?R^H`EsT)>*VLdazq|N8-*;(qwzR)^AVQI z!wSneII76X=xBUIi78l)#+1XMsnOhQWr^bGeXx8cF#Xc zQj#Q9of>~N@p|Iq@dtf`9-<%W=iLzXGm|(yQ>A@ppaHj=d1~Uts_N*M9j{u@U!jBf z>l5rIB+!Jeh^DF*oziTvroWrqUrWF))5`s|RN7O2YF0N3x?RQJ2fY1G!EUG; zSZUR;GWjb0KIi_Dg6;DcfX>7FuAd|0Gvn&Z$B#`jZ=Hd82;_}Cg?N#w&gF$^sOhg2 zd43v&`Yp(R9cf(mDt?3$I1m@E9#?tHs%mO#aXvSjH`Rh;*Ywva*kWAUkpKFrYFcx& z;5a?IHQ=u`;4g@`o+c-by_PsKQJzaVFiw8+>*g0Jg#6d52cau6uSffCCHW3)Go+uJ$*~^DUv^dST6QK>x$R38n@-@pv&2{@(d@;vZ^WU;Hb1y>$N;y4Oam=ka?_00000 z00000z!v9|4)_QF00000000000056WU3<_8nm=`jZqj9&*JlkYt=s5?jh0XmbsbWj z{JfeeSt-ZJTcpe-q@rfjpyslwepyeI99_-o8f~^arx_ONW1EQYKhOIqAB>T-eVey$ znpW!wv7Note#W)Eef-_5yz+@+FS9FKd+DBrCZ1!YR*6S-{-c*=c|=ia>%T@uM=ScD5jl$GXgsPYp#RwnP8}ORn!pEeZ+|hX znsF-ET#29QhiZsOBHZfb6tAXNUb5^3<=9S$6Hsp&GRp;|DZ%--B?Ng7aI=h|3*j|tpCkG)c;XU7S-<{SO3SJ$yBfO zLk-=Y%L=-^W?sCKZtoo5r}(sBuSK`#aDOd=y%ycxo7~?{!LF~{OBEg4u;yySSKgP| z9k9y;wO1?Y06*pJlY+fk9bmnDH9A1bdguUs@dWJE=m33j;kD}kMSpA20s6+t?eDo6 z)B!$;JiVJ(9=w@9j9!p-Da)QFj^f zce%cg?w{S>>g|VS`aW-eKi&8s>iKpAZ1J~8a6q+Jjsw(t>xcjMzapuP=wl`QUo=+! z{Wm2VRpRvbzZ69V{m*8whbz$u9DhxNZH YRC(5q!Quq~0C@cQ4{taD{{Vgf0DM)E6951J literal 0 HcmV?d00001 diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index b3f9673f7a..27287680fd 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -57,6 +57,7 @@ from easybuild.tools.modules import Lmod from easybuild.tools.py2vs3 import reload, string_type from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.version import VERSION as EASYBUILD_VERSION @@ -3238,7 +3239,7 @@ def test_toy_build_unicode_description(self): self.test_toy_build(ec_file=test_ec, raise_error=True) - def test_test_toy_build_lib64_lib_symlink(self): + def test_toy_build_lib64_lib_symlink(self): """Check whether lib64 symlink to lib subdirectory is created.""" # this is done to ensure that /lib64 is considered before /lib64 by GCC linker, # see https://github.com/easybuilders/easybuild-easyconfigs/issues/5776 @@ -3273,7 +3274,7 @@ def test_test_toy_build_lib64_lib_symlink(self): self.assertTrue(os.path.isdir(lib_path)) self.assertFalse(os.path.islink(lib_path)) - def test_test_toy_build_lib_lib64_symlink(self): + def test_toy_build_lib_lib64_symlink(self): """Check whether lib64 symlink to lib subdirectory is created.""" test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') @@ -3316,6 +3317,77 @@ def test_test_toy_build_lib_lib64_symlink(self): self.assertTrue(os.path.isdir(lib64_path)) self.assertFalse(os.path.islink(lib64_path)) + def test_toy_build_sanity_check_linked_libs(self): + """Test sanity checks for banned/requires libraries.""" + + test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + libtoy_ec = os.path.join(test_ecs, 'l', 'libtoy', 'libtoy-0.0.eb') + + libtoy_modfile_path = os.path.join(self.test_installpath, 'modules', 'all', 'libtoy', '0.0') + if get_module_syntax() == 'Lua': + libtoy_modfile_path += '.lua' + + test_ec = os.path.join(self.test_prefix, 'test.eb') + + shlib_ext = get_shared_lib_ext() + + libtoy_fn = 'libtoy.%s' % shlib_ext + error_msg = "Check for banned/required shared libraries failed for" + + # default check is done via EB_libtoy easyblock, which specifies several banned/required libraries + self.test_toy_build(ec_file=libtoy_ec, raise_error=True, verbose=False, verify=False) + remove_file(libtoy_modfile_path) + + # we can make the check fail by defining environment variables picked up by the EB_libtoy easyblock + os.environ['EB_LIBTOY_BANNED_SHARED_LIBS'] = 'libtoy' + self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False, + ec_file=libtoy_ec, extra_args=['--module-only'], raise_error=True, verbose=False) + del os.environ['EB_LIBTOY_BANNED_SHARED_LIBS'] + + os.environ['EB_LIBTOY_REQUIRED_SHARED_LIBS'] = 'thisisnottheremostlikely' + self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False, + ec_file=libtoy_ec, extra_args=['--module-only'], raise_error=True, verbose=False) + del os.environ['EB_LIBTOY_REQUIRED_SHARED_LIBS'] + + # make sure default check passes (so we know better what triggered a failing test) + self.test_toy_build(ec_file=libtoy_ec, extra_args=['--module-only'], force=False, + raise_error=True, verbose=False, verify=False) + remove_file(libtoy_modfile_path) + + # check specifying banned/required libraries via EasyBuild configuration option + args = ['--banned-linked-shared-libs=%s,foobarbaz' % libtoy_fn, '--module-only'] + self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False, + ec_file=libtoy_ec, extra_args=args, raise_error=True, verbose=False) + + args = ['--required-linked-shared=libs=foobarbazisnotthereforsure', '--module-only'] + self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False, + ec_file=libtoy_ec, extra_args=args, raise_error=True, verbose=False) + + # check specifying banned/required libraries via easyconfig parameter + test_ec_txt = read_file(libtoy_ec) + test_ec_txt += "\nbanned_linked_shared_libs = ['toy']" + write_file(test_ec, test_ec_txt) + self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False, + ec_file=test_ec, extra_args=['--module-only'], raise_error=True, verbose=False) + + test_ec_txt = read_file(libtoy_ec) + test_ec_txt += "\nrequired_linked_shared_libs = ['thereisnosuchlibraryyoudummy']" + write_file(test_ec, test_ec_txt) + self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False, + ec_file=test_ec, extra_args=['--module-only'], raise_error=True, verbose=False) + + # one last time: supercombo (with patterns that should pass the check) + test_ec_txt = read_file(libtoy_ec) + test_ec_txt += "\nbanned_linked_shared_libs = ['yeahthisisjustatest', '/usr/lib/libssl.so']" + test_ec_txt += "\nrequired_linked_shared_libs = ['%s']" % libtoy_fn + write_file(test_ec, test_ec_txt) + args = [ + '--banned-linked-shared-libs=the_forbidden_library', + '--required-linked-shared-libs=toy,%s' % libtoy_fn, + ] + self.test_toy_build(ec_file=libtoy_ec, extra_args=['--module-only'], force=False, + raise_error=True, verbose=False, verify=False) + def suite(): """ return all the tests in this file """ From 030f06ef65a6591674bb66cb4ded865ffbd8d707 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 19:51:05 +0200 Subject: [PATCH 267/864] ignore paths for which check_linked_shared_libs returns None --- easybuild/framework/easyblock.py | 11 +++++++---- test/framework/toy_build.py | 11 ++++++++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 6c0b365e94..b436e2424c 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2708,10 +2708,13 @@ def regex_for_lib(lib): libs_check = check_linked_shared_libs(path, banned_patterns=banned_lib_regexs, required_patterns=required_lib_regexs) - if libs_check: - self.log.debug("Check for banned/required linked shared libraries passed for %s", path) - else: - failed_paths.append(path) + + # None indicates the path is not a dynamically linked binary or shared library, so ignore it + if libs_check is not None: + if libs_check: + self.log.debug("Check for banned/required linked shared libraries passed for %s", path) + else: + failed_paths.append(path) fail_msg = None if failed_paths: diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 27287680fd..ae88cdbfc5 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3376,16 +3376,25 @@ def test_toy_build_sanity_check_linked_libs(self): self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False, ec_file=test_ec, extra_args=['--module-only'], raise_error=True, verbose=False) + # check behaviour when alternate subdirectories are specified + test_ec_txt = read_file(libtoy_ec) + test_ec_txt += "\nbin_lib_subdirs = ['', 'lib', 'lib64']" + write_file(test_ec, test_ec_txt) + self.test_toy_build(ec_file=test_ec, extra_args=['--module-only'], force=False, + raise_error=True, verbose=False, verify=False) + # one last time: supercombo (with patterns that should pass the check) test_ec_txt = read_file(libtoy_ec) test_ec_txt += "\nbanned_linked_shared_libs = ['yeahthisisjustatest', '/usr/lib/libssl.so']" test_ec_txt += "\nrequired_linked_shared_libs = ['%s']" % libtoy_fn + test_ec_txt += "\nbin_lib_subdirs = ['', 'lib', 'lib64']" write_file(test_ec, test_ec_txt) args = [ '--banned-linked-shared-libs=the_forbidden_library', '--required-linked-shared-libs=toy,%s' % libtoy_fn, + '--module-only', ] - self.test_toy_build(ec_file=libtoy_ec, extra_args=['--module-only'], force=False, + self.test_toy_build(ec_file=test_ec, extra_args=args, force=False, raise_error=True, verbose=False, verify=False) From f3fcd309536307b2fe798495885577baf4e84764 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 19:53:56 +0200 Subject: [PATCH 268/864] fix long line in systemtools test module --- test/framework/systemtools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index f5829c3d8e..27b693e63a 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -45,11 +45,12 @@ from easybuild.tools.systemtools import CPU_FAMILIES, POWER_LE, DARWIN, LINUX, UNKNOWN from easybuild.tools.systemtools import CPU_VENDORS, AMD, APM, ARM, CAVIUM, IBM, INTEL from easybuild.tools.systemtools import MAX_FREQ_FP, PROC_CPUINFO_FP, PROC_MEMINFO_FP -from easybuild.tools.systemtools import check_linked_shared_libs, check_os_dependency, check_python_version, pick_dep_version +from easybuild.tools.systemtools import check_linked_shared_libs, check_os_dependency, check_python_version from easybuild.tools.systemtools import det_parallelism, get_avail_core_count, get_cpu_arch_name, get_cpu_architecture from easybuild.tools.systemtools import get_cpu_family, get_cpu_features, get_cpu_model, get_cpu_speed, get_cpu_vendor from easybuild.tools.systemtools import get_gcc_version, get_glibc_version, get_os_type, get_os_name, get_os_version from easybuild.tools.systemtools import get_platform_name, get_shared_lib_ext, get_system_info, get_total_memory +from easybuild.tools.systemtools import pick_dep_version PROC_CPUINFO_TXT = None From 24bfd2fee3d929d5fb13d1ad819901eca418eafb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 20:08:12 +0200 Subject: [PATCH 269/864] only retain existing unique directories before checking for banned/required libraries --- easybuild/framework/easyblock.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index b436e2424c..93747f9565 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2697,9 +2697,18 @@ def regex_for_lib(lib): self.log.info("Using default subdirectories to check for banned/required linked shared libraries: %s", subdirs) + # filter to existing directories that are unique (after resolving symlinks) + dirpaths = [] + for subdir in subdirs: + dirpath = os.path.join(self.installdir, subdir) + if os.path.exists(dirpath) and os.path.isdir(dirpath): + dirpath = os.path.realpath(dirpath) + if dirpath not in dirpaths: + dirpaths.append(dirpath) + failed_paths = [] - for dirpath in [os.path.join(self.installdir, d) for d in subdirs]: + for dirpath in dirpaths: if os.path.exists(dirpath): self.log.debug("Checking banned/required linked shared libraries in %s", dirpath) From ff2685636834e7c0ea93e4e48fdbd2b58fca7e20 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 20:17:21 +0200 Subject: [PATCH 270/864] update FlexiBLAS toolchain support to treat backends as banned libraries --- easybuild/toolchains/linalg/flexiblas.py | 45 ++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py index c8d7755f04..f006ac8624 100644 --- a/easybuild/toolchains/linalg/flexiblas.py +++ b/easybuild/toolchains/linalg/flexiblas.py @@ -27,12 +27,40 @@ :author: Kenneth Hoste (Ghent University) """ +import re + from easybuild.tools.toolchain.linalg import LinAlg +from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import get_shared_lib_ext + TC_CONSTANT_FLEXIBLAS = 'FlexiBLAS' +def det_flexiblas_backend_libs(): + """Determine list of paths to FlexiBLAS backend libraries.""" + + # example output for 'flexiblas list': + # System-wide (config directory): + # OPENBLAS + # library = libflexiblas_openblas.so + out, _ = run_cmd("flexiblas list", simple=False, trace=False) + + shlib_ext = get_shared_lib_ext() + flexiblas_lib_regex = re.compile(r'library = (?Plib.*\.%s)' % shlib_ext, re.M) + flexiblas_libs = flexiblas_lib_regex.findall(out) + + backend_libs = [] + for flexiblas_lib in flexiblas_libs: + # assumption here is that the name of FlexiBLAS library (like 'libflexiblas_openblas.so') + # maps directly to name of the backend library ('libopenblas.so') + backend_lib = 'lib' + flexiblas_lib.lstrip('libflexiblas_') + backend_libs.append(backend_lib) + + return backend_libs + + class FlexiBLAS(LinAlg): """ Trivial class, provides FlexiBLAS support. @@ -44,3 +72,20 @@ class FlexiBLAS(LinAlg): LAPACK_MODULE_NAME = ['FlexiBLAS'] LAPACK_IS_BLAS = True LAPACK_FAMILY = TC_CONSTANT_FLEXIBLAS + + def __init__(self, *args, **kwargs): + """Constructor for FlexiBLAS toolchain component.""" + + @property + def banned_linked_shared_libs(self): + """ + List of shared libraries (names, file names, paths) which are + not allowed to be linked in any installed binary/library. + """ + banned_libs = super(FlexiBLAS, super).banned_linked_shared_libs + + # register backends are banned shared libraries, + # to avoid that anything links to them directly (rather than to libflexiblas.so) + flexiblas_banned_libs = det_flexiblas_backend_libs() + + return banned_libs + flexiblas_banned_libs From fe1f0b1a8a99c17246b1f1f2883f070d4a5d6166 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 20:23:24 +0200 Subject: [PATCH 271/864] only warn once about banned/required libraries in check_linked_shared_libs function --- easybuild/tools/systemtools.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 20d2412b8f..d6b05daae3 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -828,12 +828,10 @@ def check_linked_shared_libs(path, required_patterns=None, banned_patterns=None) missing_required_patterns = [] for regex in required_regexs: if not regex.search(linked_libs_out): - _log.warning("Required pattern '%s' not found in linked libraries output for %s", regex.pattern, path) missing_required_patterns.append(regex.pattern) for regex in banned_regexs: if regex.search(linked_libs_out): - _log.warning("Banned pattern '%s' found in linked libraries output for %s", regex.pattern, path) found_banned_patterns.append(regex.pattern) if missing_required_patterns: From edf636ba7f2e74b83ec5153c9626a08d4d787ddf Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 20:52:43 +0200 Subject: [PATCH 272/864] no need to customize FlexiBLAS constructor --- easybuild/toolchains/linalg/flexiblas.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py index f006ac8624..d62b031fb1 100644 --- a/easybuild/toolchains/linalg/flexiblas.py +++ b/easybuild/toolchains/linalg/flexiblas.py @@ -73,9 +73,6 @@ class FlexiBLAS(LinAlg): LAPACK_IS_BLAS = True LAPACK_FAMILY = TC_CONSTANT_FLEXIBLAS - def __init__(self, *args, **kwargs): - """Constructor for FlexiBLAS toolchain component.""" - @property def banned_linked_shared_libs(self): """ From 5cf08fcdfd55519530b29a8cb122877f08625440 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 21:19:16 +0200 Subject: [PATCH 273/864] use -fPIC to build libtoy --- .../test_ecs/l/libtoy/libtoy-0.0.eb | 2 +- .../sources/l/libtoy/libtoy-0.0.tar.gz | Bin 1782 -> 1788 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyconfigs/test_ecs/l/libtoy/libtoy-0.0.eb b/test/framework/easyconfigs/test_ecs/l/libtoy/libtoy-0.0.eb index 189b0cd5a9..7a1495b59a 100644 --- a/test/framework/easyconfigs/test_ecs/l/libtoy/libtoy-0.0.eb +++ b/test/framework/easyconfigs/test_ecs/l/libtoy/libtoy-0.0.eb @@ -7,7 +7,7 @@ description = "Toy C library." toolchain = SYSTEM sources = [SOURCE_TAR_GZ] -checksums = ['00331af4fd154e809beb0e3295e088b92def909da73d7edb34c1e369fa6d13cb'] +checksums = ['9978bc9837c192b8460b4a5d8089c9ffd454771fce771a171fe8527d0274dbe3'] sanity_check_paths = { 'files': ['bin/toy', 'lib/libtoy.%s' % SHLIB_EXT], diff --git a/test/framework/sandbox/sources/l/libtoy/libtoy-0.0.tar.gz b/test/framework/sandbox/sources/l/libtoy/libtoy-0.0.tar.gz index 3e77a23c0344bcf96db4ea9f8539884efe06d87a..c587c8333d664dd567317e3820308362327fd63f 100644 GIT binary patch literal 1788 zcmV}OxyxSeE-8&IP21B8 zrfu4!P^@LUx!cPv-P^nF?X?XtP%RY;7V=blsINYV4}DPZ!G}Uc5d>d+5MS(zT0x<{ ziHLD#XYO`(dv|H`4^n?07-oMvGr#%GPG)zr%$GCtSvx-;lEd;4+9+gM9*xJblaH`m z9*(h`gCj9HHaZ#~iA7^rjznXlBM8SgEFytY(N--I%6ToDFWOqodZbz=Q^yYuGyKKAF{Qe3Z z%wNA?Hz9#0bVW23rD&ICi#7e-b(TLcKsb&`f);#7xrkJb57G!iU^e2))GP$nxtRr-H7L2PqFvTfC0kRnno67P&Z)YI`q?Jp2QTt|$_Ha4ZQthZ zo2Jz|0&Hh5wV!ircOQQ@J74)kv4`1}t-W+lLle(2QmVwGIPrML@X%Nl#W_{5^owd8 z5y+E=p#M3lp4T$EvHqi%WqBkPtF8YUmE%W&DMM4ueNvm1BNwAu0~+oc=3iL1ov6B8TI_%f~jgWJ&{x<_r|3@`hRKJ59 z{U3WKQ@zp;HFSFpE9mx``RSE(d*}E*#b*V3ExJ9M`)d*GwdnTV;{J9Dc75Gms%Trf zIaedT^1jS&pIyeUy;?~J_$hCn6ztXN0PE$e(E(D{LkH-I$7iob2k40luU!Wy`df<* z&@)bMf6q<74)AH@>fOZi{+s!u=mlw)vg~T&Xg(#gb$Cl=dbw(P!SZ{2y}!4m8hXL9 z9`LhXt^Ia5`qRfhYTrcPv35MJ9&oLCzNAmjHwAh=00000003YWIC{Su`(Gx9x6yt2 zW_tGSa{s%|tLx;AUDvq2uY^4MzNBUtd7QEGIn2Hi-IsZrT~9{0n%M zbX|?;9+mdp)O9>$z=mS&eM6U(svw+_DNe|97KSZV#s!}0R^Clw;j e^FI#je^aRPtRI8L3jhG{XT?)Gv^_x7%Pdu>AuR7=Hzg*+7>^wkIPp$`f^_)w@Qg5Zk};){JzD=5@A z5i!o}%-zmz?=EfrLF(@V!|ZQ%<~P6D$?R^H`EsT)>*VLdazq|N8-*;(qwzR)^AVQI z!wSneII76X=xBUIi78l)#+1XMsnOhQWr^bGeXx8cF#Xc zQj#Q9of>~N@p|Iq@dtf`9-<%W=iLzXGm|(yQ>A@ppaHj=d1~Uts_N*M9j{u@U!jBf z>l5rIB+!Jeh^DF*oziTvroWrqUrWF))5`s|RN7O2YF0N3x?RQJ2fY1G!EUG; zSZUR;GWjb0KIi_Dg6;DcfX>7FuAd|0Gvn&Z$B#`jZ=Hd82;_}Cg?N#w&gF$^sOhg2 zd43v&`Yp(R9cf(mDt?3$I1m@E9#?tHs%mO#aXvSjH`Rh;*Ywva*kWAUkpKFrYFcx& z;5a?IHQ=u`;4g@`o+c-by_PsKQJzaVFiw8+>*g0Jg#6d52cau6uSffCCHW3)Go+uJ$*~^DUv^dST6QK>x$R38n@-@pv&2{@(d@;vZ^WU;Hb1y>$N;y4Oam=ka?_00000 z00000z!v9|4)_QF00000000000056WU3<_8nm=`jZqj9&*JlkYt=s5?jh0XmbsbWj z{JfeeSt-ZJTcpe-q@rfjpyslwepyeI99_-o8f~^arx_ONW1EQYKhOIqAB>T-eVey$ znpW!wv7Note#W)Eef-_5yz+@+FS9FKd+DBrCZ1!YR*6S-{-c*=c|=ia>%T@uM=ScD5jl$GXgsPYp#RwnP8}ORn!pEeZ+|hX znsF-ET#29QhiZsOBHZfb6tAXNUb5^3<=9S$6Hsp&GRp;|DZ%--B?Ng7aI=h|3*j|tpCkG)c;XU7S-<{SO3SJ$yBfO zLk-=Y%L=-^W?sCKZtoo5r}(sBuSK`#aDOd=y%ycxo7~?{!LF~{OBEg4u;yySSKgP| z9k9y;wO1?Y06*pJlY+fk9bmnDH9A1bdguUs@dWJE=m33j;kD}kMSpA20s6+t?eDo6 z)B!$;JiVJ(9=w@9j9!p-Da)QFj^f zce%cg?w{S>>g|VS`aW-eKi&8s>iKpAZ1J~8a6q+Jjsw(t>xcjMzapuP=wl`QUo=+! z{Wm2VRpRvbzZ69V{m*8whbz$u9DhxNZH YRC(5q!Quq~0C@cQ4{taD{{Vgf0DM)E6951J From 929f199f61c013d105a7c4c389e808d25973574c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 21:34:54 +0200 Subject: [PATCH 274/864] fix broken tests --- test/framework/filetools.py | 2 +- test/framework/options.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 0eab3d83fa..36d2bc0194 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2071,7 +2071,7 @@ def test_index_functions(self): # test with specified path with and without trailing '/'s for path in [test_ecs, test_ecs + '/', test_ecs + '//']: index = ft.create_index(path) - self.assertEqual(len(index), 83) + self.assertEqual(len(index), 84) expected = [ os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), diff --git a/test/framework/options.py b/test/framework/options.py index 98a6c95093..dbad3a5d26 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -800,7 +800,7 @@ def test_list_easyblocks(self): list_arg, '--unittest-file=%s' % self.logfile, ] - self.eb_main(args, logfile=dummylogfn) + self.eb_main(args, logfile=dummylogfn, raise_error=True) logtxt = read_file(self.logfile) expected = '\n'.join([ @@ -814,6 +814,7 @@ def test_list_easyblocks(self): r'\| \|-- EB_foofoo', r'\|-- EB_GCC', r'\|-- EB_HPL', + r'\|-- EB_libtoy', r'\|-- EB_OpenBLAS', r'\|-- EB_OpenMPI', r'\|-- EB_ScaLAPACK', @@ -936,7 +937,7 @@ def test_search(self): for search_arg in ['-S', '--search-short']: args = [ search_arg, - 'toy-0.0', + '^toy-0.0', '-r', test_easyconfigs_dir, ] From f9bff300973ba07ec953c70d03ba160535cbeff2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 21:42:10 +0200 Subject: [PATCH 275/864] do not make methods related to checking banned/required linked shared libs property methods --- easybuild/framework/easyblock.py | 13 +++++-------- easybuild/toolchains/linalg/flexiblas.py | 3 +-- easybuild/tools/toolchain/toolchain.py | 2 -- .../sandbox/easybuild/easyblocks/l/libtoy.py | 2 -- 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 93747f9565..3ddc743ed4 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2605,7 +2605,6 @@ def sanity_check_rpath(self, rpath_dirs=None): return fails - @property def bin_lib_subdirs(self): """ List of subdirectories for binaries and libraries for this software installation. @@ -2613,7 +2612,6 @@ def bin_lib_subdirs(self): """ return None - @property def banned_linked_shared_libs(self): """ List of shared libraries which are not allowed to be linked in any installed binary/library. @@ -2622,7 +2620,6 @@ def banned_linked_shared_libs(self): """ return [] - @property def required_linked_shared_libs(self): """ List of shared libraries which must be linked in all installed binaries/libraries. @@ -2639,14 +2636,14 @@ def sanity_check_linked_shared_libs(self, subdirs=None): # list of libraries that can *not* be linked in any installed binary/library banned_libs = build_option('banned_linked_shared_libs') or [] - banned_libs.extend(self.toolchain.banned_linked_shared_libs) - banned_libs.extend(self.banned_linked_shared_libs) + banned_libs.extend(self.toolchain.banned_linked_shared_libs()) + banned_libs.extend(self.banned_linked_shared_libs()) banned_libs.extend(self.cfg['banned_linked_shared_libs']) # list of libraries that *must* be linked in every installed binary/library required_libs = build_option('required_linked_shared_libs') or [] - required_libs.extend(self.toolchain.required_linked_shared_libs) - required_libs.extend(self.required_linked_shared_libs) + required_libs.extend(self.toolchain.required_linked_shared_libs()) + required_libs.extend(self.required_linked_shared_libs()) required_libs.extend(self.cfg['required_linked_shared_libs']) # early return if there are no banned/required libraries @@ -2687,7 +2684,7 @@ def regex_for_lib(lib): '\n'.join("'%s'" % regex.pattern for regex in required_lib_regexs)) if subdirs is None: - subdirs = self.cfg['bin_lib_subdirs'] or self.bin_lib_subdirs + subdirs = self.cfg['bin_lib_subdirs'] or self.bin_lib_subdirs() if subdirs: self.log.info("Using specified subdirectories to check for banned/required linked shared libraries: %s", diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py index d62b031fb1..73141f4a78 100644 --- a/easybuild/toolchains/linalg/flexiblas.py +++ b/easybuild/toolchains/linalg/flexiblas.py @@ -73,13 +73,12 @@ class FlexiBLAS(LinAlg): LAPACK_IS_BLAS = True LAPACK_FAMILY = TC_CONSTANT_FLEXIBLAS - @property def banned_linked_shared_libs(self): """ List of shared libraries (names, file names, paths) which are not allowed to be linked in any installed binary/library. """ - banned_libs = super(FlexiBLAS, super).banned_linked_shared_libs + banned_libs = super(FlexiBLAS, super).banned_linked_shared_libs() # register backends are banned shared libraries, # to avoid that anything links to them directly (rather than to libflexiblas.so) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 73d3380c81..ad046f1f5c 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1141,7 +1141,6 @@ def mpi_family(self): """Return type of MPI library used in this toolchain, or 'None' if MPI is not supported.""" return None - @property def banned_linked_shared_libs(self): """ List of shared libraries (names, file names, paths) which are @@ -1149,7 +1148,6 @@ def banned_linked_shared_libs(self): """ return [] - @property def required_linked_shared_libs(self): """ List of shared libraries (names, file names, paths) which diff --git a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py index 8c62e61bfa..e282105b95 100644 --- a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py +++ b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py @@ -39,12 +39,10 @@ class EB_libtoy(EasyBlock): """Support for building/installing libtoy.""" - @property def banned_linked_shared_libs(self): default = '/thiswillnotbethere,libtoytoytoy.%s,toytoytoy' % SHLIB_EXT return os.getenv('EB_LIBTOY_BANNED_SHARED_LIBS', default).split(',') - @property def required_linked_shared_libs(self): default = '/libtoy,toy,libtoy.%s' % SHLIB_EXT return os.getenv('EB_LIBTOY_REQUIRED_SHARED_LIBS', default).split(',') From 20bd0cf15e0c7f100d8a1ec29b8237bcd62c715c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 22:07:34 +0200 Subject: [PATCH 276/864] run check for banned/required libraries before unloading fake module in sanity check step --- easybuild/framework/easyblock.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 3ddc743ed4..4a515259f1 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3004,6 +3004,11 @@ def xs2str(xs): if not extension: self._sanity_check_step_extensions() + linked_shared_lib_fails = self.sanity_check_linked_shared_libs() + if linked_shared_lib_fails: + self.log.warning("Check for required/banned linked shared libraries failed!") + self.sanity_check_fail_msgs.append(linked_shared_lib_fails) + # cleanup if fake_mod_data: self.clean_up_fake_module(fake_mod_data) @@ -3016,11 +3021,6 @@ def xs2str(xs): else: self.log.debug("Skiping RPATH sanity check") - linked_shared_lib_fails = self.sanity_check_linked_shared_libs() - if linked_shared_lib_fails: - self.log.warning("Check for required/banned linked shared libraries failed!") - self.sanity_check_fail_msgs.append(linked_shared_lib_fails) - # pass or fail if self.sanity_check_fail_msgs: raise EasyBuildError("Sanity check failed: %s", '\n'.join(self.sanity_check_fail_msgs)) From 9d95ba7eea5adffbea94e406d7d93c1858a3c0a6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 22:16:13 +0200 Subject: [PATCH 277/864] relax test for banned/required libs sanity check (libtoy.so itself doesn't link to libtoy.so) --- test/framework/sandbox/easybuild/easyblocks/l/libtoy.py | 2 +- test/framework/toy_build.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py index e282105b95..777e8a2b40 100644 --- a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py +++ b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py @@ -44,7 +44,7 @@ def banned_linked_shared_libs(self): return os.getenv('EB_LIBTOY_BANNED_SHARED_LIBS', default).split(',') def required_linked_shared_libs(self): - default = '/libtoy,toy,libtoy.%s' % SHLIB_EXT + default = '/lib,.*' return os.getenv('EB_LIBTOY_REQUIRED_SHARED_LIBS', default).split(',') def configure_step(self, name=None): diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index ae88cdbfc5..9e5e28fddf 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3386,12 +3386,12 @@ def test_toy_build_sanity_check_linked_libs(self): # one last time: supercombo (with patterns that should pass the check) test_ec_txt = read_file(libtoy_ec) test_ec_txt += "\nbanned_linked_shared_libs = ['yeahthisisjustatest', '/usr/lib/libssl.so']" - test_ec_txt += "\nrequired_linked_shared_libs = ['%s']" % libtoy_fn + test_ec_txt += "\nrequired_linked_shared_libs = ['/lib']" test_ec_txt += "\nbin_lib_subdirs = ['', 'lib', 'lib64']" write_file(test_ec, test_ec_txt) args = [ '--banned-linked-shared-libs=the_forbidden_library', - '--required-linked-shared-libs=toy,%s' % libtoy_fn, + '--required-linked-shared-libs=.*', '--module-only', ] self.test_toy_build(ec_file=test_ec, extra_args=args, force=False, From ef52eb28b9b5945a07153c347508c305f618a362 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 22:22:08 +0200 Subject: [PATCH 278/864] fix typo in FlexiBLAS toolchain support --- easybuild/toolchains/linalg/flexiblas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py index 73141f4a78..23b4333de4 100644 --- a/easybuild/toolchains/linalg/flexiblas.py +++ b/easybuild/toolchains/linalg/flexiblas.py @@ -78,7 +78,7 @@ def banned_linked_shared_libs(self): List of shared libraries (names, file names, paths) which are not allowed to be linked in any installed binary/library. """ - banned_libs = super(FlexiBLAS, super).banned_linked_shared_libs() + banned_libs = super(FlexiBLAS, self).banned_linked_shared_libs() # register backends are banned shared libraries, # to avoid that anything links to them directly (rather than to libflexiblas.so) From d072e9ef1bed557b1c5975b522b16e2d6f650a3b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 22:34:14 +0200 Subject: [PATCH 279/864] tweak foss toolchain definition to switch from OpenBLAS to FlexiBLAS in foss/2021a --- easybuild/toolchains/foss.py | 45 ++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/easybuild/toolchains/foss.py b/easybuild/toolchains/foss.py index 794bb05485..e8e093ae09 100644 --- a/easybuild/toolchains/foss.py +++ b/easybuild/toolchains/foss.py @@ -32,27 +32,62 @@ from easybuild.toolchains.gompi import Gompi from easybuild.toolchains.golf import Golf from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.flexiblas import FlexiBLAS from easybuild.toolchains.linalg.openblas import OpenBLAS from easybuild.toolchains.linalg.scalapack import ScaLAPACK -class Foss(Gompi, OpenBLAS, ScaLAPACK, Fftw): +class Foss(Gompi, OpenBLAS, FlexiBLAS, ScaLAPACK, Fftw): """Compiler toolchain with GCC, OpenMPI, OpenBLAS, ScaLAPACK and FFTW.""" NAME = 'foss' SUBTOOLCHAIN = [Gompi.NAME, Golf.NAME] - def is_deprecated(self): - """Return whether or not this toolchain is deprecated.""" + def __init__(self, *args, **kwargs): + """Toolchain constructor.""" + super(Foss, self).__init__(*args, **kwargs) + # need to transform a version like '2016a' with something that is safe to compare with '2000' # comparing subversions that include letters causes TypeErrors in Python 3 # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) version = self.version.replace('a', '.01').replace('b', '.07') + self.looseversion = LooseVersion(version) + + constants = ('BLAS_MODULE_NAME', 'BLAS_LIB', 'BLAS_LIB_MT', 'BLAS_FAMILY', + 'LAPACK_MODULE_NAME', 'LAPACK_IS_BLAS', 'LAPACK_FAMILY') + + if self.looseversion > LooseVersion('2021.0'): + for constant in constants: + setattr(self, constant, getattr(FlexiBLAS, constant)) + else: + for constant in constants: + setattr(self, constant, getattr(OpenBLAS, constant)) + + def banned_linked_shared_libs(self): + """ + List of shared libraries (names, file names, paths) which are + not allowed to be linked in any installed binary/library. + """ + res = [] + res.extend(Gompi.banned_linked_shared_libs(self)) + + if self.looseversion >= LooseVersion('2021.0'): + res.extend(FlexiBLAS.banned_linked_shared_libs(self)) + else: + res.extend(OpenBLAS.banned_linked_shared_libs(self)) + + res.extend(ScaLAPACK.banned_linked_shared_libs(self)) + res.extend(Fftw.banned_linked_shared_libs(self)) + + return res + + def is_deprecated(self): + """Return whether or not this toolchain is deprecated.""" + # foss toolchains older than foss/2016a are deprecated # take into account that foss/2016.x is always < foss/2016a according to LooseVersion; # foss/2016.01 & co are not deprecated yet... - foss_ver = LooseVersion(version) - if foss_ver < LooseVersion('2016.01'): + if self.looseversion < LooseVersion('2016.01'): deprecated = True else: deprecated = False From c60f71ea307f92fada478da75b9885ec256b0b54 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 May 2021 22:39:41 +0200 Subject: [PATCH 280/864] appease the Hound by using .replace rather than .lstrip --- easybuild/toolchains/linalg/flexiblas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py index 23b4333de4..26250570b1 100644 --- a/easybuild/toolchains/linalg/flexiblas.py +++ b/easybuild/toolchains/linalg/flexiblas.py @@ -55,7 +55,7 @@ def det_flexiblas_backend_libs(): for flexiblas_lib in flexiblas_libs: # assumption here is that the name of FlexiBLAS library (like 'libflexiblas_openblas.so') # maps directly to name of the backend library ('libopenblas.so') - backend_lib = 'lib' + flexiblas_lib.lstrip('libflexiblas_') + backend_lib = 'lib' + flexiblas_lib.replace('libflexiblas_', '') backend_libs.append(backend_lib) return backend_libs From 59846195fd1bf0ba4a107d54a9bc569f124aedb8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 16 May 2021 10:02:06 +0200 Subject: [PATCH 281/864] fix use of self.bin_lib_subdirs() in sanity_check_rpath --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 4a515259f1..7d8e805eda 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2542,7 +2542,7 @@ def sanity_check_rpath(self, rpath_dirs=None): readelf_rpath_regex = re.compile('(RPATH)', re.M) if rpath_dirs is None: - rpath_dirs = self.cfg['bin_lib_subdirs'] or self.bin_lib_subdirs + rpath_dirs = self.cfg['bin_lib_subdirs'] or self.bin_lib_subdirs() if not rpath_dirs: rpath_dirs = DEFAULT_BIN_LIB_SUBDIRS From d4dcf343b67a89385fbcc843228b2006ca0e107a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 16 May 2021 12:57:09 +0200 Subject: [PATCH 282/864] add support to det_subtoolchain_version function for specifying alternative subtoolchain names rather than just one --- easybuild/framework/easyconfig/easyconfig.py | 63 +++++++++++++------- test/framework/easyconfig.py | 6 ++ 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index c57f55c476..5dee493ae6 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -220,7 +220,7 @@ def cache_aware_func(toolchain, incl_capabilities=False): return cache_aware_func -def det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, cands, incl_capabilities=False): +def det_subtoolchain_version(current_tc, subtoolchain_names, optional_toolchains, cands, incl_capabilities=False): """ Returns unique version for subtoolchain, in tc dict. If there is no unique version: @@ -229,30 +229,46 @@ def det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, optional toolchains or system toolchain without add_system_to_minimal_toolchains. * in all other cases, raises an exception. """ - uniq_subtc_versions = set([subtc['version'] for subtc in cands if subtc['name'] == subtoolchain_name]) # init with "skipped" subtoolchain_version = None - # system toolchain: bottom of the hierarchy - if is_system_toolchain(subtoolchain_name): - add_system_to_minimal_toolchains = build_option('add_system_to_minimal_toolchains') - if not add_system_to_minimal_toolchains and build_option('add_dummy_to_minimal_toolchains'): - depr_msg = "Use --add-system-to-minimal-toolchains instead of --add-dummy-to-minimal-toolchains" - _log.deprecated(depr_msg, '5.0') - add_system_to_minimal_toolchains = True - - if add_system_to_minimal_toolchains and not incl_capabilities: - subtoolchain_version = '' - elif len(uniq_subtc_versions) == 1: - subtoolchain_version = list(uniq_subtc_versions)[0] - elif len(uniq_subtc_versions) == 0: - if subtoolchain_name not in optional_toolchains: + # ensure we always have a tuple of alternative subtoolchain names, which makes things easier below + if isinstance(subtoolchain_names, string_type): + subtoolchain_names = (subtoolchain_names,) + + system_subtoolchain = False + + for subtoolchain_name in subtoolchain_names: + + uniq_subtc_versions = set([subtc['version'] for subtc in cands if subtc['name'] == subtoolchain_name]) + + # system toolchain: bottom of the hierarchy + if is_system_toolchain(subtoolchain_name): + add_system_to_minimal_toolchains = build_option('add_system_to_minimal_toolchains') + if not add_system_to_minimal_toolchains and build_option('add_dummy_to_minimal_toolchains'): + depr_msg = "Use --add-system-to-minimal-toolchains instead of --add-dummy-to-minimal-toolchains" + _log.deprecated(depr_msg, '5.0') + add_system_to_minimal_toolchains = True + + system_subtoolchain = True + + if add_system_to_minimal_toolchains and not incl_capabilities: + subtoolchain_version = '' + elif len(uniq_subtc_versions) == 1: + subtoolchain_version = list(uniq_subtc_versions)[0] + elif len(uniq_subtc_versions) > 1: + raise EasyBuildError("Multiple versions of %s found in dependencies of toolchain %s: %s", + subtoolchain_name, current_tc['name'], ', '.join(sorted(uniq_subtc_versions))) + + if subtoolchain_version is not None: + break + + if not system_subtoolchain and subtoolchain_version is None: + if not all(n in optional_toolchains for n in subtoolchain_names): + subtoolchain_names = ' or '.join(subtoolchain_names) # raise error if the subtoolchain considered now is not optional raise EasyBuildError("No version found for subtoolchain %s in dependencies of %s", - subtoolchain_name, current_tc['name']) - else: - raise EasyBuildError("Multiple versions of %s found in dependencies of toolchain %s: %s", - subtoolchain_name, current_tc['name'], ', '.join(sorted(uniq_subtc_versions))) + subtoolchain_names, current_tc['name']) return subtoolchain_version @@ -356,11 +372,16 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False): cands.append({'name': dep, 'version': current_tc_version}) # only retain candidates that match subtoolchain names - cands = [c for c in cands if c['name'] in subtoolchain_names] + cands = [c for c in cands if any(c['name'] == x or c['name'] in x for x in subtoolchain_names)] for subtoolchain_name in subtoolchain_names: subtoolchain_version = det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, cands, incl_capabilities=incl_capabilities) + + # narrow down alternative subtoolchain names to a single one, based on the selected version + if isinstance(subtoolchain_name, tuple): + subtoolchain_name = [cand['name'] for cand in cands if cand['version'] == subtoolchain_version][0] + # add to hierarchy and move to next if subtoolchain_version is not None and subtoolchain_name not in visited: tc = {'name': subtoolchain_name, 'version': subtoolchain_version} diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index f1a3b62efc..18c0fd1cd9 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3337,6 +3337,12 @@ def test_det_subtoolchain_version(self): for subtoolchain_name in subtoolchains[current_tc['name']]] self.assertEqual(versions, ['4.9.3', '']) + # test det_subtoolchain_version when two alternatives for subtoolchain are specified + current_tc = {'name': 'gompi', 'version': '2018b'} + cands = [{'name': 'GCC', 'version': '7.3.0-2.30'}] + subtc_version = det_subtoolchain_version(current_tc, ('GCCcore', 'GCC'), optional_toolchains, cands) + self.assertEqual(subtc_version, '7.3.0-2.30') + def test_verify_easyconfig_filename(self): """Test verify_easyconfig_filename function""" test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') From d7b0aec25685822a8cb3429817d2f44f3d717d00 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 16 May 2021 10:03:17 +0200 Subject: [PATCH 283/864] avoid picking up easyblocks outside of sandbox in framework tests (fixes #3470) --- test/framework/options.py | 24 ++++++++++++++++++++++++ test/framework/utilities.py | 16 ++++++++++++---- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 98a6c95093..3d5a1e03c2 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3144,6 +3144,14 @@ def test_xxx_include_easyblocks(self): import easybuild.easyblocks.generic reload(easybuild.easyblocks.generic) + # kick out any paths that shouldn't be there for easybuild.easyblocks and easybuild.easyblocks.generic + # to avoid that easyblocks picked up from other places cause trouble + testdir_sandbox = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') + for pkg in ('easybuild.easyblocks', 'easybuild.easyblocks.generic'): + for path in sys.modules[pkg].__path__[:]: + if testdir_sandbox not in path: + sys.modules[pkg].__path__.remove(path) + # include extra test easyblocks foo_txt = '\n'.join([ 'from easybuild.framework.easyblock import EasyBlock', @@ -3225,6 +3233,14 @@ def test_xxx_include_generic_easyblocks(self): import easybuild.easyblocks.generic reload(easybuild.easyblocks.generic) + # kick out any paths that shouldn't be there for easybuild.easyblocks and easybuild.easyblocks.generic + # to avoid that easyblocks picked up from other places cause trouble + testdir_sandbox = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') + for pkg in ('easybuild.easyblocks', 'easybuild.easyblocks.generic'): + for path in sys.modules[pkg].__path__[:]: + if testdir_sandbox not in path: + sys.modules[pkg].__path__.remove(path) + error_msg = "Failed to obtain class for FooBar easyblock" self.assertErrorRegex(EasyBuildError, error_msg, get_easyblock_class, 'FooBar') @@ -3366,6 +3382,14 @@ def test_xxx_include_easyblocks_from_pr(self): import easybuild.easyblocks.generic reload(easybuild.easyblocks.generic) + # kick out any paths that shouldn't be there for easybuild.easyblocks and easybuild.easyblocks.generic, + # to avoid that easyblocks picked up from other places cause trouble + testdir_sandbox = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') + for pkg in ('easybuild.easyblocks', 'easybuild.easyblocks.generic'): + for path in sys.modules[pkg].__path__[:]: + if testdir_sandbox not in path: + sys.modules[pkg].__path__.remove(path) + # clear log write_file(self.logfile, '') diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 39f98384ff..0088de959b 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -146,7 +146,8 @@ def setUp(self): pass # add sandbox to Python search path, update namespace packages - sys.path.append(os.path.join(testdir, 'sandbox')) + testdir_sandbox = os.path.join(testdir, 'sandbox') + sys.path.append(testdir_sandbox) # required to make sure the 'easybuild' dir in the sandbox is picked up; # this relates to the other 'reload' statements below @@ -159,14 +160,14 @@ def setUp(self): # remove any entries in Python search path that seem to provide easyblocks (except the sandbox) for path in sys.path[:]: if os.path.exists(os.path.join(path, 'easybuild', 'easyblocks', '__init__.py')): - if not os.path.samefile(path, os.path.join(testdir, 'sandbox')): + if not os.path.samefile(path, testdir_sandbox): sys.path.remove(path) # hard inject location to (generic) test easyblocks into Python search path # only prepending to sys.path is not enough due to 'pkgutil.extend_path' in easybuild/easyblocks/__init__.py - easybuild.__path__.insert(0, os.path.join(testdir, 'sandbox', 'easybuild')) + easybuild.__path__.insert(0, os.path.join(testdir_sandbox, 'easybuild')) import easybuild.easyblocks - test_easyblocks_path = os.path.join(testdir, 'sandbox', 'easybuild', 'easyblocks') + test_easyblocks_path = os.path.join(testdir_sandbox, 'easybuild', 'easyblocks') easybuild.easyblocks.__path__.insert(0, test_easyblocks_path) reload(easybuild.easyblocks) @@ -175,6 +176,13 @@ def setUp(self): easybuild.easyblocks.generic.__path__.insert(0, test_easyblocks_path) reload(easybuild.easyblocks.generic) + # kick out any paths that shouldn't be there for easybuild.easyblocks and easybuild.easyblocks.generic + # to avoid that easyblocks picked up from other places cause trouble + for pkg in ('easybuild.easyblocks', 'easybuild.easyblocks.generic'): + for path in sys.modules[pkg].__path__[:]: + if testdir_sandbox not in path: + sys.modules[pkg].__path__.remove(path) + # save values of $PATH & $PYTHONPATH, so they can be restored later # this is important in case EasyBuild was installed as a module, since that module may be unloaded, # for example due to changes to $MODULEPATH in case EasyBuild was installed in a module hierarchy From 96afa321137bd845e35f7b345ab0ff02b654619a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 16 May 2021 13:08:49 +0200 Subject: [PATCH 284/864] register intel-compilers and iccifort as alternative subtoolchain fors iimpi + make sure correct _set_compiler_vars and set_variables methods are called --- easybuild/toolchains/iimpi.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/easybuild/toolchains/iimpi.py b/easybuild/toolchains/iimpi.py index a7e374526a..235532dbf2 100644 --- a/easybuild/toolchains/iimpi.py +++ b/easybuild/toolchains/iimpi.py @@ -32,16 +32,19 @@ import re from easybuild.toolchains.iccifort import IccIfort -from easybuild.toolchains.intel_compilers import IntelCompilers +from easybuild.toolchains.intel_compilers import IntelCompilersToolchain from easybuild.toolchains.mpi.intelmpi import IntelMPI -class Iimpi(IntelCompilers, IccIfort, IntelMPI): +class Iimpi(IccIfort, IntelCompilersToolchain, IntelMPI): """ Compiler toolchain with Intel compilers (icc/ifort), Intel MPI. """ NAME = 'iimpi' - SUBTOOLCHAIN = None + # compiler-only subtoolchain can't be determine statically + # since depends on toolchain version (see below), + # so register both here as possible alternatives (which is taken into account elsewhere) + SUBTOOLCHAIN = [(IntelCompilersToolchain.NAME, IccIfort.NAME)] def __init__(self, *args, **kwargs): """Constructor for Iimpi toolchain class.""" @@ -56,8 +59,8 @@ def __init__(self, *args, **kwargs): # (good enough for this purpose) self.iimpi_ver = LooseVersion(self.version.replace('a', '.01').replace('b', '.07')) if self.iimpi_ver >= LooseVersion('2020.12'): - self.SUBTOOLCHAIN = IntelCompilers.NAME - self.COMPILER_MODULE_NAME = IntelCompilers.COMPILER_MODULE_NAME + self.SUBTOOLCHAIN = IntelCompilersToolchain.NAME + self.COMPILER_MODULE_NAME = IntelCompilersToolchain.COMPILER_MODULE_NAME else: self.SUBTOOLCHAIN = IccIfort.NAME self.COMPILER_MODULE_NAME = IccIfort.COMPILER_MODULE_NAME @@ -85,6 +88,20 @@ def is_dep_in_toolchain_module(self, *args, **kwargs): if re.match('^[0-9]', str(self.iimpi_ver)) and self.iimpi_ver < LooseVersion('2020.12'): res = IccIfort.is_dep_in_toolchain_module(self, *args, **kwargs) else: - res = super(Iimpi, self).is_dep_in_toolchain_module(*args, **kwargs) + res = IntelCompilersToolchain.is_dep_in_toolchain_module(self, *args, **kwargs) return res + + def _set_compiler_vars(self): + """Intel compilers-specific adjustments after setting compiler variables.""" + if re.match('^[0-9]', str(self.iimpi_ver)) and self.iimpi_ver < LooseVersion('2020.12'): + IccIfort._set_compiler_vars(self) + else: + IntelCompilersToolchain._set_compiler_vars(self) + + def set_variables(self): + """Intel compilers-specific adjustments after setting compiler variables.""" + if re.match('^[0-9]', str(self.iimpi_ver)) and self.iimpi_ver < LooseVersion('2020.12'): + IccIfort.set_variables(self) + else: + IntelCompilersToolchain.set_variables(self) From 3c1e6d7a0ba0faa4f77a37761ba3c38a9936d774 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 16 May 2021 22:18:05 +0200 Subject: [PATCH 285/864] clean up checks for oneAPI generation of iimpi toolchain --- easybuild/toolchains/iimpi.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/easybuild/toolchains/iimpi.py b/easybuild/toolchains/iimpi.py index 235532dbf2..0a2104872b 100644 --- a/easybuild/toolchains/iimpi.py +++ b/easybuild/toolchains/iimpi.py @@ -57,15 +57,18 @@ def __init__(self, *args, **kwargs): # comparing subversions that include letters causes TypeErrors in Python 3 # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) # (good enough for this purpose) - self.iimpi_ver = LooseVersion(self.version.replace('a', '.01').replace('b', '.07')) - if self.iimpi_ver >= LooseVersion('2020.12'): + self.iimpi_ver = self.version.replace('a', '.01').replace('b', '.07') + if LooseVersion(self.iimpi_ver) >= LooseVersion('2020.12'): + self.oneapi_gen = True self.SUBTOOLCHAIN = IntelCompilersToolchain.NAME self.COMPILER_MODULE_NAME = IntelCompilersToolchain.COMPILER_MODULE_NAME else: + self.oneapi_gen = False self.SUBTOOLCHAIN = IccIfort.NAME self.COMPILER_MODULE_NAME = IccIfort.COMPILER_MODULE_NAME else: self.iimpi_ver = self.version + self.oneapi_gen = False def is_deprecated(self): """Return whether or not this toolchain is deprecated.""" @@ -74,34 +77,35 @@ def is_deprecated(self): # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion if re.match('^[0-9]', str(self.iimpi_ver)): + loosever = LooseVersion(self.iimpi_ver) # iimpi toolchains older than iimpi/2016.01 are deprecated # iimpi 8.1.5 is an exception, since it used in intel/2016a (which is not deprecated yet) - if self.iimpi_ver < LooseVersion('8.0'): + if loosever < LooseVersion('8.0'): deprecated = True - elif self.iimpi_ver > LooseVersion('2000') and self.iimpi_ver < LooseVersion('2016.01'): + elif loosever > LooseVersion('2000') and loosever < LooseVersion('2016.01'): deprecated = True return deprecated def is_dep_in_toolchain_module(self, *args, **kwargs): """Check whether a specific software name is listed as a dependency in the module for this toolchain.""" - if re.match('^[0-9]', str(self.iimpi_ver)) and self.iimpi_ver < LooseVersion('2020.12'): - res = IccIfort.is_dep_in_toolchain_module(self, *args, **kwargs) - else: + if self.oneapi_gen: res = IntelCompilersToolchain.is_dep_in_toolchain_module(self, *args, **kwargs) + else: + res = IccIfort.is_dep_in_toolchain_module(self, *args, **kwargs) return res def _set_compiler_vars(self): """Intel compilers-specific adjustments after setting compiler variables.""" - if re.match('^[0-9]', str(self.iimpi_ver)) and self.iimpi_ver < LooseVersion('2020.12'): - IccIfort._set_compiler_vars(self) - else: + if self.oneapi_gen: IntelCompilersToolchain._set_compiler_vars(self) + else: + IccIfort._set_compiler_vars(self) def set_variables(self): """Intel compilers-specific adjustments after setting compiler variables.""" - if re.match('^[0-9]', str(self.iimpi_ver)) and self.iimpi_ver < LooseVersion('2020.12'): - IccIfort.set_variables(self) - else: + if self.oneapi_gen: IntelCompilersToolchain.set_variables(self) + else: + IccIfort.set_variables(self) From f657d64db388b14f9f6bce2f80cb5b822a5c633e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 16 May 2021 22:32:31 +0200 Subject: [PATCH 286/864] add missing __init__.py in test/framework/sandbox/easybuild/easyblocks/l/ --- test/framework/sandbox/easybuild/easyblocks/l/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 test/framework/sandbox/easybuild/easyblocks/l/__init__.py diff --git a/test/framework/sandbox/easybuild/easyblocks/l/__init__.py b/test/framework/sandbox/easybuild/easyblocks/l/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From 054689c00ba4ee9b7d9b0d4c932139e96bc640d2 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 17 May 2021 10:02:24 +0800 Subject: [PATCH 287/864] suggest missing SSH key when not able to read from remote repository in --check-github --- easybuild/tools/github.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index ecdec1a8a5..5174347329 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -2016,6 +2016,8 @@ def check_github(): ver, req_ver = git.__version__, '1.0' if LooseVersion(ver) < LooseVersion(req_ver): check_res = "FAIL (GitPython version %s is too old, should be version %s or newer)" % (ver, req_ver) + elif "Could not read from remote repository" in push_err.msg: + check_res = "FAIL (GitHub SSH key missing? %s)" % push_err else: check_res = "FAIL (unexpected exception: %s)" % push_err else: From 8d8cacdd583667676729497190bc86943272ef40 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Fri, 14 May 2021 12:18:03 +0800 Subject: [PATCH 288/864] add toolchain definition for fujitsu toolchain --- easybuild/toolchains/compiler/fujitsu.py | 80 +++++++++++++++ easybuild/toolchains/ffmpi.py | 38 +++++++ easybuild/toolchains/fft/fftw.py | 15 +-- easybuild/toolchains/fft/fujitsufftw.py | 37 +++++++ easybuild/toolchains/fujitsu.py | 37 +++++++ easybuild/toolchains/linalg/fujitsussl.py | 115 ++++++++++++++++++++++ easybuild/toolchains/mpi/fujitsumpi.py | 66 +++++++++++++ 7 files changed, 382 insertions(+), 6 deletions(-) create mode 100644 easybuild/toolchains/compiler/fujitsu.py create mode 100644 easybuild/toolchains/ffmpi.py create mode 100644 easybuild/toolchains/fft/fujitsufftw.py create mode 100644 easybuild/toolchains/fujitsu.py create mode 100644 easybuild/toolchains/linalg/fujitsussl.py create mode 100644 easybuild/toolchains/mpi/fujitsumpi.py diff --git a/easybuild/toolchains/compiler/fujitsu.py b/easybuild/toolchains/compiler/fujitsu.py new file mode 100644 index 0000000000..f598b13e44 --- /dev/null +++ b/easybuild/toolchains/compiler/fujitsu.py @@ -0,0 +1,80 @@ +## +# Copyright 2014-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for the Fujitsu compiler drivers (aka fcc, frt). + +The basic concept the same as for the Cray Programming Environment. + +:author: Miguel Dias Costa (National University of Singapore) +""" +import easybuild.tools.systemtools as systemtools +from easybuild.tools.toolchain.compiler import Compiler, DEFAULT_OPT_LEVEL + +TC_CONSTANT_FUJITSU = 'Fujitsu' + + +class FujitsuCompiler(Compiler): + """Generic support for using Fujitsu compiler drivers.""" + TOOLCHAIN_FAMILY = TC_CONSTANT_FUJITSU + + # compiler module name is lang (with version e.g. tcsds-1.2.31) + COMPILER_MODULE_NAME = ['lang'] + COMPILER_FAMILY = TC_CONSTANT_FUJITSU + + COMPILER_CC = 'fcc' + COMPILER_CXX = 'FCC' + COMPILER_F77 = 'frt' + COMPILER_F90 = 'frt' + COMPILER_FC = 'frt' + + COMPILER_UNIQUE_OPTION_MAP = { + DEFAULT_OPT_LEVEL: 'Kfast', + 'optarch': '', # Fujitsu compiler by default generates code for the arch it is running on + 'openmp': 'Kopenmp', + 'unroll': 'funroll-loops', + 'strict': ['Kfp_precision'], + 'precise': ['Kfp_precision'], + 'defaultprec': [], + 'loose': ['Kfp_relaxed'], + 'veryloose': ['Kfp_relaxed'], + 'vectorize': {False: 'KNOSVE', True: 'KSVE'}, + } + + # used when 'optarch' toolchain option is enabled (and --optarch is not specified) + COMPILER_OPTIMAL_ARCHITECTURE_OPTION = { + # -march=archi[+features]. At least on Fugaku, these are set by default (-march=armv8.3-a+sve and -mcpu=a64fx) + (systemtools.AARCH64, systemtools.ARM): '', + } + + # used with --optarch=GENERIC + COMPILER_GENERIC_OPTION = { + (systemtools.AARCH64, systemtools.ARM): '-mcpu=generic -mtune=generic', + } + + def _set_compiler_vars(self): + super(FujitsuCompiler, self)._set_compiler_vars() + + self.variables.nappend('CFLAGS', ['Nclang']) + self.variables.nappend('CXXFLAGS', ['Nclang']) diff --git a/easybuild/toolchains/ffmpi.py b/easybuild/toolchains/ffmpi.py new file mode 100644 index 0000000000..7a78df0595 --- /dev/null +++ b/easybuild/toolchains/ffmpi.py @@ -0,0 +1,38 @@ +## +# Copyright 2012-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for fompi compiler toolchain (includes Fujitsu Compiler and MPI). + +:author: Miguel Dias Costa (National University of Singapore) +""" +from easybuild.toolchains.compiler.fujitsu import FujitsuCompiler +from easybuild.toolchains.linalg.fujitsussl import FujitsuSSL +from easybuild.toolchains.mpi.fujitsumpi import FujitsuMPI + + +class ffmpi(FujitsuCompiler, FujitsuMPI, FujitsuSSL): + """Compiler toolchain with Fujitsu Compiler and MPI.""" + NAME = 'ffmpi' + SUBTOOLCHAIN = FujitsuCompiler.NAME diff --git a/easybuild/toolchains/fft/fftw.py b/easybuild/toolchains/fft/fftw.py index 6597d1ead5..4b6c32dcb9 100644 --- a/easybuild/toolchains/fft/fftw.py +++ b/easybuild/toolchains/fft/fftw.py @@ -39,15 +39,18 @@ class Fftw(Fft): """FFTW FFT library""" FFT_MODULE_NAME = ['FFTW'] + FFTW_API_VERSION = '' def _set_fftw_variables(self): - suffix = '' - version = self.get_software_version(self.FFT_MODULE_NAME)[0] - if LooseVersion(version) < LooseVersion('2') or LooseVersion(version) >= LooseVersion('4'): - raise EasyBuildError("_set_fft_variables: FFTW unsupported version %s (major should be 2 or 3)", version) - elif LooseVersion(version) > LooseVersion('2'): - suffix = '3' + suffix = self.FFTW_API_VERSION + if not suffix: + version = self.get_software_version(self.FFT_MODULE_NAME)[0] + if LooseVersion(version) < LooseVersion('2') or LooseVersion(version) >= LooseVersion('4'): + raise EasyBuildError("_set_fft_variables: FFTW unsupported version %s (major should be 2 or 3)", + version) + elif LooseVersion(version) > LooseVersion('2'): + suffix = '3' # order matters! fftw_libs = ["fftw%s" % suffix] diff --git a/easybuild/toolchains/fft/fujitsufftw.py b/easybuild/toolchains/fft/fujitsufftw.py new file mode 100644 index 0000000000..6daf035f1d --- /dev/null +++ b/easybuild/toolchains/fft/fujitsufftw.py @@ -0,0 +1,37 @@ +## +# Copyright 2012-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for Fujitsu FFTW as toolchain FFT library. + +:author: Miguel Dias Costa (National University of Singapore) +""" +from easybuild.toolchains.fft.fftw import Fftw + + +class FujitsuFFTW(Fftw): + """Fujitsu FFTW FFT library""" + + FFT_MODULE_NAME = ['FFTW3-sve'] + FFTW_API_VERSION = '3' diff --git a/easybuild/toolchains/fujitsu.py b/easybuild/toolchains/fujitsu.py new file mode 100644 index 0000000000..fb1db7d374 --- /dev/null +++ b/easybuild/toolchains/fujitsu.py @@ -0,0 +1,37 @@ +## +# Copyright 2014-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Fujitsu toolchain: Fujitsu compilers and MPI + Fujitsu SSL2 and Fujitsu FFTW + +:author: Miguel Dias Costa (National University of Singapore) +""" +from easybuild.toolchains.ffmpi import ffmpi +from easybuild.toolchains.fft.fujitsufftw import FujitsuFFTW + + +class Fujitsu(ffmpi, FujitsuFFTW): + """Compiler toolchain for Fujitsu.""" + NAME = 'Fujitsu' + SUBTOOLCHAIN = ffmpi.NAME diff --git a/easybuild/toolchains/linalg/fujitsussl.py b/easybuild/toolchains/linalg/fujitsussl.py new file mode 100644 index 0000000000..aa5c15d320 --- /dev/null +++ b/easybuild/toolchains/linalg/fujitsussl.py @@ -0,0 +1,115 @@ +## +# Copyright 2014-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for Fujitsu's SSL library, which provides BLAS/LAPACK support. + +:author: Miguel Dias Costa (National University of Singapore) +""" +import os + + +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.toolchain.constants import COMPILER_FLAGS +from easybuild.tools.toolchain.linalg import LinAlg + + +FUJITSU_SSL_MODULE_NAME = None +TC_CONSTANT_FUJITSU_SSL = 'FujitsuSSL' + + +class FujitsuSSL(LinAlg): + """Support for Fujitsu's SSL library, which provides BLAS/LAPACK support.""" + # BLAS/LAPACK support + # via lang/tcsds module + BLAS_MODULE_NAME = ['lang'] + + # no need to specify libraries, compiler driver takes care of linking the right libraries + BLAS_LIB = [''] + BLAS_LIB_MT = [''] + BLAS_FAMILY = TC_CONSTANT_FUJITSU_SSL + + LAPACK_MODULE_NAME = None + LAPACK_IS_BLAS = True + LAPACK_FAMILY = TC_CONSTANT_FUJITSU_SSL + + BLACS_MODULE_NAME = None + + SCALAPACK_MODULE_NAME = BLAS_MODULE_NAME + SCALAPACK_LIB = [''] + SCALAPACK_LIB_MT = [''] + SCALAPACK_FAMILY = TC_CONSTANT_FUJITSU_SSL + + def _get_software_root(self, name): + """Get install prefix for specified software name; special treatment for Fujitsu modules.""" + if name == 'lang': + # Fujitsu-provided module + env_var = 'FJSVXTCLANGA' + root = os.getenv(env_var, None) + if root is None: + raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var) + else: + self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root) + else: + root = super(FujitsuSSL, self)._get_software_root(name) + + return root + + def _set_blas_variables(self): + """Setting FujitsuSSL specific BLAS related variables""" + super(FujitsuSSL, self)._set_blas_variables() + if self.options.get('openmp', None): + for flags_var, _ in COMPILER_FLAGS: + self.variables.nappend(flags_var, ['SSL2BLAMP']) + else: + for flags_var, _ in COMPILER_FLAGS: + self.variables.nappend(flags_var, ['SSL2']) + + def _set_scalapack_variables(self): + """Setting FujitsuSSL specific SCALAPACK related variables""" + super(FujitsuSSL, self)._set_scalapack_variables() + for flags_var, _ in COMPILER_FLAGS: + self.variables.nappend(flags_var, ['SCALAPACK']) + + def definition(self): + """ + Filter BLAS module from toolchain definition. + The SSL2 module is loaded indirectly (and versionless) via the lang module, + and thus is not a direct toolchain component. + """ + tc_def = super(FujitsuSSL, self).definition() + tc_def['BLAS'] = [] + tc_def['LAPACK'] = [] + tc_def['SCALAPACK'] = [] + return tc_def + + def set_variables(self): + """Set the variables""" + self._set_blas_variables() + self._set_lapack_variables() + self._set_scalapack_variables() + + self.log.devel('set_variables: LinAlg variables %s', self.variables) + + super(FujitsuSSL, self).set_variables() diff --git a/easybuild/toolchains/mpi/fujitsumpi.py b/easybuild/toolchains/mpi/fujitsumpi.py new file mode 100644 index 0000000000..b28fe9af9c --- /dev/null +++ b/easybuild/toolchains/mpi/fujitsumpi.py @@ -0,0 +1,66 @@ +## +# Copyright 2014-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +MPI support for Fujitsu MPI. + +:author: Miguel Dias Costa (National University of Singapore) +""" +from easybuild.toolchains.compiler.fujitsu import FujitsuCompiler +from easybuild.toolchains.mpi.openmpi import TC_CONSTANT_OPENMPI, TC_CONSTANT_MPI_TYPE_OPENMPI +from easybuild.tools.toolchain.constants import COMPILER_VARIABLES, MPI_COMPILER_VARIABLES +from easybuild.tools.toolchain.mpi import Mpi +from easybuild.tools.toolchain.variables import CommandFlagList + + +class FujitsuMPI(Mpi): + """Generic support for using Fujitsu compiler wrappers""" + # MPI support + # no separate module, Fujitsu compiler drivers always provide MPI support + MPI_MODULE_NAME = None + MPI_FAMILY = TC_CONSTANT_OPENMPI + MPI_TYPE = TC_CONSTANT_MPI_TYPE_OPENMPI + + # OpenMPI reads from CC etc env variables + MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES]) + + MPI_LINK_INFO_OPTION = '-showme:link' + + def __init__(self, *args, **kwargs): + """Toolchain constructor""" + super(FujitsuMPI, self).__init__(*args, **kwargs) + + def _set_mpi_compiler_variables(self): + """Define MPI wrapper commands and add OMPI_* variables to set.""" + self.MPI_COMPILER_MPICC = 'mpi' + FujitsuCompiler.COMPILER_CC + self.MPI_COMPILER_MPICXX = 'mpi' + FujitsuCompiler.COMPILER_CXX + self.MPI_COMPILER_MPIF77 = 'mpi' + FujitsuCompiler.COMPILER_F77 + self.MPI_COMPILER_MPIF90 = 'mpi' + FujitsuCompiler.COMPILER_F90 + self.MPI_COMPILER_MPIFC = 'mpi' + FujitsuCompiler.COMPILER_FC + + # this needs to be done first, otherwise e.g., CC is set to MPICC if the usempi toolchain option is enabled + for var, _ in COMPILER_VARIABLES: + self.variables.nappend('OMPI_%s' % var, str(self.variables[var].get_first()), var_class=CommandFlagList) + + super(FujitsuMPI, self)._set_mpi_compiler_variables() From 2ddbb23162c6b26fb1112d3ba2ab72825a47e5e4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 17 May 2021 12:20:39 +0200 Subject: [PATCH 289/864] fix broken test by tweaking Makefile for libtoy ('{bin,include,lib}' was not getting expanded in mkdir command) --- .../test_ecs/l/libtoy/libtoy-0.0.eb | 2 +- .../sources/l/libtoy/libtoy-0.0.tar.gz | Bin 1788 -> 1784 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyconfigs/test_ecs/l/libtoy/libtoy-0.0.eb b/test/framework/easyconfigs/test_ecs/l/libtoy/libtoy-0.0.eb index 7a1495b59a..3d89f4bf2e 100644 --- a/test/framework/easyconfigs/test_ecs/l/libtoy/libtoy-0.0.eb +++ b/test/framework/easyconfigs/test_ecs/l/libtoy/libtoy-0.0.eb @@ -7,7 +7,7 @@ description = "Toy C library." toolchain = SYSTEM sources = [SOURCE_TAR_GZ] -checksums = ['9978bc9837c192b8460b4a5d8089c9ffd454771fce771a171fe8527d0274dbe3'] +checksums = ['1523195c806fb511c3f0b60de1e53890c39cb783c7061c2736e40a31a38fd017'] sanity_check_paths = { 'files': ['bin/toy', 'lib/libtoy.%s' % SHLIB_EXT], diff --git a/test/framework/sandbox/sources/l/libtoy/libtoy-0.0.tar.gz b/test/framework/sandbox/sources/l/libtoy/libtoy-0.0.tar.gz index c587c8333d664dd567317e3820308362327fd63f..a11efbdbdff5df6754af49b3c88fc0e1d036c1b6 100644 GIT binary patch literal 1784 zcmVET>*VG`a#$Wh>xC@Kqp=uv^AVQI z!wSneIHJhP=xA(2i7Hr*M3vDIgk$R#kwCHFXf_Guyq?Jw9KB{eQZ1dXA^kuK~APd1~Rss_N*Mok}&ozkC<- zHz3$8NT3B>6-`wwIK|mQO@Ftzzc!y;qMiF|E4Qcq)QoQCb-Rkc_j&v6g56X%zM^%> zuu`gFrE^vMea8JI1l!}!2c3uaTt7#~XU5gT$B#`jZ{5Cm@a4@s1$dFF&gFz@sOhgA zRs1vy_1lp5I?}lAm;DGQurDrLJ+9K2Rn^qg!hCi%XR3L}uIaB`u*JBxAn)~4)s*IF z{&809cAvj?pFcm9^)xwg?A7>@iPBukzH#!KUoXE%0pz`2eF$BZc|F>9JIS|+!{qi} znYp54qIO

**VkPm%ly#B#9@T35WL)wm;9ALmxYrl^yop<=-vA`h6M)V!JE=bz(o zoEiT7%`c~-_pS|p_`!_+$1lUYojb-Z3AKd_)w)4LND+*GW@y+xt-sThZO(-00000 zu-Q13xbais+Hc)o#s8+}jfFoGH;NB#qx&7iirh(D`=a~o-NmJW6nUwfytzXs+4%I* zz$tQeF+*GY`t#D#5_?bXK5f~H?Y%zxEN$ zzGPTIsY9si?xB}-D`li5Ni)rTnEc&I65p3!$hmenDRs;xlQ>k!Xttihq4deUU0SG)WFkHx9!h6HKEj2js@rkv*R> zY#hqt{wq@_<1bH~UKk>`48bz2q*+YqLJ1vbM>3B?n%jw|;gE2KM7+#R`TX)hczYO< zq@<~9);_7j&XVg?tpOcCqxl!szh5G;W;D|OMThwhB<^?y{8MfE$#)&H?) zGSw@6UqiR&(m=P@%8MK6_RjHricbsnYIJ)J_tz%atI_Sf!ToI)?E1RBWWliwYpzCo zrG1&bKD&fpd(}t>_z7>H5bTxe0PE!|(E(D{LI>!H$7ioZ2k40luU-cz`df_-&@)bM zf5%O~4)94-(YuM|{+s#3=mlw)vRu)`(R@l~>+pS<>E){F1xxSo_5R)#Yv=__dcaS2 z>7BR3(H}qhLH|1Xmc8v!^?<9@^Cf(GzA4c20RR910000hz}5TR-2Wmuyp8VDH`B9s zmj>Q;UtK3}?zqnNeMRKa_a$`G%;B`1%VPEssr2P~C;gyPPy2UH(Eb%c`v(9300000 z000000O8%P#&rMy00000000000NC90^fCQkPZj;&1+<84s{hN|hUKK0@-N_3()Bc> zdsNzYkL&yB{@JbVmHp66-&fh+Pd7e@dcJKwTm0=2>{IQP;sEvD+Ts2EuW)ib`e>y8 zi$qJm|E5GDN{s&gm!im^|Jew3mGo;b6`Yij3ug{U(#4#S!s5ys4B{)U_Kz*V!M@u2 zlU}#_f@9J`L){th-#}RZOdq)peLT|om!ow3$Kdz>HipXjcNM7zs1nw3rLu$cK(ptP zdvG$N*}LiS#WQDv>`vglJ&d_!w(b;d3(K(nHxIt&e`Y=UXte(2;aF+?lL`^{`5%Mz azadn4){nvB1poke{P`D$mvqbkegFVd+p}5# literal 1788 zcmV}OxyxSeE-8&IP21B8 zrfu4!P^@LUx!cPv-P^nF?X?XtP%RY;7V=blsINYV4}DPZ!G}Uc5d>d+5MS(zT0x<{ ziHLD#XYO`(dv|H`4^n?07-oMvGr#%GPG)zr%$GCtSvx-;lEd;4+9+gM9*xJblaH`m z9*(h`gCj9HHaZ#~iA7^rjznXlBM8SgEFytY(N--I%6ToDFWOqodZbz=Q^yYuGyKKAF{Qe3Z z%wNA?Hz9#0bVW23rD&ICi#7e-b(TLcKsb&`f);#7xrkJb57G!iU^e2))GP$nxtRr-H7L2PqFvTfC0kRnno67P&Z)YI`q?Jp2QTt|$_Ha4ZQthZ zo2Jz|0&Hh5wV!ircOQQ@J74)kv4`1}t-W+lLle(2QmVwGIPrML@X%Nl#W_{5^owd8 z5y+E=p#M3lp4T$EvHqi%WqBkPtF8YUmE%W&DMM4ueNvm1BNwAu0~+oc=3iL1ov6B8TI_%f~jgWJ&{x<_r|3@`hRKJ59 z{U3WKQ@zp;HFSFpE9mx``RSE(d*}E*#b*V3ExJ9M`)d*GwdnTV;{J9Dc75Gms%Trf zIaedT^1jS&pIyeUy;?~J_$hCn6ztXN0PE$e(E(D{LkH-I$7iob2k40luU!Wy`df<* z&@)bMf6q<74)AH@>fOZi{+s!u=mlw)vg~T&Xg(#gb$Cl=dbw(P!SZ{2y}!4m8hXL9 z9`LhXt^Ia5`qRfhYTrcPv35MJ9&oLCzNAmjHwAh=00000003YWIC{Su`(Gx9x6yt2 zW_tGSa{s%|tLx;AUDvq2uY^4MzNBUtd7QEGIn2Hi-IsZrT~9{0n%M zbX|?;9+mdp)O9>$z=mS&eM6U(svw+_DNe|97KSZV#s!}0R^Clw;j e^FI#je^aRPtRI8L3jhG{ Date: Mon, 17 May 2021 22:11:11 +0800 Subject: [PATCH 290/864] code cleanup --- easybuild/toolchains/compiler/fujitsu.py | 1 + easybuild/toolchains/ffmpi.py | 6 +++--- easybuild/toolchains/fujitsu.py | 6 +++--- easybuild/toolchains/linalg/fujitsussl.py | 2 +- easybuild/toolchains/mpi/fujitsumpi.py | 4 ---- 5 files changed, 8 insertions(+), 11 deletions(-) diff --git a/easybuild/toolchains/compiler/fujitsu.py b/easybuild/toolchains/compiler/fujitsu.py index f598b13e44..27efc93c6b 100644 --- a/easybuild/toolchains/compiler/fujitsu.py +++ b/easybuild/toolchains/compiler/fujitsu.py @@ -76,5 +76,6 @@ class FujitsuCompiler(Compiler): def _set_compiler_vars(self): super(FujitsuCompiler, self)._set_compiler_vars() + # enable clang compatibility mode self.variables.nappend('CFLAGS', ['Nclang']) self.variables.nappend('CXXFLAGS', ['Nclang']) diff --git a/easybuild/toolchains/ffmpi.py b/easybuild/toolchains/ffmpi.py index 7a78df0595..de7ec81664 100644 --- a/easybuild/toolchains/ffmpi.py +++ b/easybuild/toolchains/ffmpi.py @@ -23,7 +23,7 @@ # along with EasyBuild. If not, see . ## """ -EasyBuild support for fompi compiler toolchain (includes Fujitsu Compiler and MPI). +EasyBuild support for ffmpi compiler toolchain (includes Fujitsu Compiler, MPI and Scientific Libraries). :author: Miguel Dias Costa (National University of Singapore) """ @@ -32,7 +32,7 @@ from easybuild.toolchains.mpi.fujitsumpi import FujitsuMPI -class ffmpi(FujitsuCompiler, FujitsuMPI, FujitsuSSL): - """Compiler toolchain with Fujitsu Compiler and MPI.""" +class Ffmpi(FujitsuCompiler, FujitsuMPI, FujitsuSSL): + """Compiler toolchain with Fujitsu Compiler, MPI and Scientific Libraries.""" NAME = 'ffmpi' SUBTOOLCHAIN = FujitsuCompiler.NAME diff --git a/easybuild/toolchains/fujitsu.py b/easybuild/toolchains/fujitsu.py index fb1db7d374..919e268d94 100644 --- a/easybuild/toolchains/fujitsu.py +++ b/easybuild/toolchains/fujitsu.py @@ -27,11 +27,11 @@ :author: Miguel Dias Costa (National University of Singapore) """ -from easybuild.toolchains.ffmpi import ffmpi +from easybuild.toolchains.ffmpi import Ffmpi from easybuild.toolchains.fft.fujitsufftw import FujitsuFFTW -class Fujitsu(ffmpi, FujitsuFFTW): +class Fujitsu(Ffmpi, FujitsuFFTW): """Compiler toolchain for Fujitsu.""" NAME = 'Fujitsu' - SUBTOOLCHAIN = ffmpi.NAME + SUBTOOLCHAIN = Ffmpi.NAME diff --git a/easybuild/toolchains/linalg/fujitsussl.py b/easybuild/toolchains/linalg/fujitsussl.py index aa5c15d320..5422456c86 100644 --- a/easybuild/toolchains/linalg/fujitsussl.py +++ b/easybuild/toolchains/linalg/fujitsussl.py @@ -66,7 +66,7 @@ def _get_software_root(self, name): if name == 'lang': # Fujitsu-provided module env_var = 'FJSVXTCLANGA' - root = os.getenv(env_var, None) + root = os.getenv(env_var) if root is None: raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var) else: diff --git a/easybuild/toolchains/mpi/fujitsumpi.py b/easybuild/toolchains/mpi/fujitsumpi.py index b28fe9af9c..0410339ba1 100644 --- a/easybuild/toolchains/mpi/fujitsumpi.py +++ b/easybuild/toolchains/mpi/fujitsumpi.py @@ -47,10 +47,6 @@ class FujitsuMPI(Mpi): MPI_LINK_INFO_OPTION = '-showme:link' - def __init__(self, *args, **kwargs): - """Toolchain constructor""" - super(FujitsuMPI, self).__init__(*args, **kwargs) - def _set_mpi_compiler_variables(self): """Define MPI wrapper commands and add OMPI_* variables to set.""" self.MPI_COMPILER_MPICC = 'mpi' + FujitsuCompiler.COMPILER_CC From 61f8dded806460c82537cd289a610c4def23a31a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 17 May 2021 18:40:52 +0200 Subject: [PATCH 291/864] add locate_solib function to locate Linux shared libraries --- easybuild/tools/systemtools.py | 28 ++++++++++++++++++++++++++++ test/framework/systemtools.py | 12 +++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index d6b05daae3..5810dee2c3 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -845,6 +845,34 @@ def check_linked_shared_libs(path, required_patterns=None, banned_patterns=None) return not (found_banned_patterns or missing_required_patterns) +def locate_solib(libobj): + """ + Return absolute path to loaded library using dlinfo + Based on https://stackoverflow.com/a/35683698 + """ + # early return if we're not on a Linux system + if get_os_type() != LINUX: + return None + + class LINKMAP(ctypes.Structure): + _fields_ = [ + ("l_addr", ctypes.c_void_p), + ("l_name", ctypes.c_char_p) + ] + + libdl = ctypes.cdll.LoadLibrary(ctypes.util.find_library('dl')) + + dlinfo = libdl.dlinfo + dlinfo.argtypes = ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p + dlinfo.restype = ctypes.c_int + + libpointer = ctypes.c_void_p() + dlinfo(libobj._handle, 2, ctypes.byref(libpointer)) + libpath = ctypes.cast(libpointer, ctypes.POINTER(LINKMAP)).contents.l_name + + return libpath.decode('utf-8') + + def get_system_info(): """Return a dictionary with system information.""" python_version = '; '.join(sys.version.split('\n')) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 27b693e63a..aa4f27357e 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -28,6 +28,7 @@ @author: Kenneth hoste (Ghent University) @author: Ward Poelmans (Ghent University) """ +import ctypes import re import os import sys @@ -50,7 +51,7 @@ from easybuild.tools.systemtools import get_cpu_family, get_cpu_features, get_cpu_model, get_cpu_speed, get_cpu_vendor from easybuild.tools.systemtools import get_gcc_version, get_glibc_version, get_os_type, get_os_name, get_os_version from easybuild.tools.systemtools import get_platform_name, get_shared_lib_ext, get_system_info, get_total_memory -from easybuild.tools.systemtools import pick_dep_version +from easybuild.tools.systemtools import locate_solib, pick_dep_version PROC_CPUINFO_TXT = None @@ -1037,6 +1038,15 @@ def test_check_linked_shared_libs(self): error_msg = "Check on linked libs should fail for %s with %s" % (path, pattern_named_args) self.assertFalse(check_linked_shared_libs(path, **pattern_named_args), error_msg) + def test_locate_solib(self): + """Test locate_solib function (Linux only).""" + if get_os_type() == LINUX: + libname = 'libc.so.6' + obj = ctypes.cdll.LoadLibrary(libname) + libc_path = locate_solib(obj) + self.assertEqual(os.path.basename(libc_path), libname) + self.assertTrue(os.path.exists(libc_path), "%s should exist" % libname) + def suite(): """ returns all the testcases in this module """ From ff2cafc14bb3499603f81670d89f2003b4eb833f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 17 May 2021 19:02:15 +0200 Subject: [PATCH 292/864] make test for locate_solib a bit more robust, libc.so.6 may not be there everywhere --- test/framework/systemtools.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index aa4f27357e..d1be7062a9 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -1042,10 +1042,15 @@ def test_locate_solib(self): """Test locate_solib function (Linux only).""" if get_os_type() == LINUX: libname = 'libc.so.6' - obj = ctypes.cdll.LoadLibrary(libname) - libc_path = locate_solib(obj) - self.assertEqual(os.path.basename(libc_path), libname) - self.assertTrue(os.path.exists(libc_path), "%s should exist" % libname) + libc_obj = None + try: + libc_obj = ctypes.cdll.LoadLibrary(libname) + except OSError: + pass + if libc_obj: + libc_path = locate_solib(libc_obj) + self.assertEqual(os.path.basename(libc_path), libname) + self.assertTrue(os.path.exists(libc_path), "%s should exist" % libname) def suite(): From 2ebdae240d280c5248594c0dcd2a9dc568d6415e Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 18 May 2021 01:38:37 +0200 Subject: [PATCH 293/864] add system agnostic function find_library_path to locate shared libraries --- easybuild/tools/systemtools.py | 39 ++++++++++++++++++++++++++++++++++ test/framework/systemtools.py | 15 ++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 5810dee2c3..19c3bfce8f 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -41,6 +41,12 @@ from ctypes.util import find_library from socket import gethostname +try: + # only needed on macOS, may not be available on Linux + import ctypes.macholib.dyld +except ImportError: + pass + from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option @@ -849,6 +855,8 @@ def locate_solib(libobj): """ Return absolute path to loaded library using dlinfo Based on https://stackoverflow.com/a/35683698 + + :params libobj: ctypes CDLL object """ # early return if we're not on a Linux system if get_os_type() != LINUX: @@ -873,6 +881,37 @@ class LINKMAP(ctypes.Structure): return libpath.decode('utf-8') +def find_library_path(lib_filename): + """ + Search library by file name in the system + Return absolute path to existing libraries + + :params lib_filename: name of library file + """ + + lib_abspath = None + os_type = get_os_type() + + try: + lib_obj = ctypes.cdll.LoadLibrary(lib_filename) + except OSError: + _log.info("Library '%s' not found in host system", lib_filename) + else: + # ctypes.util.find_library only accepts unversioned library names + if os_type == LINUX: + # find path to library with dlinfo + lib_abspath = locate_solib(lib_obj) + elif os_type == DARWIN: + # ctypes.macholib.dyld.dyld_find accepts file names and returns full path + lib_abspath = ctypes.macholib.dyld.dyld_find(lib_filename) + else: + raise EasyBuildError("Unknown host OS type: %s", os_type) + + _log.info("Found absolute path to %s: %s" % (lib_filename, lib_abspath)) + + return lib_abspath + + def get_system_info(): """Return a dictionary with system information.""" python_version = '; '.join(sys.version.split('\n')) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index d1be7062a9..b99f9cb891 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -51,7 +51,7 @@ from easybuild.tools.systemtools import get_cpu_family, get_cpu_features, get_cpu_model, get_cpu_speed, get_cpu_vendor from easybuild.tools.systemtools import get_gcc_version, get_glibc_version, get_os_type, get_os_name, get_os_version from easybuild.tools.systemtools import get_platform_name, get_shared_lib_ext, get_system_info, get_total_memory -from easybuild.tools.systemtools import locate_solib, pick_dep_version +from easybuild.tools.systemtools import find_library_path, locate_solib, pick_dep_version PROC_CPUINFO_TXT = None @@ -1052,6 +1052,19 @@ def test_locate_solib(self): self.assertEqual(os.path.basename(libc_path), libname) self.assertTrue(os.path.exists(libc_path), "%s should exist" % libname) + def test_find_library_path(self): + """Test find_library_path function (Linux and Darwin only).""" + if get_os_type() == LINUX: + libname = 'libc.so.6' + elif get_os_type() == DARWIN: + libname = 'libSystem.dylib' + else: + libname = None + + if libname: + lib_path = find_library_path(libname) + self.assertEqual(os.path.basename(lib_path), libname) + self.assertTrue(os.path.exists(lib_path), "%s should exist" % libname) def suite(): """ returns all the testcases in this module """ From 7f5789bcfea8f2d149164c05dac94296886de209 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 18 May 2021 09:40:54 +0200 Subject: [PATCH 294/864] make log statement lazy in find_library_path Co-authored-by: Kenneth Hoste --- easybuild/tools/systemtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 19c3bfce8f..dbd7415f8a 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -907,7 +907,7 @@ def find_library_path(lib_filename): else: raise EasyBuildError("Unknown host OS type: %s", os_type) - _log.info("Found absolute path to %s: %s" % (lib_filename, lib_abspath)) + _log.info("Found absolute path to %s: %s", lib_filename, lib_abspath) return lib_abspath From 4d436878ff0d133e926f1c2bdc73aacf01412223 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 18 May 2021 10:45:17 +0200 Subject: [PATCH 295/864] add empty line before "def suite" to fix code style check fail --- test/framework/systemtools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index b99f9cb891..68fae7ea5a 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -1066,6 +1066,7 @@ def test_find_library_path(self): self.assertEqual(os.path.basename(lib_path), libname) self.assertTrue(os.path.exists(lib_path), "%s should exist" % libname) + def suite(): """ returns all the testcases in this module """ return TestLoaderFiltered().loadTestsFromTestCase(SystemToolsTest, sys.argv[1:]) From 3118cb4f0aa0c76b203dbbffdc2003aa28d0ef38 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 18 May 2021 12:03:49 +0200 Subject: [PATCH 296/864] add update_build_option function to update specific build options after initializing the EasyBuild configuration --- easybuild/tools/config.py | 12 ++++++++++++ test/framework/config.py | 9 ++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 1e7da68ced..2d14fe5800 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -558,6 +558,18 @@ def build_option(key, **kwargs): raise EasyBuildError(error_msg) +def update_build_option(key, value): + """ + Update build option with specified name to given value. + + WARNING: Use this with care, the build options are not expected to be changed during an EasyBuild session! + """ + # BuildOptions() is a (singleton) frozen dict, so this is less straightforward that it seems... + build_options = BuildOptions() + build_options._FrozenDict__dict[key] = value + _log.warning("Build option '%s' was updated to: %s", key, build_option(key)) + + def build_path(): """ Return the build path diff --git a/test/framework/config.py b/test/framework/config.py index 08a0594825..cb13d348a5 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -40,7 +40,7 @@ from easybuild.tools import run from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, build_path, get_build_log_path, get_log_filename, get_repositorypath -from easybuild.tools.config import install_path, log_file_format, log_path, source_paths +from easybuild.tools.config import install_path, log_file_format, log_path, source_paths, update_build_option from easybuild.tools.config import BuildOptions, ConfigurationVariables from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, init_build_options from easybuild.tools.filetools import copy_dir, mkdir, write_file @@ -677,6 +677,13 @@ def test_get_build_log_path(self): init_config(args=['--tmp-logdir=%s' % build_log_path]) self.assertEqual(get_build_log_path(), build_log_path) + def test_update_build_option(self): + """Test updating of a build option.""" + self.assertEqual(build_option('banned_linked_shared_libs'), None) + + update_build_option('banned_linked_shared_libs', '/usr/lib64/libssl.so.1.1') + self.assertEqual(build_option('banned_linked_shared_libs'), '/usr/lib64/libssl.so.1.1') + def suite(): return TestLoaderFiltered().loadTestsFromTestCase(EasyBuildConfigTest, sys.argv[1:]) From a8352f98548a625b7de3532b71ba170c6ca30881 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 18 May 2021 19:11:56 +0200 Subject: [PATCH 297/864] use unload/load in ModuleGeneratorLua.swap_module, since 'swap' is not supported by Lmod (fixes #3626) --- easybuild/tools/module_generator.py | 5 +++- test/framework/module_generator.py | 41 +++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 6cb2716a7c..5b48f6cf22 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -1430,7 +1430,10 @@ def swap_module(self, mod_name_out, mod_name_in, guarded=True): :param mod_name_in: name of module to load (swap in) :param guarded: guard 'swap' statement, fall back to 'load' if module being swapped out is not loaded """ - body = 'swap("%s", "%s")' % (mod_name_out, mod_name_in) + body = '\n'.join([ + 'unload("%s")' % mod_name_out, + 'load("%s")' % mod_name_in, + ]) if guarded: alt_body = self.LOAD_TEMPLATE % {'mod_name': mod_name_in} swap_statement = [self.conditional_statement(self.is_loaded(mod_name_out), body, else_body=alt_body)] diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 00ee2cd8ae..c130f8c01d 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -616,7 +616,8 @@ def test_swap(self): else: expected = '\n'.join([ '', - 'swap("foo", "bar")', + 'unload("foo")', + 'load("bar")', '', ]) @@ -637,7 +638,8 @@ def test_swap(self): expected = '\n'.join([ '', 'if isloaded("foo") then', - ' swap("foo", "bar")', + ' unload("foo")', + ' load("bar")', 'else', ' load("bar")', 'end', @@ -1460,6 +1462,41 @@ def test_det_installdir(self): self.assertEqual(self.modgen.det_installdir(test_modfile), expected) + def test_generated_module_file_swap(self): + """Test loading a generated module file that includes swap statements.""" + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorLua: + mod_ext = '.lua' + else: + mod_ext = '' + + # empty test modules + for modname in ('one/1.0', 'one/1.1'): + modfile = os.path.join(self.test_prefix, modname + mod_ext) + write_file(modfile, self.modgen.MODULE_SHEBANG) + + modulepath = os.getenv('MODULEPATH') + if modulepath: + self.modtool.unuse(modulepath) + + test_mod = os.path.join(self.test_prefix, 'test', '1.0' + mod_ext) + test_mod_txt = '\n'.join([ + self.modgen.MODULE_SHEBANG, + self.modgen.swap_module('one', 'one/1.1'), + ]) + write_file(test_mod, test_mod_txt) + + # prepare environment for loading test module + self.modtool.use(self.test_prefix) + self.modtool.load(['one/1.0']) + + self.modtool.load(['test/1.0']) + + # check whether resulting environment is correct + loaded_mods = self.modtool.list() + self.assertEqual(loaded_mods[-1]['mod_name'], 'test/1.0') + # one/1.0 module was swapped for one/1.1 + self.assertEqual(loaded_mods[-2]['mod_name'], 'one/1.1') + class TclModuleGeneratorTest(ModuleGeneratorTest): """Test for module_generator module for Tcl syntax.""" From bb43297e6e2dfb0de451a2b50cf334cf81971198 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 18 May 2021 19:44:07 +0200 Subject: [PATCH 298/864] clarify comment w.r.t. not using value of environment variable but name thereof in probe_external_module_metadata + fix test --- easybuild/framework/easyconfig/easyconfig.py | 11 +++++++---- test/framework/easyconfig.py | 7 ++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 66a20e1e78..382a1c18bf 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1326,15 +1326,18 @@ def mk_var_name_pair(var_name_pair, name): # if a version is already set in the available metadata, we retain it if 'version' not in existing_metadata: - # should not use [version], because this is a greedy behaviour - # using version_var_name instead + # Use name of environment variable as value, not the current value of that environment variable. + # This is important in case the value of the environment variables changes by the time we really + # use it, for example by a loaded module being swapped with another version of that module. + # This is particularly important w.r.t. integration with the Cray Programming Environment, + # cfr. https://github.com/easybuilders/easybuild-framework/pull/3559. res['version'] = [version_var_name] self.log.info('setting external module %s version to be %s', mod_name, version) # if a prefix is already set in the available metadata, we retain it if 'prefix' not in existing_metadata: - # should not use prefix, because this is a greedy behaviour - # using prefix_var_name instead + # Use name of environment variable as value, not the current value of that environment variable. + # (see above for more info) res['prefix'] = prefix_var_name self.log.info('setting external module %s prefix to be %s', mod_name, prefix_var_name) break diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index a1948db260..9670d53c2d 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1738,7 +1738,8 @@ def test_external_dependencies(self): 'name = TEST', '[cray-netcdf-hdf5parallel/1.10.6]', 'name = HDF5', - 'version = 1.10.6', + # purpose omit version, to see whether fallback of + # resolving $CRAY_NETCDF_HDF5PARALLEL_VERSION at runtime is used ]) write_file(metadata, metadatatxt) build_options = { @@ -1758,7 +1759,7 @@ def test_external_dependencies(self): foobar_metadata = { 'name': ['foobar'], # probed from 'foobar' module 'prefix': 'CRAY_FOOBAR_DIR', # probed from 'foobar' module - 'version': ['CRAY_FOOBAR_VERSION'], # from [foobar/1.2.3] entry in metadata file + 'version': ['1.2.3'], # from [foobar/1.2.3] entry in metadata file } self.assertEqual(deps[3]['external_module_metadata'], foobar_metadata) @@ -1772,7 +1773,7 @@ def test_external_dependencies(self): pi_metadata = { 'name': ['PI'], # from [pi/3.14] entry in metadata file 'prefix': 'PI_ROOT', # probed from 'pi/3.14' module - 'version': ['PI_VERSION'], # from [pi/3.14] entry in metadata file + 'version': ['3.14.0'], # from [pi/3.14] entry in metadata file } self.assertEqual(deps[5]['external_module_metadata'], pi_metadata) From 03b78d8aec74cff55ec0292a69a270ccc5088bb7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 18 May 2021 20:12:55 +0200 Subject: [PATCH 299/864] update HierarchicalMNS to also return 'Toolchain//' as $MODULEPATH extension for cpe* Cray toolchains (fixes #3575) --- .../tools/module_naming_scheme/hierarchical_mns.py | 11 +++++++++-- test/framework/module_generator.py | 4 ++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py index 673a31b6f5..7c9702cf71 100644 --- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py +++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py @@ -70,6 +70,12 @@ 'xlc,xlf': ('xlcxlf', '%(xlc)s'), } +# possible prefixes for Cray toolchain names +# example: CrayGNU, CrayCCE, cpeGNU, cpeCCE, ...; +# important for determining $MODULEPATH extensions in det_modpath_extensions, +# cfr. https://github.com/easybuilders/easybuild-framework/issues/3575 +CRAY_TOOLCHAIN_NAME_PREFIXES = ('Cray', 'cpe') + class HierarchicalMNS(ModuleNamingScheme): """Class implementing an example hierarchical module naming scheme.""" @@ -236,8 +242,9 @@ def det_modpath_extensions(self, ec): paths.append(os.path.join(MPI, tc_comp_name, tc_comp_ver, ec['name'], fullver)) # special case for Cray toolchains - elif modclass == MODULECLASS_TOOLCHAIN and tc_comp_info is None and ec.name.startswith('Cray'): - paths.append(os.path.join(TOOLCHAIN, ec.name, ec.version)) + elif modclass == MODULECLASS_TOOLCHAIN and tc_comp_info is None: + if any(ec.name.startswith(x) for x in CRAY_TOOLCHAIN_NAME_PREFIXES): + paths.append(os.path.join(TOOLCHAIN, ec.name, ec.version)) return paths diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 00ee2cd8ae..49c8f4a1aa 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -1326,6 +1326,10 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, ['Toolchain/CrayCCE/5.1.29'], ['Toolchain/CrayCCE/5.1.29'], ['Core']), + 'cpeGNU-21.04.eb': ('cpeGNU/21.04', 'Core', + ['Toolchain/cpeGNU/21.04'], + ['Toolchain/cpeGNU/21.04'], + ['Core']), 'HPL-2.1-CrayCCE-5.1.29.eb': ('HPL/2.1', 'Toolchain/CrayCCE/5.1.29', [], [], ['Core']), } for ecfile, mns_vals in test_ecs.items(): From c7004700fab1991797fcf0ac3d4867450d72cdcb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 19 May 2021 09:55:50 +0200 Subject: [PATCH 300/864] skip test_generated_module_file_swap when using Lua syntax is modules tool is not Lmod --- test/framework/module_generator.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index c130f8c01d..1f5e827e13 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -1464,8 +1464,15 @@ def test_det_installdir(self): def test_generated_module_file_swap(self): """Test loading a generated module file that includes swap statements.""" + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorLua: mod_ext = '.lua' + + if not isinstance(self.modtool, Lmod): + # Lua module files are only supported by Lmod, + # so early exit if that's not the case in the test setup + return + else: mod_ext = '' From 883cb265abf50b5048fe68c4411efc6a2dfa77ba Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 19 May 2021 09:57:17 +0200 Subject: [PATCH 301/864] add additional test easyconfig to check HierarchicalMNS with new cpeGNU Cray toolchain --- .../test_ecs/c/cpeGNU/cpeGNU-21.04.eb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 test/framework/easyconfigs/test_ecs/c/cpeGNU/cpeGNU-21.04.eb diff --git a/test/framework/easyconfigs/test_ecs/c/cpeGNU/cpeGNU-21.04.eb b/test/framework/easyconfigs/test_ecs/c/cpeGNU/cpeGNU-21.04.eb new file mode 100644 index 0000000000..cf9af655db --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/c/cpeGNU/cpeGNU-21.04.eb @@ -0,0 +1,16 @@ +easyblock = 'Toolchain' + +name = 'cpeGNU' +version = '21.04' + +homepage = 'https://pubs.cray.com' +description = """Toolchain using Cray compiler wrapper with gcc module (CPE release: %(version)s).""" + +toolchain = SYSTEM + +dependencies = [ + ('PrgEnv-gnu', EXTERNAL_MODULE), + ('cpe/%(version)s', EXTERNAL_MODULE), +] + +moduleclass = 'toolchain' From 9bbe29a95b429c92ddd150247816342934a23603 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 19 May 2021 16:10:20 +0200 Subject: [PATCH 302/864] fix test easyconfigs count in test_index_functions --- test/framework/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 36d2bc0194..55fa363858 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2071,7 +2071,7 @@ def test_index_functions(self): # test with specified path with and without trailing '/'s for path in [test_ecs, test_ecs + '/', test_ecs + '//']: index = ft.create_index(path) - self.assertEqual(len(index), 84) + self.assertEqual(len(index), 85) expected = [ os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), From 4e3d8580e527e0d6d56944d0f03f87a1ad8a5a72 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 19 May 2021 18:00:46 +0200 Subject: [PATCH 303/864] don't pass references to internal data structure in EasyConfig.set_keys, make a copy instead --- easybuild/framework/easyconfig/easyconfig.py | 3 ++- test/framework/easyconfig.py | 26 ++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 382a1c18bf..9a30fb8228 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -672,7 +672,8 @@ def set_keys(self, params): for key in sorted(params.keys()): # validations are skipped, just set in the config if key in self._config.keys(): - self[key] = params[key] + # don't pass mutable values like lists/dicts by reference, copy them + self[key] = copy.deepcopy(params[key]) self.log.info("setting easyconfig parameter %s: value %s (type: %s)", key, self[key], type(self[key])) else: diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 9670d53c2d..6c5548e5b2 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4314,6 +4314,32 @@ def test_det_copy_ec_specs(self): self.assertEqual(os.path.basename(paths[3]), bat_patch_fn) self.assertTrue(os.path.samefile(target_path, cwd)) + def test_pure_ec(self): + """ + Test whether we can get a 'pure' view on the easyconfig file, + which correctly reflects what's defined in the easyconfig file. + """ + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')) + + ec_dict = toy_ec.parser.get_config_dict() + self.assertEqual(ec_dict.get('version'), '0.0') + self.assertEqual(ec_dict.get('sources'), ['%(name)s-%(version)s.tar.gz']) + self.assertEqual(ec_dict.get('exts_default_options'), None) + self.assertEqual(ec_dict.get('sanity_check_paths'), {'dirs': ['bin'], 'files': [('bin/yot', 'bin/toy')]}) + + # manipulating easyconfig parameter values should not affect the result of parser.get_config_dict() + with toy_ec.disable_templating(): + toy_ec['version'] = '1.2.3' + toy_ec['sources'].append('test.tar.gz') + toy_ec['sanity_check_paths']['files'].append('bin/foobar.exe') + + ec_dict_bis = toy_ec.parser.get_config_dict() + self.assertEqual(ec_dict_bis.get('version'), '0.0') + self.assertEqual(ec_dict_bis.get('sources'), ['%(name)s-%(version)s.tar.gz']) + self.assertEqual(ec_dict_bis.get('exts_default_options'), None) + self.assertEqual(ec_dict.get('sanity_check_paths'), {'dirs': ['bin'], 'files': [('bin/yot', 'bin/toy')]}) + def suite(): """ returns all the testcases in this module """ From 791f4e10815f84b0b89299e8d6392084cc0470b9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 19 May 2021 21:13:24 +0200 Subject: [PATCH 304/864] make get_config_obj return a copy rather than a reference --- easybuild/framework/easyconfig/easyconfig.py | 3 +-- easybuild/framework/easyconfig/format/one.py | 5 ++++- easybuild/framework/easyconfig/format/yeb.py | 6 ++++-- test/framework/easyconfigparser.py | 16 ++++++++++++++++ test/framework/yeb.py | 15 +++++++++++++++ 5 files changed, 40 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 9a30fb8228..382a1c18bf 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -672,8 +672,7 @@ def set_keys(self, params): for key in sorted(params.keys()): # validations are skipped, just set in the config if key in self._config.keys(): - # don't pass mutable values like lists/dicts by reference, copy them - self[key] = copy.deepcopy(params[key]) + self[key] = params[key] self.log.info("setting easyconfig parameter %s: value %s (type: %s)", key, self[key], type(self[key])) else: diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index 71d7b929ae..d42f44c497 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -30,6 +30,7 @@ :author: Stijn De Weirdt (Ghent University) :author: Kenneth Hoste (Ghent University) """ +import copy import os import pprint import re @@ -129,7 +130,9 @@ def get_config_dict(self): if spec_tc_version is not None and not spec_tc_version == tc_version: raise EasyBuildError('Requested toolchain version %s not available, only %s', spec_tc_version, tc_version) - return cfg + # avoid passing anything by reference, so next time get_config_dict is called + # we can be sure we return a dictionary that correctly reflects the contents of the easyconfig file + return copy.deepcopy(cfg) def parse(self, txt): """ diff --git a/easybuild/framework/easyconfig/format/yeb.py b/easybuild/framework/easyconfig/format/yeb.py index 18245de7ec..4f63034ad2 100644 --- a/easybuild/framework/easyconfig/format/yeb.py +++ b/easybuild/framework/easyconfig/format/yeb.py @@ -29,7 +29,7 @@ :author: Caroline De Brouwer (Ghent University) :author: Kenneth Hoste (Ghent University) """ - +import copy import os import platform from distutils.version import LooseVersion @@ -91,7 +91,9 @@ def get_config_dict(self): """ Return parsed easyconfig as a dictionary, based on specified arguments. """ - return self.parsed_yeb + # avoid passing anything by reference, so next time get_config_dict is called + # we can be sure we return a dictionary that correctly reflects the contents of the easyconfig file + return copy.deepcopy(self.parsed_yeb) @only_if_module_is_available('yaml') def parse(self, txt): diff --git a/test/framework/easyconfigparser.py b/test/framework/easyconfigparser.py index 436858ffb2..a7eb0098d6 100644 --- a/test/framework/easyconfigparser.py +++ b/test/framework/easyconfigparser.py @@ -59,6 +59,14 @@ def test_v10(self): self.assertEqual(ec['name'], 'GCC') self.assertEqual(ec['version'], '4.6.3') + # changes to this dict should not affect the return value of the next call to get_config_dict + fn = 'test.tar.gz' + ec['sources'].append(fn) + + ec_bis = ecp.get_config_dict() + self.assertTrue(fn in ec['sources']) + self.assertFalse(fn in ec_bis['sources']) + def test_v20(self): """Test parsing of easyconfig in format v2.""" # hard enable experimental @@ -81,6 +89,14 @@ def test_v20(self): self.assertEqual(ec['name'], 'GCC') self.assertEqual(ec['version'], '4.6.2') + # changes to this dict should not affect the return value of the next call to get_config_dict + fn = 'test.tar.gz' + ec['sources'].append(fn) + + ec_bis = ecp.get_config_dict() + self.assertTrue(fn in ec['sources']) + self.assertFalse(fn in ec_bis['sources']) + # restore easybuild.tools.build_log.EXPERIMENTAL = orig_experimental diff --git a/test/framework/yeb.py b/test/framework/yeb.py index 70ea97bd75..39ce4117bb 100644 --- a/test/framework/yeb.py +++ b/test/framework/yeb.py @@ -110,6 +110,21 @@ def test_parse_yeb(self): self.assertEqual(yeb_val, eb_val) + def test_yeb_get_config_obj(self): + """Test get_config_dict method.""" + testdir = os.path.dirname(os.path.abspath(__file__)) + test_yeb_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb') + ec = EasyConfig(os.path.join(test_yeb_easyconfigs, 'toy-0.0.yeb')) + ecdict = ec.parser.get_config_dict() + + # changes to this dict should not affect the return value of the next call to get_config_dict + fn = 'test.tar.gz' + ecdict['sources'].append(fn) + + ecdict_bis = ec.parser.get_config_dict() + self.assertTrue(fn in ecdict['sources']) + self.assertFalse(fn in ecdict_bis['sources']) + def test_is_yeb_format(self): """ Test is_yeb_format function """ testdir = os.path.dirname(os.path.abspath(__file__)) From d7e79a7770d82840ed4b88f2c9a9c4a1b086972b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 20 May 2021 17:31:59 +0200 Subject: [PATCH 305/864] allow opting out of recursively unloaded of modules via recursive_module_unload easyconfig parameter --- easybuild/framework/easyconfig/default.py | 5 +- easybuild/tools/module_generator.py | 18 ++++- test/framework/easyconfig.py | 89 ++++++++++++++++++++++- 3 files changed, 106 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 414b363d59..5279332541 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -196,7 +196,10 @@ 'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES], 'module_depends_on': [False, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module ' '(implies recursive unloading of modules).', MODULES], - 'recursive_module_unload': [False, 'Recursive unload of all dependencies when unloading module', MODULES], + 'recursive_module_unload': [None, "Recursive unload of all dependencies when unloading module " + "(True/False to hard enable/disable; None implies honoring " + "the --recursive-module-unload EasyBuild configuration setting", + MODULES], # MODULES documentation easyconfig parameters # (docurls is part of MANDATORY) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 5b48f6cf22..685f158850 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -793,12 +793,13 @@ def getenv_cmd(self, envvar, default=None): cmd = '[if { [info exists %(envvar)s] } { concat $%(envvar)s } else { concat "%(default)s" } ]' % values return cmd - def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None, multi_dep_mods=None): + def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_modules=None, multi_dep_mods=None): """ Generate load statement for specified module. :param mod_name: name of module to generate load statement for :param recursive_unload: boolean indicating whether the 'load' statement should be reverted on unload + (if None: enable if recursive_mod_unload build option or depends_on is True) :param depends_on: use depends_on statements rather than (guarded) load statements :param unload_modules: name(s) of module to unload first :param multi_dep_mods: list of module names in multi_deps context, to use for guarding load statement @@ -818,7 +819,11 @@ def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload depends_on = load_template == self.LOAD_TEMPLATE_DEPENDS_ON cond_tmpl = None - if build_option('recursive_mod_unload') or recursive_unload or depends_on: + + if recursive_unload is None: + recursive_unload = build_option('recursive_mod_unload') or depends_on + + if recursive_unload: # wrapping the 'module load' statement with an 'is-loaded or mode == unload' # guard ensures recursive unloading while avoiding load storms; # when "module unload" is called on the module in which the @@ -1217,12 +1222,13 @@ def getenv_cmd(self, envvar, default=None): cmd = 'os.getenv("%s") or "%s"' % (envvar, default) return cmd - def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None, multi_dep_mods=None): + def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_modules=None, multi_dep_mods=None): """ Generate load statement for specified module. :param mod_name: name of module to generate load statement for :param recursive_unload: boolean indicating whether the 'load' statement should be reverted on unload + (if None: enable if recursive_mod_unload build option or depends_on is True) :param depends_on: use depends_on statements rather than (guarded) load statements :param unload_modules: name(s) of module to unload first :param multi_dep_mods: list of module names in multi_deps context, to use for guarding load statement @@ -1243,7 +1249,11 @@ def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload depends_on = load_template == self.LOAD_TEMPLATE_DEPENDS_ON cond_tmpl = None - if build_option('recursive_mod_unload') or recursive_unload or depends_on: + + if recursive_unload is None: + recursive_unload = build_option('recursive_mod_unload') or depends_on + + if recursive_unload: # wrapping the 'module load' statement with an 'is-loaded or mode == unload' # guard ensures recursive unloading while avoiding load storms; # when "module unload" is called on the module in which the diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 9670d53c2d..486f0145b6 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -63,7 +63,7 @@ from easybuild.framework.extension import resolve_exts_filter_template from easybuild.toolchains.system import SystemToolchain from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import module_classes +from easybuild.tools.config import build_option, get_module_syntax, module_classes, update_build_option from easybuild.tools.configobj import ConfigObj from easybuild.tools.docs import avail_easyconfig_constants, avail_easyconfig_templates from easybuild.tools.filetools import adjust_permissions, change_dir, copy_file, mkdir, read_file @@ -4314,6 +4314,93 @@ def test_det_copy_ec_specs(self): self.assertEqual(os.path.basename(paths[3]), bat_patch_fn) self.assertTrue(os.path.samefile(target_path, cwd)) + def test_recursive_module_unload(self): + """Test use of recursive_module_unload easyconfig parameter.""" + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb') + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + write_file(test_ec, test_ec_txt) + + test_module = os.path.join(self.test_installpath, 'modules', 'all', 'foss', '2018a') + gcc_modname = 'GCC/6.4.0-2.28' + if get_module_syntax() == 'Lua': + test_module += '.lua' + guarded_load_pat = r'if not \( isloaded\("%(mod)s"\) \) then\n\s*load\("%(mod)s"\)' + recursive_unload_pat = r'if mode\(\) == "unload" or not \( isloaded\("%(mod)s"\) \) then\n' + recursive_unload_pat += r'\s*load\("%(mod)s"\)' + else: + guarded_load_pat = r'if { \!\[ is-loaded %(mod)s \] } {\n\s*module load %(mod)s' + recursive_unload_pat = r'if { \[ module-info mode remove \] \|\| \!\[ is-loaded %(mod)s \] } {\n' + recursive_unload_pat += r'\s*module load %(mod)s' + + guarded_load_regex = re.compile(guarded_load_pat % {'mod': gcc_modname}, re.M) + recursive_unload_regex = re.compile(recursive_unload_pat % {'mod': gcc_modname}, re.M) + + # by default, recursive module unloading is disabled everywhere + # (--recursive-module-unload configuration option is disabled, + # recursive_module_unload easyconfig parameter is None) + self.assertFalse(build_option('recursive_mod_unload')) + ec = EasyConfig(test_ec) + self.assertFalse(ec['recursive_module_unload']) + eb = EasyBlock(ec) + eb.builddir = self.test_prefix + eb.prepare_step() + eb.make_module_step() + modtxt = read_file(test_module) + fail_msg = "Pattern '%s' should be found in: %s" % (guarded_load_regex.pattern, modtxt) + self.assertTrue(guarded_load_regex.search(modtxt), fail_msg) + fail_msg = "Pattern '%s' should not be found in: %s" % (recursive_unload_regex.pattern, modtxt) + self.assertFalse(recursive_unload_regex.search(modtxt), fail_msg) + + remove_file(test_module) + + # recursive_module_unload easyconfig parameter is honored + test_ec_bis = os.path.join(self.test_prefix, 'test_bis.eb') + test_ec_bis_txt = read_file(toy_ec) + '\nrecursive_module_unload = True' + write_file(test_ec_bis, test_ec_bis_txt) + + ec_bis = EasyConfig(test_ec_bis) + self.assertTrue(ec_bis['recursive_module_unload']) + eb_bis = EasyBlock(ec_bis) + eb_bis.builddir = self.test_prefix + eb_bis.prepare_step() + eb_bis.make_module_step() + modtxt = read_file(test_module) + fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt) + self.assertFalse(guarded_load_regex.search(modtxt), fail_msg) + fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt) + self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg) + + # recursive_mod_unload build option is honored + update_build_option('recursive_mod_unload', True) + eb = EasyBlock(ec) + eb.builddir = self.test_prefix + eb.prepare_step() + eb.make_module_step() + modtxt = read_file(test_module) + fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt) + self.assertFalse(guarded_load_regex.search(modtxt), fail_msg) + fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt) + self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg) + + # disabling via easyconfig parameter works even when recursive_mod_unload build option is enabled + self.assertTrue(build_option('recursive_mod_unload')) + test_ec_bis = os.path.join(self.test_prefix, 'test_bis.eb') + test_ec_bis_txt = read_file(toy_ec) + '\nrecursive_module_unload = False' + write_file(test_ec_bis, test_ec_bis_txt) + ec_bis = EasyConfig(test_ec_bis) + self.assertEqual(ec_bis['recursive_module_unload'], False) + eb_bis = EasyBlock(ec_bis) + eb_bis.builddir = self.test_prefix + eb_bis.prepare_step() + eb_bis.make_module_step() + modtxt = read_file(test_module) + fail_msg = "Pattern '%s' should be found in: %s" % (guarded_load_regex.pattern, modtxt) + self.assertTrue(guarded_load_regex.search(modtxt), fail_msg) + fail_msg = "Pattern '%s' should not be found in: %s" % (recursive_unload_regex.pattern, modtxt) + self.assertFalse(recursive_unload_regex.search(modtxt), fail_msg) + def suite(): """ returns all the testcases in this module """ From 1e296cfb195f1cfaabbbbac70206a84da8b081ff Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 20 May 2021 17:32:51 +0200 Subject: [PATCH 306/864] fix regex used to check for lack of 'is loaded' guard in generated module file when using Lua syntax --- test/framework/options.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index d2e806df95..691b69ef12 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1974,10 +1974,15 @@ def test_recursive_module_unload(self): self.eb_main(args, do_build=True, verbose=True) toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-deps') + if get_module_syntax() == 'Lua': toy_module += '.lua' + is_loaded_regex = re.compile(r'if not \( isloaded\("gompi/2018a"\) \)', re.M) + else: + # Tcl syntax + is_loaded_regex = re.compile(r"if { !\[is-loaded gompi/2018a\] }", re.M) + toy_module_txt = read_file(toy_module) - is_loaded_regex = re.compile(r"if { !\[is-loaded gompi/2018a\] }", re.M) self.assertFalse(is_loaded_regex.search(toy_module_txt), "Recursive unloading is used: %s" % toy_module_txt) def test_tmpdir(self): From 2138971a1c37146a240302e4f2177f1c0e89d9be Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Fri, 21 May 2021 16:14:11 +0800 Subject: [PATCH 307/864] make sure fcc is always called with -Nclang and enhance COMPILER_UNIQUE_OPTION_MAP --- easybuild/toolchains/compiler/fujitsu.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/easybuild/toolchains/compiler/fujitsu.py b/easybuild/toolchains/compiler/fujitsu.py index 27efc93c6b..4c504a23e0 100644 --- a/easybuild/toolchains/compiler/fujitsu.py +++ b/easybuild/toolchains/compiler/fujitsu.py @@ -43,23 +43,30 @@ class FujitsuCompiler(Compiler): COMPILER_MODULE_NAME = ['lang'] COMPILER_FAMILY = TC_CONSTANT_FUJITSU - COMPILER_CC = 'fcc' - COMPILER_CXX = 'FCC' + COMPILER_CC = 'fcc -Nclang' + COMPILER_CXX = 'FCC -Nclang' COMPILER_F77 = 'frt' COMPILER_F90 = 'frt' COMPILER_FC = 'frt' COMPILER_UNIQUE_OPTION_MAP = { - DEFAULT_OPT_LEVEL: 'Kfast', + DEFAULT_OPT_LEVEL: 'O2 -Kfast', + 'lowopt': 'O1', + 'noopt': 'O0', + 'opt': 'O3 -Kfast', 'optarch': '', # Fujitsu compiler by default generates code for the arch it is running on 'openmp': 'Kopenmp', 'unroll': 'funroll-loops', - 'strict': ['Kfp_precision'], - 'precise': ['Kfp_precision'], + # apparently the -Kfp_precision flag doesn't work in clang mode, will need to look into these later + # also at strict vs precise and loose vs veryloose + 'strict': ['Knoeval,nofast_matmul,nofp_contract,nofp_relaxed,noilfunc'], # ['Kfp_precision'], + 'precise': ['Knoeval,nofast_matmul,nofp_contract,nofp_relaxed,noilfunc'], # ['Kfp_precision'], 'defaultprec': [], 'loose': ['Kfp_relaxed'], 'veryloose': ['Kfp_relaxed'], - 'vectorize': {False: 'KNOSVE', True: 'KSVE'}, + # apparently the -K[NO]SVE flags don't work in clang mode + # SVE is enabled by default, -Knosimd seems to disable it + 'vectorize': {False: 'Knosimd', True: ''}, } # used when 'optarch' toolchain option is enabled (and --optarch is not specified) @@ -77,5 +84,5 @@ def _set_compiler_vars(self): super(FujitsuCompiler, self)._set_compiler_vars() # enable clang compatibility mode - self.variables.nappend('CFLAGS', ['Nclang']) - self.variables.nappend('CXXFLAGS', ['Nclang']) + # self.variables.nappend('CFLAGS', ['Nclang']) + # self.variables.nappend('CXXFLAGS', ['Nclang']) From ecaedc4cd03f892498f834f2dc4a4eb4789ed56e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 21 May 2021 13:46:42 +0200 Subject: [PATCH 308/864] don't use copy.deepcopy directly in get_config_dict --- easybuild/framework/easyconfig/format/one.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index d42f44c497..84324c734d 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -131,8 +131,14 @@ def get_config_dict(self): raise EasyBuildError('Requested toolchain version %s not available, only %s', spec_tc_version, tc_version) # avoid passing anything by reference, so next time get_config_dict is called - # we can be sure we return a dictionary that correctly reflects the contents of the easyconfig file - return copy.deepcopy(cfg) + # we can be sure we return a dictionary that correctly reflects the contents of the easyconfig file; + # we can't use copy.deepcopy() directly because in Python 2 copying the (irrelevant) __builtins__ key fails... + cfg_copy = {} + for key in cfg: + if key != '__builtins__': + cfg_copy[key] = copy.deepcopy(cfg[key]) + + return cfg_copy def parse(self, txt): """ From 829aca0b5e41ec35d5edb8fd866357abee070174 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 21 May 2021 15:14:11 +0200 Subject: [PATCH 309/864] move easystack tests to separate test module --- test/framework/easystack.py | 97 +++++++++++++++++++++++++++++++++++++ test/framework/options.py | 34 ------------- test/framework/suite.py | 5 +- 3 files changed, 100 insertions(+), 36 deletions(-) create mode 100644 test/framework/easystack.py diff --git a/test/framework/easystack.py b/test/framework/easystack.py new file mode 100644 index 0000000000..f4ae54f9a4 --- /dev/null +++ b/test/framework/easystack.py @@ -0,0 +1,97 @@ +# # +# Copyright 2013-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Unit tests for easystack files + +@author: Denis Kristak (Inuits) +@author: Kenneth Hoste (Ghent University) +""" +import os +import sys +from unittest import TextTestRunner + +import easybuild.tools.build_log +from easybuild.framework.easystack import parse_easystack +from easybuild.tools.build_log import EasyBuildError +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered + + +class EasyStackTest(EnhancedTestCase): + """Testcases for easystack files.""" + + logfile = None + + def setUp(self): + """Set up test.""" + super(EasyStackTest, self).setUp() + self.orig_experimental = easybuild.tools.build_log.EXPERIMENTAL + + def tearDown(self): + """Clean up after test.""" + easybuild.tools.build_log.EXPERIMENTAL = self.orig_experimental + super(EasyStackTest, self).tearDown() + + def test_easystack_wrong_structure(self): + """Test for --easystack when yaml easystack has wrong structure""" + easybuild.tools.build_log.EXPERIMENTAL = True + topdir = os.path.dirname(os.path.abspath(__file__)) + test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_wrong_structure.yaml') + + expected_err = r"[\S\s]*An error occurred when interpreting the data for software Bioconductor:" + expected_err += r"( 'float' object is not subscriptable[\S\s]*" + expected_err += r"| 'float' object is unsubscriptable" + expected_err += r"| 'float' object has no attribute '__getitem__'[\S\s]*)" + self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, test_easystack) + + def test_easystack_asterisk(self): + """Test for --easystack when yaml easystack contains asterisk (wildcard)""" + easybuild.tools.build_log.EXPERIMENTAL = True + topdir = os.path.dirname(os.path.abspath(__file__)) + test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_asterisk.yaml') + + expected_err = "EasyStack specifications of 'binutils' in .*/test_easystack_asterisk.yaml contain asterisk. " + expected_err += "Wildcard feature is not supported yet." + + self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, test_easystack) + + def test_easystack_labels(self): + """Test for --easystack when yaml easystack contains exclude-labels / include-labels""" + easybuild.tools.build_log.EXPERIMENTAL = True + topdir = os.path.dirname(os.path.abspath(__file__)) + test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_labels.yaml') + + error_msg = "EasyStack specifications of 'binutils' in .*/test_easystack_labels.yaml contain labels. " + error_msg += "Labels aren't supported yet." + self.assertErrorRegex(EasyBuildError, error_msg, parse_easystack, test_easystack) + + +def suite(): + """ returns all the testcases in this module """ + return TestLoaderFiltered().loadTestsFromTestCase(EasyStackTest, sys.argv[1:]) + + +if __name__ == '__main__': + res = TextTestRunner(verbosity=1).run(suite()) + sys.exit(len(res.failures)) diff --git a/test/framework/options.py b/test/framework/options.py index 691b69ef12..f201e38492 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -42,7 +42,6 @@ import easybuild.tools.toolchain from easybuild.base import fancylogger from easybuild.framework.easyblock import EasyBlock -from easybuild.framework.easystack import parse_easystack from easybuild.framework.easyconfig import BUILD, CUSTOM, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT, LICENSE from easybuild.framework.easyconfig import MANDATORY, MODULES, OTHER, TOOLCHAIN from easybuild.framework.easyconfig.easyconfig import EasyConfig, get_easyblock_class, robot_find_easyconfig @@ -6033,39 +6032,6 @@ def test_easystack_basic(self): regex = re.compile(pattern) self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) - def test_easystack_wrong_structure(self): - """Test for --easystack when yaml easystack has wrong structure""" - easybuild.tools.build_log.EXPERIMENTAL = True - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_wrong_structure.yaml') - - expected_err = r"[\S\s]*An error occurred when interpreting the data for software Bioconductor:" - expected_err += r"( 'float' object is not subscriptable[\S\s]*" - expected_err += r"| 'float' object is unsubscriptable" - expected_err += r"| 'float' object has no attribute '__getitem__'[\S\s]*)" - self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, toy_easystack) - - def test_easystack_asterisk(self): - """Test for --easystack when yaml easystack contains asterisk (wildcard)""" - easybuild.tools.build_log.EXPERIMENTAL = True - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_asterisk.yaml') - - expected_err = "EasyStack specifications of 'binutils' in .*/test_easystack_asterisk.yaml contain asterisk. " - expected_err += "Wildcard feature is not supported yet." - - self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, toy_easystack) - - def test_easystack_labels(self): - """Test for --easystack when yaml easystack contains exclude-labels / include-labels""" - easybuild.tools.build_log.EXPERIMENTAL = True - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_labels.yaml') - - error_msg = "EasyStack specifications of 'binutils' in .*/test_easystack_labels.yaml contain labels. " - error_msg += "Labels aren't supported yet." - self.assertErrorRegex(EasyBuildError, error_msg, parse_easystack, toy_easystack) - def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/suite.py b/test/framework/suite.py index 41c13d188f..1633bba103 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -49,8 +49,9 @@ import test.framework.easyconfig as e import test.framework.easyconfigparser as ep import test.framework.easyconfigformat as ef -import test.framework.ebconfigobj as ebco import test.framework.easyconfigversion as ev +import test.framework.easystack as es +import test.framework.ebconfigobj as ebco import test.framework.environment as env import test.framework.docs as d import test.framework.filetools as f @@ -119,7 +120,7 @@ # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, lic, f_c, - tw, p, i, pkg, d, env, et, y, st, h, ct, lib, u] + tw, p, i, pkg, d, env, et, y, st, h, ct, lib, u, es] SUITE = unittest.TestSuite([x.suite() for x in tests]) res = unittest.TextTestRunner().run(SUITE) From 5ec7d50e2a16d5f06b7c4492446945c13f945b4f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 21 May 2021 18:59:18 +0200 Subject: [PATCH 310/864] check for correct version values when parsing easystack file --- easybuild/framework/easystack.py | 64 +++++++---- test/framework/easystack.py | 108 +++++++++++++++++- .../easystacks/test_easystack_basic.yaml | 6 +- .../easystacks/test_easystack_labels.yaml | 2 +- 4 files changed, 148 insertions(+), 32 deletions(-) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index 8607734774..927f34ec51 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -27,12 +27,13 @@ :author: Denis Kristak (Inuits) :author: Pavel Grochal (Inuits) +:author: Kenneth Hoste (HPC-UGent) """ - from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version +from easybuild.tools.py2vs3 import string_type from easybuild.tools.utilities import only_if_module_is_available try: import yaml @@ -41,6 +42,25 @@ _log = fancylogger.getLogger('easystack', fname=False) +def check_version(value, context): + """ + Check whether specified value obtained from a YAML file in specified context represents a valid version. + The value must be a string value (not a float or an int). + """ + if not isinstance(value, string_type): + error_msg = '\n'.join([ + "Value %(value)s (of type %(type)s) obtained for %(context)s does not represent a valid version!", + "Make sure to wrap the value in single quotes (like '%(value)s') to avoid that it is interpreted " + "by the YAML parser as a non-string value.", + ]) + format_info = { + 'context': context, + 'type': type(value), + 'value': value, + } + raise EasyBuildError(error_msg % format_info) + + class EasyStack(object): """One class instance per easystack. General options + list of all SoftwareSpecs instances""" @@ -90,7 +110,12 @@ class EasyStackParser(object): def parse(filepath): """Parses YAML file and assigns obtained values to SW config instances as well as general config instance""" yaml_txt = read_file(filepath) - easystack_raw = yaml.safe_load(yaml_txt) + + try: + easystack_raw = yaml.safe_load(yaml_txt) + except yaml.YAMLError as err: + raise EasyBuildError("Failed to parse %s: %s" % (filepath, err)) + easystack = EasyStack() try: @@ -123,6 +148,8 @@ def parse(filepath): raise EasyBuildError("Incorrect toolchain specification for '%s' in %s, too many parts: %s", name, filepath, toolchain_parts) + check_version(toolchain_version, "software %s (with %s toolchain)" % (name, toolchain_name)) + try: # if version string containts asterisk or labels, raise error (asterisks not supported) versions = toolchains[toolchain]['versions'] @@ -146,6 +173,7 @@ def parse(filepath): # ======================================================================== if isinstance(versions, dict): for version in versions: + check_version(version, "%s (with %s toolchain)" % (name, toolchain_name)) if versions[version] is not None: version_spec = versions[version] if 'versionsuffix' in version_spec: @@ -172,35 +200,25 @@ def parse(filepath): easystack.software_list.append(sw) continue - # is format read as a list of versions? - # ======================================================================== - # versions: - # [2.24, 2.51] - # ======================================================================== - elif isinstance(versions, list): - versions_list = versions + elif isinstance(versions, (list, tuple)): + pass - # format = multiple lines without ':' (read as a string)? - # ======================================================================== + # multiple lines without ':' is read as a single string; example: # versions: # 2.24 # 2.51 - # ======================================================================== - elif isinstance(versions, str): - versions_list = str(versions).split() + elif isinstance(versions, string_type): + versions = versions.split() - # format read as float (containing one version only)? - # ======================================================================== - # versions: - # 2.24 - # ======================================================================== - elif isinstance(versions, float): - versions_list = [str(versions)] + # single values like 2.24 should be wrapped in a list + else: + versions = [versions] - # if no version is a dictionary, versionsuffix isn't specified + # if version is not a dictionary, versionsuffix is not specified versionsuffix = '' - for version in versions_list: + for version in versions: + check_version(version, "%s (with %s toolchain)" % (name, toolchain_name)) sw = SoftwareSpecs( name=name, version=version, versionsuffix=versionsuffix, toolchain_name=toolchain_name, toolchain_version=toolchain_version) diff --git a/test/framework/easystack.py b/test/framework/easystack.py index f4ae54f9a4..4f26b10731 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -33,8 +33,9 @@ from unittest import TextTestRunner import easybuild.tools.build_log -from easybuild.framework.easystack import parse_easystack +from easybuild.framework.easystack import check_version, parse_easystack from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import write_file from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered @@ -47,15 +48,23 @@ def setUp(self): """Set up test.""" super(EasyStackTest, self).setUp() self.orig_experimental = easybuild.tools.build_log.EXPERIMENTAL + # easystack files are an experimental feature + easybuild.tools.build_log.EXPERIMENTAL = True def tearDown(self): """Clean up after test.""" easybuild.tools.build_log.EXPERIMENTAL = self.orig_experimental super(EasyStackTest, self).tearDown() + def test_parse_fail(self): + """Test for clean error when easystack file fails to parse.""" + test_yml = os.path.join(self.test_prefix, 'test.yml') + write_file(test_yml, 'software: %s') + error_pattern = "Failed to parse .*/test.yml: while scanning for the next token" + self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_yml) + def test_easystack_wrong_structure(self): """Test for --easystack when yaml easystack has wrong structure""" - easybuild.tools.build_log.EXPERIMENTAL = True topdir = os.path.dirname(os.path.abspath(__file__)) test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_wrong_structure.yaml') @@ -67,7 +76,6 @@ def test_easystack_wrong_structure(self): def test_easystack_asterisk(self): """Test for --easystack when yaml easystack contains asterisk (wildcard)""" - easybuild.tools.build_log.EXPERIMENTAL = True topdir = os.path.dirname(os.path.abspath(__file__)) test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_asterisk.yaml') @@ -77,8 +85,6 @@ def test_easystack_asterisk(self): self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, test_easystack) def test_easystack_labels(self): - """Test for --easystack when yaml easystack contains exclude-labels / include-labels""" - easybuild.tools.build_log.EXPERIMENTAL = True topdir = os.path.dirname(os.path.abspath(__file__)) test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_labels.yaml') @@ -86,6 +92,98 @@ def test_easystack_labels(self): error_msg += "Labels aren't supported yet." self.assertErrorRegex(EasyBuildError, error_msg, parse_easystack, test_easystack) + def test_check_version(self): + """Test check_version function.""" + check_version('1.2.3', None) + check_version('1.2', None) + check_version('3.50', None) + check_version('100', None) + + context = "" + for version in (1.2, 100, None): + error_pattern = r"Value .* \(of type .*\) obtained for does not represent a valid version!" + self.assertErrorRegex(EasyBuildError, error_pattern, check_version, version, context) + + def test_easystack_versions(self): + """Test handling of versions in easystack files.""" + + test_easystack = os.path.join(self.test_prefix, 'test.yml') + tmpl_easystack_txt = '\n'.join([ + "software:", + " foo:", + " toolchains:", + " SYSTEM:", + " versions:", + ]) + + # normal versions, which are not treated special by YAML: no single quotes needed + versions = ('1.2.3', '1.2.30', '2021a', '1.2.3') + for version in versions: + write_file(test_easystack, tmpl_easystack_txt + ' ' + version) + ec_fns, _ = parse_easystack(test_easystack) + self.assertEqual(ec_fns, ['foo-%s.eb' % version]) + + # multiple versions as a list + test_easystack_txt = tmpl_easystack_txt + " [1.2.3, 3.2.1]" + write_file(test_easystack, test_easystack_txt) + ec_fns, _ = parse_easystack(test_easystack) + expected = ['foo-1.2.3.eb', 'foo-3.2.1.eb'] + self.assertEqual(sorted(ec_fns), sorted(expected)) + + # multiple versions listed with more info + test_easystack_txt = '\n'.join([ + tmpl_easystack_txt, + " 1.2.3:", + " 2021a:", + " 3.2.1:", + " versionsuffix: -foo", + ]) + write_file(test_easystack, test_easystack_txt) + ec_fns, _ = parse_easystack(test_easystack) + expected = ['foo-1.2.3.eb', 'foo-2021a.eb', 'foo-3.2.1-foo.eb'] + self.assertEqual(sorted(ec_fns), sorted(expected)) + + # versions that get interpreted by YAML as float or int, single quotes required + for version in ('1.2', '123', '3.50', '100', '2.44_01'): + error_pattern = r"Value .* \(of type .*\) obtained for foo \(with system toolchain\) " + error_pattern += r"does not represent a valid version\!" + + write_file(test_easystack, tmpl_easystack_txt + ' ' + version) + self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) + + # all is fine when wrapping the value in single quotes + write_file(test_easystack, tmpl_easystack_txt + " '" + version + "'") + ec_fns, _ = parse_easystack(test_easystack) + self.assertEqual(ec_fns, ['foo-%s.eb' % version]) + + # one rotten apple in the basket is enough + test_easystack_txt = tmpl_easystack_txt + " [1.2.3, %s, 3.2.1]" % version + write_file(test_easystack, test_easystack_txt) + self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) + + test_easystack_txt = '\n'.join([ + tmpl_easystack_txt, + " 1.2.3:", + " %s:" % version, + " 3.2.1:", + " versionsuffix: -foo", + ]) + write_file(test_easystack, test_easystack_txt) + self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) + + # single quotes to the rescue! + test_easystack_txt = '\n'.join([ + tmpl_easystack_txt, + " 1.2.3:", + " '%s':" % version, + " 3.2.1:", + " versionsuffix: -foo", + ]) + write_file(test_easystack, test_easystack_txt) + ec_fns, _ = parse_easystack(test_easystack) + expected = ['foo-1.2.3.eb', 'foo-%s.eb' % version, 'foo-3.2.1-foo.eb'] + self.assertEqual(sorted(ec_fns), sorted(expected)) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/easystacks/test_easystack_basic.yaml b/test/framework/easystacks/test_easystack_basic.yaml index 2de5dfd129..491f113f4a 100644 --- a/test/framework/easystacks/test_easystack_basic.yaml +++ b/test/framework/easystacks/test_easystack_basic.yaml @@ -3,8 +3,8 @@ software: toolchains: GCCcore-4.9.3: versions: - 2.25: - 2.26: + '2.25': + '2.26': foss: toolchains: SYSTEM: @@ -13,5 +13,5 @@ software: toolchains: gompi-2018a: versions: - 0.0: + '0.0': versionsuffix: '-test' diff --git a/test/framework/easystacks/test_easystack_labels.yaml b/test/framework/easystacks/test_easystack_labels.yaml index 51a113523f..f00db0e249 100644 --- a/test/framework/easystacks/test_easystack_labels.yaml +++ b/test/framework/easystacks/test_easystack_labels.yaml @@ -3,5 +3,5 @@ software: toolchains: GCCcore-4.9.3: versions: - 3.11: + '3.11': exclude-labels: arch:aarch64 From fc3c24edb69831259526f5c524dee2bf958f281e Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Sat, 22 May 2021 15:22:38 +0800 Subject: [PATCH 311/864] use more conservative optimization levels --- easybuild/toolchains/compiler/fujitsu.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/toolchains/compiler/fujitsu.py b/easybuild/toolchains/compiler/fujitsu.py index 4c504a23e0..f70ee3ca0e 100644 --- a/easybuild/toolchains/compiler/fujitsu.py +++ b/easybuild/toolchains/compiler/fujitsu.py @@ -43,17 +43,19 @@ class FujitsuCompiler(Compiler): COMPILER_MODULE_NAME = ['lang'] COMPILER_FAMILY = TC_CONSTANT_FUJITSU + # make sure fcc is always called in clang compatibility mode COMPILER_CC = 'fcc -Nclang' COMPILER_CXX = 'FCC -Nclang' + COMPILER_F77 = 'frt' COMPILER_F90 = 'frt' COMPILER_FC = 'frt' COMPILER_UNIQUE_OPTION_MAP = { - DEFAULT_OPT_LEVEL: 'O2 -Kfast', + DEFAULT_OPT_LEVEL: 'O2', 'lowopt': 'O1', 'noopt': 'O0', - 'opt': 'O3 -Kfast', + 'opt': 'Kfast', # -O3 -Keval,fast_matmul,fp_contract,fp_relaxed,fz,ilfunc,mfunc,omitfp,simd_packed_promotion 'optarch': '', # Fujitsu compiler by default generates code for the arch it is running on 'openmp': 'Kopenmp', 'unroll': 'funroll-loops', @@ -83,6 +85,6 @@ class FujitsuCompiler(Compiler): def _set_compiler_vars(self): super(FujitsuCompiler, self)._set_compiler_vars() - # enable clang compatibility mode + # enable clang compatibility mode; moved to compiler constants to make sure it is always used # self.variables.nappend('CFLAGS', ['Nclang']) # self.variables.nappend('CXXFLAGS', ['Nclang']) From f4f26c24bfba9434676984a96269d52e7bf00c56 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Sat, 22 May 2021 15:24:45 +0800 Subject: [PATCH 312/864] make sure linalg lib and include dirs are not set by easybuild, compiler flags take care of it --- easybuild/toolchains/linalg/fujitsussl.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/easybuild/toolchains/linalg/fujitsussl.py b/easybuild/toolchains/linalg/fujitsussl.py index 5422456c86..71a7647028 100644 --- a/easybuild/toolchains/linalg/fujitsussl.py +++ b/easybuild/toolchains/linalg/fujitsussl.py @@ -45,20 +45,28 @@ class FujitsuSSL(LinAlg): # via lang/tcsds module BLAS_MODULE_NAME = ['lang'] - # no need to specify libraries, compiler driver takes care of linking the right libraries + # no need to specify libraries nor includes, only the compiler flags below BLAS_LIB = [''] BLAS_LIB_MT = [''] + BLAS_INCLUDE_DIR = [''] BLAS_FAMILY = TC_CONSTANT_FUJITSU_SSL LAPACK_MODULE_NAME = None LAPACK_IS_BLAS = True + LAPACK_LIB = [''] + LAPACK_LIB_MT = [''] + LAPACK_INCLUDE_DIR = [''] LAPACK_FAMILY = TC_CONSTANT_FUJITSU_SSL BLACS_MODULE_NAME = None + BLACS_LIB = [''] + BLACS_LIB_MT = [''] + BLACS_INCLUDE_DIR = [''] SCALAPACK_MODULE_NAME = BLAS_MODULE_NAME SCALAPACK_LIB = [''] SCALAPACK_LIB_MT = [''] + SCALAPACK_INCLUDE_DIR = [''] SCALAPACK_FAMILY = TC_CONSTANT_FUJITSU_SSL def _get_software_root(self, name): From c91dc6c31a1208250e1c6ae373dbd7f257843bb0 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 22 May 2021 10:27:27 +0100 Subject: [PATCH 313/864] check name and toolchain are strings in easystack yaml --- easybuild/framework/easystack.py | 26 +++++++++++++------------- test/framework/easystack.py | 16 ++++++++-------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index 927f34ec51..0099e6973f 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -42,14 +42,14 @@ _log = fancylogger.getLogger('easystack', fname=False) -def check_version(value, context): +def check_value(value, context): """ - Check whether specified value obtained from a YAML file in specified context represents a valid version. - The value must be a string value (not a float or an int). + Check whether specified value obtained from a YAML file in specified context represents is valid. + The value must be a string (not a float or an int). """ if not isinstance(value, string_type): error_msg = '\n'.join([ - "Value %(value)s (of type %(type)s) obtained for %(context)s does not represent a valid version!", + "Value %(value)s (of type %(type)s) obtained for %(context)s is not valid!", "Make sure to wrap the value in single quotes (like '%(value)s') to avoid that it is interpreted " "by the YAML parser as a non-string value.", ]) @@ -128,12 +128,14 @@ def parse(filepath): for name in software: # ensure we have a string value (YAML parser returns type = dict # if levels under the current attribute are present) + check_value(toolchain, "software %s (with %s toolchain)" % (name, toolchain)) name = str(name) try: toolchains = software[name]['toolchains'] except KeyError: raise EasyBuildError("Toolchains for software '%s' are not defined in %s", name, filepath) for toolchain in toolchains: + check_value(toolchain, "software %s (with %s toolchain)" % (name, toolchain)) toolchain = str(toolchain) if toolchain == 'SYSTEM': @@ -148,8 +150,6 @@ def parse(filepath): raise EasyBuildError("Incorrect toolchain specification for '%s' in %s, too many parts: %s", name, filepath, toolchain_parts) - check_version(toolchain_version, "software %s (with %s toolchain)" % (name, toolchain_name)) - try: # if version string containts asterisk or labels, raise error (asterisks not supported) versions = toolchains[toolchain]['versions'] @@ -167,13 +167,13 @@ def parse(filepath): # Example of yaml structure: # ======================================================================== # versions: - # 2.25: - # 2.23: + # '2.25': + # '2.23': # versionsuffix: '-R-4.0.0' # ======================================================================== if isinstance(versions, dict): for version in versions: - check_version(version, "%s (with %s toolchain)" % (name, toolchain_name)) + check_value(version, "%s (with %s toolchain)" % (name, toolchain_name)) if versions[version] is not None: version_spec = versions[version] if 'versionsuffix' in version_spec: @@ -205,12 +205,12 @@ def parse(filepath): # multiple lines without ':' is read as a single string; example: # versions: - # 2.24 - # 2.51 + # '2.24' + # '2.51' elif isinstance(versions, string_type): versions = versions.split() - # single values like 2.24 should be wrapped in a list + # single values like '2.24' should be wrapped in a list else: versions = [versions] @@ -218,7 +218,7 @@ def parse(filepath): versionsuffix = '' for version in versions: - check_version(version, "%s (with %s toolchain)" % (name, toolchain_name)) + check_value(version, "%s (with %s toolchain)" % (name, toolchain_name)) sw = SoftwareSpecs( name=name, version=version, versionsuffix=versionsuffix, toolchain_name=toolchain_name, toolchain_version=toolchain_version) diff --git a/test/framework/easystack.py b/test/framework/easystack.py index 4f26b10731..3545965a37 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -33,7 +33,7 @@ from unittest import TextTestRunner import easybuild.tools.build_log -from easybuild.framework.easystack import check_version, parse_easystack +from easybuild.framework.easystack import check_value, parse_easystack from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import write_file from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered @@ -92,17 +92,17 @@ def test_easystack_labels(self): error_msg += "Labels aren't supported yet." self.assertErrorRegex(EasyBuildError, error_msg, parse_easystack, test_easystack) - def test_check_version(self): - """Test check_version function.""" - check_version('1.2.3', None) - check_version('1.2', None) - check_version('3.50', None) - check_version('100', None) + def test_check_value(self): + """Test check_value function.""" + check_value('1.2.3', None) + check_value('1.2', None) + check_value('3.50', None) + check_value('100', None) context = "" for version in (1.2, 100, None): error_pattern = r"Value .* \(of type .*\) obtained for does not represent a valid version!" - self.assertErrorRegex(EasyBuildError, error_pattern, check_version, version, context) + self.assertErrorRegex(EasyBuildError, error_pattern, check_value, version, context) def test_easystack_versions(self): """Test handling of versions in easystack files.""" From 1900f1d9b14d8bf53eb77a0501806dc08ced4214 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 22 May 2021 10:30:52 +0100 Subject: [PATCH 314/864] correct context and check --- easybuild/framework/easystack.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index 0099e6973f..d8078159ba 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -128,15 +128,13 @@ def parse(filepath): for name in software: # ensure we have a string value (YAML parser returns type = dict # if levels under the current attribute are present) - check_value(toolchain, "software %s (with %s toolchain)" % (name, toolchain)) - name = str(name) + check_value(name, "software name") try: toolchains = software[name]['toolchains'] except KeyError: raise EasyBuildError("Toolchains for software '%s' are not defined in %s", name, filepath) for toolchain in toolchains: - check_value(toolchain, "software %s (with %s toolchain)" % (name, toolchain)) - toolchain = str(toolchain) + check_value(toolchain, "software %s" % name) if toolchain == 'SYSTEM': toolchain_name, toolchain_version = 'system', '' From 4d47ae56eb96c4cebec1694a260e334e25a8d39e Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 22 May 2021 11:02:33 +0100 Subject: [PATCH 315/864] update the test to reflect the change to the error message --- test/framework/easystack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easystack.py b/test/framework/easystack.py index 3545965a37..1bd751cf0e 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -101,7 +101,7 @@ def test_check_value(self): context = "" for version in (1.2, 100, None): - error_pattern = r"Value .* \(of type .*\) obtained for does not represent a valid version!" + error_pattern = r"Value .* \(of type .*\) obtained for is not valid!" self.assertErrorRegex(EasyBuildError, error_pattern, check_value, version, context) def test_easystack_versions(self): From b0828618935ab2824bf15c4721d86820a79862bd Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 22 May 2021 11:18:16 +0100 Subject: [PATCH 316/864] update the test to reflect the change to the error message --- test/framework/easystack.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/framework/easystack.py b/test/framework/easystack.py index 1bd751cf0e..c528deae28 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -145,8 +145,7 @@ def test_easystack_versions(self): # versions that get interpreted by YAML as float or int, single quotes required for version in ('1.2', '123', '3.50', '100', '2.44_01'): - error_pattern = r"Value .* \(of type .*\) obtained for foo \(with system toolchain\) " - error_pattern += r"does not represent a valid version\!" + error_pattern = r"Value .* \(of type .*\) obtained for foo \(with system toolchain\) is not valid\!" write_file(test_easystack, tmpl_easystack_txt + ' ' + version) self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) From c02c785764a067f2f58d66f37771611cf58271db Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 22 May 2021 18:03:39 +0200 Subject: [PATCH 317/864] also check toolchain version in test_easystack_versions --- test/framework/easystack.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/framework/easystack.py b/test/framework/easystack.py index c528deae28..32313c8b06 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -183,6 +183,19 @@ def test_easystack_versions(self): expected = ['foo-1.2.3.eb', 'foo-%s.eb' % version, 'foo-3.2.1-foo.eb'] self.assertEqual(sorted(ec_fns), sorted(expected)) + # also check toolchain version that could be interpreted as a non-string value... + test_easystack_txt = '\n'.join([ + 'software:', + ' test:', + ' toolchains:', + ' intel-2021.03:', + " versions: [1.2.3, '2.3']", + ]) + write_file(test_easystack, test_easystack_txt) + ec_fns, _ = parse_easystack(test_easystack) + expected = ['test-1.2.3-intel-2021.03.eb', 'test-2.3-intel-2021.03.eb'] + self.assertEqual(sorted(ec_fns), sorted(expected)) + def suite(): """ returns all the testcases in this module """ From c095190d0dbc848e00c2efbbcd20f91f506bc128 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Sun, 23 May 2021 11:06:35 +0800 Subject: [PATCH 318/864] create FCC subtoolchain for Fujitsu Compiler only, move FujitsuSSL from ffmpi to Fujitsu full toolchain --- easybuild/toolchains/compiler/fujitsu.py | 3 +- easybuild/toolchains/fcc.py | 78 ++++++++++++++++++++++++ easybuild/toolchains/ffmpi.py | 12 ++-- easybuild/toolchains/fujitsu.py | 4 +- 4 files changed, 89 insertions(+), 8 deletions(-) create mode 100644 easybuild/toolchains/fcc.py diff --git a/easybuild/toolchains/compiler/fujitsu.py b/easybuild/toolchains/compiler/fujitsu.py index f70ee3ca0e..3d9a7e4175 100644 --- a/easybuild/toolchains/compiler/fujitsu.py +++ b/easybuild/toolchains/compiler/fujitsu.py @@ -85,6 +85,7 @@ class FujitsuCompiler(Compiler): def _set_compiler_vars(self): super(FujitsuCompiler, self)._set_compiler_vars() - # enable clang compatibility mode; moved to compiler constants to make sure it is always used + # enable clang compatibility mode + # moved to compiler constants to make sure it is always used # self.variables.nappend('CFLAGS', ['Nclang']) # self.variables.nappend('CXXFLAGS', ['Nclang']) diff --git a/easybuild/toolchains/fcc.py b/easybuild/toolchains/fcc.py new file mode 100644 index 0000000000..0ca29695fa --- /dev/null +++ b/easybuild/toolchains/fcc.py @@ -0,0 +1,78 @@ +## +# Copyright 2012-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for Fujitsu Compiler toolchain. + +:author: Miguel Dias Costa (National University of Singapore) +""" +from easybuild.toolchains.compiler.fujitsu import FujitsuCompiler +from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME + + +class FCC(FujitsuCompiler): + """Compiler toolchain with Fujitsu Compiler.""" + NAME = 'FCC' + SUBTOOLCHAIN = SYSTEM_TOOLCHAIN_NAME + OPTIONAL = False + + # override in order to add an exception for the Fujitsu lang/tcsds module + def _add_dependency_variables(self, names=None, cpp=None, ld=None): + """ Add LDFLAGS and CPPFLAGS to the self.variables based on the dependencies + names should be a list of strings containing the name of the dependency + """ + cpp_paths = ['include'] + ld_paths = ['lib'] + if not self.options.get('32bit', None): + ld_paths.insert(0, 'lib64') + + if cpp is not None: + for p in cpp: + if p not in cpp_paths: + cpp_paths.append(p) + if ld is not None: + for p in ld: + if p not in ld_paths: + ld_paths.append(p) + + if not names: + deps = self.dependencies + else: + deps = [{'name': name} for name in names if name is not None] + + # collect software install prefixes for dependencies + roots = [] + for dep in deps: + if dep.get('external_module', False): + # for software names provided via external modules, install prefix may be unknown + names = dep['external_module_metadata'].get('name', []) + roots.extend([root for root in self.get_software_root(names) if root is not None]) + else: + roots.extend(self.get_software_root(dep['name'])) + + for root in roots: + # skip Fujitsu's 'lang/tcsds' module, including the top level include breaks vectorization in clang mode + if 'tcsds' not in root: + self.variables.append_subdirs("CPPFLAGS", root, subdirs=cpp_paths) + self.variables.append_subdirs("LDFLAGS", root, subdirs=ld_paths) diff --git a/easybuild/toolchains/ffmpi.py b/easybuild/toolchains/ffmpi.py index de7ec81664..a951f8c0f1 100644 --- a/easybuild/toolchains/ffmpi.py +++ b/easybuild/toolchains/ffmpi.py @@ -23,16 +23,16 @@ # along with EasyBuild. If not, see . ## """ -EasyBuild support for ffmpi compiler toolchain (includes Fujitsu Compiler, MPI and Scientific Libraries). +EasyBuild support for ffmpi compiler toolchain (includes Fujitsu Compiler and MPI). :author: Miguel Dias Costa (National University of Singapore) """ -from easybuild.toolchains.compiler.fujitsu import FujitsuCompiler -from easybuild.toolchains.linalg.fujitsussl import FujitsuSSL +from easybuild.toolchains.fcc import FCC from easybuild.toolchains.mpi.fujitsumpi import FujitsuMPI -class Ffmpi(FujitsuCompiler, FujitsuMPI, FujitsuSSL): - """Compiler toolchain with Fujitsu Compiler, MPI and Scientific Libraries.""" +class Ffmpi(FCC, FujitsuMPI): + """Compiler toolchain with Fujitsu Compiler and MPI.""" NAME = 'ffmpi' - SUBTOOLCHAIN = FujitsuCompiler.NAME + SUBTOOLCHAIN = FCC.NAME + COMPILER_MODULE_NAME = [FCC.NAME] diff --git a/easybuild/toolchains/fujitsu.py b/easybuild/toolchains/fujitsu.py index 919e268d94..6c81342901 100644 --- a/easybuild/toolchains/fujitsu.py +++ b/easybuild/toolchains/fujitsu.py @@ -29,9 +29,11 @@ """ from easybuild.toolchains.ffmpi import Ffmpi from easybuild.toolchains.fft.fujitsufftw import FujitsuFFTW +from easybuild.toolchains.linalg.fujitsussl import FujitsuSSL -class Fujitsu(Ffmpi, FujitsuFFTW): +class Fujitsu(Ffmpi, FujitsuFFTW, FujitsuSSL): """Compiler toolchain for Fujitsu.""" NAME = 'Fujitsu' SUBTOOLCHAIN = Ffmpi.NAME + COMPILER_MODULE_NAME = [] From 558487e4b99485fa38c02a6997cac7396282a267 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 25 May 2021 12:35:10 +0800 Subject: [PATCH 319/864] fix typo in comment --- easybuild/toolchains/compiler/fujitsu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/compiler/fujitsu.py b/easybuild/toolchains/compiler/fujitsu.py index 3d9a7e4175..db8bc9ae50 100644 --- a/easybuild/toolchains/compiler/fujitsu.py +++ b/easybuild/toolchains/compiler/fujitsu.py @@ -25,7 +25,7 @@ """ Support for the Fujitsu compiler drivers (aka fcc, frt). -The basic concept the same as for the Cray Programming Environment. +The basic concept is the same as for the Cray Programming Environment. :author: Miguel Dias Costa (National University of Singapore) """ From 75630815fd070335bf2203e9858da2450514307f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 25 May 2021 09:25:00 +0200 Subject: [PATCH 320/864] add extension to test easyconfig used in test_sanity_check_only --- test/framework/options.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 6911a88174..78ddab5ba7 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5777,12 +5777,21 @@ def test_sanity_check_only(self): test_ec = os.path.join(self.test_prefix, 'test.ec') test_ec_txt = read_file(toy_ec) - test_ec_txt += "\nsanity_check_commands = ['toy']" + test_ec_txt += '\n' + '\n'.join([ + "sanity_check_commands = ['barbar', 'toy']", + "sanity_check_paths = {'files': ['bin/barbar', 'bin/toy'], 'dirs': ['bin']}", + "exts_list = [", + " ('barbar', '0.0', {", + " 'start_dir': 'src',", + " })", + "]", + ]) write_file(test_ec, test_ec_txt) # sanity check fails if software was not installed yet outtxt, error_thrown = self.eb_main([test_ec, '--sanity-check-only'], do_build=True, return_error=True) - self.assertTrue("Sanity check failed: no file found at \\'bin/yot\\' or \\'bin/toy\\'" in str(error_thrown)) + self.assertTrue("Sanity check failed: no file found at \\'bin/barbar\\'" in str(error_thrown)) + self.assertTrue("no file found at \\'bin/toy\\'" in str(error_thrown)) # actually install, then try --sanity-check-only again; # need to use --force to install toy because module already exists (but installation doesn't) @@ -5821,13 +5830,14 @@ def test_sanity_check_only(self): self.assertTrue("== sanity checking..." in stdout) self.assertTrue("COMPLETED: Installation ended successfully" in stdout) msgs = [ - "file 'bin/yot' or 'bin/toy' found: OK", + "file 'bin/barbar' found: OK", + "file 'bin/toy' found: OK", "(non-empty) directory 'bin' found: OK", "loading modules: toy/0.0...", "result for command 'toy': OK", ] for msg in msgs: - self.assertTrue(" >> %s" % msg in stdout) + self.assertTrue(" >> %s" % msg in stdout, "' >> %s' found in: %s" % (msg, stdout)) def test_fake_vsc_include(self): """Test whether fake 'vsc' namespace is triggered for modules included via --include-*.""" From 64f96e1b35d6e5822b9f0e2160e1d6405f5d833e Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 25 May 2021 16:34:06 +0800 Subject: [PATCH 321/864] also check if existing symlink is correct before skipping --- easybuild/tools/filetools.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 05b06061e8..d09750c2d9 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -298,9 +298,8 @@ def symlink(source_path, symlink_path, use_abspath_source=True): if use_abspath_source: source_path = os.path.abspath(source_path) - if os.path.exists(symlink_path): + if os.path.exists(symlink_path) and os.path.abspath(source_path) == os.path.abspath(os.readlink(symlink_path)): _log.info("Skipping symlinking %s to %s, link already exists", source_path, symlink_path) - # TODO: check if the symlink_path points to source_path else: try: os.symlink(source_path, symlink_path) From 24a9fcc21d27b787b9fa6caf65b31aa47c9ead75 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 25 May 2021 09:57:11 +0200 Subject: [PATCH 322/864] enhance test_sanity_check_only to verify that sanity check for extensions is also run when using --sanity-check-only --- test/framework/options.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 78ddab5ba7..3c93c332c4 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5783,6 +5783,7 @@ def test_sanity_check_only(self): "exts_list = [", " ('barbar', '0.0', {", " 'start_dir': 'src',", + " 'exts_filter': ('ls -l lib/lib%(ext_name)s.a', ''),", " })", "]", ]) @@ -5790,16 +5791,17 @@ def test_sanity_check_only(self): # sanity check fails if software was not installed yet outtxt, error_thrown = self.eb_main([test_ec, '--sanity-check-only'], do_build=True, return_error=True) - self.assertTrue("Sanity check failed: no file found at \\'bin/barbar\\'" in str(error_thrown)) - self.assertTrue("no file found at \\'bin/toy\\'" in str(error_thrown)) + self.assertTrue("Sanity check failed" in str(error_thrown)) # actually install, then try --sanity-check-only again; # need to use --force to install toy because module already exists (but installation doesn't) self.eb_main([test_ec, '--force'], do_build=True, raise_error=True) + args = [test_ec, '--sanity-check-only'] + self.mock_stdout(True) self.mock_stderr(True) - self.eb_main([test_ec, '--sanity-check-only', '--trace'], do_build=True, raise_error=True, testing=False) + self.eb_main(args + ['--trace'], do_build=True, raise_error=True, testing=False) stdout = self.get_stdout().strip() stderr = self.get_stderr().strip() self.mock_stdout(False) @@ -5830,14 +5832,30 @@ def test_sanity_check_only(self): self.assertTrue("== sanity checking..." in stdout) self.assertTrue("COMPLETED: Installation ended successfully" in stdout) msgs = [ - "file 'bin/barbar' found: OK", - "file 'bin/toy' found: OK", - "(non-empty) directory 'bin' found: OK", - "loading modules: toy/0.0...", - "result for command 'toy': OK", + " >> file 'bin/barbar' found: OK", + " >> file 'bin/toy' found: OK", + " >> (non-empty) directory 'bin' found: OK", + " >> loading modules: toy/0.0...", + " >> result for command 'toy': OK", + "ls -l lib/libbarbar.a", # sanity check for extension barbar (via exts_filter) ] for msg in msgs: - self.assertTrue(" >> %s" % msg in stdout, "' >> %s' found in: %s" % (msg, stdout)) + self.assertTrue(msg in stdout, "'%s' found in: %s" % (msg, stdout)) + + # check if sanity check for extension fails if a file provided by that extension, + # which is checked by the sanity check for that extension, is removed + libbarbar = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'lib', 'libbarbar.a') + remove_file(libbarbar) + + outtxt, error_thrown = self.eb_main(args + ['--debug'], do_build=True, return_error=True) + error_msg = str(error_thrown) + error_patterns = [ + r"Sanity check failed", + r'command "ls -l lib/libbarbar\.a" failed', + ] + for error_pattern in error_patterns: + regex = re.compile(error_pattern) + self.assertTrue(regex.search(error_msg), "Pattern '%s' should be found in: %s" % (regex.pattern, error_msg)) def test_fake_vsc_include(self): """Test whether fake 'vsc' namespace is triggered for modules included via --include-*.""" From 25a1fe8544f3bb6e2b2de29cfc4e75ce1b25422b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 25 May 2021 10:52:41 +0200 Subject: [PATCH 323/864] ensure sanity check is also run for extension when using --sanity-check-only or --module-only by initializing class instances for extensions if that wasn't done yet --- easybuild/framework/easyblock.py | 90 +++++++++++++++++++------------- 1 file changed, 54 insertions(+), 36 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index fc326fdb5f..e859589313 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2239,54 +2239,27 @@ def install_step(self): """Install built software (abstract method).""" raise NotImplementedError - def extensions_step(self, fetch=False, install=True): + def init_ext_instances(self): """ - After make install, run this. - - only if variable len(exts_list) > 0 - - optionally: load module that was just created using temp module file - - find source for extensions, in 'extensions' (and 'packages' for legacy reasons) - - run extra_extensions + Create class instances for all extensions. """ - if not self.cfg.get_ref('exts_list'): - self.log.debug("No extensions in exts_list") - return - - # load fake module - fake_mod_data = None - if install and not self.dry_run: - - # load modules for build dependencies as extra modules - build_dep_mods = [dep['short_mod_name'] for dep in self.cfg.dependencies(build_only=True)] - - fake_mod_data = self.load_fake_module(purge=True, extra_modules=build_dep_mods) - self.prepare_for_extensions() - - if fetch: - self.exts = self.fetch_extension_sources() - - self.exts_all = self.exts[:] # retain a copy of all extensions, regardless of filtering/skipping - - # actually install extensions - self.log.debug("Installing extensions") - exts_defaultclass = self.cfg['exts_defaultclass'] + self.ext_instances = [] exts_classmap = self.cfg['exts_classmap'] - # we really need a default class - if not exts_defaultclass and fake_mod_data: - self.clean_up_fake_module(fake_mod_data) - raise EasyBuildError("ERROR: No default extension class set for %s", self.name) + if self.cfg['exts_list'] and not self.exts: + self.exts = self.fetch_extension_sources() # obtain name and module path for default extention class + exts_defaultclass = self.cfg['exts_defaultclass'] if isinstance(exts_defaultclass, string_type): # proper way: derive module path from specified class name default_class = exts_defaultclass default_class_modpath = get_module_path(default_class, generic=True) else: - raise EasyBuildError("Improper default extension class specification, should be string.") + error_msg = "Improper default extension class specification, should be string: %s (%s)" + raise EasyBuildError(error_msg, exts_defaultclass, type(exts_defaultclass)) - # get class instances for all extensions - self.ext_instances = [] for ext in self.exts: ext_name = ext['name'] self.log.debug("Creating class instance for extension %s...", ext_name) @@ -2336,6 +2309,44 @@ def extensions_step(self, fetch=False, install=True): self.ext_instances.append(inst) + def extensions_step(self, fetch=False, install=True): + """ + After make install, run this. + - only if variable len(exts_list) > 0 + - optionally: load module that was just created using temp module file + - find source for extensions, in 'extensions' (and 'packages' for legacy reasons) + - run extra_extensions + """ + if not self.cfg.get_ref('exts_list'): + self.log.debug("No extensions in exts_list") + return + + # load fake module + fake_mod_data = None + if install and not self.dry_run: + + # load modules for build dependencies as extra modules + build_dep_mods = [dep['short_mod_name'] for dep in self.cfg.dependencies(build_only=True)] + + fake_mod_data = self.load_fake_module(purge=True, extra_modules=build_dep_mods) + + self.prepare_for_extensions() + + if fetch: + self.exts = self.fetch_extension_sources() + + self.exts_all = self.exts[:] # retain a copy of all extensions, regardless of filtering/skipping + + # actually install extensions + self.log.info("Installing extensions") + + # we really need a default class + if not self.cfg['exts_defaultclass'] and fake_mod_data: + self.clean_up_fake_module(fake_mod_data) + raise EasyBuildError("ERROR: No default extension class set for %s", self.name) + + self.init_ext_instances() + if self.skip: self.skip_extensions() @@ -2351,7 +2362,7 @@ def extensions_step(self, fetch=False, install=True): print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent) if self.dry_run: - tup = (ext.name, ext.version, cls.__name__) + tup = (ext.name, ext.version, ext.__class__.__name__) msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup self.dry_run_msg(msg) @@ -2881,6 +2892,13 @@ def _sanity_check_step_dry_run(self, custom_paths=None, custom_commands=None, ** def _sanity_check_step_extensions(self): """Sanity check on extensions (if any).""" failed_exts = [] + + # class instances for extensions may not be initialized yet here, + # for example when using --module-only or --sanity-check-only + if not self.ext_instances: + self.prepare_for_extensions() + self.init_ext_instances() + for ext in self.ext_instances: success, fail_msg = None, None res = ext.sanity_check_step() From edb5bbe119d5e5f2160e4ed3bbeb20a823f18b8c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 25 May 2021 11:57:01 +0200 Subject: [PATCH 324/864] fix broken test by doing early exit in EasyBlock.init_ext_instances if exts_list is empty --- easybuild/framework/easyblock.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e859589313..fb2cfdb904 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2243,11 +2243,16 @@ def init_ext_instances(self): """ Create class instances for all extensions. """ + exts_list = self.cfg.get_ref('exts_list') + + # early exit if there are no extensions + if not exts_list: + return self.ext_instances = [] exts_classmap = self.cfg['exts_classmap'] - if self.cfg['exts_list'] and not self.exts: + if exts_list and not self.exts: self.exts = self.fetch_extension_sources() # obtain name and module path for default extention class From 83c6208183f45c5acb2086ef2919f4f6c9cb0b19 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 9 Sep 2020 16:19:16 +0200 Subject: [PATCH 325/864] Don't modify passed regex_subs leading to '<_sre.SRE_Pattern..>' outputs --- easybuild/tools/filetools.py | 4 +--- test/framework/filetools.py | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index aec01d7000..0fbcd7307f 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1487,9 +1487,7 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb'): else: _log.info("Applying following regex substitutions to %s: %s", paths, regex_subs) - compiled_regex_subs = [] - for regex, subtxt in regex_subs: - compiled_regex_subs.append((re.compile(regex), subtxt)) + compiled_regex_subs = [(re.compile(regex), subtxt) for (regex, subtxt) in regex_subs] for path in paths: try: diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 55fa363858..f1b1ec922b 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1208,6 +1208,7 @@ def test_apply_regex_substitutions(self): (r"^(FC\s*=\s*).*$", r"\1${FC}"), (r"^(.FLAGS)\s*=\s*-O3\s-g(.*)$", r"\1 = -O2\2"), ] + regex_subs_copy = regex_subs.copy() ft.apply_regex_substitutions(testfile, regex_subs) expected_testtxt = '\n'.join([ @@ -1218,6 +1219,8 @@ def test_apply_regex_substitutions(self): ]) new_testtxt = ft.read_file(testfile) self.assertEqual(new_testtxt, expected_testtxt) + # Must not have touched the list + self.assertEqual(regex_subs_copy, regex_subs) # backup file is created by default backup = testfile + '.orig.eb' From 5cf4cac2e049cfaac662b124ba14337a18abd0cc Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 9 Sep 2020 16:34:49 +0200 Subject: [PATCH 326/864] Only print summary of replacements --- easybuild/tools/filetools.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 0fbcd7307f..1e59353a70 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1507,6 +1507,7 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb'): if backup: copy_file(path, path + backup) + replacement_msgs = [] with open_file(path, 'w') as out_file: lines = txt_utf8.split('\n') del txt_utf8 @@ -1515,11 +1516,15 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb'): match = regex.search(line) if match: origtxt = match.group(0) - _log.info("Replacing line %d in %s: '%s' -> '%s'", - (line_id + 1), path, origtxt, subtxt) + replacement_msgs.append("Replaced in line %d: '%s' -> '%s'" % + (line_id + 1, origtxt, subtxt)) line = regex.sub(subtxt, line) lines[line_id] = line out_file.write('\n'.join(lines)) + if replacement_msgs: + _log.info('Applied the following substitutions to %s:\n%s', path, '\n'.join(replacement_msgs)) + else: + _log.info('Nothing found to replace in %s:', path) except (IOError, OSError) as err: raise EasyBuildError("Failed to patch %s: %s", path, err) From 8730ea7300cf826d8a9e61e38d56289ba337569e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 9 Sep 2020 17:03:07 +0200 Subject: [PATCH 327/864] Add on_missing_match parameter --- easybuild/tools/filetools.py | 19 +++++++++++++++++-- test/framework/filetools.py | 22 ++++++++++++++++++++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 1e59353a70..2bfd3eb4cd 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1466,14 +1466,23 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=Fa return True -def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb'): +def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb', on_missing_match=None): """ Apply specified list of regex substitutions. :param paths: list of paths to files to patch (or just a single filepath) :param regex_subs: list of substitutions to apply, specified as (, ) :param backup: create backup of original file with specified suffix (no backup if value evaluates to False) + :param on_missing_match: Define what to do when the file when no match was found in the file. + Can be 'error' to raise an error, 'warn' to print a warning or 'ignore' to do nothing + Defaults to value of --strict """ + if on_missing_match is None: + on_missing_match = build_option('strict') + ALLOWED_VALUES = (run.ERROR, run.WARN, run.IGNORE) + if on_missing_match not in ALLOWED_VALUES: + raise EasyBuildError('Invalid value passed to on_missing_match: %s', on_missing_match) + if isinstance(paths, string_type): paths = [paths] @@ -1524,7 +1533,13 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb'): if replacement_msgs: _log.info('Applied the following substitutions to %s:\n%s', path, '\n'.join(replacement_msgs)) else: - _log.info('Nothing found to replace in %s:', path) + msg = 'Nothing found to replace in %s' % path + if on_missing_match == run.ERROR: + raise EasyBuildError(msg) + elif on_missing_match == run.WARN: + print_warning(msg) + else: + _log.info(msg) except (IOError, OSError) as err: raise EasyBuildError("Failed to patch %s: %s", path, err) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index f1b1ec922b..a61584c5b4 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -42,7 +42,7 @@ import time from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner - +from easybuild.tools import run import easybuild.tools.filetools as ft from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import IGNORE, ERROR @@ -1253,10 +1253,28 @@ def test_apply_regex_substitutions(self): # passing empty list of substitions is a no-op ft.write_file(testfile, testtxt) - ft.apply_regex_substitutions(testfile, []) + ft.apply_regex_substitutions(testfile, [], on_missing_match=run.IGNORE) new_testtxt = ft.read_file(testfile) self.assertEqual(new_testtxt, testtxt) + # Check handling of on_missing_match + ft.write_file(testfile, testtxt) + regex_subs_no_match = [('Not there', 'Not used')] + error_pat = 'Nothing found to replace in %s' % testfile + # Error + self.assertErrorRegex(EasyBuildError, error_pat, ft.apply_regex_substitutions, testfile, regex_subs_no_match, + on_missing_match=run.ERROR) + # Warn + self.mock_stderr(True) + ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=run.WARN) + self.assertTrue(error_pat in self.get_stderr()) + self.mock_stderr(False) + # Ignore + self.mock_stderr(True) + ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=run.IGNORE) + self.assertFalse(error_pat in self.get_stderr()) + self.mock_stderr(False) + # clean error on non-existing file error_pat = "Failed to patch .*/nosuchfile.txt: .*No such file or directory" path = os.path.join(self.test_prefix, 'nosuchfile.txt') From ae170a35cda9a5480537488cab775871ae74d67a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 16 Sep 2020 14:50:40 +0200 Subject: [PATCH 328/864] Apply suggestions from code review Co-authored-by: Kenneth Hoste --- easybuild/tools/filetools.py | 19 +++++++++++++------ test/framework/filetools.py | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 2bfd3eb4cd..88185b5350 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1473,15 +1473,16 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb', on_missing_m :param paths: list of paths to files to patch (or just a single filepath) :param regex_subs: list of substitutions to apply, specified as (, ) :param backup: create backup of original file with specified suffix (no backup if value evaluates to False) - :param on_missing_match: Define what to do when the file when no match was found in the file. + :param on_missing_match: Define what to do when no match was found in the file. Can be 'error' to raise an error, 'warn' to print a warning or 'ignore' to do nothing Defaults to value of --strict """ if on_missing_match is None: on_missing_match = build_option('strict') - ALLOWED_VALUES = (run.ERROR, run.WARN, run.IGNORE) - if on_missing_match not in ALLOWED_VALUES: - raise EasyBuildError('Invalid value passed to on_missing_match: %s', on_missing_match) + allowed_values = (run.ERROR, run.WARN, run.IGNORE) + if on_missing_match not in allowed_values: + raise EasyBuildError('Invalid value passed to on_missing_match: %s (allowed: %s)', + on_missing_match, ', '.join(allowed_values)) if isinstance(paths, string_type): paths = [paths] @@ -1537,9 +1538,15 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb', on_missing_m if on_missing_match == run.ERROR: raise EasyBuildError(msg) elif on_missing_match == run.WARN: - print_warning(msg) + _log.warning(msg) else: - _log.info(msg) + msg = 'Nothing found to replace in %s' % path + if on_missing_match == run.ERROR: + raise EasyBuildError(msg) + elif on_missing_match == run.WARN: + print_warning(msg) + else: + _log.info(msg) except (IOError, OSError) as err: raise EasyBuildError("Failed to patch %s: %s", path, err) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index a61584c5b4..c376334066 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1208,7 +1208,7 @@ def test_apply_regex_substitutions(self): (r"^(FC\s*=\s*).*$", r"\1${FC}"), (r"^(.FLAGS)\s*=\s*-O3\s-g(.*)$", r"\1 = -O2\2"), ] - regex_subs_copy = regex_subs.copy() + regex_subs_copy = regex_subs[:] ft.apply_regex_substitutions(testfile, regex_subs) expected_testtxt = '\n'.join([ From b48032d0d92f006d45356cdb377e454a0f3b6d5b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 1 Dec 2020 12:01:11 +0100 Subject: [PATCH 329/864] Fix tests by checking log file --- test/framework/filetools.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index c376334066..a6f532016f 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -48,6 +48,7 @@ from easybuild.tools.config import IGNORE, ERROR from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import std_urllib +from easybuild.base import fancylogger class FileToolsTest(EnhancedTestCase): @@ -1264,16 +1265,22 @@ def test_apply_regex_substitutions(self): # Error self.assertErrorRegex(EasyBuildError, error_pat, ft.apply_regex_substitutions, testfile, regex_subs_no_match, on_missing_match=run.ERROR) + + fancylogger.logToFile(self.logfile) + # Warn - self.mock_stderr(True) ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=run.WARN) - self.assertTrue(error_pat in self.get_stderr()) - self.mock_stderr(False) + with open(self.logfile, 'r') as f: + logtxt = f.read() + self.assertTrue('WARNING ' + error_pat in logtxt) + # Ignore - self.mock_stderr(True) ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=run.IGNORE) - self.assertFalse(error_pat in self.get_stderr()) - self.mock_stderr(False) + with open(self.logfile, 'r') as f: + logtxt = f.read() + self.assertTrue('INFO ' + error_pat in logtxt) + + fancylogger.logToFile(self.logfile, enable=False) # clean error on non-existing file error_pat = "Failed to patch .*/nosuchfile.txt: .*No such file or directory" From c6976901f49f4d85bcced518928683c1d3c4cfed Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 1 Dec 2020 13:45:35 +0100 Subject: [PATCH 330/864] Remove duplicated branch and use read_file and contextmanager for logToFile --- easybuild/tools/filetools.py | 10 ++-------- test/framework/filetools.py | 17 ++++++----------- test/framework/utilities.py | 11 +++++++++++ 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 88185b5350..4c95b7a9ce 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1475,7 +1475,7 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb', on_missing_m :param backup: create backup of original file with specified suffix (no backup if value evaluates to False) :param on_missing_match: Define what to do when no match was found in the file. Can be 'error' to raise an error, 'warn' to print a warning or 'ignore' to do nothing - Defaults to value of --strict + Defaults to the value of --strict """ if on_missing_match is None: on_missing_match = build_option('strict') @@ -1540,13 +1540,7 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb', on_missing_m elif on_missing_match == run.WARN: _log.warning(msg) else: - msg = 'Nothing found to replace in %s' % path - if on_missing_match == run.ERROR: - raise EasyBuildError(msg) - elif on_missing_match == run.WARN: - print_warning(msg) - else: - _log.info(msg) + _log.info(msg) except (IOError, OSError) as err: raise EasyBuildError("Failed to patch %s: %s", path, err) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index a6f532016f..a45a286e6e 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -48,7 +48,6 @@ from easybuild.tools.config import IGNORE, ERROR from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import std_urllib -from easybuild.base import fancylogger class FileToolsTest(EnhancedTestCase): @@ -1266,22 +1265,18 @@ def test_apply_regex_substitutions(self): self.assertErrorRegex(EasyBuildError, error_pat, ft.apply_regex_substitutions, testfile, regex_subs_no_match, on_missing_match=run.ERROR) - fancylogger.logToFile(self.logfile) - # Warn - ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=run.WARN) - with open(self.logfile, 'r') as f: - logtxt = f.read() + with self.log_to_testlogfile(): + ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=run.WARN) + logtxt = ft.read_file(self.logfile) self.assertTrue('WARNING ' + error_pat in logtxt) # Ignore - ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=run.IGNORE) - with open(self.logfile, 'r') as f: - logtxt = f.read() + with self.log_to_testlogfile(): + ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=run.IGNORE) + logtxt = ft.read_file(self.logfile) self.assertTrue('INFO ' + error_pat in logtxt) - fancylogger.logToFile(self.logfile, enable=False) - # clean error on non-existing file error_pat = "Failed to patch .*/nosuchfile.txt: .*No such file or directory" path = os.path.join(self.test_prefix, 'nosuchfile.txt') diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 0088de959b..d36c9da53a 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -36,6 +36,7 @@ import sys import tempfile import unittest +from contextlib import contextmanager from easybuild.base import fancylogger from easybuild.base.testing import TestCase @@ -205,6 +206,16 @@ def allow_deprecated_behaviour(self): del os.environ['EASYBUILD_DEPRECATED'] eb_build_log.CURRENT_VERSION = self.orig_current_version + @contextmanager + def log_to_testlogfile(self): + """Context manager class to capture log output in self.logfile for the scope used. Clears the file first""" + open(self.logfile, 'w').close() # Remove all contents + fancylogger.logToFile(self.logfile) + try: + yield self.logfile + finally: + fancylogger.logToFile(self.logfile, enable=False) + def tearDown(self): """Clean up after running testcase.""" super(EnhancedTestCase, self).tearDown() From 4b0e7ad7cbe59c31f6efe42a60afed8de57ef65b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 25 May 2021 12:22:31 +0200 Subject: [PATCH 331/864] add test to verify that sanity check for extensions is also run with --module-only, unless --force is used --- test/framework/toy_build.py | 54 ++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index f13bb0e832..2156fab04b 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -51,7 +51,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import get_module_syntax, get_repositorypath from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import adjust_permissions, change_dir, copy_file, mkdir +from easybuild.tools.filetools import adjust_permissions, change_dir, copy_file, mkdir, move_file from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file from easybuild.tools.module_generator import ModuleGeneratorTcl from easybuild.tools.modules import Lmod @@ -1682,6 +1682,58 @@ def test_module_only(self): modtxt = read_file(toy_mod + '.lua') self.assertTrue(re.search('load.*intel/2018a', modtxt), "load statement for intel/2018a found in module") + def test_module_only_extensions(self): + """ + Test use of --module-only with extensions involved. + Sanity check should catch problems with extensions, + extensions can be skipped using --skip-exts. + """ + topdir = os.path.abspath(os.path.dirname(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_mod += '.lua' + + test_ec = os.path.join(self.test_prefix, 'test.ec') + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\n' + '\n'.join([ + "sanity_check_commands = ['barbar', 'toy']", + "sanity_check_paths = {'files': ['bin/barbar', 'bin/toy'], 'dirs': ['bin']}", + "exts_list = [", + " ('barbar', '0.0', {", + " 'start_dir': 'src',", + " 'exts_filter': ('ls -l lib/lib%(ext_name)s.a', ''),", + " })", + "]", + ]) + write_file(test_ec, test_ec_txt) + + # clean up $MODULEPATH to only modules in test prefix dir are found + self.reset_modulepath([os.path.join(self.test_installpath, 'modules', 'all')]) + self.assertEqual(self.modtool.available('toy'), []) + + # install toy/0.0 + self.eb_main([test_ec], do_build=True, raise_error=True) + + # remove module file so we can try --module-only + remove_file(toy_mod) + + # rename file required for barbar extension, so we can check whether sanity check catches it + libbarbar = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'lib', 'libbarbar.a') + move_file(libbarbar, libbarbar + '.foobar') + + # check whether sanity check fails now when using --module-only + error_pattern = 'Sanity check failed: command "ls -l lib/libbarbar.a" failed' + for extra_args in (['--module-only'], ['--module-only', '--rebuild']): + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, [test_ec] + extra_args, + do_build=True, raise_error=True) + self.assertFalse(os.path.exists(toy_mod)) + + # we can force module generation via --force (which skips sanity check entirely) + self.eb_main([test_ec, '--module-only', '--force'], do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_mod)) + def test_backup_modules(self): """Test use of backing up of modules with --module-only.""" From 384e0450508f99afad2ae1f3976b6e169b48f0d1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 25 May 2021 13:02:15 +0200 Subject: [PATCH 332/864] avoid overriding local variable 'name' in Extension constructor --- easybuild/framework/extension.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 569a3bb414..407eab8c69 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -113,8 +113,8 @@ def __init__(self, mself, ext, extra_params=None): self.cfg.template_values.update(template_constant_dict({'name': name, 'version': version})) # Add install/builddir templates with values from master. - for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: - self.cfg.template_values[name[0]] = str(getattr(self.master, name[0], None)) + for key in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: + self.cfg.template_values[key[0]] = str(getattr(self.master, key[0], None)) # list of source/patch files: we use an empty list as default value like in EasyBlock self.src = resolve_template(self.ext.get('src', []), self.cfg.template_values) From 63cea59d6e77fe33d871494bb46c93ef828895ac Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 25 May 2021 19:07:43 +0800 Subject: [PATCH 333/864] make sure the top level fujitsu library dir is found by the rpath wrapper --- easybuild/toolchains/compiler/fujitsu.py | 16 ++++++++++++++-- easybuild/toolchains/linalg/fujitsussl.py | 10 ++++------ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/easybuild/toolchains/compiler/fujitsu.py b/easybuild/toolchains/compiler/fujitsu.py index db8bc9ae50..f566ecbd31 100644 --- a/easybuild/toolchains/compiler/fujitsu.py +++ b/easybuild/toolchains/compiler/fujitsu.py @@ -29,18 +29,22 @@ :author: Miguel Dias Costa (National University of Singapore) """ +import os + +import easybuild.tools.environment as env import easybuild.tools.systemtools as systemtools from easybuild.tools.toolchain.compiler import Compiler, DEFAULT_OPT_LEVEL TC_CONSTANT_FUJITSU = 'Fujitsu' +TC_CONSTANT_MODULE_NAME = 'lang' +TC_CONSTANT_MODULE_VAR = 'FJSVXTCLANGA' class FujitsuCompiler(Compiler): """Generic support for using Fujitsu compiler drivers.""" TOOLCHAIN_FAMILY = TC_CONSTANT_FUJITSU - # compiler module name is lang (with version e.g. tcsds-1.2.31) - COMPILER_MODULE_NAME = ['lang'] + COMPILER_MODULE_NAME = [TC_CONSTANT_MODULE_NAME] COMPILER_FAMILY = TC_CONSTANT_FUJITSU # make sure fcc is always called in clang compatibility mode @@ -82,6 +86,14 @@ class FujitsuCompiler(Compiler): (systemtools.AARCH64, systemtools.ARM): '-mcpu=generic -mtune=generic', } + def prepare(self, *args, **kwargs): + super(FujitsuCompiler, self).prepare(*args, **kwargs) + + # make sure the fujitsu module libraries are found (and added to rpath by wrapper) + libdir = os.path.join(os.getenv(TC_CONSTANT_MODULE_VAR), 'lib64') + self.log.debug("Adding %s to $LIBRARY_PATH" % libdir) + env.setvar('LIBRARY_PATH', os.pathsep.join([os.getenv('LIBRARY_PATH'), libdir])) + def _set_compiler_vars(self): super(FujitsuCompiler, self)._set_compiler_vars() diff --git a/easybuild/toolchains/linalg/fujitsussl.py b/easybuild/toolchains/linalg/fujitsussl.py index 71a7647028..ba09ad9a0c 100644 --- a/easybuild/toolchains/linalg/fujitsussl.py +++ b/easybuild/toolchains/linalg/fujitsussl.py @@ -29,12 +29,11 @@ """ import os - +from easybuild.toolchains.compiler.fujitsu import TC_CONSTANT_MODULE_NAME, TC_CONSTANT_MODULE_VAR from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.constants import COMPILER_FLAGS from easybuild.tools.toolchain.linalg import LinAlg - FUJITSU_SSL_MODULE_NAME = None TC_CONSTANT_FUJITSU_SSL = 'FujitsuSSL' @@ -43,7 +42,7 @@ class FujitsuSSL(LinAlg): """Support for Fujitsu's SSL library, which provides BLAS/LAPACK support.""" # BLAS/LAPACK support # via lang/tcsds module - BLAS_MODULE_NAME = ['lang'] + BLAS_MODULE_NAME = [TC_CONSTANT_MODULE_NAME] # no need to specify libraries nor includes, only the compiler flags below BLAS_LIB = [''] @@ -71,9 +70,8 @@ class FujitsuSSL(LinAlg): def _get_software_root(self, name): """Get install prefix for specified software name; special treatment for Fujitsu modules.""" - if name == 'lang': - # Fujitsu-provided module - env_var = 'FJSVXTCLANGA' + if name == TC_CONSTANT_MODULE_NAME: + env_var = TC_CONSTANT_MODULE_VAR root = os.getenv(env_var) if root is None: raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var) From efc3980b5ce44ca25d7dd387471526bed40617d5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 25 May 2021 13:06:40 +0200 Subject: [PATCH 334/864] run post-install commands specified for a specific extension --- easybuild/framework/easyblock.py | 27 ++++++++++++++++++++------- easybuild/framework/extension.py | 5 +++-- test/framework/toy_build.py | 16 ++++++++++++++-- 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 7d8e805eda..463febcfc6 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2436,21 +2436,34 @@ def fix_shebang(self): contents = shebang + '\n' + contents write_file(path, contents) - def post_install_step(self): + def run_post_install_commands(self, commands=None): """ - Do some postprocessing - - run post install commands if any were specified + Run post install commands that are specified via 'postinstallcmds' easyconfig parameter. """ + if commands is None: + commands = self.cfg['postinstallcmds'] + + if commands: + self.log.debug("Specified post install commands: %s", commands) - if self.cfg['postinstallcmds'] is not None: # make sure we have a list of commands - if not isinstance(self.cfg['postinstallcmds'], (list, tuple)): - raise EasyBuildError("Invalid value for 'postinstallcmds', should be list or tuple of strings.") - for cmd in self.cfg['postinstallcmds']: + if not isinstance(commands, (list, tuple)): + error_msg = "Invalid value for 'postinstallcmds', should be list or tuple of strings: %s" + raise EasyBuildError(error_msg, commands) + + for cmd in commands: if not isinstance(cmd, string_type): raise EasyBuildError("Invalid element in 'postinstallcmds', not a string: %s", cmd) run_cmd(cmd, simple=True, log_ok=True, log_all=True) + def post_install_step(self): + """ + Do some postprocessing + - run post install commands if any were specified + """ + + self.run_post_install_commands() + self.fix_shebang() lib_dir = os.path.join(self.installdir, 'lib') diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 407eab8c69..fcd2cc9596 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -105,9 +105,10 @@ def __init__(self, mself, ext, extra_params=None): name, version = self.ext['name'], self.ext.get('version', None) - # parent sanity check paths/commands are not relevant for extension + # parent sanity check paths/commands and postinstallcmds are not relevant for extension self.cfg['sanity_check_commands'] = [] self.cfg['sanity_check_paths'] = [] + self.cfg['postinstallcmds'] = [] # construct dict with template values that can be used self.cfg.template_values.update(template_constant_dict({'name': name, 'version': version})) @@ -169,7 +170,7 @@ def postrun(self): """ Stuff to do after installing a extension. """ - pass + self.master.run_post_install_commands(commands=self.cfg.get('postinstallcmds', [])) @property def toolchain(self): diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9e5e28fddf..2bfc985cde 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1190,8 +1190,8 @@ def test_toy_patches(self): archived_patch_file = os.path.join(repositorypath, 'toy', 'toy-0.0_fix-silly-typo-in-printf-statement.patch') self.assertTrue(os.path.isfile(archived_patch_file)) - def test_toy_extension_patches(self): - """Test install toy that includes extensions with patches.""" + def test_toy_extension_patches_postinstallcmds(self): + """Test install toy that includes extensions with patches and postinstallcmds.""" test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') toy_ec_txt = read_file(toy_ec) @@ -1210,6 +1210,7 @@ def test_toy_extension_patches(self): ' ("bar-0.0_fix-very-silly-typo-in-printf-statement.patch", 0),', # patch with patch level ' ("test.txt", "."),', # file to copy to build dir (not a real patch file) ' ],', + ' "postinstallcmds": ["touch %(installdir)s/created-via-postinstallcmds.txt"],', ' }),', ']', ]) @@ -1217,6 +1218,17 @@ def test_toy_extension_patches(self): self.test_toy_build(ec_file=test_ec) + installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') + + # make sure that patches were actually applied (without them the message producded by 'bar' is different) + bar_bin = os.path.join(installdir, 'bin', 'bar') + out, _ = run_cmd(bar_bin) + self.assertEqual(out, "I'm a bar, and very very proud of it.\n") + + # verify that post-install command for 'bar' extension was executed + fn = 'created-via-postinstallcmds.txt' + self.assertTrue(os.path.exists(os.path.join(installdir, fn))) + def test_toy_extension_sources(self): """Test install toy that includes extensions with 'sources' spec (as single-item list).""" topdir = os.path.dirname(os.path.abspath(__file__)) From ae6dcb858eb66de11876529923df0318c80505a6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 25 May 2021 13:33:35 +0200 Subject: [PATCH 335/864] guard log statement stating that extensions are being installed Co-authored-by: ocaisa --- easybuild/framework/easyblock.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index fb2cfdb904..745f3c0463 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2343,7 +2343,8 @@ def extensions_step(self, fetch=False, install=True): self.exts_all = self.exts[:] # retain a copy of all extensions, regardless of filtering/skipping # actually install extensions - self.log.info("Installing extensions") + if install: + self.log.info("Installing extensions") # we really need a default class if not self.cfg['exts_defaultclass'] and fake_mod_data: From 4863865fedc9213a9f1d5796f2d65390f6f698ad Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 25 May 2021 13:59:20 +0200 Subject: [PATCH 336/864] Improve comment --- test/framework/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 679c636b1b..8cbb977373 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1089,7 +1089,7 @@ def test_module_caches(self): """Test module caches and invalidate_module_caches_for function.""" self.assertEqual(mod.MODULE_AVAIL_CACHE, {}) - # purposely extending $MODULEPATH with non-existing path, should be handled fine + # purposely extending $MODULEPATH with an empty path, should be handled fine nonpath = os.path.join(self.test_prefix, 'nosuchfileordirectory') mkdir(nonpath) self.modtool.use(nonpath) From d3d8e4c4a35a239a7b2223ba9cfd7e15bee932a8 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 25 May 2021 20:29:30 +0800 Subject: [PATCH 337/864] raise error when symlink already exists but points to a different path, and add test for it --- easybuild/tools/filetools.py | 7 ++++++- test/framework/filetools.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index d09750c2d9..4ad50ed0be 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -298,7 +298,12 @@ def symlink(source_path, symlink_path, use_abspath_source=True): if use_abspath_source: source_path = os.path.abspath(source_path) - if os.path.exists(symlink_path) and os.path.abspath(source_path) == os.path.abspath(os.readlink(symlink_path)): + if os.path.exists(symlink_path): + abs_source_path = os.path.abspath(source_path) + symlink_target_path = os.path.abspath(os.readlink(symlink_path)) + if abs_source_path != symlink_target_path: + raise EasyBuildError("Trying to symlink %s to %s, but the symlink already exists and points to %s.", + source_path, symlink_path, symlink_target_path) _log.info("Skipping symlinking %s to %s, link already exists", source_path, symlink_path) else: try: diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 55fa363858..3effdeaa52 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -574,6 +574,17 @@ def test_symlink_resolve_path(self): self.assertTrue(os.path.samefile(os.path.join(self.test_prefix, 'test', 'test.txt'), link)) + # test symlink when it already exists and points to the same path + ft.symlink(test_file, link) + + # test symlink when it already exists but points to a different path + test_file2 = os.path.join(link_dir, 'test2.txt') + ft.write_file(test_file, "test123") + self.assertErrorRegex(EasyBuildError, + "Trying to symlink %s to %s, but the symlink already exists and points to %s." % + (test_file2, link, test_file), + ft.symlink, test_file2, link) + # test resolve_path self.assertEqual(test_dir, ft.resolve_path(link_dir)) self.assertEqual(os.path.join(os.path.realpath(self.test_prefix), 'test', 'test.txt'), ft.resolve_path(link)) From cf6050779d066fd9d247bc83979c52c48ee84e2f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 25 May 2021 15:46:25 +0200 Subject: [PATCH 338/864] Use os.pathsep --- test/framework/modules.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 8cbb977373..432570c385 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -531,14 +531,15 @@ def test_check_module_path(self): os.environ['MODULEPATH'] = test2 modtool.check_module_path() self.assertEqual(modtool.mod_paths, [mod_install_dir, test1, test2]) - self.assertEqual(os.environ['MODULEPATH'], mod_install_dir + ':' + test1 + ':' + test2) + self.assertEqual(os.environ['MODULEPATH'], os.pathsep.join([mod_install_dir, test1, test2])) # check behaviour if non-existing directories are included in $MODULEPATH os.environ['MODULEPATH'] = '%s:/does/not/exist:%s' % (test3, test2) modtool.check_module_path() # non-existing dir is filtered from mod_paths, but stays in $MODULEPATH self.assertEqual(modtool.mod_paths, [mod_install_dir, test1, test3, test2]) - self.assertEqual(os.environ['MODULEPATH'], ':'.join([mod_install_dir, test1, test3, '/does/not/exist', test2])) + self.assertEqual(os.environ['MODULEPATH'], + os.pathsep.join([mod_install_dir, test1, test3, '/does/not/exist', test2])) def test_check_module_path_hmns(self): """Test behaviour of check_module_path with HierarchicalMNS.""" @@ -1238,16 +1239,17 @@ def test_add_and_remove_module_path(self): self.modtool.add_module_path(test_dir1) self.assertEqual(os.environ['MODULEPATH'], test_dir1) self.modtool.add_module_path(test_dir2) - self.assertEqual(os.environ['MODULEPATH'], test_dir2 + ':' + test_dir1) + test_dir_2_and_1 = os.pathsep.join([test_dir2, test_dir1]) + self.assertEqual(os.environ['MODULEPATH'], test_dir_2_and_1) # Adding the same path does not change the path self.modtool.add_module_path(test_dir1) - self.assertEqual(os.environ['MODULEPATH'], test_dir2 + ':' + test_dir1) + self.assertEqual(os.environ['MODULEPATH'], test_dir_2_and_1) self.modtool.add_module_path(test_dir2) - self.assertEqual(os.environ['MODULEPATH'], test_dir2 + ':' + test_dir1) + self.assertEqual(os.environ['MODULEPATH'], test_dir_2_and_1) # Even when a (meaningless) slash is added # This occurs when using an empty modules directory name self.modtool.add_module_path(os.path.join(test_dir1, '')) - self.assertEqual(os.environ['MODULEPATH'], test_dir2 + ':' + test_dir1) + self.assertEqual(os.environ['MODULEPATH'], test_dir_2_and_1) # Similar tests for remove_module_path self.modtool.remove_module_path(test_dir2) @@ -1264,7 +1266,7 @@ def test_add_and_remove_module_path(self): # Environment-Modules 4.x seems to resolve relative paths: /foo/../foo -> /foo # Hence we can only check the real paths def get_resolved_module_path(): - return ':'.join(os.path.realpath(p) for p in os.environ['MODULEPATH'].split(':')) + return os.pathsep.join(os.path.realpath(p) for p in os.environ['MODULEPATH'].split(os.pathsep)) test_dir1_relative = os.path.join(test_dir1, '..', os.path.basename(test_dir1)) test_dir2_dot = os.path.join(os.path.dirname(test_dir2), '.', os.path.basename(test_dir2)) @@ -1273,18 +1275,18 @@ def get_resolved_module_path(): # Adding the same path, but in a different form may be possible, but may also be ignored, e.g. in EnvModules self.modtool.add_module_path(test_dir1) if get_resolved_module_path() != test_dir1: - self.assertEqual(get_resolved_module_path(), test_dir1 + ':' + test_dir1) + self.assertEqual(get_resolved_module_path(), os.pathsep.join([test_dir1, test_dir1])) self.modtool.remove_module_path(test_dir1) self.assertEqual(get_resolved_module_path(), test_dir1) self.modtool.add_module_path(test_dir2_dot) - self.assertEqual(get_resolved_module_path(), test_dir2 + ':' + test_dir1) + self.assertEqual(get_resolved_module_path(), test_dir_2_and_1) self.modtool.remove_module_path(test_dir2_dot) self.assertEqual(get_resolved_module_path(), test_dir1) # Force adding such a dot path which can be removed with either variant - os.environ['MODULEPATH'] = test_dir2_dot + ':' + test_dir1_relative + os.environ['MODULEPATH'] = os.pathsep.join([test_dir2_dot, test_dir1_relative]) self.modtool.remove_module_path(test_dir2_dot) self.assertEqual(get_resolved_module_path(), test_dir1) - os.environ['MODULEPATH'] = test_dir2_dot + ':' + test_dir1_relative + os.environ['MODULEPATH'] = os.pathsep.join([test_dir2_dot, test_dir1_relative]) self.modtool.remove_module_path(test_dir2) self.assertEqual(get_resolved_module_path(), test_dir1) From cc93c06c58e4e66cdcf1e80aec6192b0633c16df Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 25 May 2021 17:22:40 +0200 Subject: [PATCH 339/864] use ERROR/IGNORE/WARN constants directly in apply_regex_substitutions --- easybuild/tools/filetools.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 4c95b7a9ce..dbb31b3c3f 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -58,8 +58,8 @@ from easybuild.tools import run # import build_log must stay, to use of EasyBuildLog from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning -from easybuild.tools.config import (DEFAULT_WAIT_ON_LOCK_INTERVAL, GENERIC_EASYBLOCK_PKG, build_option, install_path, - IGNORE, WARN, ERROR) +from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN +from easybuild.tools.config import build_option, install_path from easybuild.tools.py2vs3 import HTMLParser, std_urllib, string_type from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars @@ -1479,7 +1479,7 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb', on_missing_m """ if on_missing_match is None: on_missing_match = build_option('strict') - allowed_values = (run.ERROR, run.WARN, run.IGNORE) + allowed_values = (ERROR, IGNORE, WARN) if on_missing_match not in allowed_values: raise EasyBuildError('Invalid value passed to on_missing_match: %s (allowed: %s)', on_missing_match, ', '.join(allowed_values)) @@ -1535,9 +1535,9 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb', on_missing_m _log.info('Applied the following substitutions to %s:\n%s', path, '\n'.join(replacement_msgs)) else: msg = 'Nothing found to replace in %s' % path - if on_missing_match == run.ERROR: + if on_missing_match == ERROR: raise EasyBuildError(msg) - elif on_missing_match == run.WARN: + elif on_missing_match == WARN: _log.warning(msg) else: _log.info(msg) From 3d5a10931cf2e751ef06d2399c598a7f9581f7da Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 25 May 2021 17:37:52 +0200 Subject: [PATCH 340/864] add missing ) in log message in EasyBlock.skip_step --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 745f3c0463..712f842eea 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3321,7 +3321,7 @@ def skip_step(self, step, skippable): else: msg = "Not skipping %s step (skippable: %s, skip: %s, skipsteps: %s, module_only: %s, force: %s, " - msg += "sanity_check_only: %s" + msg += "sanity_check_only: %s)" self.log.debug(msg, step, skippable, self.skip, skipsteps, module_only, force, sanity_check_only) return skip From bbcd8aee68515ed45611f38007129632af61478e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 25 May 2021 17:30:05 +0200 Subject: [PATCH 341/864] make sure that $TAPE is unset when using piped tar (fixes #3652) --- easybuild/tools/filetools.py | 8 +++++--- test/framework/filetools.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index aec01d7000..cd62644d57 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -141,9 +141,11 @@ '.tb2': "tar xjf %(filepath)s", '.tbz': "tar xjf %(filepath)s", '.tbz2': "tar xjf %(filepath)s", - # xzipped or xzipped tarball - '.tar.xz': "unxz %(filepath)s --stdout | tar x", - '.txz': "unxz %(filepath)s --stdout | tar x", + # xzipped or xzipped tarball; + # need to make sure that $TAPE is not set to avoid 'tar x' command failing, + # see https://github.com/easybuilders/easybuild-framework/issues/3652 + '.tar.xz': "unset TAPE; unxz %(filepath)s --stdout | tar x", + '.txz': "unset TAPE; unxz %(filepath)s --stdout | tar x", '.xz': "unxz %(filepath)s", # tarball '.tar': "tar xf %(filepath)s", diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 55fa363858..0e94211a39 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -92,8 +92,8 @@ def test_extract_cmd(self): ('untar.gz', "gunzip -c untar.gz > untar"), ("/some/path/test.gz", "gunzip -c /some/path/test.gz > test"), ('test.xz', "unxz test.xz"), - ('test.tar.xz', "unxz test.tar.xz --stdout | tar x"), - ('test.txz', "unxz test.txz --stdout | tar x"), + ('test.tar.xz', "unset TAPE; unxz test.tar.xz --stdout | tar x"), + ('test.txz', "unset TAPE; unxz test.txz --stdout | tar x"), ('test.iso', "7z x test.iso"), ('test.tar.Z', "tar xzf test.tar.Z"), ('test.foo.bar.sh', "cp -a test.foo.bar.sh ."), From e6a85de08cd7d92457f28999bfed194ace045c36 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 25 May 2021 17:42:09 +0200 Subject: [PATCH 342/864] add support for --skip-extensions --- easybuild/framework/easyblock.py | 28 +++++++++++++++++-------- easybuild/tools/config.py | 1 + easybuild/tools/options.py | 1 + test/framework/options.py | 35 ++++++++++++++++++++++++++++++++ test/framework/toy_build.py | 6 ++++++ 5 files changed, 62 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 712f842eea..43c28f0c0d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2899,9 +2899,12 @@ def _sanity_check_step_extensions(self): """Sanity check on extensions (if any).""" failed_exts = [] - # class instances for extensions may not be initialized yet here, - # for example when using --module-only or --sanity-check-only - if not self.ext_instances: + if build_option('skip_extensions'): + self.log.info("Skipping sanity check for extensions since skip-extensions is enabled...") + return + elif not self.ext_instances: + # class instances for extensions may not be initialized yet here, + # for example when using --module-only or --sanity-check-only self.prepare_for_extensions() self.init_ext_instances() @@ -3293,6 +3296,8 @@ def skip_step(self, step, skippable): force = build_option('force') module_only = build_option('module_only') sanity_check_only = build_option('sanity_check_only') + skip_extensions = build_option('skip_extensions') + skip_test_step = build_option('skip_test_step') skipsteps = self.cfg['skipsteps'] # under --skip, sanity check is not skipped @@ -3319,10 +3324,19 @@ def skip_step(self, step, skippable): self.log.info("Skipping %s step because of sanity-check-only mode", step) skip = True + elif skip_extensions and step == EXTENSIONS_STEP: + self.log.info("Skipping %s step as requested via skip-extensions", step) + skip = True + + elif skip_test_step and step == TEST_STEP: + self.log.info("Skipping %s step as requested via skip-test-step", step) + skip = True + else: msg = "Not skipping %s step (skippable: %s, skip: %s, skipsteps: %s, module_only: %s, force: %s, " - msg += "sanity_check_only: %s)" - self.log.debug(msg, step, skippable, self.skip, skipsteps, module_only, force, sanity_check_only) + msg += "sanity_check_only: %s, skip_extensions: %s, skip_test_step: %s)" + self.log.debug(msg, step, skippable, self.skip, skipsteps, module_only, force, + sanity_check_only, skip_extensions, skip_test_step) return skip @@ -3586,10 +3600,6 @@ def build_and_install_one(ecdict, init_env): _log.debug("Skip set to %s" % skip) app.cfg['skip'] = skip - if build_option('skip_test_step'): - _log.debug('Adding test_step to skipped steps') - app.cfg.update('skipsteps', TEST_STEP, allow_duplicate=False) - # build easyconfig errormsg = '(no error)' # timing info diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index ec7a28a0f2..e7bd7146bd 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -263,6 +263,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'search_paths', 'sequential', 'set_gid_bit', + 'skip_extensions', 'skip_test_cases', 'skip_test_step', 'generate_devel_module', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 2ccbf1132b..2973dcb74b 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -466,6 +466,7 @@ def override_options(self): 'set-gid-bit': ("Set group ID bit on newly created directories", None, 'store_true', False), 'silence-deprecation-warnings': ("Silence specified deprecation warnings", 'strlist', 'extend', None), 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), + 'skip-extensions': ("Skip installation of extensions", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), 'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False), 'generate-devel-module': ("Generate a develop module file, implies --force if disabled", diff --git a/test/framework/options.py b/test/framework/options.py index 3c93c332c4..0e53878423 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5857,6 +5857,41 @@ def test_sanity_check_only(self): regex = re.compile(error_pattern) self.assertTrue(regex.search(error_msg), "Pattern '%s' should be found in: %s" % (regex.pattern, error_msg)) + # failing sanity check for extension can be bypassed via --skip-extensions + self.eb_main(args + ['--skip-extensions'], do_build=True, return_error=True) + + def test_skip_extensions(self): + """Test use of --skip-extensions.""" + topdir = os.path.abspath(os.path.dirname(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + # add extension, which should be skipped + test_ec = os.path.join(self.test_prefix, 'test.ec') + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\n' + '\n'.join([ + "exts_list = [", + " ('barbar', '0.0', {", + " 'start_dir': 'src',", + " 'exts_filter': ('ls -l lib/lib%(ext_name)s.a', ''),", + " })", + "]", + ]) + write_file(test_ec, test_ec_txt) + + args = [test_ec, '--force', '--skip-extensions'] + self.eb_main(args, do_build=True, return_error=True) + + toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_mod += '.lua' + + self.assertTrue(os.path.exists(toy_mod), "%s should exist" % toy_mod) + + toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') + for path in (os.path.join('bin', 'barbar'), os.path.join('lib', 'libbarbar.a')): + path = os.path.join(toy_installdir, path) + self.assertFalse(os.path.exists(path), "Path %s should not exist" % path) + def test_fake_vsc_include(self): """Test whether fake 'vsc' namespace is triggered for modules included via --include-*.""" diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 2156fab04b..efa6fc8df0 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1730,6 +1730,12 @@ def test_module_only_extensions(self): do_build=True, raise_error=True) self.assertFalse(os.path.exists(toy_mod)) + # failing sanity check for barbar extension is ignored when using --module-only --skip-extensions + for extra_args in (['--module-only'], ['--module-only', '--rebuild']): + self.eb_main([test_ec, '--skip-extensions'] + extra_args, do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_mod)) + remove_file(toy_mod) + # we can force module generation via --force (which skips sanity check entirely) self.eb_main([test_ec, '--module-only', '--force'], do_build=True, raise_error=True) self.assertTrue(os.path.exists(toy_mod)) From b9117996809673f0222d2297a09930fcda53d886 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 25 May 2021 18:31:05 +0200 Subject: [PATCH 343/864] fix extending message for changed files in new_pr_from_branch --- easybuild/tools/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 5174347329..c5cb3bb65e 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1607,7 +1607,7 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None, msg.extend([" " + x for x in patch_paths]) if deleted_paths: msg.append("* %d deleted file(s)" % len(deleted_paths)) - msg.append([" " + x for x in deleted_paths]) + msg.extend([" " + x for x in deleted_paths]) print_msg('\n'.join(msg), log=_log) else: From 8f746b4175809f2d617ac029d224d1bd11c386e6 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 26 May 2021 11:27:41 +0200 Subject: [PATCH 344/864] Support systems with more than 1024 cores Dynamically determine the size of the cpu_set_t struct doubling it on each try --- easybuild/tools/systemtools.py | 38 ++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index dbd7415f8a..866450f2dd 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -29,6 +29,7 @@ @auther: Ward Poelmans (Ghent University) """ import ctypes +import errno import fcntl import grp # @UnresolvedImport import os @@ -160,24 +161,35 @@ class SystemToolsException(Exception): def sched_getaffinity(): """Determine list of available cores for current process.""" cpu_mask_t = ctypes.c_ulong - cpu_setsize = 1024 n_cpu_bits = 8 * ctypes.sizeof(cpu_mask_t) - n_mask_bits = cpu_setsize // n_cpu_bits - - class cpu_set_t(ctypes.Structure): - """Class that implements the cpu_set_t struct.""" - _fields_ = [('bits', cpu_mask_t * n_mask_bits)] _libc_lib = find_library('c') - _libc = ctypes.cdll.LoadLibrary(_libc_lib) + _libc = ctypes.CDLL(_libc_lib, use_errno=True) pid = os.getpid() - cs = cpu_set_t() - ec = _libc.sched_getaffinity(os.getpid(), ctypes.sizeof(cpu_set_t), ctypes.pointer(cs)) - if ec == 0: - _log.debug("sched_getaffinity for pid %s successful", pid) - else: - raise EasyBuildError("sched_getaffinity failed for pid %s ec %s", pid, ec) + + cpu_setsize = 1024 # Max number of CPUs currently detectable + max_cpu_setsize = ctypes.c_ulong(-1).value // 4 # (INT_MAX / 2) + # Limit it to something reasonable but still big enough + max_cpu_setsize = min(max_cpu_setsize, 1e9) + while cpu_setsize < max_cpu_setsize: + n_mask_bits = cpu_setsize // n_cpu_bits + + class cpu_set_t(ctypes.Structure): + """Class that implements the cpu_set_t struct.""" + _fields_ = [('bits', cpu_mask_t * n_mask_bits)] + + cs = cpu_set_t() + ec = _libc.sched_getaffinity(pid, ctypes.sizeof(cpu_set_t), ctypes.pointer(cs)) + if ec == 0: + _log.debug("sched_getaffinity for pid %s successful", pid) + break + elif ctypes.get_errno() != errno.EINVAL: + raise EasyBuildError("sched_getaffinity failed for pid %s errno %s", pid, ctypes.get_errno()) + cpu_setsize *= 2 + + if ec != 0: + raise EasyBuildError("sched_getaffinity failed finding a large enough cpuset for pid %s", pid) cpus = [] for bitmask in cs.bits: From 50ce59a53b9f950bf9ba03583ce3175698fd0867 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 26 May 2021 15:47:15 +0200 Subject: [PATCH 345/864] fix typo in comment Co-authored-by: ocaisa --- test/framework/toy_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 2156fab04b..88165c5841 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1709,7 +1709,7 @@ def test_module_only_extensions(self): ]) write_file(test_ec, test_ec_txt) - # clean up $MODULEPATH to only modules in test prefix dir are found + # clean up $MODULEPATH so only modules in test prefix dir are found self.reset_modulepath([os.path.join(self.test_installpath, 'modules', 'all')]) self.assertEqual(self.modtool.available('toy'), []) From 7153985228893f663bdf0fffda250aaebd6d6798 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 26 May 2021 20:38:51 +0200 Subject: [PATCH 346/864] avoid TypeError when comparing None with 0.0 in test_cpu_speed_native --- test/framework/systemtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 68fae7ea5a..78855b3ef7 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -454,7 +454,7 @@ def test_cpu_speed_native(self): """Test getting CPU speed.""" cpu_speed = get_cpu_speed() self.assertTrue(isinstance(cpu_speed, float) or cpu_speed is None) - self.assertTrue(cpu_speed > 0.0 or cpu_speed is None) + self.assertTrue(cpu_speed is None or cpu_speed > 0.0) def test_cpu_speed_linux(self): """Test getting CPU speed (mocked for Linux).""" From 3c1279e971ccf81d67e21e93e3996c2a09558455 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 26 May 2021 20:52:03 +0200 Subject: [PATCH 347/864] consistently use cpu_mask_t in sched_getaffinity --- easybuild/tools/systemtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 866450f2dd..26f66623cd 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -169,7 +169,7 @@ def sched_getaffinity(): pid = os.getpid() cpu_setsize = 1024 # Max number of CPUs currently detectable - max_cpu_setsize = ctypes.c_ulong(-1).value // 4 # (INT_MAX / 2) + max_cpu_setsize = cpu_mask_t(-1).value // 4 # (INT_MAX / 2) # Limit it to something reasonable but still big enough max_cpu_setsize = min(max_cpu_setsize, 1e9) while cpu_setsize < max_cpu_setsize: From 717a154574179ca90fe0a3fd7e5c7d6a3d4bbb29 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 26 May 2021 21:21:38 +0200 Subject: [PATCH 348/864] verify that --skip-extensions made the sanity check pass when using --sanity-check-only --- test/framework/options.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 0e53878423..995a4380cf 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5858,7 +5858,8 @@ def test_sanity_check_only(self): self.assertTrue(regex.search(error_msg), "Pattern '%s' should be found in: %s" % (regex.pattern, error_msg)) # failing sanity check for extension can be bypassed via --skip-extensions - self.eb_main(args + ['--skip-extensions'], do_build=True, return_error=True) + outtxt = self.eb_main(args + ['--skip-extensions'], do_build=True, raise_error=True) + self.assertTrue("Sanity check for toy successful" in outtxt) def test_skip_extensions(self): """Test use of --skip-extensions.""" From d578e736e0b44ebd8713fbfc40e1f10cbe84e55b Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 27 May 2021 08:41:21 +0800 Subject: [PATCH 349/864] append fujitsu libdir to LDFLAGS instead of LIBRARY_PATH so that it is also set by buildenv modules --- easybuild/toolchains/compiler/fujitsu.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/easybuild/toolchains/compiler/fujitsu.py b/easybuild/toolchains/compiler/fujitsu.py index f566ecbd31..042cbf436e 100644 --- a/easybuild/toolchains/compiler/fujitsu.py +++ b/easybuild/toolchains/compiler/fujitsu.py @@ -31,7 +31,6 @@ """ import os -import easybuild.tools.environment as env import easybuild.tools.systemtools as systemtools from easybuild.tools.toolchain.compiler import Compiler, DEFAULT_OPT_LEVEL @@ -86,14 +85,6 @@ class FujitsuCompiler(Compiler): (systemtools.AARCH64, systemtools.ARM): '-mcpu=generic -mtune=generic', } - def prepare(self, *args, **kwargs): - super(FujitsuCompiler, self).prepare(*args, **kwargs) - - # make sure the fujitsu module libraries are found (and added to rpath by wrapper) - libdir = os.path.join(os.getenv(TC_CONSTANT_MODULE_VAR), 'lib64') - self.log.debug("Adding %s to $LIBRARY_PATH" % libdir) - env.setvar('LIBRARY_PATH', os.pathsep.join([os.getenv('LIBRARY_PATH'), libdir])) - def _set_compiler_vars(self): super(FujitsuCompiler, self)._set_compiler_vars() @@ -101,3 +92,8 @@ def _set_compiler_vars(self): # moved to compiler constants to make sure it is always used # self.variables.nappend('CFLAGS', ['Nclang']) # self.variables.nappend('CXXFLAGS', ['Nclang']) + + # make sure the fujitsu module libraries are found (and added to rpath by wrapper) + libdir = os.path.join(os.getenv(TC_CONSTANT_MODULE_VAR), 'lib64') + self.log.debug("Adding %s to $LDFLAGS" % libdir) + self.variables.nappend('LDFLAGS', [libdir]) From 1212d0c2444c895399bc607b62534e5f1bd057a2 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 27 May 2021 09:05:30 +0800 Subject: [PATCH 350/864] add new 'required' keyword (introduced in #3555) to overriden '_get_software_root' in fujitsu toolchain --- easybuild/toolchains/linalg/fujitsussl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/toolchains/linalg/fujitsussl.py b/easybuild/toolchains/linalg/fujitsussl.py index ba09ad9a0c..3e9ec7902f 100644 --- a/easybuild/toolchains/linalg/fujitsussl.py +++ b/easybuild/toolchains/linalg/fujitsussl.py @@ -68,7 +68,7 @@ class FujitsuSSL(LinAlg): SCALAPACK_INCLUDE_DIR = [''] SCALAPACK_FAMILY = TC_CONSTANT_FUJITSU_SSL - def _get_software_root(self, name): + def _get_software_root(self, name, required=True): """Get install prefix for specified software name; special treatment for Fujitsu modules.""" if name == TC_CONSTANT_MODULE_NAME: env_var = TC_CONSTANT_MODULE_VAR @@ -78,7 +78,7 @@ def _get_software_root(self, name): else: self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root) else: - root = super(FujitsuSSL, self)._get_software_root(name) + root = super(FujitsuSSL, self)._get_software_root(name, required=required) return root From cd11c249def3645a36b3a72194589bd6003ead73 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Thu, 27 May 2021 08:23:48 +0100 Subject: [PATCH 351/864] Correctly strip the string --- easybuild/tools/filetools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index c89c37c8c9..1f5faf03da 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2005,8 +2005,8 @@ def back_up_file(src_file, backup_extension='bak', hidden=False, strip_fn=None): fn_suffix = '.%s' % backup_extension src_dir, src_fn = os.path.split(src_file) - if strip_fn: - src_fn = src_fn.rstrip(strip_fn) + if strip_fn and src_fn.endswith(strip_fn): + src_fn = src_fn[:-len(strip_fn)] backup_fp = find_backup_name_candidate(os.path.join(src_dir, fn_prefix + src_fn + fn_suffix)) From fc84287436d41e4e824448409064bf3dee10d15e Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Thu, 27 May 2021 08:36:54 +0100 Subject: [PATCH 352/864] replace another rstrip use --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 1f5faf03da..37988bca16 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1320,7 +1320,7 @@ def extract_cmd(filepath, overwrite=False): """ filename = os.path.basename(filepath) ext = find_extension(filename) - target = filename.rstrip(ext) + target = filename[:-len(ext)] cmd_tmpl = EXTRACT_CMDS[ext.lower()] if overwrite: From d18100b6002fbc3e108bfde93ad0d1db3cb0aa02 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Thu, 27 May 2021 09:16:15 +0100 Subject: [PATCH 353/864] add test --- test/framework/filetools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index c09200fb2d..b8166df22d 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -926,7 +926,7 @@ def test_back_up_file(self): self.assertEqual(ft.read_file(fp), new_txt) # check whether strip_fn works as expected - fp2 = fp + '.lua' + fp2 = fp + 'a.lua' ft.copy_file(fp, fp2) res = ft.back_up_file(fp2) self.assertTrue(fp2.endswith('.lua')) @@ -934,6 +934,8 @@ def test_back_up_file(self): res = ft.back_up_file(fp2, strip_fn='.lua') self.assertFalse('.lua' in os.path.basename(res)) + # strip_fn should not remove the first a in 'a.lua' + self.assertTrue(res.startswith(fp + 'a.bak_')) def test_move_logs(self): """Test move_logs function.""" From 30713da69a4f05b4f8125feb04731985f410f7c3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 27 May 2021 11:32:37 +0200 Subject: [PATCH 354/864] also check for correct stripping of extension by extract_cmd + better error message for extension stripping in test for back_up_file --- test/framework/filetools.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index b8166df22d..77c3c3bba1 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -97,6 +97,10 @@ def test_extract_cmd(self): ('test.iso', "7z x test.iso"), ('test.tar.Z', "tar xzf test.tar.Z"), ('test.foo.bar.sh', "cp -a test.foo.bar.sh ."), + # check whether extension is stripped correct to determine name of target file + # cfr. https://github.com/easybuilders/easybuild-framework/pull/3705 + ('testbz2.bz2', "bunzip2 -c testbz2.bz2 > testbz2"), + ('testgz.gz', "gunzip -c testgz.gz > testgz"), ] for (fn, expected_cmd) in tests: cmd = ft.extract_cmd(fn) @@ -935,7 +939,9 @@ def test_back_up_file(self): res = ft.back_up_file(fp2, strip_fn='.lua') self.assertFalse('.lua' in os.path.basename(res)) # strip_fn should not remove the first a in 'a.lua' - self.assertTrue(res.startswith(fp + 'a.bak_')) + expected = os.path.basename(fp) + 'a.bak_' + res_fn = os.path.basename(res) + self.assertTrue(res_fn.startswith(expected), "'%s' should start with with '%s'" % (res_fn, expected)) def test_move_logs(self): """Test move_logs function.""" From eb2e75799232c44b188877b242b855b1d9a78f7f Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Thu, 27 May 2021 12:48:06 +0100 Subject: [PATCH 355/864] Display the PR title in merge_pr --- easybuild/tools/github.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index c5cb3bb65e..3651e07ac1 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1381,6 +1381,7 @@ def merge_pr(pr): msg = "\n%s/%s PR #%s was submitted by %s, " % (pr_target_account, pr_target_repo, pr, pr_data['user']['login']) msg += "you are using GitHub account '%s'\n" % github_user + msg += "\nPR title: %s\n\n" % pr_data['title'] print_msg(msg, prefix=False) if pr_data['user']['login'] == github_user: raise EasyBuildError("Please do not merge your own PRs!") From 61801b4e4e83ae90aad02d9bb02f11545c845f83 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 27 May 2021 20:47:38 +0800 Subject: [PATCH 356/864] fetch easyconfigs to unique dir per PR --- easybuild/framework/easyconfig/tools.py | 3 +- .../OpenBLAS/OpenBLAS-0.3.1-GCC-7.3.0-2.30.eb | 41 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 test/framework/easyconfigs/test_ecs/o/OpenBLAS/OpenBLAS-0.3.1-GCC-7.3.0-2.30.eb diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 20aa2e9430..1610222fa3 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -344,8 +344,9 @@ def det_easyconfig_paths(orig_paths): if from_pr_list is not None: pr_files = [] + tmpdir = tempfile.mkdtemp() for pr in from_pr_list: - pr_files.extend(fetch_easyconfigs_from_pr(pr)) + pr_files.extend(fetch_easyconfigs_from_pr(pr, path=os.path.join(tmpdir, 'files_pr%s' % pr))) if ec_files: # replace paths for specified easyconfigs that are touched in PR diff --git a/test/framework/easyconfigs/test_ecs/o/OpenBLAS/OpenBLAS-0.3.1-GCC-7.3.0-2.30.eb b/test/framework/easyconfigs/test_ecs/o/OpenBLAS/OpenBLAS-0.3.1-GCC-7.3.0-2.30.eb new file mode 100644 index 0000000000..4a3957609b --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/o/OpenBLAS/OpenBLAS-0.3.1-GCC-7.3.0-2.30.eb @@ -0,0 +1,41 @@ +name = 'OpenBLAS' +version = '0.3.1' + +homepage = 'http://xianyi.github.com/OpenBLAS/' +description = "OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version." + +toolchain = {'name': 'GCC', 'version': '7.3.0-2.30'} +# need to build with -fno-tree-vectorize due to asm constraint bugs in OpenBLAS<0.3.6 +# cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/7180 +toolchainopts = {'vectorize': False} + +source_urls = [ + # order matters, trying to download the large.tgz/timing.tgz LAPACK tarballs from GitHub causes trouble + 'http://www.netlib.org/lapack/timing/', + 'https://github.com/xianyi/OpenBLAS/archive/', +] +sources = ['v%(version)s.tar.gz'] +patches = [ + ('large.tgz', '.'), + ('timing.tgz', '.'), + # this patch together with buildopts = 'USE_SIMPLE_THREADED_LEVEL3=1' + # is a workaround for the matrix multiplication issues + # https://lists.ugent.be/wws/arc/easybuild/2019-05/msg00006.html + # https://github.com/eylenth/Openblas_matrix_issue + 'OpenBLAS-%(version)s_disable-special-handling-of-OpenMP-thread-memory-allocation.patch', +] +checksums = [ + '1f5e956f35f3acdd3c74516e955d797a320c2e0135e31d838cbdb3ea94d0eb33', # v0.3.1.tar.gz + 'f328d88b7fa97722f271d7d0cfea1c220e0f8e5ed5ff01d8ef1eb51d6f4243a1', # large.tgz + '999c65f8ea8bd4eac7f1c7f3463d4946917afd20a997807300fe35d70122f3af', # timing.tgz + # OpenBLAS-0.3.1_disable-special-handling-of-OpenMP-thread-memory-allocation.patch + 'c85de436d6fff5d9cec1e24127ea9921551cdee373319dbade922d5cd3facd6a', +] + +# added as workaround for the matrices multiplication issue. see the patches above +buildopts = 'USE_SIMPLE_THREADED_LEVEL3=1' + +# extensive testing can be enabled by uncommenting the line below +# runtest = 'PATH=.:$PATH lapack-timing' + +moduleclass = 'numlib' From 894a56f3737fcc14fe917165447981927418edbb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 27 May 2021 21:33:18 +0200 Subject: [PATCH 357/864] fix support for specifying multiple PRs to --from-pr --- easybuild/framework/easyconfig/tools.py | 24 ++++++----- easybuild/tools/config.py | 2 +- easybuild/tools/github.py | 10 ++++- easybuild/tools/options.py | 14 +++---- easybuild/tools/robot.py | 11 ++--- .../OpenBLAS/OpenBLAS-0.3.1-GCC-7.3.0-2.30.eb | 41 ------------------- test/framework/options.py | 10 +++-- test/framework/robot.py | 4 +- 8 files changed, 44 insertions(+), 72 deletions(-) delete mode 100644 test/framework/easyconfigs/test_ecs/o/OpenBLAS/OpenBLAS-0.3.1-GCC-7.3.0-2.30.eb diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 1610222fa3..aae466468a 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -307,7 +307,7 @@ def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None): return paths -def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_pr=False): +def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None): """Obtain alternative paths for easyconfig files.""" # paths where tweaked easyconfigs will be placed, easyconfigs listed on the command line take priority and will be @@ -318,12 +318,13 @@ def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_pr=False): tweaked_ecs_paths = (os.path.join(tmpdir, 'tweaked_easyconfigs'), os.path.join(tmpdir, 'tweaked_dep_easyconfigs')) - # path where files touched in PR will be downloaded to - pr_path = None - if from_pr: - pr_path = os.path.join(tmpdir, "files_pr%s" % '_'.join(str(pr) for pr in from_pr)) + # paths where files touched in PRs will be downloaded to, + # which are picked up via 'pr_paths' build option in fetch_files_from_pr + pr_paths = None + if from_prs: + pr_paths = [os.path.join(tmpdir, 'files_pr%s' % pr) for pr in from_prs] - return tweaked_ecs_paths, pr_path + return tweaked_ecs_paths, pr_paths def det_easyconfig_paths(orig_paths): @@ -333,7 +334,7 @@ def det_easyconfig_paths(orig_paths): :return: list of paths to easyconfig files """ try: - from_pr_list = [int(pr_nr) for pr_nr in build_option('from_pr')] + from_prs = [int(pr_nr) for pr_nr in build_option('from_pr')] except ValueError: raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") @@ -342,11 +343,12 @@ def det_easyconfig_paths(orig_paths): # list of specified easyconfig files ec_files = orig_paths[:] - if from_pr_list is not None: + if from_prs: pr_files = [] - tmpdir = tempfile.mkdtemp() - for pr in from_pr_list: - pr_files.extend(fetch_easyconfigs_from_pr(pr, path=os.path.join(tmpdir, 'files_pr%s' % pr))) + for pr in from_prs: + # path to where easyconfig files should be downloaded is determined via 'pr_paths' build option, + # which corresponds to the list of PR paths returned by alt_easyconfig_paths + pr_files.extend(fetch_easyconfigs_from_pr(pr)) if ec_files: # replace paths for specified easyconfigs that are touched in PR diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index ec7a28a0f2..1e6cb9b66e 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -345,7 +345,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'build_specs', 'command_line', 'external_modules_metadata', - 'pr_path', + 'pr_paths', 'robot_path', 'valid_module_classes', 'valid_stops', diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index c5cb3bb65e..8d5001dc70 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -447,7 +447,15 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi if path is None: if github_repo == GITHUB_EASYCONFIGS_REPO: - path = build_option('pr_path') + pr_paths = build_option('pr_paths') + if pr_paths: + # figure out directory for this specific PR (see also alt_easyconfig_paths) + cands = [p for p in pr_paths if p.endswith('files_pr%s' % pr)] + if len(cands) == 1: + path = cands[0] + else: + raise EasyBuildError("Failed to isolate path for PR #%s from list of PR paths: %s", pr, pr_paths) + elif github_repo == GITHUB_EASYBLOCKS_REPO: path = os.path.join(tempfile.gettempdir(), 'ebs_pr%s' % pr) else: diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 98c942c444..ab9494d0db 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1461,19 +1461,19 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) - # map --from-pr strlist to list of ints + # map list of strings --from-pr value to list of integers try: - from_pr_list = [int(pr_nr) for pr_nr in eb_go.options.from_pr] + from_prs = [int(pr_nr) for pr_nr in eb_go.options.from_pr] except ValueError: raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") # determine robot path # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs tweaked_ecs = try_to_generate and build_specs - tweaked_ecs_paths, pr_path = alt_easyconfig_paths(tmpdir, tweaked_ecs=tweaked_ecs, from_pr=from_pr_list) + tweaked_ecs_paths, pr_paths = alt_easyconfig_paths(tmpdir, tweaked_ecs=tweaked_ecs, from_prs=from_prs) auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query - robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_path, auto_robot=auto_robot) - log.debug("Full robot path: %s" % robot_path) + robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_paths, auto_robot=auto_robot) + log.debug("Full robot path: %s", robot_path) if not robot_path: print_warning("Robot search path is empty!") @@ -1486,7 +1486,7 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): 'build_specs': build_specs, 'command_line': eb_cmd_line, 'external_modules_metadata': parse_external_modules_metadata(options.external_modules_metadata), - 'pr_path': pr_path, + 'pr_paths': pr_paths, 'robot_path': robot_path, 'silent': testing or new_update_opt, 'try_to_generate': try_to_generate, @@ -1543,7 +1543,7 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): sys.path.insert(0, new_fake_vsc_path) return eb_go, (build_specs, log, logfile, robot_path, search_query, tmpdir, try_to_generate, - from_pr_list, tweaked_ecs_paths) + from_prs, tweaked_ecs_paths) def process_software_build_specs(options): diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 896a213b22..a59f58bfe2 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -52,10 +52,10 @@ _log = fancylogger.getLogger('tools.robot', fname=False) -def det_robot_path(robot_paths_option, tweaked_ecs_paths, pr_path, auto_robot=False): +def det_robot_path(robot_paths_option, tweaked_ecs_paths, pr_paths, auto_robot=False): """Determine robot path.""" robot_path = robot_paths_option[:] - _log.info("Using robot path(s): %s" % robot_path) + _log.info("Using robot path(s): %s", robot_path) tweaked_ecs_path, tweaked_ecs_deps_path = None, None # paths to tweaked easyconfigs or easyconfigs downloaded from a PR have priority @@ -67,9 +67,10 @@ def det_robot_path(robot_paths_option, tweaked_ecs_paths, pr_path, auto_robot=Fa robot_path.append(tweaked_ecs_deps_path) _log.info("Prepended list of robot search paths with %s and appended with %s: %s", tweaked_ecs_path, tweaked_ecs_deps_path, robot_path) - if pr_path is not None: - robot_path.append(pr_path) - _log.info("Appended list of robot search paths with %s: %s" % (pr_path, robot_path)) + + if pr_paths is not None: + robot_path.extend(pr_paths) + _log.info("Extended list of robot search paths with %s: %s", pr_paths, robot_path) return robot_path diff --git a/test/framework/easyconfigs/test_ecs/o/OpenBLAS/OpenBLAS-0.3.1-GCC-7.3.0-2.30.eb b/test/framework/easyconfigs/test_ecs/o/OpenBLAS/OpenBLAS-0.3.1-GCC-7.3.0-2.30.eb deleted file mode 100644 index 4a3957609b..0000000000 --- a/test/framework/easyconfigs/test_ecs/o/OpenBLAS/OpenBLAS-0.3.1-GCC-7.3.0-2.30.eb +++ /dev/null @@ -1,41 +0,0 @@ -name = 'OpenBLAS' -version = '0.3.1' - -homepage = 'http://xianyi.github.com/OpenBLAS/' -description = "OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version." - -toolchain = {'name': 'GCC', 'version': '7.3.0-2.30'} -# need to build with -fno-tree-vectorize due to asm constraint bugs in OpenBLAS<0.3.6 -# cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/7180 -toolchainopts = {'vectorize': False} - -source_urls = [ - # order matters, trying to download the large.tgz/timing.tgz LAPACK tarballs from GitHub causes trouble - 'http://www.netlib.org/lapack/timing/', - 'https://github.com/xianyi/OpenBLAS/archive/', -] -sources = ['v%(version)s.tar.gz'] -patches = [ - ('large.tgz', '.'), - ('timing.tgz', '.'), - # this patch together with buildopts = 'USE_SIMPLE_THREADED_LEVEL3=1' - # is a workaround for the matrix multiplication issues - # https://lists.ugent.be/wws/arc/easybuild/2019-05/msg00006.html - # https://github.com/eylenth/Openblas_matrix_issue - 'OpenBLAS-%(version)s_disable-special-handling-of-OpenMP-thread-memory-allocation.patch', -] -checksums = [ - '1f5e956f35f3acdd3c74516e955d797a320c2e0135e31d838cbdb3ea94d0eb33', # v0.3.1.tar.gz - 'f328d88b7fa97722f271d7d0cfea1c220e0f8e5ed5ff01d8ef1eb51d6f4243a1', # large.tgz - '999c65f8ea8bd4eac7f1c7f3463d4946917afd20a997807300fe35d70122f3af', # timing.tgz - # OpenBLAS-0.3.1_disable-special-handling-of-OpenMP-thread-memory-allocation.patch - 'c85de436d6fff5d9cec1e24127ea9921551cdee373319dbade922d5cd3facd6a', -] - -# added as workaround for the matrices multiplication issue. see the patches above -buildopts = 'USE_SIMPLE_THREADED_LEVEL3=1' - -# extensive testing can be enabled by uncommenting the line below -# runtest = 'PATH=.:$PATH lapack-timing' - -moduleclass = 'numlib' diff --git a/test/framework/options.py b/test/framework/options.py index cbe8718be6..551e7ade57 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1728,7 +1728,7 @@ def test_from_pr(self): self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules)) pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr6424') - regex = re.compile("Appended list of robot search paths with %s:" % pr_tmpdir, re.M) + regex = re.compile(r"Extended list of robot search paths with \['%s'\]:" % pr_tmpdir, re.M) self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) except URLError as err: print("Ignoring URLError '%s' in test_from_pr" % err) @@ -1762,9 +1762,11 @@ def test_from_pr(self): regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules)) - pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr12150_12366') - regex = re.compile("Appended list of robot search paths with %s:" % pr_tmpdir, re.M) - self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + for pr in ('12150', '12366'): + pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr%s' % pr) + regex = re.compile(r"Extended list of robot search paths with .*%s.*:" % pr_tmpdir, re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + except URLError as err: print("Ignoring URLError '%s' in test_from_pr" % err) shutil.rmtree(tmpdir) diff --git a/test/framework/robot.py b/test/framework/robot.py index 0f44bf34e3..5e1d0fea6f 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1065,8 +1065,8 @@ def test_tweak_robotpath(self): test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') # Create directories to store the tweaked easyconfigs - tweaked_ecs_paths, pr_path = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True) - robot_path = det_robot_path([test_easyconfigs], tweaked_ecs_paths, pr_path, auto_robot=True) + tweaked_ecs_paths, pr_paths = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True) + robot_path = det_robot_path([test_easyconfigs], tweaked_ecs_paths, pr_paths, auto_robot=True) init_config(build_options={ 'valid_module_classes': module_classes(), From 3c9866d98d5b9a6de365763168a0d1c3b0d70724 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 28 May 2021 10:00:49 +0200 Subject: [PATCH 358/864] Fix upload-test-report PR numbers are integers which cannot be joined to a string directly but need to be converted first --- easybuild/tools/testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 88cb58032b..77cea10e8e 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -193,7 +193,7 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=None, gist_ descr = "(partial) EasyBuild log for failed build of %s" % ec['spec'] if pr_nrs is not None: - descr += " (PR #%s)" % ', #'.join(pr_nrs) + descr += " (PR #%s)" % ', #'.join(str(pr_nr) for pr_nr in pr_nrs) if easyblock_pr_nrs: descr += "".join(" (easyblock PR #%s)" % nr for nr in easyblock_pr_nrs) From 2ff43fe07fb7522a36189b158632bdcfd9b3fed3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 28 May 2021 10:49:31 +0200 Subject: [PATCH 359/864] add test for create_test_report function that covers bug fixed in #3708 --- test/framework/github.py | 67 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/test/framework/github.py b/test/framework/github.py index dbae54fbf6..ed94c7bf2d 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -34,8 +34,10 @@ import re import sys from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config +from time import gmtime from unittest import TextTestRunner +import easybuild.tools.testing from easybuild.base.rest import RestClient from easybuild.framework.easyconfig.tools import categorize_files_by_type from easybuild.tools.build_log import EasyBuildError @@ -45,7 +47,7 @@ from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, GITHUB_MERGEABLE_STATE_CLEAN from easybuild.tools.github import VALID_CLOSE_PR_REASONS from easybuild.tools.github import pick_default_branch -from easybuild.tools.testing import post_pr_test_report, session_state +from easybuild.tools.testing import create_test_report, post_pr_test_report, session_state from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters import easybuild.tools.github as gh @@ -71,7 +73,7 @@ class GithubTest(EnhancedTestCase): for non authenticated users of 50""" def setUp(self): - """setup""" + """Test setup.""" super(GithubTest, self).setUp() self.github_token = gh.fetch_github_token(GITHUB_TEST_ACCOUNT) @@ -85,6 +87,14 @@ def setUp(self): self.skip_github_tests = self.github_token is None and os.getenv('FORCE_EB_GITHUB_TESTS') is None + self.orig_testing_create_gist = easybuild.tools.testing.create_gist + + def tearDown(self): + """Cleanup after running test.""" + easybuild.tools.testing.create_gist = self.orig_testing_create_gist + + super(GithubTest, self).tearDown() + def test_pick_default_branch(self): """Test pick_default_branch function.""" @@ -1041,6 +1051,59 @@ def test_pr_test_report(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_create_test_report(self): + """Test create_test_report function.""" + logfile = os.path.join(self.test_prefix, 'log.txt') + write_file(logfile, "Bazel failed with: error") + ecs_with_res = [ + ({'spec': 'test.eb'}, {'success': True}), + ({'spec': 'fail.eb'}, { + 'success': False, + 'err': EasyBuildError("error: bazel"), + 'traceback': "in bazel", + 'log_file': logfile, + }), + ] + init_session_state = { + 'easybuild_configuration': ['EASYBUILD_DEBUG=1'], + 'environment': {'USER': 'test'}, + 'module_list': [{'mod_name': 'test'}], + 'system_info': {'name': 'test'}, + 'time': gmtime(0), + } + res = create_test_report("just a test", ecs_with_res, init_session_state) + patterns = [ + "**SUCCESS** _test.eb_", + "**FAIL (build issue)** _fail.eb_", + "01 Jan 1970 00:00:00", + "EASYBUILD_DEBUG=1", + ] + for pattern in patterns: + self.assertTrue(pattern in res['full'], "Pattern '%s' found in: %s" % (pattern, res['full'])) + + for pattern in patterns[:2]: + self.assertTrue(pattern in res['full'], "Pattern '%s' found in: %s" % (pattern, res['overview'])) + + # mock create_gist function, we don't want to actually create a gist every time we run this test... + def fake_create_gist(*args, **kwargs): + return 'https://gist.github.com/test' + + easybuild.tools.testing.create_gist = fake_create_gist + + res = create_test_report("just a test", ecs_with_res, init_session_state, pr_nrs=[123], gist_log=True) + + patterns.insert(2, "https://gist.github.com/test") + patterns.extend([ + "https://github.com/easybuilders/easybuild-easyconfigs/pull/123", + ]) + for pattern in patterns: + self.assertTrue(pattern in res['full'], "Pattern '%s' found in: %s" % (pattern, res['full'])) + + for pattern in patterns[:3]: + self.assertTrue(pattern in res['full'], "Pattern '%s' found in: %s" % (pattern, res['overview'])) + + self.assertTrue("**SUCCESS** _test.eb_" in res['overview']) + def suite(): """ returns all the testcases in this module """ From 916666a7a6ee0de1a6bbb1b1c234ee8ac5eae60d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 28 May 2021 11:12:35 +0200 Subject: [PATCH 360/864] Enhance detection of patch files with better error messages Instead of assuming that everything that doesn't look like a patch is an EC, assume *.patch files are patches and error out with a clear message why it was not detected as such. Avoids confusing errors such as: ERROR: Failed to process easyconfig foo.patch: Parsing easyconfig file failed: invalid syntax (, line 1) --- easybuild/framework/easyconfig/tools.py | 8 ++++++++ test/framework/easyconfig.py | 22 ++++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index aae466468a..929523ea82 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -623,6 +623,14 @@ def categorize_files_by_type(paths): # file must exist in order to check whether it's a patch file elif os.path.isfile(path) and is_patch_file(path): res['patch_files'].append(path) + elif path.endswith('.patch'): + if not os.path.exists(path): + raise EasyBuildError('File %s does not exist, did you mistype the path?', path) + elif not os.path.isfile(path): + raise EasyBuildError('File %s is expected to be a regular file, but is a folder instead', path) + else: + raise EasyBuildError('%s is not detected as a valid patch file. Please verify its contents!', + path) else: # anything else is considered to be an easyconfig file res['easyconfigs'].append(path) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 91f8502a48..f0f5ae5836 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3267,10 +3267,11 @@ def test_categorize_files_by_type(self): configuremake = os.path.join(easyblocks_dir, 'generic', 'configuremake.py') toy_easyblock = os.path.join(easyblocks_dir, 't', 'toy.py') + gzip_ec = os.path.join(test_ecs_dir, 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb') paths = [ 'bzip2-1.0.6.eb', toy_easyblock, - os.path.join(test_ecs_dir, 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb'), + gzip_ec, toy_patch, 'foo', ':toy-0.0-deps.eb', @@ -3279,7 +3280,7 @@ def test_categorize_files_by_type(self): res = categorize_files_by_type(paths) expected = [ 'bzip2-1.0.6.eb', - os.path.join(test_ecs_dir, 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb'), + gzip_ec, 'foo', ] self.assertEqual(res['easyconfigs'], expected) @@ -3287,6 +3288,23 @@ def test_categorize_files_by_type(self): self.assertEqual(res['patch_files'], [toy_patch]) self.assertEqual(res['py_files'], [toy_easyblock, configuremake]) + # Error cases + tmpdir = tempfile.mkdtemp() + non_existing = os.path.join(tmpdir, 'does_not_exist.patch') + self.assertErrorRegex(EasyBuildError, + "File %s does not exist" % non_existing, + categorize_files_by_type, [non_existing]) + patch_dir = os.path.join(tmpdir, 'folder.patch') + os.mkdir(patch_dir) + self.assertErrorRegex(EasyBuildError, + "File %s is expected to be a regular file" % patch_dir, + categorize_files_by_type, [patch_dir]) + invalid_patch = os.path.join(tmpdir, 'invalid.patch') + copy_file(gzip_ec, invalid_patch) + self.assertErrorRegex(EasyBuildError, + "%s is not detected as a valid patch file" % invalid_patch, + categorize_files_by_type, [invalid_patch]) + def test_resolve_template(self): """Test resolve_template function.""" self.assertEqual(resolve_template('', {}), '') From dd367c5b8f8163e9cba6bbc74e2ded59e0d3a4f4 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 28 May 2021 11:33:17 +0200 Subject: [PATCH 361/864] Adapt other test to use existing patch file --- test/framework/github.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/test/framework/github.py b/test/framework/github.py index dbae54fbf6..09bd6e1c42 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -916,15 +916,19 @@ def test_det_pr_target_repo(self): # no files => return default target repo (None) self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([])), None) + test_dir = os.path.dirname(os.path.abspath(__file__)) + # easyconfigs/patches (incl. files to delete) => easyconfigs repo - # this is solely based on filenames, actual files are not opened + # this is solely based on filenames, actual files are not opened, except for the patch file which must exist + toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' + toy_patch = os.path.join(test_dir, 'sandbox', 'sources', 'toy', toy_patch_fn) test_cases = [ ['toy.eb'], - ['toy.patch'], - ['toy.eb', 'toy.patch'], + [toy_patch], + ['toy.eb', toy_patch], [':toy.eb'], # deleting toy.eb ['one.eb', 'two.eb'], - ['one.eb', 'two.eb', 'toy.patch', ':todelete.eb'], + ['one.eb', 'two.eb', toy_patch, ':todelete.eb'], ] for test_case in test_cases: self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(test_case)), 'easybuild-easyconfigs') @@ -932,7 +936,6 @@ def test_det_pr_target_repo(self): # if only Python files are involved, result is easyblocks or framework repo; # all Python files are easyblocks => easyblocks repo, otherwise => framework repo; # files are opened and inspected here to discriminate between easyblocks & other Python files, so must exist! - testdir = os.path.dirname(os.path.abspath(__file__)) github_py = os.path.join(testdir, 'github.py') configuremake = os.path.join(testdir, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'configuremake.py') @@ -951,14 +954,14 @@ def test_det_pr_target_repo(self): self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(py_files)), 'easybuild-framework') # as soon as an easyconfig file or patch files is involved => result is easybuild-easyconfigs repo - for fn in ['toy.eb', 'toy.patch']: + for fn in ['toy.eb', toy_patch]: self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(py_files + [fn])), 'easybuild-easyconfigs') # if --pr-target-repo is specified, we always get this value (no guessing anymore) init_config(build_options={'pr_target_repo': 'thisisjustatest'}) self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([])), 'thisisjustatest') - self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(['toy.eb', 'toy.patch'])), 'thisisjustatest') + self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(['toy.eb', toy_patch])), 'thisisjustatest') self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(py_files)), 'thisisjustatest') self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([configuremake])), 'thisisjustatest') self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([toy_eb])), 'thisisjustatest') From ed512650a5e918a363f093c00dcbc43cb9ac6f7d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 28 May 2021 11:48:21 +0200 Subject: [PATCH 362/864] Rename testdir->test_dir --- test/framework/github.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/framework/github.py b/test/framework/github.py index 09bd6e1c42..db93540970 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -565,8 +565,8 @@ def test_find_easybuild_easyconfig(self): def test_find_patches(self): """ Test for find_software_name_for_patch """ - testdir = os.path.dirname(os.path.abspath(__file__)) - ec_path = os.path.join(testdir, 'easyconfigs') + test_dir = os.path.dirname(os.path.abspath(__file__)) + ec_path = os.path.join(test_dir, 'easyconfigs') init_config(build_options={ 'allow_modules_tool_mismatch': True, 'minimal_toolchains': True, @@ -936,11 +936,11 @@ def test_det_pr_target_repo(self): # if only Python files are involved, result is easyblocks or framework repo; # all Python files are easyblocks => easyblocks repo, otherwise => framework repo; # files are opened and inspected here to discriminate between easyblocks & other Python files, so must exist! - github_py = os.path.join(testdir, 'github.py') + github_py = os.path.join(test_dir, 'github.py') - configuremake = os.path.join(testdir, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'configuremake.py') + configuremake = os.path.join(test_dir, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'configuremake.py') self.assertTrue(os.path.exists(configuremake)) - toy_eb = os.path.join(testdir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') + toy_eb = os.path.join(test_dir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') self.assertTrue(os.path.exists(toy_eb)) self.assertEqual(build_option('pr_target_repo'), None) From 10135ed2af8cac536cf360c1c32699ec5100c762 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 31 May 2021 17:10:43 +0800 Subject: [PATCH 363/864] don't use a different module name for FFTW in the Fujitsu toolchain --- easybuild/toolchains/fft/fujitsufftw.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/toolchains/fft/fujitsufftw.py b/easybuild/toolchains/fft/fujitsufftw.py index 6daf035f1d..4b49b09932 100644 --- a/easybuild/toolchains/fft/fujitsufftw.py +++ b/easybuild/toolchains/fft/fujitsufftw.py @@ -33,5 +33,4 @@ class FujitsuFFTW(Fftw): """Fujitsu FFTW FFT library""" - FFT_MODULE_NAME = ['FFTW3-sve'] FFTW_API_VERSION = '3' From 0610e4ef82887166697f07fbae730600245d7d83 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 31 May 2021 17:23:49 +0800 Subject: [PATCH 364/864] move -Nclang back to C[XX]FLAGS in fujitsu compiler definition --- easybuild/toolchains/compiler/fujitsu.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/easybuild/toolchains/compiler/fujitsu.py b/easybuild/toolchains/compiler/fujitsu.py index 042cbf436e..63e1738407 100644 --- a/easybuild/toolchains/compiler/fujitsu.py +++ b/easybuild/toolchains/compiler/fujitsu.py @@ -46,9 +46,8 @@ class FujitsuCompiler(Compiler): COMPILER_MODULE_NAME = [TC_CONSTANT_MODULE_NAME] COMPILER_FAMILY = TC_CONSTANT_FUJITSU - # make sure fcc is always called in clang compatibility mode - COMPILER_CC = 'fcc -Nclang' - COMPILER_CXX = 'FCC -Nclang' + COMPILER_CC = 'fcc' + COMPILER_CXX = 'FCC' COMPILER_F77 = 'frt' COMPILER_F90 = 'frt' @@ -89,9 +88,8 @@ def _set_compiler_vars(self): super(FujitsuCompiler, self)._set_compiler_vars() # enable clang compatibility mode - # moved to compiler constants to make sure it is always used - # self.variables.nappend('CFLAGS', ['Nclang']) - # self.variables.nappend('CXXFLAGS', ['Nclang']) + self.variables.nappend('CFLAGS', ['Nclang']) + self.variables.nappend('CXXFLAGS', ['Nclang']) # make sure the fujitsu module libraries are found (and added to rpath by wrapper) libdir = os.path.join(os.getenv(TC_CONSTANT_MODULE_VAR), 'lib64') From 3fdaed33587451a74adb555fa3b4f673f5d62345 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 31 May 2021 17:33:41 +0800 Subject: [PATCH 365/864] add fujitsu libdir to both LIBRARY_PATH and LDFLAGS --- easybuild/toolchains/compiler/fujitsu.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/easybuild/toolchains/compiler/fujitsu.py b/easybuild/toolchains/compiler/fujitsu.py index 042cbf436e..69199d2986 100644 --- a/easybuild/toolchains/compiler/fujitsu.py +++ b/easybuild/toolchains/compiler/fujitsu.py @@ -31,6 +31,7 @@ """ import os +import easybuild.tools.environment as env import easybuild.tools.systemtools as systemtools from easybuild.tools.toolchain.compiler import Compiler, DEFAULT_OPT_LEVEL @@ -85,6 +86,14 @@ class FujitsuCompiler(Compiler): (systemtools.AARCH64, systemtools.ARM): '-mcpu=generic -mtune=generic', } + def prepare(self, *args, **kwargs): + super(FujitsuCompiler, self).prepare(*args, **kwargs) + + # make sure the fujitsu module libraries are found (and added to rpath by wrapper) + libdir = os.path.join(os.getenv(TC_CONSTANT_MODULE_VAR), 'lib64') + self.log.debug("Adding %s to $LIBRARY_PATH" % libdir) + env.setvar('LIBRARY_PATH', os.pathsep.join([os.getenv('LIBRARY_PATH', ''), libdir])) + def _set_compiler_vars(self): super(FujitsuCompiler, self)._set_compiler_vars() @@ -93,7 +102,7 @@ def _set_compiler_vars(self): # self.variables.nappend('CFLAGS', ['Nclang']) # self.variables.nappend('CXXFLAGS', ['Nclang']) - # make sure the fujitsu module libraries are found (and added to rpath by wrapper) + # also add fujitsu module library path to LDFLAGS libdir = os.path.join(os.getenv(TC_CONSTANT_MODULE_VAR), 'lib64') self.log.debug("Adding %s to $LDFLAGS" % libdir) self.variables.nappend('LDFLAGS', [libdir]) From 8ba55f6c619e11cba81ea0708c5baea12adc01f2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 31 May 2021 15:49:33 +0200 Subject: [PATCH 366/864] drop support for Python 2.6 --- easybuild/tools/systemtools.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 26f66623cd..f487875c27 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -1032,17 +1032,9 @@ def check_python_version(): python_ver = '%d.%d' % (python_maj_ver, python_min_ver) _log.info("Found Python version %s", python_ver) - silence_deprecation_warnings = build_option('silence_deprecation_warnings') or [] - if python_maj_ver == 2: - if python_min_ver < 6: - raise EasyBuildError("Python 2.6 or higher is required when using Python 2, found Python %s", python_ver) - elif python_min_ver == 6: - depr_msg = "Running EasyBuild with Python 2.6 is deprecated" - if 'Python26' in silence_deprecation_warnings: - _log.warning(depr_msg) - else: - _log.deprecated(depr_msg, '5.0') + if python_min_ver < 7: + raise EasyBuildError("Python 2.7 is required when using Python 2, found Python %s", python_ver) else: _log.info("Running EasyBuild with Python 2 (version %s)", python_ver) From 9353c65f9525c1a97ac197178272625e759a365d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 31 May 2021 17:11:19 +0200 Subject: [PATCH 367/864] drop import for build_option, now unused --- easybuild/tools/systemtools.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index f487875c27..108cdb4571 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -50,7 +50,6 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import build_option from easybuild.tools.filetools import is_readable, read_file, which from easybuild.tools.py2vs3 import string_type from easybuild.tools.run import run_cmd From e0b7c517f97ac456b12abe500a77175d15b970f5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 31 May 2021 18:51:55 +0200 Subject: [PATCH 368/864] fix test_check_python_version --- test/framework/systemtools.py | 34 ++++------------------------------ 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 78855b3ef7..c0e2fbf881 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -850,8 +850,8 @@ def mock_python_ver(py_maj_ver, py_min_ver): error_pattern = r"EasyBuild is not compatible \(yet\) with Python 4.0" self.assertErrorRegex(EasyBuildError, error_pattern, check_python_version) - mock_python_ver(2, 5) - error_pattern = r"Python 2.6 or higher is required when using Python 2, found Python 2.5" + mock_python_ver(2, 6) + error_pattern = r"Python 2.7 is required when using Python 2, found Python 2.6" self.assertErrorRegex(EasyBuildError, error_pattern, check_python_version) # no problems when running with a supported Python version @@ -859,45 +859,19 @@ def mock_python_ver(py_maj_ver, py_min_ver): mock_python_ver(*pyver) self.assertEqual(check_python_version(), pyver) - mock_python_ver(2, 6) - # deprecation warning triggers an error in test environment - error_pattern = r"Running EasyBuild with Python 2.6 is deprecated" - self.assertErrorRegex(EasyBuildError, error_pattern, check_python_version) - - # we may trigger a deprecation warning below (when testing with Python 2.6) - py26_depr_warning = "\nWARNING: Deprecated functionality, will no longer work in v5.0: " - py26_depr_warning += "Running EasyBuild with Python 2.6 is deprecated" - - self.allow_deprecated_behaviour() - - # first test with mocked Python 2.6 - self.mock_stderr(True) - check_python_version() - stderr = self.get_stderr() - self.mock_stderr(False) - - # we should always get a deprecation warning here - self.assertTrue(stderr.startswith(py26_depr_warning)) - - # restore Python version info to check with Python version used to run tests - st.sys.version_info = self.orig_sys_version_info - # shouldn't raise any errors, since Python version used to run tests should be supported; self.mock_stderr(True) (py_maj_ver, py_min_ver) = check_python_version() stderr = self.get_stderr() self.mock_stderr(False) + self.assertFalse(stderr) self.assertTrue(py_maj_ver in [2, 3]) if py_maj_ver == 2: - self.assertTrue(py_min_ver in [6, 7]) + self.assertTrue(py_min_ver == 7) else: self.assertTrue(py_min_ver >= 5) - # only deprecation warning when actually testing with Python 2.6 - if sys.version_info[:2] == (2, 6): - self.assertTrue(stderr.startswith(py26_depr_warning)) - def test_pick_dep_version(self): """Test pick_dep_version function.""" From 3f587696d586b5a603e2e2f6fdb43dbf1d2a22fa Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 1 Jun 2021 11:12:58 +0800 Subject: [PATCH 369/864] only add fujitsu libdir to LIBRARY_PATH if it's not already there --- easybuild/toolchains/compiler/fujitsu.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/toolchains/compiler/fujitsu.py b/easybuild/toolchains/compiler/fujitsu.py index 23ec59ce4f..d5e6d09808 100644 --- a/easybuild/toolchains/compiler/fujitsu.py +++ b/easybuild/toolchains/compiler/fujitsu.py @@ -89,9 +89,11 @@ def prepare(self, *args, **kwargs): super(FujitsuCompiler, self).prepare(*args, **kwargs) # make sure the fujitsu module libraries are found (and added to rpath by wrapper) + library_path = os.getenv('LIBRARY_PATH', '') libdir = os.path.join(os.getenv(TC_CONSTANT_MODULE_VAR), 'lib64') - self.log.debug("Adding %s to $LIBRARY_PATH" % libdir) - env.setvar('LIBRARY_PATH', os.pathsep.join([os.getenv('LIBRARY_PATH', ''), libdir])) + if libdir not in library_path: + self.log.debug("Adding %s to $LIBRARY_PATH" % libdir) + env.setvar('LIBRARY_PATH', os.pathsep.join([library_path, libdir])) def _set_compiler_vars(self): super(FujitsuCompiler, self)._set_compiler_vars() From 6b029778185316f9460b714eb475e4391c055ac2 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 1 Jun 2021 16:16:41 +0800 Subject: [PATCH 370/864] prepare release notes for EasyBuild v4.4.0 + bump version to 4.4.0 --- RELEASE_NOTES | 58 ++++++++++++++++++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 21d810f690..fb0555bedc 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,64 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.4.0 (June 2nd 2021) +---------------------- + +feature release + +- various enhancements, including: + - performance improvements for easyconfig parsing and eb startup (#3555) + - add support for multiple PRs in --from-pr (#3605, #3707) + - update setup.py to indicate compatibility with Python 3.8 and 3.9 (#3647) + - add support for prepending custom library paths in RPATH section (#3650) + - allow amending easyconfig parameters which are not the default (#3651) + - update HMNS for Intel OneAPI compilers (#3653) + - add support for --sanity-check-only + also run sanity check for extensions when using --module-only (#3655) + - add support for running commands asynchronously via run_cmd + complete_cmd functions (#3656) + - add support for using oneAPI versions of 'intel' toolchain components (#3665) + - add toolchain definition for gofbf (foss with FlexiBLAS rather than OpenBLAS) (#3666) + - add support for using ARCH constant in easyconfig files (#3670) + - honor keyboard interrupt in 'eb' command (#3674) + - add toolchain definition for Fujitsu toolchain (#3677, #3704, #3712, #3713, #3714, #3717) + - extend sanity check step to check whether specific libraries are not linked into installed binaries/libraries (#3678) + - add locate_solib function to locate Linux shared libraries (#3682) + - add system agnostic function to locate shared libraries (#3683) + - add update_build_option function to update specific build options after initializing the EasyBuild configuration (#3684) + - use unload/load in ModuleGeneratorLua.swap_module, since 'swap' is not supported by Lmod (#3685) + - update HierarchicalMNS to also return 'Toolchain//' as $MODULEPATH extension for cpe* Cray toolchains (#3686) + - allow opting out of recursively unloaded of modules via recursive_module_unload easyconfig parameter (#3689) + - check for correct version values when parsing easystack file (#3693) + - run post-install commands specified for a specific extension (#3696) + - add support for --skip-extensions (#3702) + - include PR title in output produced by --merge-pr (#3706) +- various bug fixes, including: + - enhance apply_regex_substitutions to allow specifying action to take in case there are no matches (#3440) + - avoid metadata greedy behaviour when probing for external module metadata (#3559) + - catch problems early on if --github-user is not specified for --new-pr & co (#3644) + - re-enable write permissions when installing with read-only-installdir (#3649) + - improve logging when failing to load class from exts_classmap (#3657) + - fix use of --module-only on existing installations without write permissions (#3659) + - correct help text for subdir-user-modules (#3660) + - pin GitPython version for Python<3.6, don't test bootstrap script against Python 3.8/3.9 (#3675) + - avoid picking up easyblocks outside of sandbox in framework tests (#3680) + - make get_config_obj return a copy rather than a reference (#3692) + - make sure that $TAPE is unset when using piped tar (#3698) + - fix extending message for changed files in new_pr_from_branch (#3699) + - enhance sched_getaffinity function to avoid early crash when counting available cores on systems with more than 1024 cores (#3701) + - correctly strip extension from filename in extract_cmd and back_up_file functions (#3705) + - fix crash in create_test_report with --upload-test-report --from-pr when one or more test installations failed (#3708) +- other changes: + - deprecate adding a non-existing path to $MODULEPATH (#3637) + - bump cryptography requirement from 3.2.1 to 3.3.2 (#3643, #3648) + - test bootstrap script in separate workflow, and limit test configurations a bit (#3646) + - replace log_error parameter of which() by on_error (#3661, #3664) + - don't skip sanity check for --module-only --rebuild (#3645) + - move disable_templating function into the EasyConfig class (#3668) + - tweak foss toolchain definition to switch from OpenBLAS to FlexiBLAS in foss/2021a (#3679) + - suggest missing SSH key when not able to read from remote repository in --check-github (#3681) + - drop support for Python 2.6 (#3715) + + v4.3.4 (April 9th 2021) ----------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 089f123624..bd33a5c163 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.3.5.dev0') +VERSION = LooseVersion('4.4.0') UNKNOWN = 'UNKNOWN' From d98727c12bf6a26dc3b9c0c68b28ce4e21a99722 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 1 Jun 2021 20:47:01 +0200 Subject: [PATCH 371/864] minor tweaks to v4.4.0 release notes --- RELEASE_NOTES | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index fb0555bedc..cfd7e6f31d 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -10,13 +10,13 @@ v4.4.0 (June 2nd 2021) feature release - various enhancements, including: + - enhance apply_regex_substitutions to allow specifying action to take in case there are no matches (#3440) - performance improvements for easyconfig parsing and eb startup (#3555) - - add support for multiple PRs in --from-pr (#3605, #3707) - - update setup.py to indicate compatibility with Python 3.8 and 3.9 (#3647) - - add support for prepending custom library paths in RPATH section (#3650) + - add support for downloading easyconfigs from multiple PRs with --from-pr (#3605, #3707, #3708) + - add support for prepending custom library paths in RPATH section via --rpath-override-dirs (#3650) - allow amending easyconfig parameters which are not the default (#3651) - - update HMNS for Intel OneAPI compilers (#3653) - - add support for --sanity-check-only + also run sanity check for extensions when using --module-only (#3655) + - update HierarchicalMNS for Intel OneAPI compilers (#3653) + - add support for --sanity-check-only (#3655) - add support for running commands asynchronously via run_cmd + complete_cmd functions (#3656) - add support for using oneAPI versions of 'intel' toolchain components (#3665) - add toolchain definition for gofbf (foss with FlexiBLAS rather than OpenBLAS) (#3666) @@ -24,39 +24,43 @@ feature release - honor keyboard interrupt in 'eb' command (#3674) - add toolchain definition for Fujitsu toolchain (#3677, #3704, #3712, #3713, #3714, #3717) - extend sanity check step to check whether specific libraries are not linked into installed binaries/libraries (#3678) + - via banned-linked-shared-libs and required-linked-shared-libs EasyBuild configuration options + - via banned_linked_shared_libs and required_linked_shared_libs methods in toolchain support + - via banned_linked_shared_libs and required_linked_shared_libs methods in easyblock + - via banned_linked_shared_libs and required_linked_shared_libs easyconfig parameters - add locate_solib function to locate Linux shared libraries (#3682) - add system agnostic function to locate shared libraries (#3683) - add update_build_option function to update specific build options after initializing the EasyBuild configuration (#3684) - - use unload/load in ModuleGeneratorLua.swap_module, since 'swap' is not supported by Lmod (#3685) - - update HierarchicalMNS to also return 'Toolchain//' as $MODULEPATH extension for cpe* Cray toolchains (#3686) - allow opting out of recursively unloaded of modules via recursive_module_unload easyconfig parameter (#3689) - check for correct version values when parsing easystack file (#3693) - run post-install commands specified for a specific extension (#3696) - add support for --skip-extensions (#3702) - include PR title in output produced by --merge-pr (#3706) - various bug fixes, including: - - enhance apply_regex_substitutions to allow specifying action to take in case there are no matches (#3440) - - avoid metadata greedy behaviour when probing for external module metadata (#3559) + - avoid metadata greedy behaviour when probing for external module metadata (mostly relevant for integration with Cray Programming Environment) (#3559) - catch problems early on if --github-user is not specified for --new-pr & co (#3644) - re-enable write permissions when installing with read-only-installdir (#3649) + - also run sanity check for extensions when using --module-only (#3655) - improve logging when failing to load class from exts_classmap (#3657) - fix use of --module-only on existing installations without write permissions (#3659) - correct help text for subdir-user-modules (#3660) - - pin GitPython version for Python<3.6, don't test bootstrap script against Python 3.8/3.9 (#3675) - avoid picking up easyblocks outside of sandbox in framework tests (#3680) - - make get_config_obj return a copy rather than a reference (#3692) + - use unload/load in ModuleGeneratorLua.swap_module, since 'swap' is not supported by Lmod (#3685) + - update HierarchicalMNS to also return 'Toolchain//' as $MODULEPATH extension for cpe* Cray toolchains (#3686) + - make EasyConfigParser.get_config_dict return a copy rather than a reference (#3692) - make sure that $TAPE is unset when using piped tar (#3698) - fix extending message for changed files in new_pr_from_branch (#3699) - enhance sched_getaffinity function to avoid early crash when counting available cores on systems with more than 1024 cores (#3701) - correctly strip extension from filename in extract_cmd and back_up_file functions (#3705) - - fix crash in create_test_report with --upload-test-report --from-pr when one or more test installations failed (#3708) - other changes: - deprecate adding a non-existing path to $MODULEPATH (#3637) - bump cryptography requirement from 3.2.1 to 3.3.2 (#3643, #3648) - test bootstrap script in separate workflow, and limit test configurations a bit (#3646) + - update setup.py to indicate compatibility with Python 3.8 and 3.9 (#3647) - replace log_error parameter of which() by on_error (#3661, #3664) - don't skip sanity check for --module-only --rebuild (#3645) - move disable_templating function into the EasyConfig class (#3668) + - pin GitPython version for Python<3.6, don't test bootstrap script against Python 3.8/3.9 (#3675) - tweak foss toolchain definition to switch from OpenBLAS to FlexiBLAS in foss/2021a (#3679) - suggest missing SSH key when not able to read from remote repository in --check-github (#3681) - drop support for Python 2.6 (#3715) From 478713bb4b7cc8ef4e72b41fa1ece057773a0091 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 2 Jun 2021 12:42:52 +0800 Subject: [PATCH 372/864] don't override COMPILER_MODULE_NAME, inherited from Ffmpi, in Fujitsu toolchain --- easybuild/toolchains/fujitsu.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/toolchains/fujitsu.py b/easybuild/toolchains/fujitsu.py index 6c81342901..a4fedfa1a3 100644 --- a/easybuild/toolchains/fujitsu.py +++ b/easybuild/toolchains/fujitsu.py @@ -36,4 +36,3 @@ class Fujitsu(Ffmpi, FujitsuFFTW, FujitsuSSL): """Compiler toolchain for Fujitsu.""" NAME = 'Fujitsu' SUBTOOLCHAIN = Ffmpi.NAME - COMPILER_MODULE_NAME = [] From fcec79b02705e15077028ebd37b2b95f3d8aab02 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 2 Jun 2021 13:40:58 +0200 Subject: [PATCH 373/864] bump version to 4.4.1dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index bd33a5c163..47cc16313a 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.4.0') +VERSION = LooseVersion('4.4.1.dev0') UNKNOWN = 'UNKNOWN' From c118ec65a0fe4a47a09ea705f7c291497c06b3f2 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Sat, 5 Jun 2021 11:24:40 +0200 Subject: [PATCH 374/864] avoid overwritting pr_nr in post_pr_test_report for reports with --include-easyblocks-from-pr --- easybuild/tools/testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 77cea10e8e..e176fa0c9f 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -295,7 +295,7 @@ def post_pr_test_report(pr_nr, repo_type, test_report, msg, init_session_state, if build_option('include_easyblocks_from_pr'): if repo_type == GITHUB_EASYCONFIGS_REPO: - easyblocks_pr_nrs = [int(pr_nr) for pr_nr in build_option('include_easyblocks_from_pr')] + easyblocks_pr_nrs = [int(eb_pr_nr) for eb_pr_nr in build_option('include_easyblocks_from_pr')] comment_lines.append("Using easyblocks from PR(s) %s" % ", ".join(["https://github.com/%s/%s/pull/%s" % (pr_target_account, GITHUB_EASYBLOCKS_REPO, easyblocks_pr_nr) From 5b9b7558f1f105d9ab2cdbfe88e37e8fcbf488ad Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Sun, 6 Jun 2021 13:01:00 +0200 Subject: [PATCH 375/864] Update linting.yml --- .github/workflows/linting.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 55ada2170c..3048de67f4 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -3,13 +3,17 @@ on: [push, pull_request] jobs: python-linting: runs-on: ubuntu-18.04 + strategy: + matrix: + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] + steps: - uses: actions/checkout@v2 - name: set up Python uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: ${{ matrix.python-version }} - name: install Python packages run: | From ee878caa4f0eaf22df100ab03105877f0f006fef Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 6 Jun 2021 17:32:05 +0200 Subject: [PATCH 376/864] enhance test for post_pr_test_report to cover combination of --from-pr and --include-easyblocks-from-pr --- easybuild/tools/testing.py | 2 +- test/framework/github.py | 23 ++++++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index e176fa0c9f..3fa74ebeb0 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -295,7 +295,7 @@ def post_pr_test_report(pr_nr, repo_type, test_report, msg, init_session_state, if build_option('include_easyblocks_from_pr'): if repo_type == GITHUB_EASYCONFIGS_REPO: - easyblocks_pr_nrs = [int(eb_pr_nr) for eb_pr_nr in build_option('include_easyblocks_from_pr')] + easyblocks_pr_nrs = [int(x) for x in build_option('include_easyblocks_from_pr')] comment_lines.append("Using easyblocks from PR(s) %s" % ", ".join(["https://github.com/%s/%s/pull/%s" % (pr_target_account, GITHUB_EASYBLOCKS_REPO, easyblocks_pr_nr) diff --git a/test/framework/github.py b/test/framework/github.py index ed94c7bf2d..f14ce10e95 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -41,7 +41,7 @@ from easybuild.base.rest import RestClient from easybuild.framework.easyconfig.tools import categorize_files_by_type from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import build_option, module_classes +from easybuild.tools.config import build_option, module_classes, update_build_option from easybuild.tools.configobj import ConfigObj from easybuild.tools.filetools import read_file, write_file from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, GITHUB_MERGEABLE_STATE_CLEAN @@ -1051,6 +1051,27 @@ def test_pr_test_report(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + # also test combination of --from-pr and --include-easyblocks-from-pr + update_build_option('include_easyblocks_from_pr', ['6789']) + + self.mock_stderr(True) + self.mock_stdout(True) + post_pr_test_report('1234', gh.GITHUB_EASYCONFIGS_REPO, test_report, "OK!", init_session_state, True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertEqual(stderr, '') + + patterns = [ + r"^\[DRY RUN\] Adding comment to easybuild-easyconfigs issue #1234: 'Test report by @easybuild_test", + r"^See https://gist.github.com/DRY_RUN for a full test report.'", + r"Using easyblocks from PR\(s\) https://github.com/easybuilders/easybuild-easyblocks/pull/6789", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_create_test_report(self): """Test create_test_report function.""" logfile = os.path.join(self.test_prefix, 'log.txt') From 14715cb831d2d98bdf309236f06ed04d740e74e2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 7 Jun 2021 09:36:30 +0200 Subject: [PATCH 377/864] use more single-letter variables in list comprehensions to avoid overwriting variables outside of list comprehension when using Python 2 --- easybuild/tools/github.py | 8 ++++---- easybuild/tools/testing.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index dd07ef07d8..000568a479 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1232,7 +1232,7 @@ def reasons_for_closing(pr_data): robot_paths = build_option('robot_path') - pr_files = [path for path in fetch_easyconfigs_from_pr(pr_data['number']) if path.endswith('.eb')] + pr_files = [p for p in fetch_easyconfigs_from_pr(pr_data['number']) if p.endswith('.eb')] obsoleted = [] uses_archived_tc = [] @@ -1491,17 +1491,17 @@ def add_pr_labels(pr, branch=GITHUB_DEVELOP_BRANCH): download_repo_path = download_repo(branch=branch, path=tmpdir) - pr_files = [path for path in fetch_easyconfigs_from_pr(pr) if path.endswith('.eb')] + pr_files = [p for p in fetch_easyconfigs_from_pr(pr) if p.endswith('.eb')] file_info = det_file_info(pr_files, download_repo_path) pr_target_account = build_option('pr_target_account') github_user = build_option('github_user') pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user) - pr_labels = [label['name'] for label in pr_data['labels']] + pr_labels = [x['name'] for x in pr_data['labels']] expected_labels = det_pr_labels(file_info, pr_target_repo) - missing_labels = [label for label in expected_labels if label not in pr_labels] + missing_labels = [x for x in expected_labels if x not in pr_labels] dry_run = build_option('dry_run') or build_option('extended_dry_run') diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 3fa74ebeb0..7ec605415f 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -151,7 +151,7 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=None, gist_ test_report = [] if pr_nrs is not None: repo = pr_target_repo or GITHUB_EASYCONFIGS_REPO - pr_urls = ["https://github.com/%s/%s/pull/%s" % (pr_target_account, repo, pr_nr) for pr_nr in pr_nrs] + pr_urls = ["https://github.com/%s/%s/pull/%s" % (pr_target_account, repo, x) for x in pr_nrs] test_report.extend([ "Test report for %s" % ', '.join(pr_urls), "", @@ -333,12 +333,12 @@ def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state dump_path = build_option('dump_test_report') try: - pr_nrs = [int(pr_nr) for pr_nr in build_option('from_pr')] + pr_nrs = [int(x) for x in build_option('from_pr')] except ValueError: raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") try: - easyblock_pr_nrs = [int(pr_nr) for pr_nr in build_option('include_easyblocks_from_pr')] + easyblock_pr_nrs = [int(x) for x in build_option('include_easyblocks_from_pr')] except ValueError: raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s.") From b61190d9d18685f4351991407461d0904dd61105 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 7 Jun 2021 17:30:56 +0200 Subject: [PATCH 378/864] Add module-write hook This hook will be called anytime a module is written, so even for the fake modules used for sanity checks etc. It gets passed the easyblock as the first argument and the filename and content to be written as the next 2 and may return a new text to be used for the module file. If nothing or None is returned the original text will be used. --- easybuild/framework/easyblock.py | 6 +- easybuild/tools/hooks.py | 26 ++++---- test/framework/toy_build.py | 107 +++++++++++++++++++------------ 3 files changed, 82 insertions(+), 57 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index c100e88d88..1fe20a5a9b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -81,7 +81,7 @@ from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP -from easybuild.tools.hooks import load_hooks, run_hook +from easybuild.tools.hooks import MODULE_WRITE, load_hooks, run_hook from easybuild.tools.run import run_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for @@ -3165,6 +3165,10 @@ def make_module_step(self, fake=False): txt += self.make_module_extra() txt += self.make_module_footer() + hook_txt = run_hook(MODULE_WRITE, self.hooks, args=[self, mod_filepath, txt]) + if hook_txt is not None: + txt = hook_txt + if self.dry_run: # only report generating actual module file during dry run, don't mention temporary module files if not fake: diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 85407fb993..001d191b0b 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -58,6 +58,7 @@ START = 'start' PARSE = 'parse' +MODULE_WRITE = 'module_write' END = 'end' PRE_PREF = 'pre_' @@ -69,7 +70,7 @@ INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP, PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP] -HOOK_NAMES = [START, PARSE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END] +HOOK_NAMES = [START, PARSE, MODULE_WRITE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END] KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] @@ -99,7 +100,7 @@ def load_hooks(hooks_path): if attr.endswith(HOOK_SUFF): hook = getattr(imported_hooks, attr) if callable(hook): - hooks.update({attr: hook}) + hooks[attr] = hook else: _log.debug("Skipping non-callable attribute '%s' when loading hooks", attr) _log.info("Found hooks: %s", sorted(hooks.keys())) @@ -120,10 +121,7 @@ def load_hooks(hooks_path): def verify_hooks(hooks): """Check whether list of obtained hooks only includes known hooks.""" - unknown_hooks = [] - for key in sorted(hooks): - if key not in KNOWN_HOOKS: - unknown_hooks.append(key) + unknown_hooks = [key for key in sorted(hooks) if key not in KNOWN_HOOKS] if unknown_hooks: error_lines = ["Found one or more unknown hooks:"] @@ -147,7 +145,7 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False): Find hook with specified label. :param label: name of hook - :param hooks: list of defined hooks + :param hooks: dict of defined hooks :param pre_step_hook: indicates whether hook to run is a pre-step hook :param post_step_hook: indicates whether hook to run is a post-step hook """ @@ -162,18 +160,16 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False): hook_name = hook_prefix + label + HOOK_SUFF - for key in hooks: - if key == hook_name: - _log.info("Found %s hook", hook_name) - res = hooks[key] - break + res = hooks.get(hook_name) + if res: + _log.info("Found %s hook", hook_name) return res def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, msg=None): """ - Run hook with specified label. + Run hook with specified label and return result of hook or None. :param label: name of hook :param hooks: list of defined hooks @@ -183,6 +179,7 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, :param msg: custom message that is printed when hook is called """ hook = find_hook(label, hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook) + res = None if hook: if args is None: args = [] @@ -197,4 +194,5 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, print_msg(msg) _log.info("Running '%s' hook function (arguments: %s)...", hook.__name__, args) - hook(*args) + res = hook(*args) + return res diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index c339194ad3..b8d58ae64b 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -39,6 +39,7 @@ import stat import sys import tempfile +import textwrap from distutils.version import LooseVersion from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from test.framework.package import mock_fpm @@ -2638,34 +2639,38 @@ def test_toy_build_trace(self): def test_toy_build_hooks(self): """Test use of --hooks.""" hooks_file = os.path.join(self.test_prefix, 'my_hooks.py') - hooks_file_txt = '\n'.join([ - "import os", - '', - "def start_hook():", - " print('start hook triggered')", - '', - "def parse_hook(ec):", - " print('%s %s' % (ec.name, ec.version))", + hooks_file_txt = textwrap.dedent(""" + import os + + def start_hook(): + print('start hook triggered') + + def parse_hook(ec): + print('%s %s' % (ec.name, ec.version)) # print sources value to check that raw untemplated strings are exposed in parse_hook - " print(ec['sources'])", + print(ec['sources']) # try appending to postinstallcmd to see whether the modification is actually picked up # (required templating to be disabled before parse_hook is called) - " ec['postinstallcmds'].append('echo toy')", - " print(ec['postinstallcmds'][-1])", - '', - "def pre_configure_hook(self):", - " print('pre-configure: toy.source: %s' % os.path.exists('toy.source'))", - '', - "def post_configure_hook(self):", - " print('post-configure: toy.source: %s' % os.path.exists('toy.source'))", - '', - "def post_install_hook(self):", - " print('in post-install hook for %s v%s' % (self.name, self.version))", - " print(', '.join(sorted(os.listdir(self.installdir))))", - '', - "def end_hook():", - " print('end hook triggered, all done!')", - ]) + ec['postinstallcmds'].append('echo toy') + print(ec['postinstallcmds'][-1]) + + def pre_configure_hook(self): + print('pre-configure: toy.source: %s' % os.path.exists('toy.source')) + + def post_configure_hook(self): + print('post-configure: toy.source: %s' % os.path.exists('toy.source')) + + def post_install_hook(self): + print('in post-install hook for %s v%s' % (self.name, self.version)) + print(', '.join(sorted(os.listdir(self.installdir)))) + + def module_write_hook(self, module_path, module_txt): + print('in module-write hook hook for %s' % os.path.basename(module_path)) + return module_txt.replace('Toy C program, 100% toy.', 'Not a toy anymore') + + def end_hook(): + print('end hook triggered, all done!') + """) write_file(hooks_file, hooks_file_txt) self.mock_stderr(True) @@ -2676,26 +2681,44 @@ def test_toy_build_hooks(self): self.mock_stderr(False) self.mock_stdout(False) + test_mod_path = os.path.join(self.test_installpath, 'modules', 'all') + toy_mod_file = os.path.join(test_mod_path, 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_mod_file += '.lua' + self.assertEqual(stderr, '') - expected_output = '\n'.join([ - "== Running start hook...", - "start hook triggered", - "== Running parse hook for toy-0.0.eb...", - "toy 0.0", - "['%(name)s-%(version)s.tar.gz']", - "echo toy", - "== Running pre-configure hook...", - "pre-configure: toy.source: True", - "== Running post-configure hook...", - "post-configure: toy.source: False", - "== Running post-install hook...", - "in post-install hook for toy v0.0", - "bin, lib", - "== Running end hook...", - "end hook triggered, all done!", - ]) + # There are 4 modules written: + # Sanitycheck for extensions and main easyblock (1 each), main and devel module + expected_output = textwrap.dedent(""" + == Running start hook... + start hook triggered + == Running parse hook for toy-0.0.eb... + toy 0.0 + ['%(name)s-%(version)s.tar.gz'] + echo toy + == Running pre-configure hook... + pre-configure: toy.source: True + == Running post-configure hook... + post-configure: toy.source: False + == Running post-install hook... + in post-install hook for toy v0.0 + bin, lib + == Running module_write hook... + in module-write hook hook for {mod_name} + == Running module_write hook... + in module-write hook hook for {mod_name} + == Running module_write hook... + in module-write hook hook for {mod_name} + == Running module_write hook... + in module-write hook hook for {mod_name} + == Running end hook... + end hook triggered, all done! + """).strip().format(mod_name=os.path.basename(toy_mod_file)) self.assertEqual(stdout.strip(), expected_output) + toy_mod = read_file(toy_mod_file) + self.assertIn('Not a toy anymore', toy_mod) + def test_toy_multi_deps(self): """Test installation of toy easyconfig that uses multi_deps.""" test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') From d8f95df928b0c405de6f36fd303a3d34c6bde308 Mon Sep 17 00:00:00 2001 From: Damian Alvarez Date: Mon, 7 Jun 2021 18:03:26 +0200 Subject: [PATCH 379/864] This generalizes the avoidance of attempting to pickle objects that can't be serialized. For instance `os`, if it is imported in a eb file --- easybuild/framework/easyconfig/format/one.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index 84324c734d..0466c893c3 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -135,8 +135,10 @@ def get_config_dict(self): # we can't use copy.deepcopy() directly because in Python 2 copying the (irrelevant) __builtins__ key fails... cfg_copy = {} for key in cfg: - if key != '__builtins__': + try: cfg_copy[key] = copy.deepcopy(cfg[key]) + except TypeError: + self.log.info("Contents of key '%s' can't be pickled: '%s'", key, cfg[key]) return cfg_copy From 4a206b30cc10fa910d1d1122acde6244364a0a64 Mon Sep 17 00:00:00 2001 From: Damian Alvarez Date: Mon, 7 Jun 2021 19:48:38 +0200 Subject: [PATCH 380/864] Explicitely avoid __builtins__, otherwise the tests fail --- easybuild/framework/easyconfig/format/one.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index 0466c893c3..50a7f8d07b 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -135,10 +135,11 @@ def get_config_dict(self): # we can't use copy.deepcopy() directly because in Python 2 copying the (irrelevant) __builtins__ key fails... cfg_copy = {} for key in cfg: - try: - cfg_copy[key] = copy.deepcopy(cfg[key]) - except TypeError: - self.log.info("Contents of key '%s' can't be pickled: '%s'", key, cfg[key]) + if key != '__builtins__': + try: + cfg_copy[key] = copy.deepcopy(cfg[key]) + except TypeError: + self.log.info("Contents of key '%s' can't be pickled: '%s'", key, cfg[key]) return cfg_copy From 2f74ccc2bd30d807300c2ff3f3dc2e7d18fb64d9 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 8 Jun 2021 09:44:01 +0800 Subject: [PATCH 381/864] make sure that tests requiring a github token have 'github' in the test name so that they can be easily filtered --- test/framework/options.py | 34 +++++++++++++++++----------------- test/framework/robot.py | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 87d2d5b195..71a75130a6 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1196,7 +1196,7 @@ def check_copied_files(): error_pattern = "One or more files to copy should be specified!" self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) - def test_copy_ec_from_pr(self): + def test_github_copy_ec_from_pr(self): """Test combination of --copy-ec with --from-pr.""" if self.github_token is None: print("Skipping test_copy_ec_from_pr, no GitHub token available?") @@ -1684,7 +1684,7 @@ def test_dry_run_categorized(self): if os.path.exists(dummylogfn): os.remove(dummylogfn) - def test_from_pr(self): + def test_github_from_pr(self): """Test fetching easyconfigs from a PR.""" if self.github_token is None: print("Skipping test_from_pr, no GitHub token available?") @@ -1771,7 +1771,7 @@ def test_from_pr(self): print("Ignoring URLError '%s' in test_from_pr" % err) shutil.rmtree(tmpdir) - def test_from_pr_token_log(self): + def test_github_from_pr_token_log(self): """Check that --from-pr doesn't leak GitHub token in log.""" if self.github_token is None: print("Skipping test_from_pr_token_log, no GitHub token available?") @@ -1804,7 +1804,7 @@ def test_from_pr_token_log(self): except URLError as err: print("Ignoring URLError '%s' in test_from_pr" % err) - def test_from_pr_listed_ecs(self): + def test_github_from_pr_listed_ecs(self): """Test --from-pr in combination with specifying easyconfigs on the command line.""" if self.github_token is None: print("Skipping test_from_pr, no GitHub token available?") @@ -1859,7 +1859,7 @@ def test_from_pr_listed_ecs(self): print("Ignoring URLError '%s' in test_from_pr" % err) shutil.rmtree(tmpdir) - def test_from_pr_x(self): + def test_github_from_pr_x(self): """Test combination of --from-pr with --extended-dry-run.""" if self.github_token is None: print("Skipping test_from_pr_x, no GitHub token available?") @@ -3318,7 +3318,7 @@ def test_xxx_include_generic_easyblocks(self): # must be run after test for --list-easyblocks, hence the '_xxx_' # cleaning up the imported easyblocks is quite difficult... - def test_xxx_include_easyblocks_from_pr(self): + def test_github_xxx_include_easyblocks_from_pr(self): """Test --include-easyblocks-from-pr.""" if self.github_token is None: print("Skipping test_preview_pr, no GitHub token available?") @@ -3660,7 +3660,7 @@ def test_cleanup_tmpdir(self): tweaked_dir = os.path.join(tmpdir, tmpdir_files[0], 'tweaked_easyconfigs') self.assertTrue(os.path.exists(os.path.join(tweaked_dir, 'toy-1.0.eb'))) - def test_preview_pr(self): + def test_github_preview_pr(self): """Test --preview-pr.""" if self.github_token is None: print("Skipping test_preview_pr, no GitHub token available?") @@ -3682,7 +3682,7 @@ def test_preview_pr(self): regex = re.compile(r"^Comparing bzip2-1.0.6\S* with bzip2-1.0.6") self.assertTrue(regex.search(txt), "Pattern '%s' not found in: %s" % (regex.pattern, txt)) - def test_review_pr(self): + def test_github_review_pr(self): """Test --review-pr.""" if self.github_token is None: print("Skipping test_review_pr, no GitHub token available?") @@ -4004,7 +4004,7 @@ def test_new_branch_github(self): ] self._assert_regexs(regexs, txt) - def test_new_pr_from_branch(self): + def test_github_new_pr_from_branch(self): """Test --new-pr-from-branch.""" if self.github_token is None: print("Skipping test_new_pr_from_branch, no GitHub token available?") @@ -4073,7 +4073,7 @@ def test_update_branch_github(self): ] self._assert_regexs(regexs, txt) - def test_new_update_pr(self): + def test_github_new_update_pr(self): """Test use of --new-pr (dry run only).""" if self.github_token is None: print("Skipping test_new_update_pr, no GitHub token available?") @@ -4283,7 +4283,7 @@ def test_new_update_pr(self): ] self._assert_regexs(regexs, txt, assert_true=False) - def test_sync_pr_with_develop(self): + def test_github_sync_pr_with_develop(self): """Test use of --sync-pr-with-develop (dry run only).""" if self.github_token is None: print("Skipping test_sync_pr_with_develop, no GitHub token available?") @@ -4313,7 +4313,7 @@ def test_sync_pr_with_develop(self): regex = re.compile(pattern) self.assertTrue(regex.match(txt), "Pattern '%s' doesn't match: %s" % (regex.pattern, txt)) - def test_sync_branch_with_develop(self): + def test_github_sync_branch_with_develop(self): """Test use of --sync-branch-with-develop (dry run only).""" if self.github_token is None: print("Skipping test_sync_pr_with_develop, no GitHub token available?") @@ -4343,7 +4343,7 @@ def test_sync_branch_with_develop(self): regex = re.compile(pattern) self.assertTrue(regex.match(stdout), "Pattern '%s' doesn't match: %s" % (regex.pattern, stdout)) - def test_new_pr_python(self): + def test_github_new_pr_python(self): """Check generated PR title for --new-pr on easyconfig that includes Python dependency.""" if self.github_token is None: print("Skipping test_new_pr_python, no GitHub token available?") @@ -4388,7 +4388,7 @@ def test_new_pr_python(self): regex = re.compile(r"^\* title: \"\{tools\}\[system/system\] toy v0.0 w/ Python 2.7.15 \+ 3.7.2\"$", re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) - def test_new_pr_delete(self): + def test_github_new_pr_delete(self): """Test use of --new-pr to delete easyconfigs.""" if self.github_token is None: @@ -4413,7 +4413,7 @@ def test_new_pr_delete(self): ] self._assert_regexs(regexs, txt) - def test_new_pr_dependencies(self): + def test_github_new_pr_dependencies(self): """Test use of --new-pr with automatic dependency lookup.""" if self.github_token is None: @@ -4461,7 +4461,7 @@ def test_new_pr_dependencies(self): self._assert_regexs(regexs, txt) - def test_merge_pr(self): + def test_github_merge_pr(self): """ Test use of --merge-pr (dry run)""" if self.github_token is None: @@ -4566,7 +4566,7 @@ def test_merge_pr(self): ]) self.assertTrue(expected_stdout in stdout) - def test_empty_pr(self): + def test_github_empty_pr(self): """Test use of --new-pr (dry run only) with no changes""" if self.github_token is None: print("Skipping test_empty_pr, no GitHub token available?") diff --git a/test/framework/robot.py b/test/framework/robot.py index 5e1d0fea6f..d2c82e013b 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -696,7 +696,7 @@ def test_search_paths(self): regex = re.compile(r"^ \* %s$" % os.path.join(self.test_prefix, test_ec), re.M) self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) - def test_det_easyconfig_paths_from_pr(self): + def test_github_det_easyconfig_paths_from_pr(self): """Test det_easyconfig_paths function, with --from-pr enabled as well.""" if self.github_token is None: print("Skipping test_from_pr, no GitHub token available?") From 26657cdffc80d28a82991cd564b3be01a6b6e2c0 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 8 Jun 2021 10:29:31 +0800 Subject: [PATCH 382/864] use even more single-letter variables in list comprehensions to avoid overwriting variables outside of list comprehensions when using Python 2 --- easybuild/framework/easyconfig/tools.py | 2 +- easybuild/tools/options.py | 4 ++-- easybuild/tools/testing.py | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index aae466468a..37c59b887f 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -334,7 +334,7 @@ def det_easyconfig_paths(orig_paths): :return: list of paths to easyconfig files """ try: - from_prs = [int(pr_nr) for pr_nr in build_option('from_pr')] + from_prs = [int(x) for x in build_option('from_pr')] except ValueError: raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index ecf627504d..2ba011b2f3 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1464,7 +1464,7 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): # map list of strings --from-pr value to list of integers try: - from_prs = [int(pr_nr) for pr_nr in eb_go.options.from_pr] + from_prs = [int(x) for x in eb_go.options.from_pr] except ValueError: raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") @@ -1500,7 +1500,7 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): # done here instead of in _postprocess_include because github integration requires build_options to be initialized if eb_go.options.include_easyblocks_from_pr: try: - easyblock_prs = [int(pr_nr) for pr_nr in eb_go.options.include_easyblocks_from_pr] + easyblock_prs = [int(x) for x in eb_go.options.include_easyblocks_from_pr] except ValueError: raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s.") diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index e176fa0c9f..e584e0c589 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -151,7 +151,7 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=None, gist_ test_report = [] if pr_nrs is not None: repo = pr_target_repo or GITHUB_EASYCONFIGS_REPO - pr_urls = ["https://github.com/%s/%s/pull/%s" % (pr_target_account, repo, pr_nr) for pr_nr in pr_nrs] + pr_urls = ["https://github.com/%s/%s/pull/%s" % (pr_target_account, repo, x) for x in pr_nrs] test_report.extend([ "Test report for %s" % ', '.join(pr_urls), "", @@ -193,7 +193,7 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=None, gist_ descr = "(partial) EasyBuild log for failed build of %s" % ec['spec'] if pr_nrs is not None: - descr += " (PR #%s)" % ', #'.join(str(pr_nr) for pr_nr in pr_nrs) + descr += " (PR #%s)" % ', #'.join(str(x) for x in pr_nrs) if easyblock_pr_nrs: descr += "".join(" (easyblock PR #%s)" % nr for nr in easyblock_pr_nrs) @@ -295,7 +295,7 @@ def post_pr_test_report(pr_nr, repo_type, test_report, msg, init_session_state, if build_option('include_easyblocks_from_pr'): if repo_type == GITHUB_EASYCONFIGS_REPO: - easyblocks_pr_nrs = [int(eb_pr_nr) for eb_pr_nr in build_option('include_easyblocks_from_pr')] + easyblocks_pr_nrs = [int(x) for x in build_option('include_easyblocks_from_pr')] comment_lines.append("Using easyblocks from PR(s) %s" % ", ".join(["https://github.com/%s/%s/pull/%s" % (pr_target_account, GITHUB_EASYBLOCKS_REPO, easyblocks_pr_nr) @@ -333,12 +333,12 @@ def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state dump_path = build_option('dump_test_report') try: - pr_nrs = [int(pr_nr) for pr_nr in build_option('from_pr')] + pr_nrs = [int(x) for x in build_option('from_pr')] except ValueError: raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") try: - easyblock_pr_nrs = [int(pr_nr) for pr_nr in build_option('include_easyblocks_from_pr')] + easyblock_pr_nrs = [int(x) for x in build_option('include_easyblocks_from_pr')] except ValueError: raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s.") From 16ad408d702cee3620138d5441d3203bc4368998 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 8 Jun 2021 13:40:46 +0800 Subject: [PATCH 383/864] parse C standard flags in CFLAGS for fujitsu compiler --- easybuild/toolchains/compiler/fujitsu.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/easybuild/toolchains/compiler/fujitsu.py b/easybuild/toolchains/compiler/fujitsu.py index d5e6d09808..e12d9e060e 100644 --- a/easybuild/toolchains/compiler/fujitsu.py +++ b/easybuild/toolchains/compiler/fujitsu.py @@ -30,6 +30,7 @@ :author: Miguel Dias Costa (National University of Singapore) """ import os +import re import easybuild.tools.environment as env import easybuild.tools.systemtools as systemtools @@ -88,6 +89,13 @@ class FujitsuCompiler(Compiler): def prepare(self, *args, **kwargs): super(FujitsuCompiler, self).prepare(*args, **kwargs) + # fcc doesn't accept e.g. -std=c++11 or -std=gnu++11, only -std=c11 or -std=gnu11 + pattern = r'-std=(gnu|c)\+\+(\d+)' + if re.search(pattern, self.vars['CFLAGS']): + self.log.debug("Found '-std=(gnu|c)++' in CFLAGS, fcc doesn't accept '++' here, removing it") + self.vars['CFLAGS'] = re.sub(pattern, r'-std=\1\2', self.vars['CFLAGS']) + self._setenv_variables() + # make sure the fujitsu module libraries are found (and added to rpath by wrapper) library_path = os.getenv('LIBRARY_PATH', '') libdir = os.path.join(os.getenv(TC_CONSTANT_MODULE_VAR), 'lib64') From e0eed8c0ece378f2a0904abae6a69f6bf5df4b71 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 31 May 2021 16:17:16 +0200 Subject: [PATCH 384/864] Add per-step timing information Shows e.g.: == fetching files... == creating build dir, resetting environment... == ... (took 2 secs) == unpacking... == ... (took 15 secs) == patching... == ... (took 7 secs) --- easybuild/framework/easyblock.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index c100e88d88..0bb18f4137 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3528,7 +3528,13 @@ def run_all_steps(self, run_test_cases): else: print_msg("%s..." % descr, log=self.log, silent=self.silent) self.current_step = step_name + start_time = datetime.now() self.run_step(step_name, step_methods) + step_duration = datetime.now() - start_time + if step_duration.total_seconds() >= 1: + print_msg("... (took %s)", time2str(step_duration)) + elif self.logdebug or build_option('trace'): + print_msg("... (took < 1 sec)") except StopException: pass From 20cbc0c74b030a91e224e813848b58853270b0cc Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 31 May 2021 16:24:46 +0200 Subject: [PATCH 385/864] Use correct singluar/plural forms in time2str --- easybuild/tools/utilities.py | 29 ++++++++++++++--------------- test/framework/utilities_test.py | 32 +++++++++++++++++--------------- 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index 15c501de8b..0fd3b1dc1c 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -300,22 +300,21 @@ def time2str(delta): if not isinstance(delta, datetime.timedelta): raise EasyBuildError("Incorrect value type provided to time2str, should be datetime.timedelta: %s", type(delta)) - delta_secs = delta.days * 3600 * 24 + delta.seconds + delta.microseconds / 10**6 - - if delta_secs < 60: - res = '%d sec' % int(delta_secs) - elif delta_secs < 3600: - mins = int(delta_secs / 60) - secs = int(delta_secs - (mins * 60)) - res = '%d min %d sec' % (mins, secs) - else: - hours = int(delta_secs / 3600) - mins = int((delta_secs - hours * 3600) / 60) - secs = int(delta_secs - (hours * 3600) - (mins * 60)) - hours_str = 'hours' if hours > 1 else 'hour' - res = '%d %s %d min %d sec' % (hours, hours_str, mins, secs) + delta_secs = delta.total_seconds() - return res + hours = int(delta_secs / 3600) + delta_secs -= hours * 3600 + mins = int(delta_secs / 60) + secs = int(delta_secs - mins * 60) + + res = [] + if hours: + res.append('%d %s' % (hours, 'hour' if hours == 1 else 'hours')) + if mins or hours: + res.append('%d %s' % (mins, 'min' if mins == 1 else 'mins')) + res.append('%d %s' % (secs, 'sec' if secs == 1 else 'secs')) + + return ' '.join(res) def natural_keys(key): diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index efe9fb7fd3..c05ef90539 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -57,21 +57,23 @@ def test_time2str(self): start = datetime(2019, 7, 30, 5, 14, 23) test_cases = [ - (start, "0 sec"), - (datetime(2019, 7, 30, 5, 14, 37), "14 sec"), - (datetime(2019, 7, 30, 5, 15, 22), "59 sec"), - (datetime(2019, 7, 30, 5, 15, 23), "1 min 0 sec"), - (datetime(2019, 7, 30, 5, 16, 22), "1 min 59 sec"), - (datetime(2019, 7, 30, 5, 37, 26), "23 min 3 sec"), - (datetime(2019, 7, 30, 6, 14, 22), "59 min 59 sec"), - (datetime(2019, 7, 30, 6, 14, 23), "1 hour 0 min 0 sec"), - (datetime(2019, 7, 30, 6, 49, 14), "1 hour 34 min 51 sec"), - (datetime(2019, 7, 30, 7, 14, 23), "2 hours 0 min 0 sec"), - (datetime(2019, 7, 30, 8, 35, 59), "3 hours 21 min 36 sec"), - (datetime(2019, 7, 30, 16, 29, 24), "11 hours 15 min 1 sec"), - (datetime(2019, 7, 31, 5, 14, 22), "23 hours 59 min 59 sec"), - (datetime(2019, 7, 31, 5, 14, 23), "24 hours 0 min 0 sec"), - (datetime(2019, 8, 5, 20, 39, 44), "159 hours 25 min 21 sec"), + (start, "0 secs"), + (datetime(2019, 7, 30, 5, 14, 37), "14 secs"), + (datetime(2019, 7, 30, 5, 15, 22), "59 secs"), + (datetime(2019, 7, 30, 5, 15, 23), "1 min 0 secs"), + (datetime(2019, 7, 30, 5, 16, 22), "1 min 59 secs"), + (datetime(2019, 7, 30, 5, 16, 24), "2 mins 1 sec"), + (datetime(2019, 7, 30, 5, 37, 26), "23 mins 3 secs"), + (datetime(2019, 7, 30, 6, 14, 22), "59 mins 59 secs"), + (datetime(2019, 7, 30, 6, 14, 23), "1 hour 0 mins 0 secs"), + (datetime(2019, 7, 30, 6, 49, 14), "1 hour 34 mins 51 secs"), + (datetime(2019, 7, 30, 7, 14, 23), "2 hours 0 mins 0 secs"), + (datetime(2019, 7, 30, 8, 35, 59), "3 hours 21 mins 36 secs"), + (datetime(2019, 7, 30, 16, 29, 24), "11 hours 15 mins 1 sec"), + (datetime(2019, 7, 31, 5, 14, 22), "23 hours 59 mins 59 secs"), + (datetime(2019, 7, 31, 5, 14, 23), "24 hours 0 mins 0 secs"), + (datetime(2019, 7, 31, 5, 15, 24), "24 hours 1 min 1 sec"), + (datetime(2019, 8, 5, 20, 39, 44), "159 hours 25 mins 21 secs"), ] for end, expected in test_cases: self.assertEqual(time2str(end - start), expected) From 0e4847c708862943d2c1566c7fb99426ee3a05a4 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 1 Jun 2021 10:44:05 +0200 Subject: [PATCH 386/864] Honor silent, log and dry-run options for step-timing --- easybuild/framework/easyblock.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 0bb18f4137..06aa7ee140 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3530,11 +3530,12 @@ def run_all_steps(self, run_test_cases): self.current_step = step_name start_time = datetime.now() self.run_step(step_name, step_methods) - step_duration = datetime.now() - start_time - if step_duration.total_seconds() >= 1: - print_msg("... (took %s)", time2str(step_duration)) - elif self.logdebug or build_option('trace'): - print_msg("... (took < 1 sec)") + if not self.dry_run: + step_duration = datetime.now() - start_time + if step_duration.total_seconds() >= 1: + print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent) + elif self.logdebug or build_option('trace'): + print_msg("... (took < 1 sec)", log=self.log, silent=self.silent) except StopException: pass From a9e882ef440ea8853f2927524c8d4e732d68721f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 1 Jun 2021 10:44:27 +0200 Subject: [PATCH 387/864] Fix tests for plural of sec/secs --- test/framework/options.py | 8 ++++---- test/framework/toy_build.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 87d2d5b195..176e052f48 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1895,7 +1895,7 @@ def test_from_pr_x(self): re.compile(r"^\*\*\* DRY RUN using 'EB_FFTW' easyblock", re.M), re.compile(r"^== building and installing FFTW/3.3.8-gompi-2018b\.\.\.", re.M), re.compile(r"^building... \[DRY RUN\]", re.M), - re.compile(r"^== COMPLETED: Installation ended successfully \(took .* sec\)", re.M), + re.compile(r"^== COMPLETED: Installation ended successfully \(took .* secs?\)", re.M), ] for msg_regex in msg_regexs: @@ -3820,7 +3820,7 @@ def test_extended_dry_run(self): msg_regexs = [ re.compile(r"the actual build \& install procedure that will be performed may diverge", re.M), re.compile(r"^\*\*\* DRY RUN using 'EB_toy' easyblock", re.M), - re.compile(r"^== COMPLETED: Installation ended successfully \(took .* sec\)", re.M), + re.compile(r"^== COMPLETED: Installation ended successfully \(took .* secs?\)", re.M), re.compile(r"^\(no ignored errors during dry run\)", re.M), ] ignoring_error_regex = re.compile(r"WARNING: ignoring error", re.M) @@ -4804,7 +4804,7 @@ def test_stop(self): args = ['toy-0.0.eb', '--force', '--stop=configure'] txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) - regex = re.compile(r"COMPLETED: Installation STOPPED successfully \(took .* sec\)", re.M) + regex = re.compile(r"COMPLETED: Installation STOPPED successfully \(took .* secs?\)", re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) def test_fetch(self): @@ -4829,7 +4829,7 @@ def test_fetch(self): patterns = [ r"^== fetching files\.\.\.$", - r"^== COMPLETED: Installation STOPPED successfully \(took .* sec\)$", + r"^== COMPLETED: Installation STOPPED successfully \(took .* secs?\)$", ] for pattern in patterns: regex = re.compile(pattern, re.M) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index c339194ad3..302b673a8a 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -111,7 +111,7 @@ def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versio full_version = ''.join([versionprefix, version, versionsuffix]) # check for success - success = re.compile(r"COMPLETED: Installation ended successfully \(took .* sec\)") + success = re.compile(r"COMPLETED: Installation ended successfully \(took .* secs?\)") self.assertTrue(success.search(outtxt), "COMPLETED message found in '%s" % outtxt) # if the module exists, it should be fine From 30d29baa4379575901ef3ce4ca565c646470b184 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 1 Jun 2021 11:10:11 +0200 Subject: [PATCH 388/864] Also print step timing when step failed --- easybuild/framework/easyblock.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 06aa7ee140..605c3ab5f8 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3529,13 +3529,15 @@ def run_all_steps(self, run_test_cases): print_msg("%s..." % descr, log=self.log, silent=self.silent) self.current_step = step_name start_time = datetime.now() - self.run_step(step_name, step_methods) - if not self.dry_run: - step_duration = datetime.now() - start_time - if step_duration.total_seconds() >= 1: - print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent) - elif self.logdebug or build_option('trace'): - print_msg("... (took < 1 sec)", log=self.log, silent=self.silent) + try: + self.run_step(step_name, step_methods) + finally: + if not self.dry_run: + step_duration = datetime.now() - start_time + if step_duration.total_seconds() >= 1: + print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent) + elif self.logdebug or build_option('trace'): + print_msg("... (took < 1 sec)", log=self.log, silent=self.silent) except StopException: pass From cbed99ab3b353754347518486b22253f606921c9 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 8 Jun 2021 15:32:55 +0200 Subject: [PATCH 389/864] Add option to ignore failing test step Users can now use --ignore-test-failure instead of --skip-test-step which will run the tests but does not abort on error and just prints it instead. Using both options doesn't make sense and is hence disallowed and a warning is show to make users move to ignoring failures instead of skipping the test completely. --- easybuild/base/testing.py | 21 +++++++++++++++++++++ easybuild/framework/easyblock.py | 21 ++++++++++++++++++++- easybuild/main.py | 9 +++++++++ easybuild/tools/config.py | 1 + easybuild/tools/options.py | 1 + test/framework/options.py | 22 ++++++++++++++++++++++ 6 files changed, 74 insertions(+), 1 deletion(-) diff --git a/easybuild/base/testing.py b/easybuild/base/testing.py index 0df4c537ad..ff855ae346 100644 --- a/easybuild/base/testing.py +++ b/easybuild/base/testing.py @@ -35,6 +35,7 @@ import pprint import re import sys +from contextlib import contextmanager try: from cStringIO import StringIO # Python 2 @@ -185,6 +186,26 @@ def get_stderr(self): """Return output captured from stderr until now.""" return sys.stderr.getvalue() + @contextmanager + def mocked_stdout_stderr(self, mock_stdout=True, mock_stderr=True): + """Context manager to mock stdout and stderr""" + if mock_stdout: + self.mock_stdout(True) + if mock_stderr: + self.mock_stderr(True) + try: + if mock_stdout and mock_stderr: + yield sys.stdout, sys.stderr + elif mock_stdout: + yield sys.stdout + else: + yield sys.stderr + finally: + if mock_stdout: + self.mock_stdout(False) + if mock_stderr: + self.mock_stderr(False) + def tearDown(self): """Cleanup after running a test.""" self.mock_stdout(False) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index c100e88d88..4c27e3f185 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -258,6 +258,8 @@ def __init__(self, ec): if group_name is not None: self.group = use_group(group_name) + self.ignore_test_failure = build_option('ignore_test_failure') + # generate build/install directories self.gen_builddir() self.gen_installdir() @@ -1814,6 +1816,16 @@ def remove_module_file(self): self.log.info("Removing existing module file %s", self.mod_filepath) remove_file(self.mod_filepath) + def report_test_failure(self, msgOrError): + """Report a failing test either via an exception or warning depending on ignore-test-failure + + msgOrError - Failure description. Either a string or an EasyBuildError""" + if self.ignore_test_failure: + print_warning("Test failure ignored: " + str(msgOrError), log=self.log) + else: + exception = msgOrError if isinstance(msgOrError, EasyBuildError) else EasyBuildError(msgOrError) + raise exception + # # STEP FUNCTIONS # @@ -2229,6 +2241,13 @@ def test_step(self): return out + def _test_step(self): + """Run the test_step and handles failures""" + try: + self.test_step() + except EasyBuildError as e: + self.report_test_failure(e) + def stage_install_step(self): """ Install in a stage directory before actual installation. @@ -3439,7 +3458,7 @@ def install_step_spec(initial): prepare_step_spec = (PREPARE_STEP, 'preparing', [lambda x: x.prepare_step], False) configure_step_spec = (CONFIGURE_STEP, 'configuring', [lambda x: x.configure_step], True) build_step_spec = (BUILD_STEP, 'building', [lambda x: x.build_step], True) - test_step_spec = (TEST_STEP, 'testing', [lambda x: x.test_step], True) + test_step_spec = (TEST_STEP, 'testing', [lambda x: x._test_step], True) extensions_step_spec = (EXTENSIONS_STEP, 'taking care of extensions', [lambda x: x.extensions_step], False) # part 1: pre-iteration + first iteration diff --git a/easybuild/main.py b/easybuild/main.py index 039b400d2f..6f271cad6b 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -55,6 +55,7 @@ from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak +from easybuild.tools.build_log import print_warning from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.containers.common import containerize from easybuild.tools.docs import list_software @@ -304,6 +305,14 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) + if options.skip_test_step: + if options.ignore_test_failure: + raise EasyBuildError("Found both ignore-test-failure and skip-test-step being set. " + "Please use only one of them.") + else: + print_warning("Will not run the test step as requested via skip-test-step. " + "Consider using ignore-test-failure instead and verify the results afterwards") + # determine easybuild-easyconfigs package install path easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) if not easyconfigs_pkg_paths: diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 52537e04e6..23f3c97f54 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -248,6 +248,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'ignore_checksums', 'ignore_index', 'ignore_locks', + 'ignore_test_failure', 'install_latest_eb_release', 'logtostdout', 'minimal_toolchains', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index ecf627504d..96f1018707 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -418,6 +418,7 @@ def override_options(self): "\"client[A-z0-9]*.example.com': ['Authorization: Basic token']\".", None, 'append', None, {'metavar': '[URLPAT::][HEADER:]FILE|FIELD'}), 'ignore-checksums': ("Ignore failing checksum verification", None, 'store_true', False), + 'ignore-test-failure': ("Ignore a failing test step", None, 'store_true', False), 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), 'install-latest-eb-release': ("Install latest known version of easybuild", None, 'store_true', False), 'lib-lib64-symlink': ("Automatically create symlinks for lib/ pointing to lib64/ if the former is missing", diff --git a/test/framework/options.py b/test/framework/options.py index 87d2d5b195..f384da8a4e 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -394,6 +394,28 @@ def test_skip_test_step(self): found = re.search(test_run_msg, outtxt) self.assertFalse(found, "Test execution command is NOT present, outtxt: %s" % outtxt) + def test_ignore_test_failure(self): + """Test ignore failing tests (--ignore-test-failure).""" + + topdir = os.path.abspath(os.path.dirname(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb') + + args = [toy_ec, '--ignore-test-failure', '--force'] + + with self.mocked_stdout_stderr() as (_, stderr): + outtxt = self.eb_main(args, do_build=True) + + msg = 'Test failure ignored' + self.assertTrue(re.search(msg, outtxt), + "Ignored test failure message in log not found, outtxt: %s" % outtxt) + self.assertTrue(re.search(msg, stderr.getvalue()), + "Ignored test failure message in stderr not found, stderr: %s" % stderr.getvalue()) + + # Passing skip and ignore options is disallowed + args.append('--skip-test-step') + error_pattern = 'Found both ignore-test-failure and skip-test-step being set' + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True) + def test_job(self): """Test submitting build as a job.""" From 7234b8405c641fe82841fca979cea6dabfac013d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 8 Jun 2021 16:51:37 +0200 Subject: [PATCH 390/864] Remove leading underscores from logged step name --- easybuild/framework/easyblock.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 4c27e3f185..6d1a7f157e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3382,10 +3382,12 @@ def run_step(self, step, step_methods): run_hook(step, self.hooks, pre_step_hook=True, args=[self]) for step_method in step_methods: - self.log.info("Running method %s part of step %s" % (extract_method_name(step_method), step)) + # Remove leading underscore from e.g. "_test_step" + method_name = extract_method_name(step_method).lstrip('_') + self.log.info("Running method %s part of step %s" % (method_name, step)) if self.dry_run: - self.dry_run_msg("[%s method]", step_method(self).__name__) + self.dry_run_msg("[%s method]", method_name) # if an known possible error occurs, just report it and continue try: From 5c521a9fc6c71acece84443d1b76b13cee3445ea Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 9 Jun 2021 10:29:49 +0800 Subject: [PATCH 391/864] also make sure that all tests in test/framework/github.py have 'github' in the test name so that they can be easily filtered --- test/framework/github.py | 52 ++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/test/framework/github.py b/test/framework/github.py index ed94c7bf2d..e5d488ce85 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -95,13 +95,13 @@ def tearDown(self): super(GithubTest, self).tearDown() - def test_pick_default_branch(self): + def test_github_pick_default_branch(self): """Test pick_default_branch function.""" self.assertEqual(pick_default_branch('easybuilders'), 'main') self.assertEqual(pick_default_branch('foobar'), 'master') - def test_walk(self): + def test_github_walk(self): """test the gitubfs walk function""" if self.skip_github_tests: print("Skipping test_walk, no GitHub token available?") @@ -117,7 +117,7 @@ def test_walk(self): except IOError: pass - def test_read_api(self): + def test_github_read_api(self): """Test the githubfs read function""" if self.skip_github_tests: print("Skipping test_read_api, no GitHub token available?") @@ -128,7 +128,7 @@ def test_read_api(self): except IOError: pass - def test_read(self): + def test_github_read(self): """Test the githubfs read function without using the api""" if self.skip_github_tests: print("Skipping test_read, no GitHub token available?") @@ -141,7 +141,7 @@ def test_read(self): except (IOError, OSError): pass - def test_add_pr_labels(self): + def test_github_add_pr_labels(self): """Test add_pr_labels function.""" if self.skip_github_tests: print("Skipping test_add_pr_labels, no GitHub token available?") @@ -183,7 +183,7 @@ def test_add_pr_labels(self): self.mock_stderr(False) self.assertTrue("PR #8006 should be labelled 'update'" in stdout) - def test_fetch_pr_data(self): + def test_github_fetch_pr_data(self): """Test fetch_pr_data function.""" if self.skip_github_tests: print("Skipping test_fetch_pr_data, no GitHub token available?") @@ -205,7 +205,7 @@ def test_fetch_pr_data(self): self.assertEqual(pr_data['reviews'][0]['user']['login'], 'boegel') self.assertEqual(pr_data['status_last_commit'], None) - def test_list_prs(self): + def test_github_list_prs(self): """Test list_prs function.""" if self.skip_github_tests: print("Skipping test_list_prs, no GitHub token available?") @@ -227,7 +227,7 @@ def test_list_prs(self): self.assertEqual(expected, output) - def test_reasons_for_closing(self): + def test_github_reasons_for_closing(self): """Test reasons_for_closing function.""" if self.skip_github_tests: print("Skipping test_reasons_for_closing, no GitHub token available?") @@ -267,7 +267,7 @@ def test_reasons_for_closing(self): for pattern in patterns: self.assertTrue(pattern in stdout, "Pattern '%s' found in: %s" % (pattern, stdout)) - def test_close_pr(self): + def test_github_close_pr(self): """Test close_pr function.""" if self.skip_github_tests: print("Skipping test_close_pr, no GitHub token available?") @@ -312,7 +312,7 @@ def test_close_pr(self): for pattern in patterns: self.assertTrue(pattern in stdout, "Pattern '%s' found in: %s" % (pattern, stdout)) - def test_fetch_easyblocks_from_pr(self): + def test_github_fetch_easyblocks_from_pr(self): """Test fetch_easyblocks_from_pr function.""" if self.skip_github_tests: print("Skipping test_fetch_easyblocks_from_pr, no GitHub token available?") @@ -339,7 +339,7 @@ def test_fetch_easyblocks_from_pr(self): except URLError as err: print("Ignoring URLError '%s' in test_fetch_easyblocks_from_pr" % err) - def test_fetch_easyconfigs_from_pr(self): + def test_github_fetch_easyconfigs_from_pr(self): """Test fetch_easyconfigs_from_pr function.""" if self.skip_github_tests: print("Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?") @@ -390,7 +390,7 @@ def test_fetch_easyconfigs_from_pr(self): except URLError as err: print("Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err) - def test_fetch_files_from_pr_cache(self): + def test_github_fetch_files_from_pr_cache(self): """Test caching for fetch_files_from_pr.""" if self.skip_github_tests: print("Skipping test_fetch_files_from_pr_cache, no GitHub token available?") @@ -451,7 +451,7 @@ def test_fetch_files_from_pr_cache(self): res = gh.fetch_easyblocks_from_pr(12345, tmpdir) self.assertEqual(sorted(pr12345_files), sorted(res)) - def test_fetch_latest_commit_sha(self): + def test_github_fetch_latest_commit_sha(self): """Test fetch_latest_commit_sha function.""" if self.skip_github_tests: print("Skipping test_fetch_latest_commit_sha, no GitHub token available?") @@ -463,7 +463,7 @@ def test_fetch_latest_commit_sha(self): branch='develop') self.assertTrue(re.match('^[0-9a-f]{40}$', sha)) - def test_download_repo(self): + def test_github_download_repo(self): """Test download_repo function.""" if self.skip_github_tests: print("Skipping test_download_repo, no GitHub token available?") @@ -562,7 +562,7 @@ def test_validate_github_token(self): if token_old_format: self.assertTrue(gh.validate_github_token(token_old_format, GITHUB_TEST_ACCOUNT)) - def test_find_easybuild_easyconfig(self): + def test_github_find_easybuild_easyconfig(self): """Test for find_easybuild_easyconfig function""" if self.skip_github_tests: print("Skipping test_find_easybuild_easyconfig, no GitHub token available?") @@ -573,7 +573,7 @@ def test_find_easybuild_easyconfig(self): self.assertTrue(regex.search(path), "Pattern '%s' found in '%s'" % (regex.pattern, path)) self.assertTrue(os.path.exists(path), "Path %s exists" % path) - def test_find_patches(self): + def test_github_find_patches(self): """ Test for find_software_name_for_patch """ testdir = os.path.dirname(os.path.abspath(__file__)) ec_path = os.path.join(testdir, 'easyconfigs') @@ -595,7 +595,7 @@ def test_find_patches(self): reg = re.compile(r'[1-9]+ of [1-9]+ easyconfigs checked') self.assertTrue(re.search(reg, txt)) - def test_det_commit_status(self): + def test_github_det_commit_status(self): """Test det_commit_status function.""" if self.skip_github_tests: @@ -642,7 +642,7 @@ def test_det_commit_status(self): res = gh.det_commit_status('easybuilders', GITHUB_REPO, commit_sha, GITHUB_TEST_ACCOUNT) self.assertEqual(res, None) - def test_check_pr_eligible_to_merge(self): + def test_github_check_pr_eligible_to_merge(self): """Test check_pr_eligible_to_merge function""" def run_check(expected_result=False): """Helper function to check result of check_pr_eligible_to_merge""" @@ -770,7 +770,7 @@ def run_check(expected_result=False): expected_warning = '' self.assertEqual(run_check(True), '') - def test_det_pr_labels(self): + def test_github_det_pr_labels(self): """Test for det_pr_labels function.""" file_info = {'new_folder': [False], 'new_file_in_existing_folder': [True]} @@ -789,7 +789,7 @@ def test_det_pr_labels(self): res = gh.det_pr_labels(file_info, GITHUB_EASYBLOCKS_REPO) self.assertEqual(res, ['new']) - def test_det_patch_specs(self): + def test_github_det_patch_specs(self): """Test for det_patch_specs function.""" patch_paths = [os.path.join(self.test_prefix, p) for p in ['1.patch', '2.patch', '3.patch']] @@ -841,7 +841,7 @@ def test_det_patch_specs(self): self.assertEqual(os.path.basename(res[2][0]), '3.patch') self.assertEqual(res[2][1], 'patched_ext') - def test_restclient(self): + def test_github_restclient(self): """Test use of RestClient.""" if self.skip_github_tests: print("Skipping test_restclient, no GitHub token available?") @@ -876,7 +876,7 @@ def test_restclient(self): httperror_hit = True self.assertTrue(httperror_hit, "expected HTTPError not encountered") - def test_create_delete_gist(self): + def test_github_create_delete_gist(self): """Test create_gist and delete_gist.""" if self.skip_github_tests: print("Skipping test_restclient, no GitHub token available?") @@ -888,7 +888,7 @@ def test_create_delete_gist(self): gist_id = gist_url.split('/')[-1] gh.delete_gist(gist_id, github_user=GITHUB_TEST_ACCOUNT, github_token=self.github_token) - def test_det_account_branch_for_pr(self): + def test_github_det_account_branch_for_pr(self): """Test det_account_branch_for_pr.""" if self.skip_github_tests: print("Skipping test_det_account_branch_for_pr, no GitHub token available?") @@ -918,7 +918,7 @@ def test_det_account_branch_for_pr(self): self.assertEqual(account, 'migueldiascosta') self.assertEqual(branch, 'fix_inject_checksums') - def test_det_pr_target_repo(self): + def test_github_det_pr_target_repo(self): """Test det_pr_target_repo.""" self.assertEqual(build_option('pr_target_repo'), None) @@ -1002,7 +1002,7 @@ def test_push_branch_to_github(self): regex = re.compile(pattern) self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' doesn't match: %s" % (regex.pattern, stdout)) - def test_pr_test_report(self): + def test_github_pr_test_report(self): """Test for post_pr_test_report function.""" if self.skip_github_tests: print("Skipping test_post_pr_test_report, no GitHub token available?") @@ -1051,7 +1051,7 @@ def test_pr_test_report(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) - def test_create_test_report(self): + def test_github_create_test_report(self): """Test create_test_report function.""" logfile = os.path.join(self.test_prefix, 'log.txt') write_file(logfile, "Bazel failed with: error") From a5ff50abcd91067e2cde4bd2ca2519b4849738ff Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 9 Jun 2021 12:01:52 +0200 Subject: [PATCH 392/864] Fix comments claiming hooks was a list --- easybuild/framework/easyblock.py | 2 +- easybuild/tools/hooks.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 1fe20a5a9b..b9da26043a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -144,7 +144,7 @@ def __init__(self, ec): # keep track of original working directory, so we can go back there self.orig_workdir = os.getcwd() - # list of pre- and post-step hooks + # dict of all hooks (mapping of name to function) self.hooks = load_hooks(build_option('hooks')) # list of patch/source files, along with checksums diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 001d191b0b..cb2d72c472 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -120,7 +120,7 @@ def load_hooks(hooks_path): def verify_hooks(hooks): - """Check whether list of obtained hooks only includes known hooks.""" + """Check whether obtained hooks only includes known hooks.""" unknown_hooks = [key for key in sorted(hooks) if key not in KNOWN_HOOKS] if unknown_hooks: @@ -169,10 +169,10 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False): def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, msg=None): """ - Run hook with specified label and return result of hook or None. + Run hook with specified label and return result of calling the hook or None. :param label: name of hook - :param hooks: list of defined hooks + :param hooks: dict of defined hooks :param pre_step_hook: indicates whether hook to run is a pre-step hook :param post_step_hook: indicates whether hook to run is a post-step hook :param args: arguments to pass to hook function From 600631d8d05374df233f8c120b249aaf71eb68ee Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 9 Jun 2021 12:30:32 +0200 Subject: [PATCH 393/864] Fix error message for --use-ccache --- easybuild/tools/toolchain/toolchain.py | 2 +- test/framework/toolchain.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index a8961a8dbb..782d6634b3 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -927,7 +927,7 @@ def prepare_compiler_cache(self, cache_tool): cache_path = which(cache_tool) if cache_path is None: - raise EasyBuildError("%s binary not found in $PATH, required by --use-compiler-cache", cache_tool) + raise EasyBuildError("%s binary not found in $PATH, required by --use-ccache", cache_tool) else: self.symlink_commands({cache_tool: (cache_path, compilers)}) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 69bc14350a..0fa2ff2d56 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1699,7 +1699,7 @@ def test_compiler_cache(self): ccache = which('ccache') if ccache is None: - msg = r"ccache binary not found in \$PATH, required by --use-compiler-cache" + msg = r"ccache binary not found in \$PATH, required by --use-ccache" self.assertErrorRegex(EasyBuildError, msg, self.eb_main, args, raise_error=True, do_build=True) # generate shell script to mock ccache/f90cache From 39d3d19278762509048ab6c6e5ee2c661736bea2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 9 Jun 2021 13:32:48 +0200 Subject: [PATCH 394/864] add test for parsing an easyconfig file that includes an import statement --- test/framework/easyconfig.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 91f8502a48..cfa48e53dd 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4428,6 +4428,28 @@ def test_pure_ec(self): self.assertEqual(ec_dict_bis.get('exts_default_options'), None) self.assertEqual(ec_dict.get('sanity_check_paths'), {'dirs': ['bin'], 'files': [('bin/yot', 'bin/toy')]}) + def test_easyconfig_import(self): + """ + Test parsing of an easyconfig file that includes import statements. + """ + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\n' + '\n'.join([ + "import os", + "local_test = os.getenv('TEST_TOY')", + "sanity_check_commands = ['toy | grep %s' % local_test]", + ]) + write_file(test_ec, test_ec_txt) + + os.environ['TEST_TOY'] = '123' + + ec = EasyConfig(test_ec) + + self.assertEqual(ec['sanity_check_commands'], ['toy | grep 123']) + def suite(): """ returns all the testcases in this module """ From bbc6c8e8ffc17cf3067a7464c88fed9879f283e1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 9 Jun 2021 13:35:03 +0200 Subject: [PATCH 395/864] explicitly check for imported modules in get_config_dict rather than just skipping failing copies --- easybuild/framework/easyconfig/format/one.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index 50a7f8d07b..d63081e27c 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -135,11 +135,9 @@ def get_config_dict(self): # we can't use copy.deepcopy() directly because in Python 2 copying the (irrelevant) __builtins__ key fails... cfg_copy = {} for key in cfg: - if key != '__builtins__': - try: - cfg_copy[key] = copy.deepcopy(cfg[key]) - except TypeError: - self.log.info("Contents of key '%s' can't be pickled: '%s'", key, cfg[key]) + # skip special variables like __builtins__, and imported modules (like 'os') + if key != '__builtins__' and "'module'" not in str(type(cfg[key])): + cfg_copy[key] = copy.deepcopy(cfg[key]) return cfg_copy From 7d53ee3f1fe6dafa8ccde0a799af446cde27f6eb Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 9 Jun 2021 15:58:19 +0200 Subject: [PATCH 396/864] Use divmod instead of manual divide and subtract Co-authored-by: Sam Moors --- easybuild/tools/utilities.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index 0fd3b1dc1c..c6a4512c82 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -302,10 +302,8 @@ def time2str(delta): delta_secs = delta.total_seconds() - hours = int(delta_secs / 3600) - delta_secs -= hours * 3600 - mins = int(delta_secs / 60) - secs = int(delta_secs - mins * 60) + hours, remainder = divmod(delta_secs, 3600) + mins, secs = divmod(remainder, 60) res = [] if hours: From 558829b1d7227529a6e252339d7383d91280512f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 9 Jun 2021 16:55:37 +0200 Subject: [PATCH 397/864] avoid traceback in get_config_obj, report a clean error when copying a variable fails --- easybuild/framework/easyconfig/format/one.py | 7 ++++++- test/framework/easyconfig.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index d63081e27c..2956d6a009 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -137,7 +137,12 @@ def get_config_dict(self): for key in cfg: # skip special variables like __builtins__, and imported modules (like 'os') if key != '__builtins__' and "'module'" not in str(type(cfg[key])): - cfg_copy[key] = copy.deepcopy(cfg[key]) + try: + cfg_copy[key] = copy.deepcopy(cfg[key]) + except Exception as err: + raise EasyBuildError("Failed to copy '%s' easyconfig parameter: %s" % (key, err)) + else: + self.log.debug("Not copying '%s' variable from parsed easyconfig", key) return cfg_copy diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index cfa48e53dd..c3a1050788 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4450,6 +4450,19 @@ def test_easyconfig_import(self): self.assertEqual(ec['sanity_check_commands'], ['toy | grep 123']) + # inject weird stuff, like a class definition that creates a logger instance, to check clean error handling + test_ec_txt += '\n' + '\n'.join([ + "import logging", + "class _TestClass(object):", + " def __init__(self):", + " self.log = logging.Logger('alogger')", + "local_test = _TestClass()", + ]) + write_file(test_ec, test_ec_txt) + + error_pattern = r"Failed to copy '.*' easyconfig parameter" + self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, test_ec) + def suite(): """ returns all the testcases in this module """ From 3abbd19be949f363e2d1008ecbce49a321beb27c Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 10 Jun 2021 08:53:39 +0200 Subject: [PATCH 398/864] Add per-extension timing --- easybuild/framework/easyblock.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index d561df3ed7..90eeda1ac8 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2366,6 +2366,7 @@ def extensions_step(self, fetch=False, install=True): tup = (ext.name, ext.version or '', idx + 1, exts_cnt) print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent) + start_time = datetime.now() if self.dry_run: tup = (ext.name, ext.version, ext.__class__.__name__) @@ -2386,11 +2387,19 @@ def extensions_step(self, fetch=False, install=True): # real work if install: - ext.prerun() - txt = ext.run() - if txt: - self.module_extra_extensions += txt - ext.postrun() + try: + ext.prerun() + txt = ext.run() + if txt: + self.module_extra_extensions += txt + ext.postrun() + finally: + if not self.dry_run: + ext_duration = datetime.now() - start_time + if ext_duration.total_seconds() >= 1: + print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent) + elif self.logdebug or build_option('trace'): + print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent) # cleanup (unload fake module, remove fake module dir) if fake_mod_data: From 0012889d8c0b3b48e2c2ca5b04a1a8578a606700 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 10 Jun 2021 09:02:21 +0200 Subject: [PATCH 399/864] enhance test easyconfig in test_easyconfig_import to try to always trigger pickle failure --- test/framework/easyconfig.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index c3a1050788..5393b0f3c6 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4450,13 +4450,15 @@ def test_easyconfig_import(self): self.assertEqual(ec['sanity_check_commands'], ['toy | grep 123']) - # inject weird stuff, like a class definition that creates a logger instance, to check clean error handling + # inject weird stuff, like a class definition that creates a logger instance + # and a local variable with a list of imported modules, to check clean error handling test_ec_txt += '\n' + '\n'.join([ "import logging", "class _TestClass(object):", " def __init__(self):", " self.log = logging.Logger('alogger')", "local_test = _TestClass()", + "local_modules = [logging, os]", ]) write_file(test_ec, test_ec_txt) From 9df4e7da4e6cad0003bd899667fc4c8f5c3ccd23 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Thu, 10 Jun 2021 16:51:17 +0200 Subject: [PATCH 400/864] add toolchain nompi --- easybuild/toolchains/nompi.py | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 easybuild/toolchains/nompi.py diff --git a/easybuild/toolchains/nompi.py b/easybuild/toolchains/nompi.py new file mode 100644 index 0000000000..debcce03b0 --- /dev/null +++ b/easybuild/toolchains/nompi.py @@ -0,0 +1,41 @@ +## +# Copyright 2016-2016 Ghent University +# Copyright 2016-2016 Forschungszentrum Juelich +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for nompi compiler toolchain (includes NVHPC and ParaStationMPI, and CUDA as dependency). + +@author: Damian Alvarez (Forschungszentrum Juelich) +""" + +from easybuild.toolchains.nvhpc import NvhpcToolchain +# We pull in MPI and CUDA at once so this maps nicely to HMNS +from easybuild.toolchains.mpi.openmpi import OpenMPI +from easybuild.toolchains.compiler.cuda import Cuda + +# Order matters! +class Ompi(NvhpcToolchain, Cuda, OpenMPI): + """Compiler toolchain with NVHPC and OpenMPI, with CUDA as dependency.""" + NAME = 'nompi' + SUBTOOLCHAIN = NvhpcToolchain.NAME From 665bf2f294bcf2ba9e6a99352ff23028c2144bfb Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Thu, 10 Jun 2021 16:51:47 +0200 Subject: [PATCH 401/864] add toolchain npsmpic --- easybuild/toolchains/npsmpic.py | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 easybuild/toolchains/npsmpic.py diff --git a/easybuild/toolchains/npsmpic.py b/easybuild/toolchains/npsmpic.py new file mode 100644 index 0000000000..33ee2bb06a --- /dev/null +++ b/easybuild/toolchains/npsmpic.py @@ -0,0 +1,41 @@ +## +# Copyright 2016-2016 Ghent University +# Copyright 2016-2016 Forschungszentrum Juelich +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for npsmpi compiler toolchain (includes NVHPC and ParaStationMPI, and CUDA as dependency). + +@author: Damian Alvarez (Forschungszentrum Juelich) +""" + +from easybuild.toolchains.nvhpc import NvhpcToolchain +# We pull in MPI and CUDA at once so this maps nicely to HMNS +from easybuild.toolchains.mpi.psmpi import Psmpi +from easybuild.toolchains.compiler.cuda import Cuda + +# Order matters! +class Npsmpic(NvhpcToolchain, Cuda, Psmpi): + """Compiler toolchain with NVHPC and ParaStationMPI, with CUDA as dependency.""" + NAME = 'npsmpic' + SUBTOOLCHAIN = NvhpcToolchain.NAME From d35139e3c5896e0f46e62fa6891254139ee7a206 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Thu, 10 Jun 2021 17:16:39 +0200 Subject: [PATCH 402/864] update nompi toolchain --- easybuild/toolchains/nompi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/toolchains/nompi.py b/easybuild/toolchains/nompi.py index debcce03b0..15122a190a 100644 --- a/easybuild/toolchains/nompi.py +++ b/easybuild/toolchains/nompi.py @@ -24,7 +24,7 @@ # along with EasyBuild. If not, see . ## """ -EasyBuild support for nompi compiler toolchain (includes NVHPC and ParaStationMPI, and CUDA as dependency). +EasyBuild support for nompi compiler toolchain (includes NVHPC and OpenMPI, and CUDA as dependency). @author: Damian Alvarez (Forschungszentrum Juelich) """ @@ -34,6 +34,7 @@ from easybuild.toolchains.mpi.openmpi import OpenMPI from easybuild.toolchains.compiler.cuda import Cuda + # Order matters! class Ompi(NvhpcToolchain, Cuda, OpenMPI): """Compiler toolchain with NVHPC and OpenMPI, with CUDA as dependency.""" From f3c05415fd0cf413c353273980ec4001d5c570d1 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Thu, 10 Jun 2021 17:19:17 +0200 Subject: [PATCH 403/864] update npsmpic toolchain --- easybuild/toolchains/npsmpic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/toolchains/npsmpic.py b/easybuild/toolchains/npsmpic.py index 33ee2bb06a..ed79ce2306 100644 --- a/easybuild/toolchains/npsmpic.py +++ b/easybuild/toolchains/npsmpic.py @@ -34,6 +34,7 @@ from easybuild.toolchains.mpi.psmpi import Psmpi from easybuild.toolchains.compiler.cuda import Cuda + # Order matters! class Npsmpic(NvhpcToolchain, Cuda, Psmpi): """Compiler toolchain with NVHPC and ParaStationMPI, with CUDA as dependency.""" From fabc0238df6c313357158be60562b0a9106e1a81 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Fri, 11 Jun 2021 09:50:59 +0200 Subject: [PATCH 404/864] update nvompic toolchain --- easybuild/toolchains/{nompi.py => nvompic.py} | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) rename easybuild/toolchains/{nompi.py => nvompic.py} (87%) diff --git a/easybuild/toolchains/nompi.py b/easybuild/toolchains/nvompic.py similarity index 87% rename from easybuild/toolchains/nompi.py rename to easybuild/toolchains/nvompic.py index 15122a190a..d49f4540af 100644 --- a/easybuild/toolchains/nompi.py +++ b/easybuild/toolchains/nvompic.py @@ -1,6 +1,6 @@ ## -# Copyright 2016-2016 Ghent University -# Copyright 2016-2016 Forschungszentrum Juelich +# Copyright 2016-2021 Ghent University +# Copyright 2016-2021 Forschungszentrum Juelich # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -27,6 +27,7 @@ EasyBuild support for nompi compiler toolchain (includes NVHPC and OpenMPI, and CUDA as dependency). @author: Damian Alvarez (Forschungszentrum Juelich) +@author: Sebastian Achilles (Forschungszentrum Juelich) """ from easybuild.toolchains.nvhpc import NvhpcToolchain @@ -36,7 +37,7 @@ # Order matters! -class Ompi(NvhpcToolchain, Cuda, OpenMPI): +class NVompic(NvhpcToolchain, Cuda, OpenMPI): """Compiler toolchain with NVHPC and OpenMPI, with CUDA as dependency.""" - NAME = 'nompi' + NAME = 'nvompic' SUBTOOLCHAIN = NvhpcToolchain.NAME From 419879e39e1c784e2de82036b2be6993d2bdbce7 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Fri, 11 Jun 2021 09:52:41 +0200 Subject: [PATCH 405/864] update nvpsmpic toolchain --- easybuild/toolchains/{npsmpic.py => nvpsmpic.py} | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) rename easybuild/toolchains/{npsmpic.py => nvpsmpic.py} (87%) diff --git a/easybuild/toolchains/npsmpic.py b/easybuild/toolchains/nvpsmpic.py similarity index 87% rename from easybuild/toolchains/npsmpic.py rename to easybuild/toolchains/nvpsmpic.py index ed79ce2306..de530f055c 100644 --- a/easybuild/toolchains/npsmpic.py +++ b/easybuild/toolchains/nvpsmpic.py @@ -1,6 +1,6 @@ ## -# Copyright 2016-2016 Ghent University -# Copyright 2016-2016 Forschungszentrum Juelich +# Copyright 2016-2021 Ghent University +# Copyright 2016-2021 Forschungszentrum Juelich # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -27,6 +27,7 @@ EasyBuild support for npsmpi compiler toolchain (includes NVHPC and ParaStationMPI, and CUDA as dependency). @author: Damian Alvarez (Forschungszentrum Juelich) +@author: Sebastian Achilles (Forschungszentrum Juelich) """ from easybuild.toolchains.nvhpc import NvhpcToolchain @@ -36,7 +37,7 @@ # Order matters! -class Npsmpic(NvhpcToolchain, Cuda, Psmpi): +class NVpsmpic(NvhpcToolchain, Cuda, Psmpi): """Compiler toolchain with NVHPC and ParaStationMPI, with CUDA as dependency.""" - NAME = 'npsmpic' + NAME = 'nvpsmpic' SUBTOOLCHAIN = NvhpcToolchain.NAME From a5a182a22b3a8e0096c29d1ce07832bf3c6c9051 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 16 Jun 2021 12:00:21 +0800 Subject: [PATCH 406/864] minor fixes to output of test reports --- easybuild/tools/testing.py | 62 +++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index e584e0c589..5d765b6b82 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -149,20 +149,21 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=None, gist_ # create a gist with a full test report test_report = [] - if pr_nrs is not None: + pr_list = [] + if pr_nrs: repo = pr_target_repo or GITHUB_EASYCONFIGS_REPO pr_urls = ["https://github.com/%s/%s/pull/%s" % (pr_target_account, repo, x) for x in pr_nrs] - test_report.extend([ - "Test report for %s" % ', '.join(pr_urls), - "", - ]) + pr_list.append("PR(s) %s" % ', '.join(pr_urls)) if easyblock_pr_nrs: repo = pr_target_repo or GITHUB_EASYBLOCKS_REPO + easyblock_pr_urls = ["https://github.com/%s/%s/pull/%s" % (pr_target_account, repo, x) + for x in easyblock_pr_nrs] + pr_list.append("easyblock PR(s) %s" % ', '.join(easyblock_pr_urls)) + if pr_list: test_report.extend([ - "Test report for https://github.com/%s/%s/pull/%s" % (pr_target_account, repo, nr) - for nr in easyblock_pr_nrs + "Test report for %s" % ', '.join(pr_list), + "", ]) - test_report.append("") test_report.extend([ "#### Test result", "%s" % msg, @@ -192,11 +193,11 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=None, gist_ partial_log_txt = '\n'.join(logtxt.split('\n')[-500:]) descr = "(partial) EasyBuild log for failed build of %s" % ec['spec'] - if pr_nrs is not None: - descr += " (PR #%s)" % ', #'.join(str(x) for x in pr_nrs) + if pr_nrs: + descr += " (PR(s) #%s)" % ', #'.join(str(x) for x in pr_nrs) if easyblock_pr_nrs: - descr += "".join(" (easyblock PR #%s)" % nr for nr in easyblock_pr_nrs) + descr += " (easyblock PR(s) #%s)" % ', #'.join(str(x) for x in easyblock_pr_nrs) fn = '%s_partial.log' % os.path.basename(ec['spec'])[:-3] gist_url = create_gist(partial_log_txt, fn, descr=descr, github_user=github_user) @@ -263,21 +264,32 @@ def upload_test_report_as_gist(test_report, descr=None, fn=None): return gist_url -def post_pr_test_report(pr_nr, repo_type, test_report, msg, init_session_state, success): +def post_pr_test_report(pr_nrs, repo_type, test_report, msg, init_session_state, success): """Post test report in a gist, and submit comment in easyconfigs or easyblocks PR.""" + # make sure pr_nrs is a list of strings + if isinstance(pr_nrs, str): + pr_nrs = [pr_nrs] + elif isinstance(pr_nrs, int): + pr_nrs = [str(pr_nrs)] + else: + try: + pr_nrs = [str(x) for x in pr_nrs] + except ValueError: + raise EasyBuildError("Can't convert %s to a list of PR #s." % pr_nrs) + github_user = build_option('github_user') pr_target_account = build_option('pr_target_account') pr_target_repo = build_option('pr_target_repo') or repo_type # create gist with test report - descr = "EasyBuild test report for %s/%s PR #%s" % (pr_target_account, pr_target_repo, pr_nr) + descr = "EasyBuild test report for %s/%s PR(s) #%s" % (pr_target_account, pr_target_repo, ', #'.join(pr_nrs)) timestamp = strftime("%Y%M%d-UTC-%H-%M-%S", gmtime()) - fn = 'easybuild_test_report_%s_%s_pr%s_%s.md' % (pr_nr, pr_target_account, pr_target_repo, timestamp) + fn = 'easybuild_test_report_%s_%s_pr%s_%s.md' % ('_'.join(pr_nrs), pr_target_account, pr_target_repo, timestamp) gist_url = upload_test_report_as_gist(test_report['full'], descr=descr, fn=fn) # post comment to report test result - system_info = init_session_state['system_info'] + system_info = init_session_state['system_info'].copy() # also mention CPU architecture name, but only if it's known if system_info['cpu_arch_name'] != UNKNOWN: @@ -315,9 +327,11 @@ def post_pr_test_report(pr_nr, repo_type, test_report, msg, init_session_state, ]) comment = '\n'.join(comment_lines) - post_comment_in_issue(pr_nr, comment, account=pr_target_account, repo=pr_target_repo, github_user=github_user) + for pr_nr in pr_nrs: + post_comment_in_issue(pr_nr, comment, account=pr_target_account, repo=pr_target_repo, github_user=github_user) - msg = "Test report uploaded to %s and mentioned in a comment in %s PR#%s" % (gist_url, pr_target_repo, pr_nr) + msg = "Test report uploaded to %s and mentioned in a comment in %s PR(s) #%s" % (gist_url, pr_target_repo, + ', #'.join(pr_nrs)) return msg @@ -350,15 +364,13 @@ def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=pr_nrs, gist_log=True, easyblock_pr_nrs=easyblock_pr_nrs) if pr_nrs: - # upload test report to gist and issue a comment in the PR to notify - for pr_nr in pr_nrs: - txt = post_pr_test_report(pr_nr, GITHUB_EASYCONFIGS_REPO, test_report, msg, init_session_state, - success) + # upload test report to gist and issue a comment in the PR(s) to notify + txt = post_pr_test_report(pr_nrs, GITHUB_EASYCONFIGS_REPO, test_report, msg, init_session_state, + success) elif easyblock_pr_nrs: - # upload test report to gist and issue a comment in the easyblocks PR to notify - for easyblock_pr_nr in easyblock_pr_nrs: - txt = post_pr_test_report(easyblock_pr_nr, GITHUB_EASYBLOCKS_REPO, test_report, msg, - init_session_state, success) + # upload test report to gist and issue a comment in the easyblocks PR(s) to notify + txt = post_pr_test_report(easyblock_pr_nrs, GITHUB_EASYBLOCKS_REPO, test_report, msg, + init_session_state, success) else: # only upload test report as a gist From 80b05b8a6e94c903f876fdd9698130ab74850420 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 16 Jun 2021 15:25:56 +0100 Subject: [PATCH 407/864] point people to other install methods than the boostrap --- easybuild/scripts/bootstrap_eb.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 1ccc317466..abc848283b 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -854,6 +854,8 @@ def main(): self_txt = open(__file__).read() if IS_PY3: self_txt = self_txt.encode('utf-8') + info("DEPRECATED: Use of the EasyBuild boostrap script is deprecated and may not work.") + info("Recommended install methods: https://docs.easybuild.io/en/latest/Installation.html\n"). info("EasyBuild bootstrap script (version %s, MD5: %s)" % (EB_BOOTSTRAP_VERSION, md5(self_txt).hexdigest())) info("Found Python %s\n" % '; '.join(sys.version.split('\n'))) From 28c9de2b1c09f8c5b4304356d4f722fc51e7570f Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 16 Jun 2021 15:30:04 +0100 Subject: [PATCH 408/864] typo --- easybuild/scripts/bootstrap_eb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index abc848283b..1b2422e831 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -855,7 +855,7 @@ def main(): if IS_PY3: self_txt = self_txt.encode('utf-8') info("DEPRECATED: Use of the EasyBuild boostrap script is deprecated and may not work.") - info("Recommended install methods: https://docs.easybuild.io/en/latest/Installation.html\n"). + info("Recommended install methods: https://docs.easybuild.io/en/latest/Installation.html\n") info("EasyBuild bootstrap script (version %s, MD5: %s)" % (EB_BOOTSTRAP_VERSION, md5(self_txt).hexdigest())) info("Found Python %s\n" % '; '.join(sys.version.split('\n'))) From 92fcf99ca3417a798337aac78ef8b0bdb82836f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Wed, 16 Jun 2021 15:13:42 -0400 Subject: [PATCH 409/864] Fix typo reproducability -> reproducibility --- easybuild/framework/easyblock.py | 18 +++++++++--------- test/framework/toy_build.py | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index c100e88d88..d48fb6bdaf 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -107,7 +107,7 @@ # string part of URL for Python packages on PyPI that indicates needs to be rewritten (see derive_alt_pypi_url) PYPI_PKG_URL_PATTERN = 'pypi.python.org/packages/source/' -# Directory name in which to store reproducability files +# Directory name in which to store reproducibility files REPROD = 'reprod' _log = fancylogger.getLogger('easyblock') @@ -3589,7 +3589,7 @@ def build_and_install_one(ecdict, init_env): # load easyblock easyblock = build_option('easyblock') if easyblock: - # set the value in the dict so this is included in the reproducability dump of the easyconfig + # set the value in the dict so this is included in the reproducibility dump of the easyconfig ecdict['ec']['easyblock'] = easyblock else: easyblock = fetch_parameters_from_easyconfig(rawtxt, ['easyblock'])[0] @@ -3621,7 +3621,7 @@ def build_and_install_one(ecdict, init_env): run_test_cases = not build_option('skip_test_cases') and app.cfg['tests'] if not dry_run: - # create our reproducability files before carrying out the easyblock steps + # create our reproducibility files before carrying out the easyblock steps reprod_dir_root = os.path.dirname(app.logfile) reprod_dir = reproduce_build(app, reprod_dir_root) @@ -3633,7 +3633,7 @@ def build_and_install_one(ecdict, init_env): result = app.run_all_steps(run_test_cases=run_test_cases) if not dry_run: - # also add any extension easyblocks used during the build for reproducability + # also add any extension easyblocks used during the build for reproducibility if app.ext_instances: copy_easyblocks_for_reprod(app.ext_instances, reprod_dir) @@ -3677,14 +3677,14 @@ def build_and_install_one(ecdict, init_env): _log.info("Build stats: %s" % buildstats) try: - # move the reproducability files to the final log directory + # move the reproducibility files to the final log directory archive_reprod_dir = os.path.join(new_log_dir, REPROD) if os.path.exists(archive_reprod_dir): backup_dir = find_backup_name_candidate(archive_reprod_dir) move_file(archive_reprod_dir, backup_dir) - _log.info("Existing reproducability directory %s backed up to %s", archive_reprod_dir, backup_dir) + _log.info("Existing reproducibility directory %s backed up to %s", archive_reprod_dir, backup_dir) move_file(reprod_dir, archive_reprod_dir) - _log.info("Wrote files for reproducability to %s", archive_reprod_dir) + _log.info("Wrote files for reproducibility to %s", archive_reprod_dir) except EasyBuildError as error: if build_option('module_only'): _log.info("Using --module-only so can recover from error: %s", error) @@ -3803,12 +3803,12 @@ def copy_easyblocks_for_reprod(easyblock_instances, reprod_dir): def reproduce_build(app, reprod_dir_root): """ - Create reproducability files (processed easyconfig and easyblocks used) from class instance + Create reproducibility files (processed easyconfig and easyblocks used) from class instance :param app: easyblock class instance :param reprod_dir_root: root directory in which to create the 'reprod' directory - :return reprod_dir: directory containing reproducability files + :return reprod_dir: directory containing reproducibility files """ ec_filename = app.cfg.filename() diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index c339194ad3..e9060d29e5 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1972,8 +1972,8 @@ def test_minimal_toolchains(self): # this test doesn't check for anything specific to using minimal toolchains, only side-effects self.test_toy_build(extra_args=['--minimal-toolchains']) - def test_reproducability(self): - """Test toy build produces expected reproducability files""" + def test_reproducibility(self): + """Test toy build produces expected reproducibility files""" # We need hooks for a complete test hooks_filename = 'my_hooks.py' @@ -2025,8 +2025,8 @@ def test_reproducability(self): reprod_hooks = os.path.join(reprod_dir, 'hooks', hooks_filename) self.assertTrue(os.path.exists(reprod_hooks)) - def test_reproducability_ext_easyblocks(self): - """Test toy build produces expected reproducability files also when extensions are used""" + def test_reproducibility_ext_easyblocks(self): + """Test toy build produces expected reproducibility files also when extensions are used""" topdir = os.path.dirname(os.path.abspath(__file__)) toy_ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') From 973e1a7fc4c9005985ee0edfcf5b592b25cb1439 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 17 Jun 2021 14:02:17 +0200 Subject: [PATCH 410/864] hierarchical_mns.py: Always check toolchain compiler name against template map. This fixes a problem with modules ending up in the wrong place for intel-compilers --- .../module_naming_scheme/hierarchical_mns.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py index 7c9702cf71..0b050e34cd 100644 --- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py +++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py @@ -116,12 +116,6 @@ def det_toolchain_compilers_name_version(self, tc_comps): if tc_comps is None: # no compiler in toolchain, system toolchain res = None - elif len(tc_comps) == 1: - tc_comp = tc_comps[0] - if tc_comp is None: - res = None - else: - res = (tc_comp['name'], self.det_full_version(tc_comp)) else: comp_versions = dict([(comp['name'], self.det_full_version(comp)) for comp in tc_comps]) comp_names = comp_versions.keys() @@ -132,10 +126,17 @@ def det_toolchain_compilers_name_version(self, tc_comps): # make sure that icc/ifort versions match (unless not existing as separate modules) if tc_comp_name == 'intel' and comp_versions.get('icc') != comp_versions.get('ifort'): raise EasyBuildError("Bumped into different versions for Intel compilers: %s", comp_versions) + res = (tc_comp_name, tc_comp_ver) else: - raise EasyBuildError("Unknown set of toolchain compilers, module naming scheme needs work: %s", + if len(tc_comps) == 1: + tc_comp = tc_comps[0] + if tc_comp is None: + res = None + else: + res = (tc_comp['name'], self.det_full_version(tc_comp)) + else: + raise EasyBuildError("Unknown set of toolchain compilers, module naming scheme needs work: %s", comp_names) - res = (tc_comp_name, tc_comp_ver) return res def det_module_subdir(self, ec): From 137495d1c19376e60b92db0ef15bbc1bde59ffc4 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 17 Jun 2021 14:54:20 +0200 Subject: [PATCH 411/864] hierarchical_mns.py: Adjust code so it doesn't try to access a None object. --- .../module_naming_scheme/hierarchical_mns.py | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py index 0b050e34cd..fd50cb5635 100644 --- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py +++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py @@ -117,26 +117,26 @@ def det_toolchain_compilers_name_version(self, tc_comps): # no compiler in toolchain, system toolchain res = None else: - comp_versions = dict([(comp['name'], self.det_full_version(comp)) for comp in tc_comps]) - comp_names = comp_versions.keys() - key = ','.join(sorted(comp_names)) - if key in COMP_NAME_VERSION_TEMPLATES: - tc_comp_name, tc_comp_ver_tmpl = COMP_NAME_VERSION_TEMPLATES[key] - tc_comp_ver = tc_comp_ver_tmpl % comp_versions - # make sure that icc/ifort versions match (unless not existing as separate modules) - if tc_comp_name == 'intel' and comp_versions.get('icc') != comp_versions.get('ifort'): - raise EasyBuildError("Bumped into different versions for Intel compilers: %s", comp_versions) - res = (tc_comp_name, tc_comp_ver) - else: - if len(tc_comps) == 1: - tc_comp = tc_comps[0] - if tc_comp is None: - res = None - else: - res = (tc_comp['name'], self.det_full_version(tc_comp)) + if len(tc_comps) > 0 and tc_comps[0]: + comp_versions = dict([(comp['name'], self.det_full_version(comp)) for comp in tc_comps]) + comp_names = comp_versions.keys() + key = ','.join(sorted(comp_names)) + if key in COMP_NAME_VERSION_TEMPLATES: + tc_comp_name, tc_comp_ver_tmpl = COMP_NAME_VERSION_TEMPLATES[key] + tc_comp_ver = tc_comp_ver_tmpl % comp_versions + # make sure that icc/ifort versions match (unless not existing as separate modules) + if tc_comp_name == 'intel' and comp_versions.get('icc') != comp_versions.get('ifort'): + raise EasyBuildError("Bumped into different versions for Intel compilers: %s", comp_versions) + res = (tc_comp_name, tc_comp_ver) else: - raise EasyBuildError("Unknown set of toolchain compilers, module naming scheme needs work: %s", - comp_names) + if len(tc_comps) == 1: + res = (tc_comp['name'], self.det_full_version(tc_comp)) + else: + raise EasyBuildError("Unknown set of toolchain compilers, module naming scheme needs work: %s", + comp_names) + else: + res = None + return res def det_module_subdir(self, ec): From 30aefc4f814e3550f4ff5191c1a14ad7fcffa20d Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 17 Jun 2021 14:56:30 +0200 Subject: [PATCH 412/864] erarchical_mns.py: Re-add dropped tc_comp = tc_comps[0] --- easybuild/tools/module_naming_scheme/hierarchical_mns.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py index fd50cb5635..f9839532bc 100644 --- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py +++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py @@ -130,6 +130,7 @@ def det_toolchain_compilers_name_version(self, tc_comps): res = (tc_comp_name, tc_comp_ver) else: if len(tc_comps) == 1: + tc_comp = tc_comps[0] res = (tc_comp['name'], self.det_full_version(tc_comp)) else: raise EasyBuildError("Unknown set of toolchain compilers, module naming scheme needs work: %s", From 4257628704580e4fdb4168dd435e5ea4739c4a97 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 18 Jun 2021 10:53:13 +0200 Subject: [PATCH 413/864] require that $EASYBUILD_BOOTSTRAP_DEPRECATED is defined to run EasyBuild bootstrap script --- .github/workflows/bootstrap_script.yml | 3 ++- easybuild/scripts/bootstrap_eb.py | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bootstrap_script.yml b/.github/workflows/bootstrap_script.yml index e46f5cb41e..3eddc7dcf3 100644 --- a/.github/workflows/bootstrap_script.yml +++ b/.github/workflows/bootstrap_script.yml @@ -107,11 +107,12 @@ jobs: EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - EB_BOOTSTRAP_EXPECTED="20210106.01 c2d93de0dd91123eb4f51cfc16d1f5efb80f1d238b3d6cd100994086887a1ae0" + EB_BOOTSTRAP_EXPECTED="20210618.01 e5d477d717c6d3648ba2027ab735713ba5804fbf52f4b4adcca0bc1379b44618" test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) # test bootstrap script export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap + export EASYBUILD_BOOTSTRAP_DEPRECATED=1 python easybuild/scripts/bootstrap_eb.py $PREFIX # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module unset PYTHONPATH diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 1b2422e831..ed3390ea93 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -62,7 +62,7 @@ import urllib.request as std_urllib -EB_BOOTSTRAP_VERSION = '20210106.01' +EB_BOOTSTRAP_VERSION = '20210618.01' # argparse preferrred, optparse deprecated >=2.7 HAVE_ARGPARSE = False @@ -82,6 +82,9 @@ STAGE1_SUBDIR = 'eb_stage1' +# the EasyBuild bootstrap script is deprecated, and will only run if $EASYBUILD_BOOTSTRAP_DEPRECATED is defined +EASYBUILD_BOOTSTRAP_DEPRECATED = os.environ.pop('EASYBUILD_BOOTSTRAP_DEPRECATED', None) + # set print_debug to True for detailed progress info print_debug = os.environ.pop('EASYBUILD_BOOTSTRAP_DEBUG', False) @@ -854,8 +857,17 @@ def main(): self_txt = open(__file__).read() if IS_PY3: self_txt = self_txt.encode('utf-8') - info("DEPRECATED: Use of the EasyBuild boostrap script is deprecated and may not work.") - info("Recommended install methods: https://docs.easybuild.io/en/latest/Installation.html\n") + + url = 'https://docs.easybuild.io/en/latest/Installation.html' + info("Use of the EasyBuild boostrap script is DEPRECATED (since June 2021).") + info("It is strongly recommended to use one of the installation methods outlined at %s instead!\n" % url) + if not EASYBUILD_BOOTSTRAP_DEPRECATED: + error("The EasyBuild bootstrap script will only run if $EASYBUILD_BOOTSTRAP_DEPRECATED is defined.") + else: + msg = "You have opted to continue with the EasyBuild bootstrap script by defining " + msg += "$EASYBUILD_BOOTSTRAP_DEPRECATED. Good luck!\n" + info(msg) + info("EasyBuild bootstrap script (version %s, MD5: %s)" % (EB_BOOTSTRAP_VERSION, md5(self_txt).hexdigest())) info("Found Python %s\n" % '; '.join(sys.version.split('\n'))) From 2cea28e6f255a6fb37b4e955c862b9498a56f17e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 18 Jun 2021 10:59:40 +0200 Subject: [PATCH 414/864] need to unset $EASYBUILD_BOOTSTRAP_DEPRECATED or EasyBuild gets upset about not knowing this configuration option --- .github/workflows/bootstrap_script.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/bootstrap_script.yml b/.github/workflows/bootstrap_script.yml index 3eddc7dcf3..c0ade9843b 100644 --- a/.github/workflows/bootstrap_script.yml +++ b/.github/workflows/bootstrap_script.yml @@ -114,6 +114,7 @@ jobs: export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap export EASYBUILD_BOOTSTRAP_DEPRECATED=1 python easybuild/scripts/bootstrap_eb.py $PREFIX + unset EASYBUILD_BOOTSTRAP_DEPRECATED # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module unset PYTHONPATH # simple sanity check on bootstrapped EasyBuild module From 9c508687ed01c6c5e2686f1841201dbf9257d0f1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 18 Jun 2021 14:55:04 +0200 Subject: [PATCH 415/864] add test to cover fix for intel/2021a in HierarchicalMNS --- .../test_ecs/i/iimpi/iimpi-2021a.eb | 18 ++++++++++ .../i/imkl/imkl-2021.2.0-iimpi-2021a.eb | 17 +++++++++ .../impi-2021.2.0-intel-compilers-2021.2.0.eb | 22 ++++++++++++ .../intel-compilers-2021.2.0.eb | 36 +++++++++++++++++++ test/framework/module_generator.py | 8 +++++ 5 files changed, 101 insertions(+) create mode 100644 test/framework/easyconfigs/test_ecs/i/iimpi/iimpi-2021a.eb create mode 100644 test/framework/easyconfigs/test_ecs/i/imkl/imkl-2021.2.0-iimpi-2021a.eb create mode 100644 test/framework/easyconfigs/test_ecs/i/impi/impi-2021.2.0-intel-compilers-2021.2.0.eb create mode 100644 test/framework/easyconfigs/test_ecs/i/intel-compilers/intel-compilers-2021.2.0.eb diff --git a/test/framework/easyconfigs/test_ecs/i/iimpi/iimpi-2021a.eb b/test/framework/easyconfigs/test_ecs/i/iimpi/iimpi-2021a.eb new file mode 100644 index 0000000000..b5ebf60d37 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/i/iimpi/iimpi-2021a.eb @@ -0,0 +1,18 @@ +# This is an easyconfig file for EasyBuild, see http://easybuilders.github.io/easybuild +easyblock = 'Toolchain' + +name = 'iimpi' +version = '2021a' + +homepage = 'https://software.intel.com/parallel-studio-xe' +description = """Intel C/C++ and Fortran compilers, alongside Intel MPI.""" + +toolchain = SYSTEM + +local_comp_ver = '2021.2.0' +dependencies = [ + ('intel-compilers', local_comp_ver), + ('impi', local_comp_ver, '', ('intel-compilers', local_comp_ver)), +] + +moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/test_ecs/i/imkl/imkl-2021.2.0-iimpi-2021a.eb b/test/framework/easyconfigs/test_ecs/i/imkl/imkl-2021.2.0-iimpi-2021a.eb new file mode 100644 index 0000000000..4d047da06e --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/i/imkl/imkl-2021.2.0-iimpi-2021a.eb @@ -0,0 +1,17 @@ +# dummy easyconfig, only for use with easybuild-framwork test suite! +easyblock = 'Toolchain' + +name = 'imkl' +version = '2021.2.0' + +homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html' +description = "Intel oneAPI Math Kernel Library" + +toolchain = {'name': 'iimpi', 'version': '2021a'} + +# see https://software.intel.com/content/www/us/en/develop/articles/oneapi-standalone-components.html +source_urls = ['https://registrationcenter-download.intel.com/akdlm/irc_nas/tec/17757/'] +sources = ['l_onemkl_p_%(version)s.296_offline.sh'] +checksums = ['816e9df26ff331d6c0751b86ed5f7d243f9f172e76f14e83b32bf4d1d619dbae'] + +moduleclass = 'numlib' diff --git a/test/framework/easyconfigs/test_ecs/i/impi/impi-2021.2.0-intel-compilers-2021.2.0.eb b/test/framework/easyconfigs/test_ecs/i/impi/impi-2021.2.0-intel-compilers-2021.2.0.eb new file mode 100644 index 0000000000..acbee9cd10 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/i/impi/impi-2021.2.0-intel-compilers-2021.2.0.eb @@ -0,0 +1,22 @@ +# dummy easyconfig, only for use with easybuild-framwork test suite! +easyblock = 'Toolchain' + +name = 'impi' +version = '2021.2.0' + +homepage = 'https://software.intel.com/content/www/us/en/develop/tools/mpi-library.html' +description = "Intel MPI Library, compatible with MPICH ABI" + +toolchain = {'name': 'intel-compilers', 'version': '2021.2.0'} + +# see https://software.intel.com/content/www/us/en/develop/articles/oneapi-standalone-components.html +source_urls = ['https://registrationcenter-download.intel.com/akdlm/irc_nas/17729/'] +sources = ['l_mpi_oneapi_p_%(version)s.215_offline.sh'] +checksums = ['d0d4cdd11edaff2e7285e38f537defccff38e37a3067c02f4af43a3629ad4aa3'] + +# dummy easyconfig, only for use with easybuild-framwork test suite! +dependencies = [ + # ('UCX', '1.10.0'), +] + +moduleclass = 'mpi' diff --git a/test/framework/easyconfigs/test_ecs/i/intel-compilers/intel-compilers-2021.2.0.eb b/test/framework/easyconfigs/test_ecs/i/intel-compilers/intel-compilers-2021.2.0.eb new file mode 100644 index 0000000000..fe9e39c7f2 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/i/intel-compilers/intel-compilers-2021.2.0.eb @@ -0,0 +1,36 @@ +# dummy easyconfig, only for use with easybuild-framwork test suite! +easyblock = 'Toolchain' + +name = 'intel-compilers' +version = '2021.2.0' + +homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html' +description = "Intel C, C++ & Fortran compilers (classic and oneAPI)" + +toolchain = SYSTEM + +# see https://software.intel.com/content/www/us/en/develop/articles/oneapi-standalone-components.html +sources = [ + { + 'source_urls': ['https://registrationcenter-download.intel.com/akdlm/irc_nas/17749/'], + 'filename': 'l_dpcpp-cpp-compiler_p_%(version)s.118_offline.sh', + }, + { + 'source_urls': ['https://registrationcenter-download.intel.com/akdlm/irc_nas/17756/'], + 'filename': 'l_fortran-compiler_p_%(version)s.136_offline.sh', + }, +] +checksums = [ + # l_dpcpp-cpp-compiler_p_2021.2.0.118_offline.sh + '5d01cbff1a574c3775510cd97ffddd27fdf56d06a6b0c89a826fb23da4336d59', + 'a62e04a80f6d2f05e67cd5acb03fa58857ee22c6bd581ec0651c0ccd5bdec5a1', # l_fortran-compiler_p_2021.2.0.136_offline.sh +] + +# dummy easyconfig, only for use with easybuild-framwork test suite! +local_gccver = '10.3.0' +dependencies = [ + # ('GCCcore', local_gccver), + # ('binutils', '2.36.1', '', ('GCCcore', local_gccver)), +] + +moduleclass = 'compiler' diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 652b48d89b..517245e741 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -1301,6 +1301,14 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, ['Compiler/intel/2019.4.243'], ['Core']), 'imkl-2019.4.243-iimpi-2019.08.eb': ('imkl/2019.4.243', 'MPI/intel/2019.4.243/impi/2019.4.243', [], [], ['Core']), + 'intel-compilers-2021.2.0.eb': ('intel-compilers/2021.2.0', 'Core', + ['Compiler/intel/2021.2.0'], ['Compiler/intel/2021.2.0'], ['Core']), + 'impi-2021.2.0-intel-compilers-2021.2.0.eb': ('impi/2021.2.0', 'Compiler/intel/2021.2.0', + ['MPI/intel/2021.2.0/impi/2021.2.0'], + ['MPI/intel/2021.2.0/impi/2021.2.0'], + ['Core']), + 'imkl-2021.2.0-iimpi-2021a.eb': ('imkl/2021.2.0', 'MPI/intel/2021.2.0/impi/2021.2.0', + [], [], ['Core']), 'CUDA-9.1.85-GCC-6.4.0-2.28.eb': ('CUDA/9.1.85', 'Compiler/GCC/6.4.0-2.28', ['Compiler/GCC-CUDA/6.4.0-2.28-9.1.85'], ['Compiler/GCC-CUDA/6.4.0-2.28-9.1.85'], ['Core']), From 4564278f3659d9fe08e7dcef67a2e9af43264036 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 18 Jun 2021 15:18:09 +0200 Subject: [PATCH 416/864] fix count in test_index_functions after adding 4 test easyconfigs --- test/framework/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 77c3c3bba1..12692ba147 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2128,7 +2128,7 @@ def test_index_functions(self): # test with specified path with and without trailing '/'s for path in [test_ecs, test_ecs + '/', test_ecs + '//']: index = ft.create_index(path) - self.assertEqual(len(index), 85) + self.assertEqual(len(index), 89) expected = [ os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), From 763d7279e33ab0ad96b7eb25799a6d46e7221c1f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 18 Jun 2021 16:57:42 +0200 Subject: [PATCH 417/864] fix test_search_archived after adding additional 'intel*' test easyconfig --- test/framework/options.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 176e052f48..da58ac8fff 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1031,6 +1031,7 @@ def test_search_archived(self): txt = self.get_stdout().rstrip() self.mock_stdout(False) expected = '\n'.join([ + ' * intel-compilers-2021.2.0.eb', ' * intel-2018a.eb', '', "Note: 1 matching archived easyconfig(s) found, use --consider-archived-easyconfigs to see them", @@ -1043,6 +1044,7 @@ def test_search_archived(self): txt = self.get_stdout().rstrip() self.mock_stdout(False) expected = '\n'.join([ + ' * intel-compilers-2021.2.0.eb', ' * intel-2018a.eb', '', "Matching archived easyconfigs:", From 3afb062f3b766cf8e8279bdda304de89f067d0f8 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 21 Jun 2021 19:24:08 +0800 Subject: [PATCH 418/864] warn about generic milestone in --review-pr and --merge-pr --- easybuild/framework/easyconfig/tools.py | 6 ++++++ easybuild/tools/github.py | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index fadbf03f50..d78ff96e7c 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -539,6 +539,12 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False if missing_labels: lines.extend(['', "This PR should be labelled with %s" % ', '.join(["'%s'" % ml for ml in missing_labels])]) + if not pr_data['milestone']: + lines.extend(['', "This PR should be associated with a milestone"]) + elif '.x' in pr_data['milestone']['title']: + lines.extend(['', "This PR is associated with a generic '.x' milestone, " + "it should be associated to the next release milestone once merged"]) + return '\n'.join(lines) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 000568a479..5ac8cb071f 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1183,7 +1183,10 @@ def not_eligible(msg): # check whether a milestone is set msg_tmpl = "* milestone is set: %s" if pr_data['milestone']: - print_msg(msg_tmpl % "OK (%s)" % pr_data['milestone']['title'], prefix=False) + milestone = pr_data['milestone']['title'] + if '.x' in milestone: + milestone += ", please change to the next release milestone once the PR is merged" + print_msg(msg_tmpl % "OK (%s)" % milestone, prefix=False) else: res = not_eligible(msg_tmpl % 'no milestone found') From 4450fdf2b1cde2fab7c9f7b0eb6bdbf7ba54b0e8 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 22 Jun 2021 15:46:04 +0800 Subject: [PATCH 419/864] add --review-pr-max and --review-pr-filter options to limit easyconfigs shown in multi-diff --- easybuild/framework/easyconfig/tools.py | 6 +++++- easybuild/main.py | 3 ++- easybuild/tools/options.py | 3 +++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index fadbf03f50..ff6882f33a 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -486,7 +486,7 @@ def find_related_easyconfigs(path, ec): return sorted(res) -def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False): +def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False, max_ecs=None, filter_ecs=None): """ Print multi-diff overview between specified easyconfigs or PR and specified branch. :param pr: pull request number in easybuild-easyconfigs repo to review @@ -521,6 +521,10 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False pr_msg = "new PR" _log.debug("File in %s %s has these related easyconfigs: %s" % (pr_msg, ec['spec'], files)) if files: + if filter_ecs is not None: + files = [x for x in files if filter_ecs.search(x)] + if max_ecs is not None: + files = files[:max_ecs] lines.append(multidiff(ec['spec'], files, colored=colored)) else: lines.extend(['', "(no related easyconfigs found for %s)\n" % os.path.basename(ec['spec'])]) diff --git a/easybuild/main.py b/easybuild/main.py index 039b400d2f..9b0d8131d3 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -261,7 +261,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): merge_pr(options.merge_pr) elif options.review_pr: - print(review_pr(pr=options.review_pr, colored=use_color(options.color), testing=testing)) + print(review_pr(pr=options.review_pr, colored=use_color(options.color), testing=testing, + max_ecs=options.review_pr_max, filter_ecs=options.review_pr_filter)) elif options.add_pr_labels: add_pr_labels(options.add_pr_labels) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 2ba011b2f3..22645e3ce8 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -690,6 +690,9 @@ def github_options(self): 'sync-pr-with-develop': ("Sync pull request with current 'develop' branch", int, 'store', None, {'metavar': 'PR#'}), 'review-pr': ("Review specified pull request", int, 'store', None, {'metavar': 'PR#'}), + 'review-pr-filter': ("Regex used to filter out easyconfigs to diff against in --review-pr", + None, 'regex', None), + 'review-pr-max': ("Maximum number of easyconfigs to diff against in --review-pr", int, 'store', None), 'test-report-env-filter': ("Regex used to filter out variables in environment dump of test report", None, 'regex', None), 'update-branch-github': ("Update specified branch in GitHub", str, 'store', None), From 7cd95bef8d7081167e4cdbd2dada440beafc20ae Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 23 Jun 2021 13:36:22 +0800 Subject: [PATCH 420/864] use temporary download folder in test_http_header_fields_urlpat --- test/framework/options.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index bec20a6e11..618d22e9be 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2724,6 +2724,7 @@ def test_parse_http_header_fields_urlpat(self): def test_http_header_fields_urlpat(self): """Test use of --http-header-fields-urlpat.""" + tmpdir = tempfile.mkdtemp() test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') ec_file = os.path.join(test_ecs_dir, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb') common_args = [ @@ -2733,6 +2734,7 @@ def test_http_header_fields_urlpat(self): '--force', '--force-download', '--logtostdout', + '--sourcepath=%s' % tmpdir, ] # define header fields:values that should (not) show up in the logs, either @@ -2829,6 +2831,9 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None): not_expected = [testdoval, testdonthdr, testdontval] run_and_assert(args, "case E", expected, not_expected) + # cleanup downloads + shutil.rmtree(tmpdir) + def test_test_report_env_filter(self): """Test use of --test-report-env-filter.""" From 2a763735946452ef836cc2287c9d93c091d82fff Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 23 Jun 2021 16:45:25 +0200 Subject: [PATCH 421/864] Work in review --- easybuild/framework/easyblock.py | 18 ++++++++++-------- easybuild/main.py | 2 +- test/framework/options.py | 7 ++++--- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 6d1a7f157e..8f1cb55183 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1816,14 +1816,16 @@ def remove_module_file(self): self.log.info("Removing existing module file %s", self.mod_filepath) remove_file(self.mod_filepath) - def report_test_failure(self, msgOrError): - """Report a failing test either via an exception or warning depending on ignore-test-failure + def report_test_failure(self, msg_or_error): + """ + Report a failing test either via an exception or warning depending on ignore-test-failure - msgOrError - Failure description. Either a string or an EasyBuildError""" + :param msg_or_error: failure description (string value or an EasyBuildError instance) + """ if self.ignore_test_failure: - print_warning("Test failure ignored: " + str(msgOrError), log=self.log) + print_warning("Test failure ignored: " + str(msg_or_error), log=self.log) else: - exception = msgOrError if isinstance(msgOrError, EasyBuildError) else EasyBuildError(msgOrError) + exception = msg_or_error if isinstance(msg_or_error, EasyBuildError) else EasyBuildError(msg_or_error) raise exception # @@ -2245,8 +2247,8 @@ def _test_step(self): """Run the test_step and handles failures""" try: self.test_step() - except EasyBuildError as e: - self.report_test_failure(e) + except EasyBuildError as err: + self.report_test_failure(err) def stage_install_step(self): """ @@ -3384,7 +3386,7 @@ def run_step(self, step, step_methods): for step_method in step_methods: # Remove leading underscore from e.g. "_test_step" method_name = extract_method_name(step_method).lstrip('_') - self.log.info("Running method %s part of step %s" % (method_name, step)) + self.log.info("Running method %s part of step %s", method_name, step) if self.dry_run: self.dry_run_msg("[%s method]", method_name) diff --git a/easybuild/main.py b/easybuild/main.py index 6f271cad6b..8cbff81b41 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -307,7 +307,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if options.skip_test_step: if options.ignore_test_failure: - raise EasyBuildError("Found both ignore-test-failure and skip-test-step being set. " + raise EasyBuildError("Found both ignore-test-failure and skip-test-step enabled. " "Please use only one of them.") else: print_warning("Will not run the test step as requested via skip-test-step. " diff --git a/test/framework/options.py b/test/framework/options.py index f384da8a4e..fd4ee305fb 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -398,6 +398,7 @@ def test_ignore_test_failure(self): """Test ignore failing tests (--ignore-test-failure).""" topdir = os.path.abspath(os.path.dirname(__file__)) + # This EC uses a `runtest` command which does not exist and hence will make the test step fail toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb') args = [toy_ec, '--ignore-test-failure', '--force'] @@ -407,13 +408,13 @@ def test_ignore_test_failure(self): msg = 'Test failure ignored' self.assertTrue(re.search(msg, outtxt), - "Ignored test failure message in log not found, outtxt: %s" % outtxt) + "Ignored test failure message in log should be found, outtxt: %s" % outtxt) self.assertTrue(re.search(msg, stderr.getvalue()), - "Ignored test failure message in stderr not found, stderr: %s" % stderr.getvalue()) + "Ignored test failure message in stderr should be found, stderr: %s" % stderr.getvalue()) # Passing skip and ignore options is disallowed args.append('--skip-test-step') - error_pattern = 'Found both ignore-test-failure and skip-test-step being set' + error_pattern = 'Found both ignore-test-failure and skip-test-step enabled' self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True) def test_job(self): From 0bde82d07d8b63c75861cabe5b13a85193f4d17f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 24 Jun 2021 11:07:12 +0200 Subject: [PATCH 422/864] avoid checking msg attribute of GitCommandError --- easybuild/tools/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 000568a479..a86a9fa988 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -2025,7 +2025,7 @@ def check_github(): ver, req_ver = git.__version__, '1.0' if LooseVersion(ver) < LooseVersion(req_ver): check_res = "FAIL (GitPython version %s is too old, should be version %s or newer)" % (ver, req_ver) - elif "Could not read from remote repository" in push_err.msg: + elif "Could not read from remote repository" in str(push_err): check_res = "FAIL (GitHub SSH key missing? %s)" % push_err else: check_res = "FAIL (unexpected exception: %s)" % push_err From 29c83d7a3b2e71d5664acffe62f8272e09d6d0a2 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 11 Jun 2021 10:42:57 +0200 Subject: [PATCH 423/864] Error out when passing a list to run_cmd* This is a serious pitfall as e.g. run_cmd(['python', '-c' , 'print(1)']) would be interpreted as `bash -c python -c 'print(1)'`, i.e. the arguments are passed to bash and not to python! Allow to opt-in into that by passing shell=True explicitely. --- easybuild/tools/run.py | 16 ++++++++++++++-- test/framework/run.py | 5 ++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 477390d7ba..ce04900204 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -128,7 +128,7 @@ def get_output_from_process(proc, read_size=None, asynchronous=False): @run_cmd_cache def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None, - force_in_dry_run=False, verbose=True, shell=True, trace=True, stream_output=None, asynchronous=False): + force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False): """ Run specified command (in a subshell) :param cmd: command to run @@ -141,7 +141,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True :param path: path to execute the command in; current working directory is used if unspecified :param force_in_dry_run: force running the command during dry run :param verbose: include message on running the command in dry run output - :param shell: allow commands to not run in a shell (especially useful for cmd lists) + :param shell: allow commands to not run in a shell (especially useful for cmd lists), defaults to True :param trace: print command being executed as part of trace output :param stream_output: enable streaming command output to stdout :param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True) @@ -155,6 +155,13 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True else: raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd) + if shell is None: + shell = True + if isinstance(cmd, list): + raise EasyBuildError("When passing cmd as a list then `shell` must be set explictely! " + "Note that all elements of the list but the first are treated as arguments " + "to the shell and NOT to the command to be executed!") + if log_output or (trace and build_option('trace')): # collect output of running command in temporary log file, if desired fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd-') @@ -318,6 +325,11 @@ def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, re """ cwd = os.getcwd() + if not isinstance(cmd, string_type) and len(cmd) > 1: + # We use shell=True and hence we should really pass the command as a string + # When using a list then every element past the first is passed to the shell itself, not the command! + raise EasyBuildError("The command passed must be a string!") + if log_all or (trace and build_option('trace')): # collect output of running command in temporary log file, if desired fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd_qa-') diff --git a/test/framework/run.py b/test/framework/run.py index ea04d789f0..368a4b14ed 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -524,7 +524,10 @@ def test_dry_run(self): def test_run_cmd_list(self): """Test run_cmd with command specified as a list rather than a string""" - (out, ec) = run_cmd(['/bin/sh', '-c', "echo hello"], shell=False) + cmd = ['/bin/sh', '-c', "echo hello"] + self.assertErrorRegex(EasyBuildError, "When passing cmd as a list then `shell` must be set explictely!", + run_cmd, cmd) + (out, ec) = run_cmd(cmd, shell=False) self.assertEqual(out, "hello\n") # no reason echo hello could fail self.assertEqual(ec, 0) From ddff401f0b5b5b304c01700720ee89f8692c7c20 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 24 Jun 2021 17:41:42 +0200 Subject: [PATCH 424/864] also consider sources provided as dict in EasyBlock.check_checksums_for --- easybuild/framework/easyblock.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 61be9ac2f1..ae99718752 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2012,7 +2012,11 @@ def check_checksums_for(self, ent, sub='', source_cnt=None): for fn, checksum in zip(sources + patches, checksums): if isinstance(checksum, dict): - checksum = checksum.get(fn) + if isinstance(fn, dict): + filename = fn['filename'] + else: + filename = fn + checksum = checksum.get(filename) # take into account that we may encounter a tuple of valid SHA256 checksums # (see https://github.com/easybuilders/easybuild-framework/pull/2958) From 853f990e0f548237f87e2d793ba76cd298c3471b Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Fri, 25 Jun 2021 16:05:54 +0200 Subject: [PATCH 425/864] update nvompic toolchain --- easybuild/toolchains/nvompic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/nvompic.py b/easybuild/toolchains/nvompic.py index d49f4540af..688f5c3f79 100644 --- a/easybuild/toolchains/nvompic.py +++ b/easybuild/toolchains/nvompic.py @@ -24,7 +24,7 @@ # along with EasyBuild. If not, see . ## """ -EasyBuild support for nompi compiler toolchain (includes NVHPC and OpenMPI, and CUDA as dependency). +EasyBuild support for nvompic compiler toolchain (includes NVHPC and OpenMPI, and CUDA as dependency). @author: Damian Alvarez (Forschungszentrum Juelich) @author: Sebastian Achilles (Forschungszentrum Juelich) From 78ecc14025f39cc5ebd73f5f3c69855ba6d4c198 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 28 Jun 2021 15:51:11 +0800 Subject: [PATCH 426/864] warn about potentially missing patches in --new-pr --- easybuild/tools/github.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index a86a9fa988..45602791ea 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1751,6 +1751,11 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): res = new_branch_github(paths, ecs, commit_msg=commit_msg) file_info, deleted_paths, _, branch_name, diff_stat, pr_target_repo = res + for ec in file_info['ecs']: + for patch in ec.asdict()['patches']: + if patch not in paths['patch_files']: + print_warning("%s, referenced by %s, is not included in this PR" % (patch, ec.filename())) + new_pr_from_branch(branch_name, title=title, descr=descr, pr_target_repo=pr_target_repo, pr_metadata=(file_info, deleted_paths, diff_stat), commit_msg=commit_msg) From a82abc4a61a00501547232de6b117bdc5aa8235b Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Mon, 28 Jun 2021 10:41:02 +0200 Subject: [PATCH 427/864] update nvompic toolchain --- easybuild/toolchains/nvompic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/toolchains/nvompic.py b/easybuild/toolchains/nvompic.py index 688f5c3f79..55bcabbf28 100644 --- a/easybuild/toolchains/nvompic.py +++ b/easybuild/toolchains/nvompic.py @@ -30,14 +30,14 @@ @author: Sebastian Achilles (Forschungszentrum Juelich) """ -from easybuild.toolchains.nvhpc import NvhpcToolchain +from easybuild.toolchains.nvhpc import NVHPCToolchain # We pull in MPI and CUDA at once so this maps nicely to HMNS from easybuild.toolchains.mpi.openmpi import OpenMPI from easybuild.toolchains.compiler.cuda import Cuda # Order matters! -class NVompic(NvhpcToolchain, Cuda, OpenMPI): +class NVompic(NVHPCToolchain, Cuda, OpenMPI): """Compiler toolchain with NVHPC and OpenMPI, with CUDA as dependency.""" NAME = 'nvompic' - SUBTOOLCHAIN = NvhpcToolchain.NAME + SUBTOOLCHAIN = NVHPCToolchain.NAME From 5415187fa37db2620e345540062c1794331250ef Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Mon, 28 Jun 2021 10:42:26 +0200 Subject: [PATCH 428/864] update nvpsmpic toolchain --- easybuild/toolchains/nvpsmpic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/toolchains/nvpsmpic.py b/easybuild/toolchains/nvpsmpic.py index de530f055c..538651076d 100644 --- a/easybuild/toolchains/nvpsmpic.py +++ b/easybuild/toolchains/nvpsmpic.py @@ -30,14 +30,14 @@ @author: Sebastian Achilles (Forschungszentrum Juelich) """ -from easybuild.toolchains.nvhpc import NvhpcToolchain +from easybuild.toolchains.nvhpc import NVHPCToolchain # We pull in MPI and CUDA at once so this maps nicely to HMNS from easybuild.toolchains.mpi.psmpi import Psmpi from easybuild.toolchains.compiler.cuda import Cuda # Order matters! -class NVpsmpic(NvhpcToolchain, Cuda, Psmpi): +class NVpsmpic(NVHPCToolchain, Cuda, Psmpi): """Compiler toolchain with NVHPC and ParaStationMPI, with CUDA as dependency.""" NAME = 'nvpsmpic' - SUBTOOLCHAIN = NvhpcToolchain.NAME + SUBTOOLCHAIN = NVHPCToolchain.NAME From 984888f980b64ba2bb155f61f918932e8cfb113b Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Mon, 28 Jun 2021 11:40:13 +0200 Subject: [PATCH 429/864] update nvompic toolchain --- easybuild/toolchains/nvompic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/toolchains/nvompic.py b/easybuild/toolchains/nvompic.py index 55bcabbf28..b411c4120f 100644 --- a/easybuild/toolchains/nvompic.py +++ b/easybuild/toolchains/nvompic.py @@ -26,8 +26,8 @@ """ EasyBuild support for nvompic compiler toolchain (includes NVHPC and OpenMPI, and CUDA as dependency). -@author: Damian Alvarez (Forschungszentrum Juelich) -@author: Sebastian Achilles (Forschungszentrum Juelich) +:author: Damian Alvarez (Forschungszentrum Juelich) +:author: Sebastian Achilles (Forschungszentrum Juelich) """ from easybuild.toolchains.nvhpc import NVHPCToolchain From 2bd3b537c62d40d5404fb7a92440f139f15c9cf5 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Mon, 28 Jun 2021 11:41:04 +0200 Subject: [PATCH 430/864] update nvpsmpic toolchain --- easybuild/toolchains/nvpsmpic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/toolchains/nvpsmpic.py b/easybuild/toolchains/nvpsmpic.py index 538651076d..e4120344c6 100644 --- a/easybuild/toolchains/nvpsmpic.py +++ b/easybuild/toolchains/nvpsmpic.py @@ -26,8 +26,8 @@ """ EasyBuild support for npsmpi compiler toolchain (includes NVHPC and ParaStationMPI, and CUDA as dependency). -@author: Damian Alvarez (Forschungszentrum Juelich) -@author: Sebastian Achilles (Forschungszentrum Juelich) +:author: Damian Alvarez (Forschungszentrum Juelich) +:author: Sebastian Achilles (Forschungszentrum Juelich) """ from easybuild.toolchains.nvhpc import NVHPCToolchain From 865f1e5863a585e2ab4f7e734e83e9c14edb7e40 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 2 Jul 2021 18:29:24 +0200 Subject: [PATCH 431/864] add test for handling sources/checksums as dict in check_checksums_for --- easybuild/framework/easyblock.py | 1 + test/framework/easyblock.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index ae99718752..416647aceb 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2012,6 +2012,7 @@ def check_checksums_for(self, ent, sub='', source_cnt=None): for fn, checksum in zip(sources + patches, checksums): if isinstance(checksum, dict): + # sources entry may be a dictionary rather than just a string value with filename if isinstance(fn, dict): filename = fn['filename'] else: diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 7bb665984b..d6b2a1d57a 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2073,6 +2073,21 @@ def run_checks(): # no checksum issues self.assertEqual(eb.check_checksums(), []) + # checksums as dict for some files + eb.cfg['checksums'] = [ + { + 'toy-0.0.tar.gz': '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', + 'toy-0.1.tar.gz': '123456789abcdef123456789abcdef123456789abcdef123456789abcdef1234', + }, + '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487', # toy-*.patch + '4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458', # toy-extra.txt + ] + self.assertEqual(eb.check_checksums(), []) + + # sources can also have dict entries + eb.cfg['sources'] = [{'filename': 'toy-0.0.tar.gz', 'download_fileame': 'toy.tar.gz'}] + self.assertEqual(eb.check_checksums(), []) + def test_this_is_easybuild(self): """Test 'this_is_easybuild' function (and get_git_revision function used by it).""" # make sure both return a non-Unicode string From b7fce957dff304ce9b4553e13ec02365347c6471 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 2 Jul 2021 18:51:37 +0200 Subject: [PATCH 432/864] don't make changes to software installation directory when using --sanity-check-only (fixes #3757) --- easybuild/framework/easyblock.py | 52 +++++++++++++++++++------------- test/framework/options.py | 40 ++++++++++++++---------- 2 files changed, 55 insertions(+), 37 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 533407d811..4f53b604a5 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3695,6 +3695,11 @@ def build_and_install_one(ecdict, init_env): new_log_dir = os.path.join(app.builddir, config.log_path(ec=app.cfg)) else: new_log_dir = os.path.dirname(app.logfile) + + # if we're only running the sanity check, we should copy anything new to the installation directory + elif build_option('sanity_check_only'): + _log.info("Only running sanity check, so skipping build stats, easyconfigs archive, reprod files...") + else: new_log_dir = os.path.join(app.installdir, config.log_path(ec=app.cfg)) if build_option('read_only_installdir'): @@ -3744,30 +3749,35 @@ def build_and_install_one(ecdict, init_env): # cleanup logs app.close_log() - log_fn = os.path.basename(get_log_filename(app.name, app.version)) - try: - application_log = os.path.join(new_log_dir, log_fn) - move_logs(app.logfile, application_log) - newspec = os.path.join(new_log_dir, app.cfg.filename()) - copy_file(spec, newspec) - _log.debug("Copied easyconfig file %s to %s", spec, newspec) + if build_option('sanity_check_only'): + _log.info("Only running sanity check, so not copying anything to software install directory...") + else: + log_fn = os.path.basename(get_log_filename(app.name, app.version)) + try: + application_log = os.path.join(new_log_dir, log_fn) + move_logs(app.logfile, application_log) - # copy patches - for patch in app.patches: - target = os.path.join(new_log_dir, os.path.basename(patch['path'])) - copy_file(patch['path'], target) - _log.debug("Copied patch %s to %s", patch['path'], target) + newspec = os.path.join(new_log_dir, app.cfg.filename()) + copy_file(spec, newspec) + _log.debug("Copied easyconfig file %s to %s", spec, newspec) - if build_option('read_only_installdir'): - # take away user write permissions (again) - adjust_permissions(new_log_dir, stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False, recursive=True) - except EasyBuildError as error: - if build_option('module_only'): - application_log = None - _log.debug("Using --module-only so can recover from error: %s", error) - else: - raise error + # copy patches + for patch in app.patches: + target = os.path.join(new_log_dir, os.path.basename(patch['path'])) + copy_file(patch['path'], target) + _log.debug("Copied patch %s to %s", patch['path'], target) + + if build_option('read_only_installdir'): + # take away user write permissions (again) + perms = stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH + adjust_permissions(new_log_dir, perms, add=False, recursive=True) + except EasyBuildError as error: + if build_option('module_only'): + application_log = None + _log.debug("Using --module-only so can recover from error: %s", error) + else: + raise error end_timestamp = datetime.now() diff --git a/test/framework/options.py b/test/framework/options.py index 327ec0f753..597d700b79 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -31,6 +31,7 @@ import os import re import shutil +import stat import sys import tempfile from distutils.version import LooseVersion @@ -50,9 +51,9 @@ from easybuild.tools.config import DEFAULT_MODULECLASSES from easybuild.tools.config import find_last_log, get_build_log_path, get_module_syntax, module_classes from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import change_dir, copy_dir, copy_file, download_file, is_patch_file, mkdir -from easybuild.tools.filetools import parse_http_header_fields_urlpat, read_file, remove_dir, remove_file -from easybuild.tools.filetools import which, write_file +from easybuild.tools.filetools import adjust_permissions, change_dir, copy_dir, copy_file, download_file +from easybuild.tools.filetools import is_patch_file, mkdir, move_file, parse_http_header_fields_urlpat +from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file from easybuild.tools.github import GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO from easybuild.tools.github import URL_SEPARATOR, fetch_github_token from easybuild.tools.modules import Lmod @@ -1105,11 +1106,14 @@ def test_show_ec(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) - def mocked_main(self, args): + def mocked_main(self, args, **kwargs): """Run eb_main with mocked stdout/stderr.""" + if not kwargs: + kwargs = {'raise_error': True} + self.mock_stderr(True) self.mock_stdout(True) - self.eb_main(args, raise_error=True) + self.eb_main(args, **kwargs) stderr, stdout = self.get_stderr(), self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) @@ -5867,15 +5871,8 @@ def test_sanity_check_only(self): args = [test_ec, '--sanity-check-only'] - self.mock_stdout(True) - self.mock_stderr(True) - self.eb_main(args + ['--trace'], do_build=True, raise_error=True, testing=False) - stdout = self.get_stdout().strip() - stderr = self.get_stderr().strip() - self.mock_stdout(False) - self.mock_stderr(False) + stdout = self.mocked_main(args + ['--trace'], do_build=True, raise_error=True, testing=False) - self.assertFalse(stderr) skipped = [ "fetching files", "creating build dir, resetting environment", @@ -5910,10 +5907,12 @@ def test_sanity_check_only(self): for msg in msgs: self.assertTrue(msg in stdout, "'%s' found in: %s" % (msg, stdout)) + ebroottoy = os.path.join(self.test_installpath, 'software', 'toy', '0.0') + # check if sanity check for extension fails if a file provided by that extension, - # which is checked by the sanity check for that extension, is removed - libbarbar = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'lib', 'libbarbar.a') - remove_file(libbarbar) + # which is checked by the sanity check for that extension, is no longer there + libbarbar = os.path.join(ebroottoy, 'lib', 'libbarbar.a') + move_file(libbarbar, libbarbar + '.moved') outtxt, error_thrown = self.eb_main(args + ['--debug'], do_build=True, return_error=True) error_msg = str(error_thrown) @@ -5929,6 +5928,15 @@ def test_sanity_check_only(self): outtxt = self.eb_main(args + ['--skip-extensions'], do_build=True, raise_error=True) self.assertTrue("Sanity check for toy successful" in outtxt) + # restore fail, we want a passing sanity check for the next check + move_file(libbarbar + '.moved', libbarbar) + + # check use of --sanity-check-only when installation directory is read-only; + # cfr. https://github.com/easybuilders/easybuild-framework/issues/3757 + adjust_permissions(ebroottoy, stat.S_IWUSR, add=False, recursive=True) + + stdout = self.mocked_main(args + ['--trace'], do_build=True, raise_error=True, testing=False) + def test_skip_extensions(self): """Test use of --skip-extensions.""" topdir = os.path.abspath(os.path.dirname(__file__)) From 7de54038d152573dc7710f976ac700daa97f68ba Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 2 Jul 2021 19:49:35 +0200 Subject: [PATCH 433/864] fix comment + ensure all write permissions are removed in test for --sanity-check-only --- easybuild/framework/easyblock.py | 2 +- test/framework/options.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 4f53b604a5..006c9b6b75 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3696,7 +3696,7 @@ def build_and_install_one(ecdict, init_env): else: new_log_dir = os.path.dirname(app.logfile) - # if we're only running the sanity check, we should copy anything new to the installation directory + # if we're only running the sanity check, we should not copy anything new to the installation directory elif build_option('sanity_check_only'): _log.info("Only running sanity check, so skipping build stats, easyconfigs archive, reprod files...") diff --git a/test/framework/options.py b/test/framework/options.py index 597d700b79..5ace2e0934 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5933,7 +5933,7 @@ def test_sanity_check_only(self): # check use of --sanity-check-only when installation directory is read-only; # cfr. https://github.com/easybuilders/easybuild-framework/issues/3757 - adjust_permissions(ebroottoy, stat.S_IWUSR, add=False, recursive=True) + adjust_permissions(ebroottoy, stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False, recursive=True) stdout = self.mocked_main(args + ['--trace'], do_build=True, raise_error=True, testing=False) From e2d70bd2196c8e7f41fd314803b7ebaf1364da18 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 3 Jul 2021 10:36:36 +0200 Subject: [PATCH 434/864] honor specified easyblock for extensions (fixes #3710) --- easybuild/framework/easyblock.py | 20 ++++++++++++++++---- test/framework/easyblock.py | 24 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 99471eb48a..d6599e166e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -540,6 +540,10 @@ def fetch_extension_sources(self, skip_checksums=False): 'options': ext_options, } + # if a particular easyblock is specified, make sure it's used + # (this is picked up by init_ext_instances) + ext_src['easyblock'] = ext_options.get('easyblock', None) + # construct dictionary with template values; # inherited from parent, except for name/version templates which are specific to this extension template_values = copy.deepcopy(self.cfg.template_values) @@ -2295,15 +2299,23 @@ def init_ext_instances(self): ext_name = ext['name'] self.log.debug("Creating class instance for extension %s...", ext_name) + # if a specific easyblock is specified for this extension, honor it; + # just passing this to get_easyblock_class is sufficient + easyblock = ext.get('easyblock', None) + if easyblock: + class_name = easyblock + mod_path = get_module_path(class_name) + else: + class_name = encode_class_name(ext_name) + mod_path = get_module_path(class_name, generic=False) + cls, inst = None, None - class_name = encode_class_name(ext_name) - mod_path = get_module_path(class_name, generic=False) - # try instantiating extension-specific class + # try instantiating extension-specific class, or honor specified easyblock try: # no error when importing class fails, in case we run into an existing easyblock # with a similar name (e.g., Perl Extension 'GO' vs 'Go' for which 'EB_Go' is available) - cls = get_easyblock_class(None, name=ext_name, error_on_failed_import=False, + cls = get_easyblock_class(easyblock, name=ext_name, error_on_failed_import=False, error_on_missing_easyblock=False) self.log.debug("Obtained class %s for extension %s", cls, ext_name) if cls is not None: diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index d6b2a1d57a..ed44889feb 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -993,6 +993,30 @@ def test_extensions_step(self): eb.close_log() os.remove(eb.logfile) + def test_init_extensions(self): + """Test creating extension instances.""" + + testdir = os.path.abspath(os.path.dirname(__file__)) + toy_ec_file = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec_txt = read_file(toy_ec_file) + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = toy_ec_txt.replace("('barbar', '0.0', {", "('barbar', '0.0', {'easyblock': 'DummyExtension',") + write_file(test_ec, test_ec_txt) + ec = process_easyconfig(test_ec)[0] + eb = get_easyblock_instance(ec) + + eb.prepare_for_extensions() + eb.init_ext_instances() + ext_inst_class_names = [x.__class__.__name__ for x in eb.ext_instances] + expected = [ + 'Toy_Extension', # 'ls' extension + 'Toy_Extension', # 'bar' extension + 'DummyExtension', # 'barbar' extension + 'EB_toy', # 'toy' extension + ] + self.assertEqual(ext_inst_class_names, expected) + def test_skip_extensions_step(self): """Test the skip_extensions_step""" From 6f3ff806a1a593cb24a31dd59efcf287a2921dec Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 3 Jul 2021 19:51:10 +0200 Subject: [PATCH 435/864] make sure that specified easyblock can be used to install extensions (should derive from Extension class) --- easybuild/framework/easyblock.py | 7 ++++++- test/framework/easyblock.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index d6599e166e..93810300b4 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -61,7 +61,7 @@ from easybuild.framework.easyconfig.style import MAX_LINE_LENGTH from easybuild.framework.easyconfig.tools import get_paths_for from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict -from easybuild.framework.extension import resolve_exts_filter_template +from easybuild.framework.extension import Extension, resolve_exts_filter_template from easybuild.tools import config, run from easybuild.tools.build_details import get_build_stats from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs @@ -2317,8 +2317,13 @@ def init_ext_instances(self): # with a similar name (e.g., Perl Extension 'GO' vs 'Go' for which 'EB_Go' is available) cls = get_easyblock_class(easyblock, name=ext_name, error_on_failed_import=False, error_on_missing_easyblock=False) + self.log.debug("Obtained class %s for extension %s", cls, ext_name) if cls is not None: + # make sure that this easyblock can be used to install extensions + if not issubclass(cls, Extension): + raise EasyBuildError("%s easyblock can not be used to install extensions!", cls.__name__) + inst = cls(self, ext) except (ImportError, NameError) as err: self.log.debug("Failed to use extension-specific class for extension %s: %s", ext_name, err) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index ed44889feb..0c9f899223 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1017,6 +1017,18 @@ def test_init_extensions(self): ] self.assertEqual(ext_inst_class_names, expected) + # check what happen if we specify an easyblock that doesn't derive from Extension, + # and hence can't be used to install extensions... + test_ec = os.path.join(self.test_prefix, 'test_broken.eb') + test_ec_txt = test_ec_txt.replace('DummyExtension', 'ConfigureMake') + write_file(test_ec, test_ec_txt) + ec = process_easyconfig(test_ec)[0] + eb = get_easyblock_instance(ec) + + eb.prepare_for_extensions() + error_pattern = "ConfigureMake easyblock can not be used to install extensions" + self.assertErrorRegex(EasyBuildError, error_pattern, eb.init_ext_instances) + def test_skip_extensions_step(self): """Test the skip_extensions_step""" From 27d49a5b25065652a9368c4b8abefec75587775e Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 5 Jul 2021 07:51:05 +0800 Subject: [PATCH 436/864] prepare release notes for EasyBuild v4.4.1 + bump version to 4.4.1 --- RELEASE_NOTES | 32 ++++++++++++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index cfd7e6f31d..0efb91ea09 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,38 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.4.1 (July 6th 2021) +---------------------- + +update/bugfix release + +- various enhancements, including: + - enhance detection of patch files supplied to 'eb' command with better error messages (#3709) + - add per-step timing information (#3716) + - add module-write hook (#3728) + - add option to ignore failing test step (#3732) + - add toolchain definition for nvompic (#3735) + - warn about generic milestone in --review-pr and --merge-pr (#3751) +- various bug fixes, including: + - don't override COMPILER_MODULE_NAME, inherited from Ffmpi, in Fujitsu toolchain definition (#3721) + - avoid overwritting pr_nr in post_pr_test_report for reports with --include-easyblocks-from-pr (#3724, #3726) + - fix crash in get_config_dict when copying modules that were imported in easyconfig file (like 'import os') (#3729) + - parse C standard flags in CFLAGS for fujitsu compiler (#3731) + - fix error message for --use-ccache (#3733) + - error out when passing a list to run_cmd* to avoid running wrong command unintended, unless shell=True is used (#3737) + - minor fixes to output of test reports when using multiple PRs (#3741) + - fix typo reproducability -> reproducibility (#3743) + - fix location for modules installed with intel-compilers toolchain in HierarchicalMNS by always checking toolchain compiler name against template map (#3745) + - use temporary download folder in test_http_header_fields_urlpat (#3755) + - avoid checking msg attribute of GitCommandError (#3756) + - consider sources provided as dict in EasyBlock.check_checksums_for (#3758) + - don't make changes to software installation directory when using --sanity-check-only (#3761) + - honor specified easyblock for extensions (#3762) +- other changes: + - make sure that tests requiring a github token have 'github' in the test name so that they can be easily filtered (#3730) + - deprecate EasyBuild bootstrap script (#3742) + + v4.4.0 (June 2nd 2021) ---------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 47cc16313a..fa6ec500b5 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.4.1.dev0') +VERSION = LooseVersion('4.4.1') UNKNOWN = 'UNKNOWN' From 69498044265bb664f614396c0bc49194febaa777 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 5 Jul 2021 08:41:58 +0200 Subject: [PATCH 437/864] minor tweaks to v4.4.1 release notes --- RELEASE_NOTES | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 0efb91ea09..402f9a8e9a 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -13,20 +13,18 @@ update/bugfix release - enhance detection of patch files supplied to 'eb' command with better error messages (#3709) - add per-step timing information (#3716) - add module-write hook (#3728) - - add option to ignore failing test step (#3732) - - add toolchain definition for nvompic (#3735) + - add ignore-test-failure configuration option to ignore failing test step (#3732) + - add toolchain definition for nvompic (NVHPC + OpenMPI) (#3735) - warn about generic milestone in --review-pr and --merge-pr (#3751) - various bug fixes, including: - don't override COMPILER_MODULE_NAME, inherited from Ffmpi, in Fujitsu toolchain definition (#3721) - avoid overwritting pr_nr in post_pr_test_report for reports with --include-easyblocks-from-pr (#3724, #3726) - fix crash in get_config_dict when copying modules that were imported in easyconfig file (like 'import os') (#3729) - - parse C standard flags in CFLAGS for fujitsu compiler (#3731) + - parse C standard flags in CFLAGS for Fujitsu compiler (#3731) - fix error message for --use-ccache (#3733) - error out when passing a list to run_cmd* to avoid running wrong command unintended, unless shell=True is used (#3737) - minor fixes to output of test reports when using multiple PRs (#3741) - - fix typo reproducability -> reproducibility (#3743) - fix location for modules installed with intel-compilers toolchain in HierarchicalMNS by always checking toolchain compiler name against template map (#3745) - - use temporary download folder in test_http_header_fields_urlpat (#3755) - avoid checking msg attribute of GitCommandError (#3756) - consider sources provided as dict in EasyBlock.check_checksums_for (#3758) - don't make changes to software installation directory when using --sanity-check-only (#3761) @@ -34,6 +32,7 @@ update/bugfix release - other changes: - make sure that tests requiring a github token have 'github' in the test name so that they can be easily filtered (#3730) - deprecate EasyBuild bootstrap script (#3742) + - use temporary download folder in test_http_header_fields_urlpat (#3755) v4.4.0 (June 2nd 2021) From 39e74cef52ffc8401377eb59b3a94df2ee3828ac Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 5 Jul 2021 14:08:12 +0200 Subject: [PATCH 438/864] make sure that test for --list-easyblocks is run first, before other tests that pick up additional easyblocks --- test/framework/options.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 5ace2e0934..b001b0546e 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -807,7 +807,9 @@ def test_avail_cfgfile_constants(self): os.remove(dummylogfn) sys.path[:] = orig_sys_path - def test_list_easyblocks(self): + # use test_000_* to ensure this test is run *first*, + # before any tests that pick up additional easyblocks (which are difficult to clean up) + def test_000_list_easyblocks(self): """Test listing easyblock hierarchy.""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') From df3b1ffcf8c7361ccbcb95966ea41e2c8fa76595 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 6 Jul 2021 17:02:53 +0800 Subject: [PATCH 439/864] bump version to 4.4.2dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index fa6ec500b5..6e9468b67c 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.4.1') +VERSION = LooseVersion('4.4.2.dev0') UNKNOWN = 'UNKNOWN' From 8745eb20d0ae69ef91bfbdf64f7131a14a286c10 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 6 Jul 2021 12:33:20 +0200 Subject: [PATCH 440/864] remove Python 2.6 from list of supported Python versions in setup.py --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 2d5543aa37..e95b620a65 100644 --- a/setup.py +++ b/setup.py @@ -109,7 +109,6 @@ def find_rel_test(): "Intended Audience :: System Administrators", "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", From c6b9bdd896f22ed3dbd13a2f7faf3d7ec3cb440a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 7 Jul 2021 11:21:49 +0200 Subject: [PATCH 441/864] Add recursive param to dir_contains_files --- easybuild/tools/filetools.py | 12 +++++++++--- test/framework/filetools.py | 5 +++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 37988bca16..7bd668b675 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1096,9 +1096,15 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen return var_defs, hits -def dir_contains_files(path): - """Return True if the given directory does contain any file in itself or any subdirectory""" - return any(files for _root, _dirs, files in os.walk(path)) +def dir_contains_files(path, recursive=True): + """ + Return True if the given directory does contain any file + + :recursive If False only the path itself is considered, else all subdirectories are also searched""" + if recursive: + return any(files for _root, _dirs, files in os.walk(path)) + else: + return any(os.path.isfile(os.path.join(path, x)) for x in os.listdir(path)) def find_eb_script(script_name): diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 12692ba147..1a5b6b3e32 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2330,21 +2330,26 @@ def makedirs_in_test(*paths): empty_dir = makedirs_in_test('empty_dir') self.assertFalse(ft.dir_contains_files(empty_dir)) + self.assertFalse(ft.dir_contains_files(empty_dir, recursive=False)) dir_w_subdir = makedirs_in_test('dir_w_subdir', 'sub_dir') self.assertFalse(ft.dir_contains_files(dir_w_subdir)) + self.assertFalse(ft.dir_contains_files(dir_w_subdir, recursive=False)) dir_subdir_file = makedirs_in_test('dir_subdir_file', 'sub_dir_w_file') ft.write_file(os.path.join(dir_subdir_file, 'sub_dir_w_file', 'file.h'), '') self.assertTrue(ft.dir_contains_files(dir_subdir_file)) + self.assertFalse(ft.dir_contains_files(dir_subdir_file, recursive=False)) dir_w_file = makedirs_in_test('dir_w_file') ft.write_file(os.path.join(dir_w_file, 'file.h'), '') self.assertTrue(ft.dir_contains_files(dir_w_file)) + self.assertTrue(ft.dir_contains_files(dir_w_file, recursive=False)) dir_w_dir_and_file = makedirs_in_test('dir_w_dir_and_file', 'sub_dir') ft.write_file(os.path.join(dir_w_dir_and_file, 'file.h'), '') self.assertTrue(ft.dir_contains_files(dir_w_dir_and_file)) + self.assertTrue(ft.dir_contains_files(dir_w_dir_and_file, recursive=False)) def test_find_eb_script(self): """Test find_eb_script function.""" From d9cf1b1bbf84241a7c71490c32654d485f624d53 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 7 Jul 2021 12:11:17 +0200 Subject: [PATCH 442/864] Don't add empty PATH or LD_LIBRARY_PATH --- easybuild/framework/easyblock.py | 33 ++++++++++++++++++-------------- easybuild/tools/filetools.py | 5 +++-- test/framework/easyblock.py | 19 ++++++++++++++++++ 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 93810300b4..015aa18691 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -93,7 +93,7 @@ from easybuild.tools.py2vs3 import extract_method_name, string_type from easybuild.tools.repository.repository import init_repository from easybuild.tools.systemtools import check_linked_shared_libs, det_parallelism, get_shared_lib_ext, use_group -from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, quote_str +from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, nub, quote_str from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION @@ -1429,9 +1429,17 @@ def make_module_req(self): note += "for paths are skipped for the statements below due to dry run" lines.append(self.module_generator.comment(note)) - # for these environment variables, the corresponding subdirectory must include at least one file - keys_requiring_files = set(('PATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH', 'CPATH', - 'CMAKE_PREFIX_PATH', 'CMAKE_LIBRARY_PATH')) + # For these environment variables, the corresponding directory must include at least one file. + # The values determine if detection is done recursively, i.e. if it accepts directories where files + # are only in subdirectories. + keys_requiring_files = { + 'PATH': False, + 'LD_LIBRARY_PATH': False, + 'LIBRARY_PATH': True, + 'CPATH': True, + 'CMAKE_PREFIX_PATH': True, + 'CMAKE_LIBRARY_PATH': True, + } for key, reqs in sorted(requirements.items()): if isinstance(reqs, string_type): @@ -1461,19 +1469,16 @@ def make_module_req(self): if fixed_paths != paths: self.log.info("Fixed symlink lib64 in paths for %s: %s -> %s", key, paths, fixed_paths) paths = fixed_paths - # remove duplicate paths - # don't use 'set' here, since order in 'paths' is important! - uniq_paths = [] - for path in paths: - if path not in uniq_paths: - uniq_paths.append(path) - paths = uniq_paths + # remove duplicate paths preserving order + paths = nub(paths) if key in keys_requiring_files: # only retain paths that contain at least one file + recursive = keys_requiring_files[key] retained_paths = [ - path for path in paths - if os.path.isdir(os.path.join(self.installdir, path)) - and dir_contains_files(os.path.join(self.installdir, path)) + path + for path, fullpath in ((path, os.path.join(self.installdir, path)) for path in paths) + if os.path.isdir(fullpath) + and dir_contains_files(fullpath, recursive=recursive) ] if retained_paths != paths: self.log.info("Only retaining paths for %s that contain at least one file: %s -> %s", diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 7bd668b675..d043394e3e 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1099,8 +1099,9 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen def dir_contains_files(path, recursive=True): """ Return True if the given directory does contain any file - - :recursive If False only the path itself is considered, else all subdirectories are also searched""" + + :recursive If False only the path itself is considered, else all subdirectories are also searched + """ if recursive: return any(files for _root, _dirs, files in os.walk(path)) else: diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 0c9f899223..510ac24dfb 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -551,6 +551,25 @@ def test_make_module_req(self): else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + # If PATH or LD_LIBRARY_PATH contain only folders, do not add an entry + sub_lib_path = os.path.join('lib', 'path_folders') + sub_path_path = os.path.join('bin', 'path_folders') + eb.make_module_req_guess = lambda: {'LD_LIBRARY_PATH': sub_lib_path, 'PATH': sub_path_path} + for path in (sub_lib_path, sub_path_path): + full_path = os.path.join(eb.installdir, path, 'subpath') + os.makedirs(full_path) + write_file(os.path.join(full_path, 'any.file'), 'test') + txt = eb.make_module_req() + if get_module_syntax() == 'Tcl': + self.assertFalse(re.search(r"prepend-path\s+LD_LIBRARY_PATH\s+\$%s\n" % sub_lib_path, + txt, re.M)) + self.assertFalse(re.search(r"prepend-path\s+PATH\s+\$%d\n" % sub_path_path, txt, re.M)) + else: + assert get_module_syntax() == 'Lua' + self.assertFalse(re.search(r'prepend_path\("LD_LIBRARY_PATH", pathJoin\(root, "%s"\)\)\n' % sub_lib_path, + txt, re.M)) + self.assertFalse(re.search(r'prepend_path\("PATH", pathJoin\(root, "%s"\)\)\n' % sub_path_path, txt, re.M)) + # cleanup eb.close_log() os.remove(eb.logfile) From 7b6bb0300ec89675b808f312dec112a3e211c0ae Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 7 Jul 2021 13:42:34 +0200 Subject: [PATCH 443/864] Fix placeholder Co-authored-by: ocaisa --- test/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 510ac24dfb..5b90a527d5 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -563,7 +563,7 @@ def test_make_module_req(self): if get_module_syntax() == 'Tcl': self.assertFalse(re.search(r"prepend-path\s+LD_LIBRARY_PATH\s+\$%s\n" % sub_lib_path, txt, re.M)) - self.assertFalse(re.search(r"prepend-path\s+PATH\s+\$%d\n" % sub_path_path, txt, re.M)) + self.assertFalse(re.search(r"prepend-path\s+PATH\s+\$%s\n" % sub_path_path, txt, re.M)) else: assert get_module_syntax() == 'Lua' self.assertFalse(re.search(r'prepend_path\("LD_LIBRARY_PATH", pathJoin\(root, "%s"\)\)\n' % sub_lib_path, From 068bad130bd1194e6bc37e7144a09420505780e9 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Jul 2021 12:46:09 +0200 Subject: [PATCH 444/864] Make logdir writable also when --stop/--fetch is given --- easybuild/framework/easyblock.py | 31 +++++++++++++++++++++++-------- test/framework/toy_build.py | 13 +++++++++++-- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 015aa18691..415d66da40 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3690,8 +3690,11 @@ def build_and_install_one(ecdict, init_env): if os.path.exists(app.installdir) and build_option('read_only_installdir') and ( build_option('rebuild') or build_option('force')): + enabled_write_permissions = True # re-enable write permissions so we can install additional modules adjust_permissions(app.installdir, stat.S_IWUSR, add=True, recursive=True) + else: + enabled_write_permissions = False result = app.run_all_steps(run_test_cases=run_test_cases) @@ -3699,6 +3702,9 @@ def build_and_install_one(ecdict, init_env): # also add any extension easyblocks used during the build for reproducibility if app.ext_instances: copy_easyblocks_for_reprod(app.ext_instances, reprod_dir) + # If not already done remove the granted write permissions if we did so + if enabled_write_permissions and os.lstat(app.installdir)[stat.ST_MODE] & stat.S_IWUSR: + adjust_permissions(app.installdir, stat.S_IWUSR, add=False, recursive=True) except EasyBuildError as err: first_n = 300 @@ -3715,6 +3721,21 @@ def build_and_install_one(ecdict, init_env): # successful (non-dry-run) build if result and not dry_run: + def ensure_writable_log_dir(log_dir): + """Make sure we can write into the log dir""" + if build_option('read_only_installdir'): + # temporarily re-enable write permissions for copying log/easyconfig to install dir + if os.path.exists(log_dir): + adjust_permissions(log_dir, stat.S_IWUSR, add=True, recursive=True) + else: + parent_dir = os.path.dirname(log_dir) + if os.path.exists(parent_dir): + adjust_permissions(parent_dir, stat.S_IWUSR, add=True, recursive=False) + mkdir(log_dir, parents=True) + adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False) + else: + mkdir(log_dir, parents=True) + adjust_permissions(log_dir, stat.S_IWUSR, add=True, recursive=True) if app.cfg['stop']: ended = 'STOPPED' @@ -3722,6 +3743,7 @@ def build_and_install_one(ecdict, init_env): new_log_dir = os.path.join(app.builddir, config.log_path(ec=app.cfg)) else: new_log_dir = os.path.dirname(app.logfile) + ensure_writable_log_dir(new_log_dir) # if we're only running the sanity check, we should not copy anything new to the installation directory elif build_option('sanity_check_only'): @@ -3729,14 +3751,7 @@ def build_and_install_one(ecdict, init_env): else: new_log_dir = os.path.join(app.installdir, config.log_path(ec=app.cfg)) - if build_option('read_only_installdir'): - # temporarily re-enable write permissions for copying log/easyconfig to install dir - if os.path.exists(new_log_dir): - adjust_permissions(new_log_dir, stat.S_IWUSR, add=True, recursive=True) - else: - adjust_permissions(app.installdir, stat.S_IWUSR, add=True, recursive=False) - mkdir(new_log_dir, parents=True) - adjust_permissions(app.installdir, stat.S_IWUSR, add=False, recursive=False) + ensure_writable_log_dir(new_log_dir) # collect build stats _log.info("Collecting build stats...") diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 51d3f1a977..10c855aeaf 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -112,7 +112,7 @@ def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versio full_version = ''.join([versionprefix, version, versionsuffix]) # check for success - success = re.compile(r"COMPLETED: Installation ended successfully \(took .* secs?\)") + success = re.compile(r"COMPLETED: Installation (ended|STOPPED) successfully \(took .* secs?\)") self.assertTrue(success.search(outtxt), "COMPLETED message found in '%s" % outtxt) # if the module exists, it should be fine @@ -615,7 +615,16 @@ def test_toy_permissions_installdir(self): # 2. Existing build with --rebuild -> Reinstall and set read-only # 3. Existing build with --force -> Reinstall and set read-only # 4-5: Same as 2-3 but with --skip - for extra_args in ([], ['--rebuild'], ['--force'], ['--skip', '--rebuild'], ['--skip', '--force']): + # 6. Existing build with --fetch -> Test that logs can be written + test_cases = ( + [], + ['--rebuild'], + ['--force'], + ['--skip', '--rebuild'], + ['--skip', '--force'], + ['--rebuild', '--fetch'], + ) + for extra_args in test_cases: self.mock_stdout(True) self.test_toy_build(ec_file=test_ec, extra_args=['--read-only-installdir'] + extra_args, force=False) self.mock_stdout(False) From 65a7d91291bb97a64c05b0bbf30468d10c6becd4 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Jul 2021 14:46:42 +0200 Subject: [PATCH 445/864] Add __str__ method to EasyConfig We often print an easyconfig which is currently shown as "" This is not helpful at all. Instead show something like "GCC EasyConfig @ /tmp/GCC.eb" --- easybuild/framework/easyconfig/easyconfig.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 382a1c18bf..47be31903d 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -559,6 +559,13 @@ def disable_templating(self): finally: self.enable_templating = old_enable_templating + def __str__(self): + """Return a string representation of this EasyConfig instance""" + if self.path: + return '%s EasyConfig @ %s' % (self.name, self.path) + else: + return 'Raw %s EasyConfig' % self.name + def filename(self): """Determine correct filename for this easyconfig file.""" From 003b7f58624b724979f00ba6d752e990c0282ef4 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Jul 2021 14:52:56 +0200 Subject: [PATCH 446/864] Explicitely print for unset MODULEPATH --- easybuild/tools/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 4ee0cd8674..d8df2b0db2 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -801,7 +801,7 @@ def run_module(self, *args, **kwargs): else: args = list(args) - self.log.debug('Current MODULEPATH: %s' % os.environ.get('MODULEPATH', '')) + self.log.debug('Current MODULEPATH: %s' % os.environ.get('MODULEPATH', '')) # restore selected original environment variables before running module command environ = os.environ.copy() From 7a5b63fa1b9f2269400f2ceb634a7fbe3eb665d1 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Jul 2021 16:12:34 +0200 Subject: [PATCH 447/864] Print caught error for failed toy build --- test/framework/toy_build.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 10c855aeaf..cad57faaa7 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -106,21 +106,26 @@ def tearDown(self): if os.path.exists(self.dummylogfn): os.remove(self.dummylogfn) - def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versionsuffix=''): + def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versionsuffix='', error=None): """Check whether toy build succeeded.""" full_version = ''.join([versionprefix, version, versionsuffix]) + if error is not None: + error_msg = '\nNote: Caught error: %s' % error + else: + error_msg = '' + # check for success success = re.compile(r"COMPLETED: Installation (ended|STOPPED) successfully \(took .* secs?\)") - self.assertTrue(success.search(outtxt), "COMPLETED message found in '%s" % outtxt) + self.assertTrue(success.search(outtxt), "COMPLETED message found in '%s'%s" % (outtxt, error_msg)) # if the module exists, it should be fine toy_module = os.path.join(installpath, 'modules', 'all', 'toy', full_version) msg = "module for toy build toy/%s found (path %s)" % (full_version, toy_module) if get_module_syntax() == 'Lua': toy_module += '.lua' - self.assertTrue(os.path.exists(toy_module), msg) + self.assertTrue(os.path.exists(toy_module), msg + error_msg) # module file is symlinked according to moduleclass toy_module_symlink = os.path.join(installpath, 'modules', 'tools', 'toy', full_version) @@ -183,7 +188,7 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True raise myerr if verify: - self.check_toy(self.test_installpath, outtxt, versionsuffix=versionsuffix) + self.check_toy(self.test_installpath, outtxt, versionsuffix=versionsuffix, error=myerr) if test_readme: # make sure postinstallcmds were used From c866662b91739f11936928690eabadf8d68422b7 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Jul 2021 16:42:53 +0200 Subject: [PATCH 448/864] Pass modtool=None for --fetch Avoids failure with modules-env due to having no $MODULEPATH but an (otherwise inexistant) modules tool --- test/framework/utilities.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index d36c9da53a..4a82aaf6a8 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -295,7 +295,13 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos env_before = copy.deepcopy(os.environ) try: - main(args=args, logfile=logfile, do_build=do_build, testing=testing, modtool=self.modtool) + if '--fetch' in args: + # The config sets modules_tool to None if --fetch is specified, + # so do the same here to keep the behavior consistent + modtool = None + else: + modtool = self.modtool + main(args=args, logfile=logfile, do_build=do_build, testing=testing, modtool=modtool) except SystemExit as err: if raise_systemexit: raise err From a7bdee40a25687b3ebfcba0610a5407859e1bfdc Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 9 Jul 2021 14:33:10 +0200 Subject: [PATCH 449/864] Fix forgotten renaming of 'l' to 'char' --- easybuild/tools/include.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index 58464a4d25..9f0939be24 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -72,7 +72,7 @@ for subdir in subdirs: __path__ = pkgutil.extend_path(__path__, '%s.%s' % (__name__, subdir)) -del l, subdir, subdirs +del char, subdir, subdirs __path__ = __import__('pkgutil').extend_path(__path__, __name__) """ From 0677bc210cf92fde37d5da0838e1635c3ae3f14c Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 9 Jul 2021 14:52:52 +0200 Subject: [PATCH 450/864] Conditionally delete --- easybuild/tools/include.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index 9f0939be24..1b5ba26d13 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -72,7 +72,9 @@ for subdir in subdirs: __path__ = pkgutil.extend_path(__path__, '%s.%s' % (__name__, subdir)) -del char, subdir, subdirs +del subdir, subdirs +if 'char' in dir(): + del char __path__ = __import__('pkgutil').extend_path(__path__, __name__) """ From f598c2e90c8cd6080db7971fa73d647dc7bcb9de Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 9 Jul 2021 14:54:29 +0200 Subject: [PATCH 451/864] Remove superflous import --- easybuild/tools/include.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index 1b5ba26d13..34390d632a 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -76,7 +76,7 @@ if 'char' in dir(): del char -__path__ = __import__('pkgutil').extend_path(__path__, __name__) +__path__ = pkgutil.extend_path(__path__, __name__) """ From 11612082213a58499e0f691f41abc000c7f56e34 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 14 Jul 2021 10:21:10 +0200 Subject: [PATCH 452/864] Print the hook messages only for debug-mode The output is VERY noisy, especially with parse_hook and module_write hook The log already has this info and if you want it back you can use --debug --- easybuild/tools/hooks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index cb2d72c472..6046e933f1 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -33,6 +33,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, print_msg +from easybuild.tools.config import build_option _log = fancylogger.getLogger('hooks', fname=False) @@ -191,7 +192,8 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, if msg is None: msg = "Running %s hook..." % label - print_msg(msg) + if build_option('debug'): + print_msg(msg) _log.info("Running '%s' hook function (arguments: %s)...", hook.__name__, args) res = hook(*args) From 7910ca94c10c28f5aefcee89e3f660d7e59902e8 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 14 Jul 2021 13:36:09 +0200 Subject: [PATCH 453/864] Fix test --- test/framework/hooks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/framework/hooks.py b/test/framework/hooks.py index f51cd0fc91..1e3c44ea5f 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -29,7 +29,7 @@ """ import os import sys -from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner import easybuild.tools.hooks @@ -124,6 +124,8 @@ def test_run_hook(self): hooks = load_hooks(self.test_hooks_pymod) + init_config(build_options={'debug': True}) + self.mock_stdout(True) self.mock_stderr(True) run_hook('start', hooks) From 3da42628823184c25770ba88ecf7e00b85caa6c7 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 15 Jul 2021 11:33:18 +0200 Subject: [PATCH 454/864] Improve warning about missing intel fftw libs --- easybuild/toolchains/fft/intelfftw.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py index dca03a0476..6367efad4d 100644 --- a/easybuild/toolchains/fft/intelfftw.py +++ b/easybuild/toolchains/fft/intelfftw.py @@ -112,14 +112,15 @@ def fftw_lib_exists(libname): # filter out libraries from list of FFTW libraries to check for if they are not provided by Intel MKL check_fftw_libs = [lib for lib in fftw_libs if lib not in ['dl', 'gfortran']] - if all([fftw_lib_exists(lib) for lib in check_fftw_libs]): - self.FFT_LIB = fftw_libs - else: + missing_fftw_libs = [lib for lib in check_fftw_libs if not fftw_lib_exists(lib)] + if missing_fftw_libs: msg = "Not all FFTW interface libraries %s are found in %s" % (check_fftw_libs, fft_lib_dirs) - msg += ", can't set $FFT_LIB." + msg += ", can't set $FFT_LIB. Missing: %s" % (missing_fftw_libs) if self.dry_run: dry_run_warning(msg, silent=build_option('silent')) else: raise EasyBuildError(msg) + else: + self.FFT_LIB = fftw_libs self.FFT_LIB_MT = fftw_mt_libs From 297af992c7301963d17915b7dc465d7b87005eff Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 15 Jul 2021 11:37:09 +0200 Subject: [PATCH 455/864] Don't create lists when using all/any --- easybuild/framework/easyconfig/easyconfig.py | 2 +- easybuild/framework/easyconfig/format/format.py | 2 +- easybuild/scripts/bootstrap_eb.py | 2 +- easybuild/toolchains/fft/intelfftw.py | 2 +- easybuild/tools/filetools.py | 2 +- easybuild/tools/github.py | 4 ++-- easybuild/tools/options.py | 4 ++-- test/framework/docs.py | 4 ++-- test/framework/filetools.py | 2 +- test/framework/modules.py | 6 +++--- test/framework/robot.py | 2 +- test/framework/systemtools.py | 2 +- 12 files changed, 17 insertions(+), 17 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 382a1c18bf..55948e11a2 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -881,7 +881,7 @@ def validate_os_deps(self): raise EasyBuildError("Non-tuple value type for OS dependency specification: %s (type %s)", dep, type(dep)) - if not any([check_os_dependency(cand_dep) for cand_dep in dep]): + if not any(check_os_dependency(cand_dep) for cand_dep in dep): not_found.append(dep) if not_found: diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 073cc1bdc5..4fa31c84e6 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -577,7 +577,7 @@ def get_version_toolchain(self, version=None, tcname=None, tcversion=None): self.log.debug("No toolchain version specified, using default %s" % tcversion) else: raise EasyBuildError("No toolchain version specified, no default found.") - elif any([tc.test(tcname, tcversion) for tc in tcs]): + elif any(tc.test(tcname, tcversion) for tc in tcs): self.log.debug("Toolchain '%s' version '%s' is supported in easyconfig" % (tcname, tcversion)) else: raise EasyBuildError("Toolchain '%s' version '%s' not supported in easyconfig (only %s)", diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index ed3390ea93..646ac65db0 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -925,7 +925,7 @@ def main(): for path in orig_sys_path: include_path = True # exclude path if it's potentially an EasyBuild/VSC package, providing the 'easybuild'/'vsc' namespace, resp. - if any([os.path.exists(os.path.join(path, pkg, '__init__.py')) for pkg in ['easyblocks', 'easybuild', 'vsc']]): + if any(os.path.exists(os.path.join(path, pkg, '__init__.py')) for pkg in ['easyblocks', 'easybuild', 'vsc']): include_path = False # exclude any .egg paths if path.endswith('.egg'): diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py index 6367efad4d..b748fd5e7e 100644 --- a/easybuild/toolchains/fft/intelfftw.py +++ b/easybuild/toolchains/fft/intelfftw.py @@ -98,7 +98,7 @@ def _set_fftw_variables(self): def fftw_lib_exists(libname): """Helper function to check whether FFTW library with specified name exists.""" - return any([os.path.exists(os.path.join(d, "lib%s.a" % libname)) for d in fft_lib_dirs]) + return any(os.path.exists(os.path.join(d, "lib%s.a" % libname)) for d in fft_lib_dirs) if not fftw_lib_exists(interface_lib) and LooseVersion(imklver) >= LooseVersion("10.2"): # interface libs can be optional: diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index d043394e3e..1cb65f826f 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -528,7 +528,7 @@ def det_common_path_prefix(paths): found_common = False while not found_common and prefix != os.path.dirname(prefix): prefix = os.path.dirname(prefix) - found_common = all([p.startswith(prefix) for p in paths]) + found_common = all(p.startswith(prefix) for p in paths) if found_common: # prefix may be empty string for relative paths with a non-common prefix diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 5daeeec120..8259de1587 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1275,7 +1275,7 @@ def reasons_for_closing(pr_data): if uses_archived_tc: possible_reasons.append('archived') - if any([e['name'] in pr_data['title'] for e in obsoleted]): + if any(e['name'] in pr_data['title'] for e in obsoleted): possible_reasons.append('obsolete') return possible_reasons @@ -1804,7 +1804,7 @@ def det_pr_target_repo(paths): # if all Python files are easyblocks, target repo should be easyblocks; # otherwise, target repo is assumed to be framework - if all([get_easyblock_class_name(path) for path in py_files]): + if all(get_easyblock_class_name(path) for path in py_files): pr_target_repo = GITHUB_EASYBLOCKS_REPO _log.info("All Python files are easyblocks, target repository is assumed to be %s", pr_target_repo) else: diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 37d06a6396..7dadcd415d 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -883,14 +883,14 @@ def postprocess(self): self._postprocess_include() # prepare for --list/--avail - if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates, + if any((self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates, self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_cfgfile_constants, self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses, self.options.avail_repositories, self.options.show_default_moduleclasses, self.options.avail_modules_tools, self.options.avail_module_naming_schemes, self.options.show_default_configfiles, self.options.avail_toolchain_opts, self.options.avail_hooks, self.options.show_system_info, - ]): + )): build_easyconfig_constants_dict() # runs the easyconfig constants sanity check self._postprocess_list_avail() diff --git a/test/framework/docs.py b/test/framework/docs.py index 075ec58908..b4b2496d88 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -260,7 +260,7 @@ def test_list_software(self): ] txt = list_software(output_format='txt', detailed=True) lines = txt.split('\n') - expected_found = any([lines[i:i + len(expected)] == expected for i in range(len(lines))]) + expected_found = any(lines[i:i + len(expected)] == expected for i in range(len(lines))) self.assertTrue(expected_found, "%s found in: %s" % (expected, lines)) expected = [ @@ -283,7 +283,7 @@ def test_list_software(self): ] txt = list_software(output_format='rst', detailed=True) lines = txt.split('\n') - expected_found = any([lines[i:i + len(expected)] == expected for i in range(len(lines))]) + expected_found = any(lines[i:i + len(expected)] == expected for i in range(len(lines))) self.assertTrue(expected_found, "%s found in: %s" % (expected, lines)) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 1a5b6b3e32..444a6986e3 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1009,7 +1009,7 @@ def test_multidiff(self): # no postinstallcmds in toy-0.0-deps.eb expected = "29 %s+ postinstallcmds = " % green - self.assertTrue(any([line.startswith(expected) for line in lines])) + self.assertTrue(any(line.startswith(expected) for line in lines)) expected = "30 %s+%s (1/2) toy-0.0" % (green, endcol) self.assertTrue(any(line.startswith(expected) for line in lines), "Found '%s' in: %s" % (expected, lines)) self.assertEqual(lines[-1], "=====") diff --git a/test/framework/modules.py b/test/framework/modules.py index 28772dc87e..b56c08bb2f 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1096,7 +1096,7 @@ def test_module_caches(self): mkdir(nonpath) self.modtool.use(nonpath) modulepaths = [p for p in os.environ.get('MODULEPATH', '').split(os.pathsep) if p] - self.assertTrue(any([os.path.samefile(nonpath, mp) for mp in modulepaths])) + self.assertTrue(any(os.path.samefile(nonpath, mp) for mp in modulepaths)) shutil.rmtree(nonpath) # create symlink to entry in $MODULEPATH we're going to use, and add it to $MODULEPATH @@ -1136,9 +1136,9 @@ def test_module_caches(self): # invalidate caches with correct path modulepaths = [p for p in os.environ.get('MODULEPATH', '').split(os.pathsep) if p] - self.assertTrue(any([os.path.exists(mp) and os.path.samefile(test_mods_path, mp) for mp in modulepaths])) + self.assertTrue(any(os.path.exists(mp) and os.path.samefile(test_mods_path, mp) for mp in modulepaths)) paths_in_key = [p for p in avail_cache_key[0].split('=')[1].split(os.pathsep) if p] - self.assertTrue(any([os.path.exists(p) and os.path.samefile(test_mods_path, p) for p in paths_in_key])) + self.assertTrue(any(os.path.exists(p) and os.path.samefile(test_mods_path, p) for p in paths_in_key)) # verify cache invalidation, caches should be empty again invalidate_module_caches_for(test_mods_path) diff --git a/test/framework/robot.py b/test/framework/robot.py index d2c82e013b..f55a6e822b 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1029,7 +1029,7 @@ def test_find_resolved_modules(self): ordered_ecs, new_easyconfigs, new_avail_modules = find_resolved_modules(ecs, mods, self.modtool) # all dependencies are resolved for easyconfigs included in ordered_ecs - self.assertFalse(any([ec['dependencies'] for ec in ordered_ecs])) + self.assertFalse(any(ec['dependencies'] for ec in ordered_ecs)) # nodeps/ondep easyconfigs have all dependencies resolved self.assertEqual(len(ordered_ecs), 2) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index c0e2fbf881..fa49515eda 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -489,7 +489,7 @@ def test_cpu_features_native(self): cpu_feat = get_cpu_features() self.assertTrue(isinstance(cpu_feat, list)) self.assertTrue(len(cpu_feat) > 0) - self.assertTrue(all([isinstance(x, string_type) for x in cpu_feat])) + self.assertTrue(all(isinstance(x, string_type) for x in cpu_feat)) def test_cpu_features_linux(self): """Test getting CPU features (mocked for Linux).""" From 78f609f6c1fa89b224636a2b0a7b277d90c851ef Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 15 Jul 2021 12:32:55 +0200 Subject: [PATCH 456/864] Update bootstrap checksum/version --- .github/workflows/bootstrap_script.yml | 2 +- easybuild/scripts/bootstrap_eb.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bootstrap_script.yml b/.github/workflows/bootstrap_script.yml index c0ade9843b..58c3b0ddfa 100644 --- a/.github/workflows/bootstrap_script.yml +++ b/.github/workflows/bootstrap_script.yml @@ -107,7 +107,7 @@ jobs: EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - EB_BOOTSTRAP_EXPECTED="20210618.01 e5d477d717c6d3648ba2027ab735713ba5804fbf52f4b4adcca0bc1379b44618" + EB_BOOTSTRAP_EXPECTED="20210715.01 784dd29063d941be2d8b70e4c2eec12a9afe360f3ef8f753dcb518abf43ca7de" test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) # test bootstrap script diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 646ac65db0..3274730412 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -62,7 +62,7 @@ import urllib.request as std_urllib -EB_BOOTSTRAP_VERSION = '20210618.01' +EB_BOOTSTRAP_VERSION = '20210715.01' # argparse preferrred, optparse deprecated >=2.7 HAVE_ARGPARSE = False From 661a52adfcd63710ce7c97964bbb9a1ce4e40802 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 21 Jul 2021 09:31:25 +0200 Subject: [PATCH 457/864] Fix ignore_test_failure not set for Extensions The ExtensionEasyBlock does not call __init__ from EasyBlock and hence that value is not set making uses of report_test_failure fail. --- easybuild/framework/easyblock.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 015aa18691..3b1ffa3096 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -258,8 +258,6 @@ def __init__(self, ec): if group_name is not None: self.group = use_group(group_name) - self.ignore_test_failure = build_option('ignore_test_failure') - # generate build/install directories self.gen_builddir() self.gen_installdir() @@ -1831,7 +1829,7 @@ def report_test_failure(self, msg_or_error): :param msg_or_error: failure description (string value or an EasyBuildError instance) """ - if self.ignore_test_failure: + if build_option('ignore_test_failure'): print_warning("Test failure ignored: " + str(msg_or_error), log=self.log) else: exception = msg_or_error if isinstance(msg_or_error, EasyBuildError) else EasyBuildError(msg_or_error) From 23640975c5b8b4c73a23f95c828cf2315aa40be0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 21 Jul 2021 09:39:30 +0200 Subject: [PATCH 458/864] Add reproducer for ExtensionEB.report_test_failure error --- test/framework/easyblock.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 5b90a527d5..21873a1662 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -127,6 +127,8 @@ def check_extra_options_format(extra_options): extra_options = exeb1.extra_options() check_extra_options_format(extra_options) self.assertTrue('options' in extra_options) + # Reporting test failure should work also for the extension EB + self.assertRaises(EasyBuildError, exeb1.report_test_failure, "Fails") # test extensioneasyblock, as easyblock exeb2 = ExtensionEasyBlock(ec) @@ -135,6 +137,8 @@ def check_extra_options_format(extra_options): extra_options = exeb2.extra_options() check_extra_options_format(extra_options) self.assertTrue('options' in extra_options) + # Reporting test failure should work also for the extension EB + self.assertRaises(EasyBuildError, exeb2.report_test_failure, "Fails") class TestExtension(ExtensionEasyBlock): @staticmethod From 4e83468df97cafbd78cd13deffb3f7fa7897932b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 22 Jul 2021 16:11:04 +0200 Subject: [PATCH 459/864] Add has_recursive_symlinks function --- easybuild/tools/filetools.py | 20 ++++++++++++++++++ test/framework/filetools.py | 40 ++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 1cb65f826f..ef021455a2 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -44,6 +44,7 @@ import hashlib import imp import inspect +import itertools import os import re import shutil @@ -2340,6 +2341,25 @@ def copy_files(paths, target_path, force_in_dry_run=False, target_single_file=Fa raise EasyBuildError("One or more files to copy should be specified!") +def has_recursive_symlinks(path): + """ + Check the given directory for recursive symlinks. + + That means symlinks to folders inside the path which would cause infinite loops when traversed regularily. + + :param path: Path to directory to check + """ + for dirpath, dirnames, filenames in os.walk(path, followlinks=True): + for name in itertools.chain(dirnames, filenames): + fullpath = os.path.join(dirpath, name) + if os.path.islink(fullpath): + linkpath = os.path.realpath(fullpath) + fullpath += os.sep # To catch the case where both are equal + if fullpath.startswith(linkpath + os.sep): + return True + return False + + def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **kwargs): """ Copy a directory from specified location to specified location diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 444a6986e3..5d959eb091 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1790,6 +1790,46 @@ def test_copy_files(self): regex = re.compile("^copied 2 files to .*/target") self.assertTrue(regex.match(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_has_recursive_symlinks(self): + """Test has_recursive_symlinks function""" + test_folder = tempfile.mkdtemp() + self.assertFalse(ft.has_recursive_symlinks(test_folder)) + # Clasic Loop: Symlink to . + os.symlink('.', os.path.join(test_folder, 'self_link_dot')) + self.assertTrue(ft.has_recursive_symlinks(test_folder)) + # Symlink to self + test_folder = tempfile.mkdtemp() + os.symlink('self_link', os.path.join(test_folder, 'self_link')) + self.assertTrue(ft.has_recursive_symlinks(test_folder)) + # Symlink from 2 folders up + test_folder = tempfile.mkdtemp() + sub_folder = os.path.join(test_folder, 'sub1', 'sub2') + os.makedirs(sub_folder) + os.symlink(os.path.join('..', '..'), os.path.join(sub_folder, 'uplink')) + self.assertTrue(ft.has_recursive_symlinks(test_folder)) + # Non-issue: Symlink to sibling folders + test_folder = tempfile.mkdtemp() + sub_folder = os.path.join(test_folder, 'sub1', 'sub2') + os.makedirs(sub_folder) + sibling_folder = os.path.join(test_folder, 'sub1', 'sibling') + os.mkdir(sibling_folder) + os.symlink('sibling', os.path.join(test_folder, 'sub1', 'sibling_link')) + os.symlink(os.path.join('..', 'sibling'), os.path.join(test_folder, sub_folder, 'sibling_link')) + self.assertFalse(ft.has_recursive_symlinks(test_folder)) + # Tricky case: Sibling symlink to folder starting with the same name + os.mkdir(os.path.join(test_folder, 'sub11')) + os.symlink(os.path.join('..', 'sub11'), os.path.join(test_folder, 'sub1', 'trick_link')) + self.assertFalse(ft.has_recursive_symlinks(test_folder)) + # Symlink cycle: sub1/cycle_2 -> sub2, sub2/cycle_1 -> sub1, ... + test_folder = tempfile.mkdtemp() + sub_folder1 = os.path.join(test_folder, 'sub1') + sub_folder2 = sub_folder = os.path.join(test_folder, 'sub2') + os.mkdir(sub_folder1) + os.mkdir(sub_folder2) + os.symlink(os.path.join('..', 'sub2'), os.path.join(sub_folder1, 'cycle_1')) + os.symlink(os.path.join('..', 'sub1'), os.path.join(sub_folder2, 'cycle_2')) + self.assertTrue(ft.has_recursive_symlinks(test_folder)) + def test_copy_dir(self): """Test copy_dir function.""" testdir = os.path.dirname(os.path.abspath(__file__)) From 9fd71570f18da696b2b7185ecbfcaf7a588dedf5 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 23 Jul 2021 14:05:47 +0200 Subject: [PATCH 460/864] Check for recursive symlinks by default before copying a folder Avoids endless loops and creating infinite length paths when `symlinks=True` is NOT also passed to copy_dir --- easybuild/tools/filetools.py | 16 ++++++++++++++-- test/framework/filetools.py | 9 +++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index ef021455a2..28d825709b 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2356,11 +2356,13 @@ def has_recursive_symlinks(path): linkpath = os.path.realpath(fullpath) fullpath += os.sep # To catch the case where both are equal if fullpath.startswith(linkpath + os.sep): + _log.info("Recursive symlink detected at %s", fullpath) return True return False -def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **kwargs): +def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, check_for_recursive_symlinks=True, + **kwargs): """ Copy a directory from specified location to specified location @@ -2368,6 +2370,7 @@ def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **k :param target_path: path to copy the directory to :param force_in_dry_run: force running the command during dry run :param dirs_exist_ok: boolean indicating whether it's OK if the target directory already exists + :param check_for_recursive_symlinks: If symlink arg is not given or False check for recursive symlinks first shutil.copytree is used if the target path does not exist yet; if the target path already exists, the 'copy' function will be used to copy the contents of @@ -2379,6 +2382,13 @@ def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **k dry_run_msg("copied directory %s to %s" % (path, target_path)) else: try: + if check_for_recursive_symlinks and not kwargs.get('symlinks'): + if has_recursive_symlinks(path): + raise EasyBuildError("Recursive symlinks detected in %s. " + "Will not try copying this unless `symlinks=True` is passed", + path) + else: + _log.debug("No recursive symlinks in %s", path) if not dirs_exist_ok and os.path.exists(target_path): raise EasyBuildError("Target location %s to copy %s to already exists", target_path, path) @@ -2406,7 +2416,9 @@ def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **k paths_to_copy = [os.path.join(path, x) for x in entries] copy(paths_to_copy, target_path, - force_in_dry_run=force_in_dry_run, dirs_exist_ok=dirs_exist_ok, **kwargs) + force_in_dry_run=force_in_dry_run, dirs_exist_ok=dirs_exist_ok, + check_for_recursive_symlinks=False, # Don't check again + **kwargs) else: # if dirs_exist_ok is not enabled or target directory doesn't exist, just use shutil.copytree diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 5d959eb091..6e5d0748a6 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1901,6 +1901,15 @@ def ignore_func(_, names): ft.mkdir(subdir) ft.copy_dir(srcdir, target_dir, symlinks=True, dirs_exist_ok=True) + # Detect recursive symlinks by default instead of infinite loop during copy + ft.remove_dir(target_dir) + os.symlink('.', os.path.join(subdir, 'recursive_link')) + self.assertErrorRegex(EasyBuildError, 'Recursive symlinks detected', ft.copy_dir, srcdir, target_dir) + self.assertFalse(os.path.exists(target_dir)) + # Ok for symlinks=True + ft.copy_dir(srcdir, target_dir, symlinks=True) + self.assertTrue(os.path.exists(target_dir)) + # also test behaviour of copy_file under --dry-run build_options = { 'extended_dry_run': True, From 7fbe6f6124eafadf397f576b1fd5f676ef52c806 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Mon, 26 Jul 2021 14:49:46 +0000 Subject: [PATCH 461/864] update iompi toolchain to intel-compiler subtoolchain for oneAPI versions (>= iompi 2020.12) This ports commits c4d780a, 96afa32, and 3c1e6d7 for iimpi from @boegel to iompi. --- easybuild/toolchains/iompi.py | 57 +++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/easybuild/toolchains/iompi.py b/easybuild/toolchains/iompi.py index 240d5d29d0..05664cb484 100644 --- a/easybuild/toolchains/iompi.py +++ b/easybuild/toolchains/iompi.py @@ -28,14 +28,67 @@ :author: Stijn De Weirdt (Ghent University) :author: Kenneth Hoste (Ghent University) """ +from distutils.version import LooseVersion +import re from easybuild.toolchains.iccifort import IccIfort +from easybuild.toolchains.intel_compilers import IntelCompilersToolchain from easybuild.toolchains.mpi.openmpi import OpenMPI -class Iompi(IccIfort, OpenMPI): +class Iompi(IccIfort, IntelCompilersToolchain, OpenMPI): """ Compiler toolchain with Intel compilers (icc/ifort) and OpenMPI. """ NAME = 'iompi' - SUBTOOLCHAIN = IccIfort.NAME + # compiler-only subtoolchain can't be determine statically + # since depends on toolchain version (see below), + # so register both here as possible alternatives (which is taken into account elsewhere) + SUBTOOLCHAIN = [(IntelCompilersToolchain.NAME, IccIfort.NAME)] + + def __init__(self, *args, **kwargs): + """Constructor for Iompi toolchain class.""" + + super(Iompi, self).__init__(*args, **kwargs) + + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', self.version): + # need to transform a version like '2016a' with something that is safe to compare with '8.0', '2016.01' + # comparing subversions that include letters causes TypeErrors in Python 3 + # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) + # (good enough for this purpose) + self.iompi_ver = self.version.replace('a', '.01').replace('b', '.07') + if LooseVersion(self.iompi_ver) >= LooseVersion('2020.12'): + self.oneapi_gen = True + self.SUBTOOLCHAIN = IntelCompilersToolchain.NAME + self.COMPILER_MODULE_NAME = IntelCompilersToolchain.COMPILER_MODULE_NAME + else: + self.oneapi_gen = False + self.SUBTOOLCHAIN = IccIfort.NAME + self.COMPILER_MODULE_NAME = IccIfort.COMPILER_MODULE_NAME + else: + self.iompi_ver = self.version + self.oneapi_gen = False + + def is_dep_in_toolchain_module(self, *args, **kwargs): + """Check whether a specific software name is listed as a dependency in the module for this toolchain.""" + if self.oneapi_gen: + res = IntelCompilersToolchain.is_dep_in_toolchain_module(self, *args, **kwargs) + else: + res = IccIfort.is_dep_in_toolchain_module(self, *args, **kwargs) + + return res + + def _set_compiler_vars(self): + """Intel compilers-specific adjustments after setting compiler variables.""" + if self.oneapi_gen: + IntelCompilersToolchain._set_compiler_vars(self) + else: + IccIfort._set_compiler_vars(self) + + def set_variables(self): + """Intel compilers-specific adjustments after setting compiler variables.""" + if self.oneapi_gen: + IntelCompilersToolchain.set_variables(self) + else: + IccIfort.set_variables(self) From 1ae90cccc24077d32878ed9aecfec4905f29c25d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 28 Jul 2021 09:38:18 +0200 Subject: [PATCH 462/864] Don't parse patch files as EasyConfigs when searching for patch usage Exclude *.patch files during search Remove C&P bug that needlessly reads the file a third time --- easybuild/tools/github.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 8259de1587..750f77163b 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1073,7 +1073,8 @@ def find_software_name_for_patch(patch_name, ec_dirs): for ec_dir in ec_dirs: for (dirpath, _, filenames) in os.walk(ec_dir): for fn in filenames: - if fn != 'TEMPLATE.eb' and not fn.endswith('.py'): + # TODO: In EasyBuild 5.x only check for '*.eb' files + if fn != 'TEMPLATE.eb' and os.path.splitext(fn) not in ('.py', '.patch'): path = os.path.join(dirpath, fn) rawtxt = read_file(path) if 'patches' in rawtxt: @@ -1083,7 +1084,6 @@ def find_software_name_for_patch(patch_name, ec_dirs): for idx, path in enumerate(all_ecs): if soft_name: break - rawtxt = read_file(path) try: ecs = process_easyconfig(path, validate=False) for ec in ecs: From ffcd3715a6a980d30d22fef1bc8cc5400d3d499e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 28 Jul 2021 10:04:23 +0200 Subject: [PATCH 463/864] Speed up EC-for-patch search by smart sorting ECs --- easybuild/tools/github.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 750f77163b..dab4b5c522 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1080,6 +1080,24 @@ def find_software_name_for_patch(patch_name, ec_dirs): if 'patches' in rawtxt: all_ecs.append(path) + # Usual patch names are -_fix_foo.patch + # So search those ECs first + patch_stem = os.path.splitext(patch_name)[0] + # Extract possible sw name and version according to above scheme + # Those might be the same as the whole patch stem, which is OK + possible_sw_name = patch_stem.split('-')[0].lower() + possible_sw_name_version = patch_stem.split('_')[0].lower() + + def ec_key(path): + filename = os.path.basename(path).lower() + # Put files with one of those as the prefix first, then sort by name + return ( + not filename.startswith(possible_sw_name_version), + not filename.startswith(possible_sw_name), + filename + ) + all_ecs.sort(key=ec_key) + nr_of_ecs = len(all_ecs) for idx, path in enumerate(all_ecs): if soft_name: From f6c16190b175f07ccf6fb1350970e370df042894 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 19 Jul 2021 16:23:12 +0200 Subject: [PATCH 464/864] Add reproducer test for #3779 --- easybuild/tools/include.py | 4 ++-- test/framework/options.py | 48 +++++++++++++++++++++++++++++++------- 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index 34390d632a..7de349e5b3 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -180,8 +180,8 @@ def include_easyblocks(tmpdir, paths): if not os.path.exists(target_path): symlink(easyblock_module, target_path) - included_ebs = [x for x in os.listdir(easyblocks_dir) if x not in ['__init__.py', 'generic']] - included_generic_ebs = [x for x in os.listdir(os.path.join(easyblocks_dir, 'generic')) if x != '__init__.py'] + included_ebs = sorted(x for x in os.listdir(easyblocks_dir) if x not in ['__init__.py', 'generic']) + included_generic_ebs = sorted(x for x in os.listdir(os.path.join(easyblocks_dir, 'generic')) if x != '__init__.py') _log.debug("Included generic easyblocks: %s", included_generic_ebs) _log.debug("Included software-specific easyblocks: %s", included_ebs) diff --git a/test/framework/options.py b/test/framework/options.py index b001b0546e..e2a017823e 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -34,6 +34,7 @@ import stat import sys import tempfile +import textwrap from distutils.version import LooseVersion from unittest import TextTestRunner @@ -3232,13 +3233,29 @@ def test_xxx_include_easyblocks(self): sys.modules[pkg].__path__.remove(path) # include extra test easyblocks - foo_txt = '\n'.join([ - 'from easybuild.framework.easyblock import EasyBlock', - 'class EB_foo(EasyBlock):', - ' pass', - '' - ]) + # Make them inherit from each other to trigger a known issue with changed imports, see #3779 + # Choose naming so that order of naming is different than inheritance order + afoo_txt = textwrap.dedent(""" + from easybuild.framework.easyblock import EasyBlock + class EB_afoo(EasyBlock): + def __init__(self, *args, **kwargs): + super(EB_afoo, self).__init__(*args, **kwargs) + """) + write_file(os.path.join(self.test_prefix, 'afoo.py'), afoo_txt) + foo_txt = textwrap.dedent(""" + from easybuild.easyblocks.zfoo import EB_zfoo + class EB_foo(EB_zfoo): + def __init__(self, *args, **kwargs): + super(EB_foo, self).__init__(*args, **kwargs) + """) write_file(os.path.join(self.test_prefix, 'foo.py'), foo_txt) + zfoo_txt = textwrap.dedent(""" + from easybuild.easyblocks.afoo import EB_afoo + class EB_zfoo(EB_afoo): + def __init__(self, *args, **kwargs): + super(EB_zfoo, self).__init__(*args, **kwargs) + """) + write_file(os.path.join(self.test_prefix, 'zfoo.py'), zfoo_txt) # clear log write_file(self.logfile, '') @@ -3256,11 +3273,26 @@ def test_xxx_include_easyblocks(self): foo_regex = re.compile(r"^\|-- EB_foo \(easybuild.easyblocks.foo @ %s\)" % path_pattern, re.M) self.assertTrue(foo_regex.search(logtxt), "Pattern '%s' found in: %s" % (foo_regex.pattern, logtxt)) + ec_txt = '\n'.join([ + 'easyblock = "EB_foo"', + 'name = "pi"', + 'version = "3.14"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = SYSTEM', + ]) + ec = EasyConfig(path=None, rawtxt=ec_txt) + # easyblock is found via get_easyblock_class - klass = get_easyblock_class('EB_foo') - self.assertTrue(issubclass(klass, EasyBlock), "%s is an EasyBlock derivative class" % klass) + for name in ('EB_afoo', 'EB_foo', 'EB_zfoo'): + klass = get_easyblock_class(name) + self.assertTrue(issubclass(klass, EasyBlock), "%s (%s) is an EasyBlock derivative class" % (klass, name)) + + eb_inst = klass(ec) + self.assertTrue(eb_inst is not None, "Instantiating the injected class %s works" % name) # 'undo' import of foo easyblock + del sys.modules['easybuild.easyblocks.afoo'] del sys.modules['easybuild.easyblocks.foo'] # must be run after test for --list-easyblocks, hence the '_xxx_' From 993d43e02cd42d0d1d4c8de19074d60ed47c363b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 19 Jul 2021 16:26:59 +0200 Subject: [PATCH 465/864] Delete all modules to be imported first before importing them one by one Avoids errors on on Python2 when we delete a subclass after importing a module with a class depending on it. Fixes #3779 --- easybuild/tools/include.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index 7de349e5b3..31aecb9997 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -128,15 +128,17 @@ def set_up_eb_package(parent_path, eb_pkg_name, subpkgs=None, pkg_init_body=None def verify_imports(pymods, pypkg, from_path): """Verify that import of specified modules from specified package and expected location works.""" - for pymod in pymods: - pymod_spec = '%s.%s' % (pypkg, pymod) - + pymod_specs = ['%s.%s' % (pypkg, pymod) for pymod in pymods] + for pymod_spec in pymod_specs: # force re-import if the specified modules was already imported; # this is required to ensure that an easyblock that is included via --include-easyblocks-from-pr # gets preference over one that is included via --include-easyblocks if pymod_spec in sys.modules: del sys.modules[pymod_spec] + # After all modules to be reloaded have been removed, import them again + # Note that removing them here may delete transitively loaded modules and not import them again + for pymod_spec in pymod_specs: try: pymod = __import__(pymod_spec, fromlist=[pypkg]) # different types of exceptions may be thrown, not only ImportErrors From c61efb438d6ee3a43bd30919e73e80d656ebdada Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 22 Jul 2021 09:00:40 +0200 Subject: [PATCH 466/864] Delete all imported easyblocks from test --- test/framework/options.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index e2a017823e..f40829b34b 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3291,9 +3291,9 @@ def __init__(self, *args, **kwargs): eb_inst = klass(ec) self.assertTrue(eb_inst is not None, "Instantiating the injected class %s works" % name) - # 'undo' import of foo easyblock - del sys.modules['easybuild.easyblocks.afoo'] - del sys.modules['easybuild.easyblocks.foo'] + # 'undo' import of the easyblocks + for name in ('afoo', 'foo', 'zfoo'): + del sys.modules['easybuild.easyblocks.' + name] # must be run after test for --list-easyblocks, hence the '_xxx_' # cleaning up the imported easyblocks is quite difficult... From 07b671aa4227de2f19674d6e460d479ab96b07a6 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Aug 2021 12:37:47 +0200 Subject: [PATCH 467/864] Clone with tags as the explicit target Avoids accidentally cloning a branch with the same name as that would take preference for the --branch option of git clone --- easybuild/tools/filetools.py | 2 +- test/framework/filetools.py | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 1cb65f826f..7c1e201674 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2472,7 +2472,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): clone_cmd = ['git', 'clone'] if tag: - clone_cmd.extend(['--branch', tag]) + clone_cmd.extend(['--branch', 'refs/tags/' + tag]) if recursive: clone_cmd.append('--recursive') diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 444a6986e3..58cfbb3c6a 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2491,16 +2491,22 @@ def test_get_source_tarball_from_git(self): git_config = { 'repo_name': 'testrepository', 'url': 'https://github.com/easybuilders', - 'tag': 'main', + 'tag': 'tag_for_tests', } target_dir = os.path.join(self.test_prefix, 'target') try: ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) # (only) tarball is created in specified target dir - self.assertTrue(os.path.isfile(os.path.join(target_dir, 'test.tar.gz'))) + test_file = os.path.join(target_dir, 'test.tar.gz') + self.assertTrue(os.path.isfile(test_file)) self.assertEqual(os.listdir(target_dir), ['test.tar.gz']) + # Check that we indeed downloaded the tag and not a branch + extracted_dir = tempfile.mkdtemp(prefix='extracted_dir') + target_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False) + self.assertTrue(os.path.isfile(os.path.join(target_dir, 'this-is-a-tag.txt'))) + del git_config['tag'] git_config['commit'] = '8456f86' ft.get_source_tarball_from_git('test2.tar.gz', target_dir, git_config) @@ -2516,7 +2522,7 @@ def test_get_source_tarball_from_git(self): git_config = { 'repo_name': 'testrepository', 'url': 'git@github.com:easybuilders', - 'tag': 'master', + 'tag': 'tag_for_tests', } args = ['test.tar.gz', self.test_prefix, git_config] @@ -2570,10 +2576,10 @@ def run_check(): git_config = { 'repo_name': 'testrepository', 'url': 'git@github.com:easybuilders', - 'tag': 'master', + 'tag': 'tag_for_tests', } expected = '\n'.join([ - r' running command "git clone --branch master git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --branch refs/tags/tag_for_tests git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", @@ -2582,7 +2588,7 @@ def run_check(): git_config['recursive'] = True expected = '\n'.join([ - r' running command "git clone --branch master --recursive git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --branch refs/tags/tag_for_tests --recursive git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", @@ -2591,7 +2597,7 @@ def run_check(): git_config['keep_git_dir'] = True expected = '\n'.join([ - r' running command "git clone --branch master --recursive git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --branch refs/tags/tag_for_tests --recursive git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz testrepository"', r" \(in .*/tmp.*\)", From 956fa9c49a7572fdafc810a2f48262bc7eeacc19 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Aug 2021 12:46:00 +0200 Subject: [PATCH 468/864] Speed up git checkouts Use shallow checkouts if .git folder is not required Don't download submodules if we do that again for a potentially other version --- easybuild/tools/filetools.py | 12 +++++++++--- test/framework/filetools.py | 8 ++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 7c1e201674..dfd2012e56 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2471,11 +2471,17 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # compose 'git clone' command, and run it clone_cmd = ['git', 'clone'] + if not keep_git_dir: + # Speed up cloning by only fetching the most recent commit, not the whole history + # When we don't want to keep the .git folder there won't be a difference in the result + clone_cmd.extend(['--depth', '1']) + if tag: clone_cmd.extend(['--branch', 'refs/tags/' + tag]) - - if recursive: - clone_cmd.append('--recursive') + if recursive: + clone_cmd.append('--recursive') + else: + clone_cmd.append('--no-checkout') # We do that manually below clone_cmd.append('%s/%s.git' % (url, repo_name)) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 58cfbb3c6a..6e4a9c06e5 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2579,7 +2579,7 @@ def run_check(): 'tag': 'tag_for_tests', } expected = '\n'.join([ - r' running command "git clone --branch refs/tags/tag_for_tests git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --depth 1 --branch refs/tags/tag_for_tests git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", @@ -2588,7 +2588,7 @@ def run_check(): git_config['recursive'] = True expected = '\n'.join([ - r' running command "git clone --branch refs/tags/tag_for_tests --recursive git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --depth 1 --branch refs/tags/tag_for_tests --recursive git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", @@ -2608,7 +2608,7 @@ def run_check(): del git_config['tag'] git_config['commit'] = '8456f86' expected = '\n'.join([ - r' running command "git clone --recursive git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --depth 1 --no-checkout git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "git checkout 8456f86 && git submodule update --init --recursive"', r" \(in testrepository\)", @@ -2619,7 +2619,7 @@ def run_check(): del git_config['recursive'] expected = '\n'.join([ - r' running command "git clone git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --depth 1 --no-checkout git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "git checkout 8456f86"', r" \(in testrepository\)", From d1fa2a4669112c4dcc86450ba1c2394668b3525e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Aug 2021 12:50:12 +0200 Subject: [PATCH 469/864] Slightly enhance test Check for return value of get_source_tarball_from_git and use context manager --- test/framework/filetools.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 6e4a9c06e5..41c88b0afb 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2496,9 +2496,10 @@ def test_get_source_tarball_from_git(self): target_dir = os.path.join(self.test_prefix, 'target') try: - ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) + res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) # (only) tarball is created in specified target dir test_file = os.path.join(target_dir, 'test.tar.gz') + self.assertEqual(res, test_file) self.assertTrue(os.path.isfile(test_file)) self.assertEqual(os.listdir(target_dir), ['test.tar.gz']) @@ -2509,8 +2510,10 @@ def test_get_source_tarball_from_git(self): del git_config['tag'] git_config['commit'] = '8456f86' - ft.get_source_tarball_from_git('test2.tar.gz', target_dir, git_config) - self.assertTrue(os.path.isfile(os.path.join(target_dir, 'test2.tar.gz'))) + res = ft.get_source_tarball_from_git('test2.tar.gz', target_dir, git_config) + test_file = os.path.join(target_dir, 'test2.tar.gz') + self.assertEqual(res, test_file) + self.assertTrue(os.path.isfile(test_file)) self.assertEqual(sorted(os.listdir(target_dir)), ['test.tar.gz', 'test2.tar.gz']) except EasyBuildError as err: @@ -2559,13 +2562,10 @@ def test_get_source_tarball_from_git(self): def run_check(): """Helper function to run get_source_tarball_from_git & check dry run output""" - self.mock_stdout(True) - self.mock_stderr(True) - res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) - stdout = self.get_stdout() - stderr = self.get_stderr() - self.mock_stdout(False) - self.mock_stderr(False) + with self.mocked_stdout_stderr(): + res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) + stdout = self.get_stdout() + stderr = self.get_stderr() self.assertEqual(stderr, '') regex = re.compile(expected) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) From 15799c0e33bb4a4629dca4baeeceec41a5285f5d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Aug 2021 15:16:57 +0200 Subject: [PATCH 470/864] Move dry-run tests before real tests --- test/framework/filetools.py | 130 +++++++++++++++++++----------------- 1 file changed, 67 insertions(+), 63 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 41c88b0afb..245eab6981 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2488,71 +2488,8 @@ def test_diff_files(self): def test_get_source_tarball_from_git(self): """Test get_source_tarball_from_git function.""" - git_config = { - 'repo_name': 'testrepository', - 'url': 'https://github.com/easybuilders', - 'tag': 'tag_for_tests', - } target_dir = os.path.join(self.test_prefix, 'target') - try: - res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) - # (only) tarball is created in specified target dir - test_file = os.path.join(target_dir, 'test.tar.gz') - self.assertEqual(res, test_file) - self.assertTrue(os.path.isfile(test_file)) - self.assertEqual(os.listdir(target_dir), ['test.tar.gz']) - - # Check that we indeed downloaded the tag and not a branch - extracted_dir = tempfile.mkdtemp(prefix='extracted_dir') - target_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False) - self.assertTrue(os.path.isfile(os.path.join(target_dir, 'this-is-a-tag.txt'))) - - del git_config['tag'] - git_config['commit'] = '8456f86' - res = ft.get_source_tarball_from_git('test2.tar.gz', target_dir, git_config) - test_file = os.path.join(target_dir, 'test2.tar.gz') - self.assertEqual(res, test_file) - self.assertTrue(os.path.isfile(test_file)) - self.assertEqual(sorted(os.listdir(target_dir)), ['test.tar.gz', 'test2.tar.gz']) - - except EasyBuildError as err: - if "Network is down" in str(err): - print("Ignoring download error in test_get_source_tarball_from_git, working offline?") - else: - raise err - - git_config = { - 'repo_name': 'testrepository', - 'url': 'git@github.com:easybuilders', - 'tag': 'tag_for_tests', - } - args = ['test.tar.gz', self.test_prefix, git_config] - - for key in ['repo_name', 'url', 'tag']: - orig_value = git_config.pop(key) - if key == 'tag': - error_pattern = "Neither tag nor commit found in git_config parameter" - else: - error_pattern = "%s not specified in git_config parameter" % key - self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) - git_config[key] = orig_value - - git_config['commit'] = '8456f86' - error_pattern = "Tag and commit are mutually exclusive in git_config parameter" - self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) - del git_config['commit'] - - git_config['unknown'] = 'foobar' - error_pattern = "Found one or more unexpected keys in 'git_config' specification" - self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) - del git_config['unknown'] - - args[0] = 'test.txt' - error_pattern = "git_config currently only supports filename ending in .tar.gz" - self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) - args[0] = 'test.tar.gz' - # only test in dry run mode, i.e. check which commands would be executed without actually running them build_options = { 'extended_dry_run': True, @@ -2628,6 +2565,73 @@ def run_check(): ]) run_check() + # Test with real data + init_config() + git_config = { + 'repo_name': 'testrepository', + 'url': 'https://github.com/easybuilders', + 'tag': 'tag_for_tests', + } + + try: + res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) + # (only) tarball is created in specified target dir + test_file = os.path.join(target_dir, 'test.tar.gz') + self.assertEqual(res, test_file) + self.assertTrue(os.path.isfile(test_file)) + self.assertEqual(os.listdir(target_dir), ['test.tar.gz']) + + # Check that we indeed downloaded the tag and not a branch + extracted_dir = tempfile.mkdtemp(prefix='extracted_dir') + target_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False) + self.assertTrue(os.path.isfile(os.path.join(target_dir, 'this-is-a-tag.txt'))) + + del git_config['tag'] + git_config['commit'] = '8456f86' + res = ft.get_source_tarball_from_git('test2.tar.gz', target_dir, git_config) + test_file = os.path.join(target_dir, 'test2.tar.gz') + self.assertEqual(res, test_file) + self.assertTrue(os.path.isfile(test_file)) + self.assertEqual(sorted(os.listdir(target_dir)), ['test.tar.gz', 'test2.tar.gz']) + + except EasyBuildError as err: + if "Network is down" in str(err): + print("Ignoring download error in test_get_source_tarball_from_git, working offline?") + else: + raise err + + git_config = { + 'repo_name': 'testrepository', + 'url': 'git@github.com:easybuilders', + 'tag': 'tag_for_tests', + } + args = ['test.tar.gz', self.test_prefix, git_config] + + for key in ['repo_name', 'url', 'tag']: + orig_value = git_config.pop(key) + if key == 'tag': + error_pattern = "Neither tag nor commit found in git_config parameter" + else: + error_pattern = "%s not specified in git_config parameter" % key + self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) + git_config[key] = orig_value + + git_config['commit'] = '8456f86' + error_pattern = "Tag and commit are mutually exclusive in git_config parameter" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) + del git_config['commit'] + + git_config['unknown'] = 'foobar' + error_pattern = "Found one or more unexpected keys in 'git_config' specification" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) + del git_config['unknown'] + + args[0] = 'test.txt' + error_pattern = "git_config currently only supports filename ending in .tar.gz" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) + args[0] = 'test.tar.gz' + + def test_is_sha256_checksum(self): """Test for is_sha256_checksum function.""" a_sha256_checksum = '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc' From 95e05c88d3089b48225d73bc036dd24d366b45cb Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Aug 2021 15:22:37 +0200 Subject: [PATCH 471/864] Fix line length --- test/framework/filetools.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 245eab6981..542a7c91e2 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2515,54 +2515,55 @@ def run_check(): 'url': 'git@github.com:easybuilders', 'tag': 'tag_for_tests', } + git_repo = {'git_repo': 'git@github.com:easybuilders/testrepository.git'} # Just to make the below shorter expected = '\n'.join([ - r' running command "git clone --depth 1 --branch refs/tags/tag_for_tests git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --depth 1 --branch refs/tags/tag_for_tests %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", - ]) + ]) % git_repo run_check() git_config['recursive'] = True expected = '\n'.join([ - r' running command "git clone --depth 1 --branch refs/tags/tag_for_tests --recursive git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --depth 1 --branch refs/tags/tag_for_tests --recursive %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", - ]) + ]) % git_repo run_check() git_config['keep_git_dir'] = True expected = '\n'.join([ - r' running command "git clone --branch refs/tags/tag_for_tests --recursive git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --branch refs/tags/tag_for_tests --recursive %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz testrepository"', r" \(in .*/tmp.*\)", - ]) + ]) % git_repo run_check() del git_config['keep_git_dir'] del git_config['tag'] git_config['commit'] = '8456f86' expected = '\n'.join([ - r' running command "git clone --depth 1 --no-checkout git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --depth 1 --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "git checkout 8456f86 && git submodule update --init --recursive"', r" \(in testrepository\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", - ]) + ]) % git_repo run_check() del git_config['recursive'] expected = '\n'.join([ - r' running command "git clone --depth 1 --no-checkout git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --depth 1 --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "git checkout 8456f86"', r" \(in testrepository\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", - ]) + ]) % git_repo run_check() # Test with real data @@ -2631,7 +2632,6 @@ def run_check(): self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) args[0] = 'test.tar.gz' - def test_is_sha256_checksum(self): """Test for is_sha256_checksum function.""" a_sha256_checksum = '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc' From fafaf309a082fe9baf01d4006fd8034acef1f842 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Aug 2021 17:16:25 +0200 Subject: [PATCH 472/864] Fix the cloning of tags Can't use refs/tags/xxx for git clone so clone it assuming it is a tag and check afterwards. Fall back to fetching the full history and checking out the tag and submodules manually --- easybuild/tools/filetools.py | 27 +++++++++++++++++++++++---- test/framework/filetools.py | 26 ++++++++++++++++++-------- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index dfd2012e56..df23ec331d 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2477,7 +2477,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): clone_cmd.extend(['--depth', '1']) if tag: - clone_cmd.extend(['--branch', 'refs/tags/' + tag]) + clone_cmd.extend(['--branch', tag]) if recursive: clone_cmd.append('--recursive') else: @@ -2487,7 +2487,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): tmpdir = tempfile.mkdtemp() cwd = change_dir(tmpdir) - run.run_cmd(' '.join(clone_cmd), log_all=True, log_ok=False, simple=False, regexp=False) + run.run_cmd(' '.join(clone_cmd), log_all=True, simple=True, regexp=False) # if a specific commit is asked for, check it out if commit: @@ -2495,14 +2495,33 @@ def get_source_tarball_from_git(filename, targetdir, git_config): if recursive: checkout_cmd.extend(['&&', 'git', 'submodule', 'update', '--init', '--recursive']) - run.run_cmd(' '.join(checkout_cmd), log_all=True, log_ok=False, simple=False, regexp=False, path=repo_name) + run.run_cmd(' '.join(checkout_cmd), log_all=True, simple=True, regexp=False, path=repo_name) + elif not build_option('extended_dry_run'): + # If we wanted to get a tag make sure we actually got a tag and not a branch with the same name + # This doesn't make sense in dry-run mode as we don't have anything to check + cmd = 'git describe --exact-match --tags HEAD' + # Note: Disable logging to also disable the error handling in run_cmd + (out, ec) = run.run_cmd(cmd, log_ok=False, log_all=False, regexp=False, path=repo_name) + if ec != 0 or tag not in out.splitlines(): + cmds = [] + if not keep_git_dir: + # Make the repo unshallow, same as git fetch --unshallow in git 1.8.3+ + # The first fetch seemingly does nothing, no idea why. + cmds.append('git fetch --depth=2147483647 && git fetch --depth=2147483647') + cmds.append('git checkout refs/tags/' + tag) + # Clean all untracked files, e.g. from left-over submodules + cmds.append('git clean --force -d -x') + if recursive: + cmds.append('git submodule update --init --recursive') + for cmd in cmds: + run.run_cmd(cmd, log_all=True, simple=True, regexp=False, path=repo_name) # create an archive and delete the git repo directory if keep_git_dir: tar_cmd = ['tar', 'cfvz', targetpath, repo_name] else: tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name] - run.run_cmd(' '.join(tar_cmd), log_all=True, log_ok=False, simple=False, regexp=False) + run.run_cmd(' '.join(tar_cmd), log_all=True, simple=True, regexp=False) # cleanup (repo_name dir does not exist in dry run mode) change_dir(cwd) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 542a7c91e2..722fbf54da 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2517,7 +2517,7 @@ def run_check(): } git_repo = {'git_repo': 'git@github.com:easybuilders/testrepository.git'} # Just to make the below shorter expected = '\n'.join([ - r' running command "git clone --depth 1 --branch refs/tags/tag_for_tests %(git_repo)s"', + r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", @@ -2526,7 +2526,7 @@ def run_check(): git_config['recursive'] = True expected = '\n'.join([ - r' running command "git clone --depth 1 --branch refs/tags/tag_for_tests --recursive %(git_repo)s"', + r' running command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", @@ -2535,7 +2535,7 @@ def run_check(): git_config['keep_git_dir'] = True expected = '\n'.join([ - r' running command "git clone --branch refs/tags/tag_for_tests --recursive %(git_repo)s"', + r' running command "git clone --branch tag_for_tests --recursive %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz testrepository"', r" \(in .*/tmp.*\)", @@ -2566,12 +2566,12 @@ def run_check(): ]) % git_repo run_check() - # Test with real data + # Test with real data. init_config() git_config = { 'repo_name': 'testrepository', 'url': 'https://github.com/easybuilders', - 'tag': 'tag_for_tests', + 'tag': 'branch_tag_for_test', } try: @@ -2581,11 +2581,21 @@ def run_check(): self.assertEqual(res, test_file) self.assertTrue(os.path.isfile(test_file)) self.assertEqual(os.listdir(target_dir), ['test.tar.gz']) + # Check that we indeed downloaded the right tag + extracted_dir = tempfile.mkdtemp(prefix='extracted_dir') + extracted_repo_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False) + self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'this-is-a-branch.txt'))) + os.remove(test_file) - # Check that we indeed downloaded the tag and not a branch + # use a tag that clashes with a branch name and make sure this is handled correctly + git_config['tag'] = 'tag_for_tests' + res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) + self.assertEqual(res, test_file) + self.assertTrue(os.path.isfile(test_file)) + # Check that we indeed downloaded the tag and not the branch extracted_dir = tempfile.mkdtemp(prefix='extracted_dir') - target_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False) - self.assertTrue(os.path.isfile(os.path.join(target_dir, 'this-is-a-tag.txt'))) + extracted_repo_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False) + self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'this-is-a-tag.txt'))) del git_config['tag'] git_config['commit'] = '8456f86' From c352a9a588c964fd73e557c37f8584f77937886f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Aug 2021 17:36:40 +0200 Subject: [PATCH 473/864] Print a warning if the slow path is entered --- easybuild/tools/filetools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index df23ec331d..983abdcb76 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2503,6 +2503,9 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # Note: Disable logging to also disable the error handling in run_cmd (out, ec) = run.run_cmd(cmd, log_ok=False, log_all=False, regexp=False, path=repo_name) if ec != 0 or tag not in out.splitlines(): + print_warning('Tag %s was not downloaded in the first try due to %s/%s containing a branch' + ' with the same name. You might want to alert the maintainers of %s about that issue.', + tag, url, repo_name, repo_name) cmds = [] if not keep_git_dir: # Make the repo unshallow, same as git fetch --unshallow in git 1.8.3+ From 6276d912e188c2ab35238eba3040e9c3493e39d3 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Aug 2021 17:40:29 +0200 Subject: [PATCH 474/864] Fix getting extension from a file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mikael Öhman --- easybuild/tools/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index dab4b5c522..8c261c65df 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1074,7 +1074,7 @@ def find_software_name_for_patch(patch_name, ec_dirs): for (dirpath, _, filenames) in os.walk(ec_dir): for fn in filenames: # TODO: In EasyBuild 5.x only check for '*.eb' files - if fn != 'TEMPLATE.eb' and os.path.splitext(fn) not in ('.py', '.patch'): + if fn != 'TEMPLATE.eb' and os.path.splitext(fn)[1] not in ('.py', '.patch'): path = os.path.join(dirpath, fn) rawtxt = read_file(path) if 'patches' in rawtxt: From fa60ab455c89bb60cb03b19147b6795f5cc77fc9 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Aug 2021 18:19:28 +0200 Subject: [PATCH 475/864] Don't search hidden folders for ECs --- easybuild/tools/github.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 8c261c65df..c9e5e3d623 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -839,7 +839,7 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ # copy easyconfig files to right place target_dir = os.path.join(git_working_dir, pr_target_repo) print_msg("copying files to %s..." % target_dir) - file_info = COPY_FUNCTIONS[pr_target_repo](ec_paths, os.path.join(git_working_dir, pr_target_repo)) + file_info = COPY_FUNCTIONS[pr_target_repo](ec_paths, target_dir) # figure out commit message to use if commit_msg: @@ -1071,7 +1071,9 @@ def find_software_name_for_patch(patch_name, ec_dirs): all_ecs = [] for ec_dir in ec_dirs: - for (dirpath, _, filenames) in os.walk(ec_dir): + for (dirpath, dirnames, filenames) in os.walk(ec_dir): + # Don't visit any hidden folders, such as .git + dirnames[:] = [i for i in dirnames if not i.startswith('.')] for fn in filenames: # TODO: In EasyBuild 5.x only check for '*.eb' files if fn != 'TEMPLATE.eb' and os.path.splitext(fn)[1] not in ('.py', '.patch'): From 6850f104efc9775322ec25d081a7c50e9b67d830 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 5 Aug 2021 16:03:18 +0200 Subject: [PATCH 476/864] Use the ignore-dirs option instead --- easybuild/tools/github.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index c9e5e3d623..d49cbc2f5e 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1069,11 +1069,13 @@ def find_software_name_for_patch(patch_name, ec_dirs): soft_name = None + ignore_dirs = build_option('ignore_dirs') all_ecs = [] for ec_dir in ec_dirs: for (dirpath, dirnames, filenames) in os.walk(ec_dir): - # Don't visit any hidden folders, such as .git - dirnames[:] = [i for i in dirnames if not i.startswith('.')] + # Exclude ignored dirs + if ignore_dirs: + dirnames[:] = [i for i in dirnames if i not in ignore_dirs] for fn in filenames: # TODO: In EasyBuild 5.x only check for '*.eb' files if fn != 'TEMPLATE.eb' and os.path.splitext(fn)[1] not in ('.py', '.patch'): From afd47593574c3a84b18a9cb61c90a1a40a0d22f1 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 5 Aug 2021 16:26:34 +0200 Subject: [PATCH 477/864] Add --filter-ecs parameter to remove ECs from the build set Removes ECs to be build from the ECs passed on the cmdline or implicitely (and more useful) selected via the --from-pr option Accepts a list of glob style patterns --- easybuild/framework/easyconfig/tools.py | 5 +++++ easybuild/tools/config.py | 1 + easybuild/tools/options.py | 3 +++ test/framework/robot.py | 17 +++++++++++++++++ 4 files changed, 26 insertions(+) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index d78ff96e7c..5f2acb9aa5 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -37,6 +37,7 @@ :author: Ward Poelmans (Ghent University) """ import copy +import fnmatch import glob import os import re @@ -360,6 +361,10 @@ def det_easyconfig_paths(orig_paths): # if no easyconfigs are specified, use all the ones touched in the PR ec_files = [path for path in pr_files if path.endswith('.eb')] + filter_ecs = build_option('filter_ecs') + if filter_ecs: + ec_files = [ec for ec in ec_files + if not any(fnmatch.fnmatch(ec, filter_spec) for filter_spec in filter_ecs)] if ec_files and robot_path: ignore_subdirs = build_option('ignore_dirs') if not build_option('consider_archived_easyconfigs'): diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 23f3c97f54..8f660331f5 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -180,6 +180,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'envvars_user_modules', 'extra_modules', 'filter_deps', + 'filter_ecs', 'filter_env_vars', 'hide_deps', 'hide_toolchains', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 7dadcd415d..647c10c383 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -396,6 +396,9 @@ def override_options(self): 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, " "because equivalent OS packages are installed. (e.g. --filter-deps=zlib,ncurses)", 'strlist', 'extend', None), + 'filter-ecs': ("List of easyconfigs (given as glob patterns) to *ignore* when given on command line " + "or auto-selected when building with --from-pr. (e.g. --filter-ecs=*intel*)", + 'strlist', 'extend', None), 'filter-env-vars': ("List of names of environment variables that should *not* be defined/updated by " "module files generated by EasyBuild", 'strlist', 'extend', None), 'fixed-installdir-naming-scheme': ("Use fixed naming scheme for installation directories", None, diff --git a/test/framework/robot.py b/test/framework/robot.py index f55a6e822b..4add2ada9f 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -670,6 +670,23 @@ def test_det_easyconfig_paths(self): regex = re.compile(r"^ \* \[.\] .*/__archive__/.*/intel-2012a.eb \(module: intel/2012a\)", re.M) self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + args = [ + os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), + os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), + os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0-gompi-2018a.eb'), + '--dry-run', + '--robot', + '--tmpdir=%s' % self.test_prefix, + '--filter-ecs=*oy-0.0.eb,*-test.eb', + ] + outtxt = self.eb_main(args, raise_error=True) + + regex = re.compile(r"^ \* \[.\] .*/toy-0.0-gompi-2018a.eb \(module: toy/0.0-gompi-2018a\)", re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + for ec in ('toy-0.0.eb', 'toy-0.0-gompi-2018a-test.eb'): + regex = re.compile(r"^ \* \[.\] .*/%s \(module:" % ec, re.M) + self.assertFalse(regex.search(outtxt), "%s should be fitered in %s" % (ec, outtxt)) + def test_search_paths(self): """Test search_paths command line argument.""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') From 999deb36e9c398060952e490c09132335a717161 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 5 Aug 2021 16:32:19 +0200 Subject: [PATCH 478/864] Mock and catch warning message --- test/framework/filetools.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 722fbf54da..4ea4738881 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2589,7 +2589,10 @@ def run_check(): # use a tag that clashes with a branch name and make sure this is handled correctly git_config['tag'] = 'tag_for_tests' - res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) + with self.mocked_stdout_stderr(): + res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) + stderr = self.get_stderr() + self.assertIn('Tag tag_for_tests was not downloaded in the first try', stderr) self.assertEqual(res, test_file) self.assertTrue(os.path.isfile(test_file)) # Check that we indeed downloaded the tag and not the branch From 2c84fea1d3608c50de03538c939f9f7944da940a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 10 Aug 2021 16:41:15 +0200 Subject: [PATCH 479/864] Only catch our own errors when failing to parse config options --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 7dadcd415d..678363ab35 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1392,7 +1392,7 @@ def parse_options(args=None, with_include=True): eb_go = EasyBuildOptions(usage=usage, description=description, prog='eb', envvar_prefix=CONFIG_ENV_VAR_PREFIX, go_args=eb_args, error_env_options=True, error_env_option_method=raise_easybuilderror, with_include=with_include) - except Exception as err: + except EasyBuildError as err: raise EasyBuildError("Failed to parse configuration options: %s" % err) return eb_go From 055b28a2a11070692bf9251e68748a2a95ef40bf Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 10 Aug 2021 16:42:32 +0200 Subject: [PATCH 480/864] Fix CI on Python 2 --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e5d99f3e1e..8730d3f102 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,7 +49,8 @@ PyYAML; python_version >= '2.7' pycodestyle; python_version < '2.7' flake8; python_version >= '2.7' -GC3Pie +# 2.6.7 uses invalid Python 2 syntax +GC3Pie!=2.6.7 python-graph-dot python-hglib requests From be485dc47664054e72afa3215f9fc873a0f4d7ff Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 11 Aug 2021 11:38:40 +0200 Subject: [PATCH 481/864] Only exclude latest GC3Pie for python2 --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8730d3f102..e63085eb51 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,7 +50,8 @@ pycodestyle; python_version < '2.7' flake8; python_version >= '2.7' # 2.6.7 uses invalid Python 2 syntax -GC3Pie!=2.6.7 +GC3Pie!=2.6.7; python_version < '3.0' +GC3Pie; python_version >= '3.0' python-graph-dot python-hglib requests From 88acb30df50c3acff684528d0bf907cc53138dab Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Aug 2021 16:50:44 +0200 Subject: [PATCH 482/864] check type of source_tmpl value for extensions, ensure it's a string value (not a list) --- easybuild/framework/easyblock.py | 8 +++++++- test/framework/easyblock.py | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 3b1ffa3096..a0e496cd9a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -593,7 +593,13 @@ def fetch_extension_sources(self, skip_checksums=False): default_source_tmpl = resolve_template('%(name)s-%(version)s.tar.gz', template_values) # if no sources are specified via 'sources', fall back to 'source_tmpl' - src_fn = ext_options.get('source_tmpl', default_source_tmpl) + src_fn = ext_options.get('source_tmpl') + if src_fn is None: + src_fn = default_source_tmpl + elif not isinstance(src_fn, string_type): + error_msg = "source_tmpl value must be a string! (found value of type '%s'): %s" + raise EasyBuildError(error_msg, type(src_fn).__name__, src_fn) + src_path = self.obtain_file(src_fn, extension=True, urls=source_urls, force_download=force_download) if src_path: diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 21873a1662..18bfcf07b0 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1052,6 +1052,33 @@ def test_init_extensions(self): error_pattern = "ConfigureMake easyblock can not be used to install extensions" self.assertErrorRegex(EasyBuildError, error_pattern, eb.init_ext_instances) + def test_extension_source_tmpl(self): + """Test type checking for 'source_tmpl' value of an extension.""" + self.contents = '\n'.join([ + "easyblock = 'ConfigureMake'", + "name = 'toy'", + "version = '0.0'", + "homepage = 'https://example.com'", + "description = 'test'", + "toolchain = SYSTEM", + "exts_list = [", + " ('bar', '0.0', {", + " 'source_tmpl': [SOURCE_TAR_GZ],", + " }),", + "]", + ]) + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + + error_pattern = r"source_tmpl value must be a string! " + error_pattern += r"\(found value of type 'list'\): \['bar-0\.0\.tar\.gz'\]" + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + + self.contents = self.contents.replace("'source_tmpl': [SOURCE_TAR_GZ]", "'source_tmpl': SOURCE_TAR_GZ") + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + eb.fetch_step() + def test_skip_extensions_step(self): """Test the skip_extensions_step""" From 9ed8415afb710e4cab5fb107fdf91bc0d92c1247 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Aug 2021 18:33:50 +0200 Subject: [PATCH 483/864] enhance test to check for fixed bug reported in #3781 --- test/framework/github.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/framework/github.py b/test/framework/github.py index 34f2f7f885..b280fc5448 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -595,6 +595,15 @@ def test_github_find_patches(self): reg = re.compile(r'[1-9]+ of [1-9]+ easyconfigs checked') self.assertTrue(re.search(reg, txt)) + self.assertEqual(gh.find_software_name_for_patch('test.patch', []), None) + + # check behaviour of find_software_name_for_patch when non-UTF8 patch files are present + non_utf8_patch = os.path.join(self.test_prefix, 'problem.patch') + with open(non_utf8_patch, 'wb') as fp: + fp.write(bytes("+ ximage->byte_order=T1_byte_order; /* Set t1lib\xb4s byteorder */\n", 'iso_8859_1')) + + self.assertEqual(gh.find_software_name_for_patch('test.patch', [self.test_prefix]), None) + def test_github_det_commit_status(self): """Test det_commit_status function.""" From 822750dc0b2d01729e561cf154d4837adcfdbef1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Aug 2021 19:30:03 +0200 Subject: [PATCH 484/864] also define $BLAS_SHARED_LIBS & co in build environment (analogous to $BLAS_STATIC_LIBS) --- easybuild/toolchains/fft/fftw.py | 8 +- easybuild/tools/toolchain/constants.py | 26 +++++- easybuild/tools/toolchain/fft.py | 2 + easybuild/tools/toolchain/linalg.py | 10 +++ easybuild/tools/toolchain/variables.py | 10 ++- test/framework/toolchain.py | 106 ++++++++++++++++++++++++- 6 files changed, 152 insertions(+), 10 deletions(-) diff --git a/easybuild/toolchains/fft/fftw.py b/easybuild/toolchains/fft/fftw.py index 4b6c32dcb9..ece375253a 100644 --- a/easybuild/toolchains/fft/fftw.py +++ b/easybuild/toolchains/fft/fftw.py @@ -71,7 +71,7 @@ def _set_fft_variables(self): # TODO can these be replaced with the FFT ones? self.variables.join('FFTW_INC_DIR', 'FFT_INC_DIR') self.variables.join('FFTW_LIB_DIR', 'FFT_LIB_DIR') - if 'FFT_STATIC_LIBS' in self.variables: - self.variables.join('FFTW_STATIC_LIBS', 'FFT_STATIC_LIBS') - if 'FFT_STATIC_LIBS_MT' in self.variables: - self.variables.join('FFTW_STATIC_LIBS_MT', 'FFT_STATIC_LIBS_MT') + + for key in ('SHARED_LIBS', 'SHARED_LIBS_MT', 'STATIC_LIBS', 'STATIC_LIBS_MT'): + if 'FFT_' + key in self.variables: + self.variables.join('FFTW_' + key, 'FFT_' + key) diff --git a/easybuild/tools/toolchain/constants.py b/easybuild/tools/toolchain/constants.py index 307827dd1e..b334d17ae9 100644 --- a/easybuild/tools/toolchain/constants.py +++ b/easybuild/tools/toolchain/constants.py @@ -30,8 +30,8 @@ """ from easybuild.tools.variables import AbsPathList -from easybuild.tools.toolchain.variables import LinkLibraryPaths, IncludePaths, CommandFlagList, CommaStaticLibs -from easybuild.tools.toolchain.variables import FlagList, LibraryList +from easybuild.tools.toolchain.variables import CommandFlagList, CommaSharedLibs, CommaStaticLibs +from easybuild.tools.toolchain.variables import FlagList, IncludePaths, LibraryList, LinkLibraryPaths COMPILER_VARIABLES = [ @@ -114,6 +114,10 @@ ('LIBBLAS', 'BLAS libraries'), ('LIBBLAS_MT', 'multithreaded BLAS libraries'), ], + CommaSharedLibs: [ + ('BLAS_SHARED_LIBS', 'Comma-separated list of shared BLAS libraries'), + ('BLAS_MT_SHARED_LIBS', 'Comma-separated list of shared multithreaded BLAS libraries'), + ], CommaStaticLibs: [ ('BLAS_STATIC_LIBS', 'Comma-separated list of static BLAS libraries'), ('BLAS_MT_STATIC_LIBS', 'Comma-separated list of static multithreaded BLAS libraries'), @@ -132,6 +136,12 @@ ('LIBLAPACK', 'LAPACK libraries'), ('LIBLAPACK_MT', 'multithreaded LAPACK libraries'), ], + CommaSharedLibs: [ + ('LAPACK_SHARED_LIBS', 'Comma-separated list of shared LAPACK libraries'), + ('LAPACK_MT_SHARED_LIBS', 'Comma-separated list of shared LAPACK libraries'), + ('BLAS_LAPACK_SHARED_LIBS', 'Comma-separated list of shared BLAS and LAPACK libraries'), + ('BLAS_LAPACK_MT_SHARED_LIBS', 'Comma-separated list of shared BLAS and LAPACK libraries'), + ], CommaStaticLibs: [ ('LAPACK_STATIC_LIBS', 'Comma-separated list of static LAPACK libraries'), ('LAPACK_MT_STATIC_LIBS', 'Comma-separated list of static LAPACK libraries'), @@ -166,6 +176,10 @@ ('LIBSCALAPACK', 'SCALAPACK libraries'), ('LIBSCALAPACK_MT', 'multithreaded SCALAPACK libraries'), ], + CommaSharedLibs: [ + ('SCALAPACK_SHARED_LIBS', 'Comma-separated list of shared SCALAPACK libraries'), + ('SCALAPACK_MT_SHARED_LIBS', 'Comma-separated list of shared SCALAPACK libraries'), + ], CommaStaticLibs: [ ('SCALAPACK_STATIC_LIBS', 'Comma-separated list of static SCALAPACK libraries'), ('SCALAPACK_MT_STATIC_LIBS', 'Comma-separated list of static SCALAPACK libraries'), @@ -181,6 +195,10 @@ ('LIBFFT', 'FFT libraries'), ('LIBFFT_MT', 'Multithreaded FFT libraries'), ], + CommaSharedLibs: [ + ('FFT_SHARED_LIBS', 'Comma-separated list of shared FFT libraries'), + ('FFT_SHARED_LIBS_MT', 'Comma-separated list of shared multithreaded FFT libraries'), + ], CommaStaticLibs: [ ('FFT_STATIC_LIBS', 'Comma-separated list of static FFT libraries'), ('FFT_STATIC_LIBS_MT', 'Comma-separated list of static multithreaded FFT libraries'), @@ -192,6 +210,10 @@ ('FFTW_LIB_DIR', 'FFTW library directory'), ('FFTW_INC_DIR', 'FFTW include directory'), ], + CommaSharedLibs: [ + ('FFTW_SHARED_LIBS', 'Comma-separated list of shared FFTW libraries'), + ('FFTW_SHARED_LIBS_MT', 'Comma-separated list of shared multithreaded FFTW libraries'), + ], CommaStaticLibs: [ ('FFTW_STATIC_LIBS', 'Comma-separated list of static FFTW libraries'), ('FFTW_STATIC_LIBS_MT', 'Comma-separated list of static multithreaded FFTW libraries'), diff --git a/easybuild/tools/toolchain/fft.py b/easybuild/tools/toolchain/fft.py index 4facafc7c2..1e39953ba7 100644 --- a/easybuild/tools/toolchain/fft.py +++ b/easybuild/tools/toolchain/fft.py @@ -68,7 +68,9 @@ def _set_fft_variables(self): if getattr(self, 'LIB_MULTITHREAD', None) is not None: self.variables.nappend('LIBFFT_MT', self.LIB_MULTITHREAD) + self.variables.join('FFT_SHARED_LIBS', 'LIBFFT') self.variables.join('FFT_STATIC_LIBS', 'LIBFFT') + self.variables.join('FFT_SHARED_LIBS_MT', 'LIBFFT_MT') self.variables.join('FFT_STATIC_LIBS_MT', 'LIBFFT_MT') for root in self.get_software_root(self.FFT_MODULE_NAME): diff --git a/easybuild/tools/toolchain/linalg.py b/easybuild/tools/toolchain/linalg.py index 7d27350996..2e1a46db41 100644 --- a/easybuild/tools/toolchain/linalg.py +++ b/easybuild/tools/toolchain/linalg.py @@ -127,7 +127,9 @@ def _set_blas_variables(self): self.variables.nappend('LIBBLAS', self.LIB_EXTRA, position=20) self.variables.nappend('LIBBLAS_MT', self.LIB_EXTRA, position=20) + self.variables.join('BLAS_SHARED_LIBS', 'LIBBLAS') self.variables.join('BLAS_STATIC_LIBS', 'LIBBLAS') + self.variables.join('BLAS_MT_SHARED_LIBS', 'LIBBLAS_MT') self.variables.join('BLAS_MT_STATIC_LIBS', 'LIBBLAS_MT') for root in self.get_software_root(self.BLAS_MODULE_NAME): self.variables.append_exists('BLAS_LIB_DIR', root, self.BLAS_LIB_DIR) @@ -147,7 +149,9 @@ def _set_lapack_variables(self): self.variables.join('LIBLAPACK_MT_ONLY', 'LIBBLAS_MT') self.variables.join('LIBLAPACK', 'LIBBLAS') self.variables.join('LIBLAPACK_MT', 'LIBBLAS_MT') + self.variables.join('LAPACK_SHARED_LIBS', 'BLAS_SHARED_LIBS') self.variables.join('LAPACK_STATIC_LIBS', 'BLAS_STATIC_LIBS') + self.variables.join('LAPACK_MT_SHARED_LIBS', 'BLAS_MT_SHARED_LIBS') self.variables.join('LAPACK_MT_STATIC_LIBS', 'BLAS_MT_STATIC_LIBS') self.variables.join('LAPACK_LIB_DIR', 'BLAS_LIB_DIR') self.variables.join('LAPACK_INC_DIR', 'BLAS_INC_DIR') @@ -183,7 +187,9 @@ def _set_lapack_variables(self): self.variables.nappend('LIBLAPACK', self.LIB_EXTRA, position=20) self.variables.nappend('LIBLAPACK_MT', self.LIB_EXTRA, position=20) + self.variables.join('LAPACK_SHARED_LIBS', 'LIBLAPACK') self.variables.join('LAPACK_STATIC_LIBS', 'LIBLAPACK') + self.variables.join('LAPACK_MT_SHARED_LIBS', 'LIBLAPACK_MT') self.variables.join('LAPACK_MT_STATIC_LIBS', 'LIBLAPACK_MT') for root in self.get_software_root(self.LAPACK_MODULE_NAME): @@ -192,7 +198,9 @@ def _set_lapack_variables(self): self.variables.join('BLAS_LAPACK_LIB_DIR', 'LAPACK_LIB_DIR', 'BLAS_LIB_DIR') self.variables.join('BLAS_LAPACK_INC_DIR', 'LAPACK_INC_DIR', 'BLAS_INC_DIR') + self.variables.join('BLAS_LAPACK_SHARED_LIBS', 'LAPACK_SHARED_LIBS', 'BLAS_SHARED_LIBS') self.variables.join('BLAS_LAPACK_STATIC_LIBS', 'LAPACK_STATIC_LIBS', 'BLAS_STATIC_LIBS') + self.variables.join('BLAS_LAPACK_MT_SHARED_LIBS', 'LAPACK_MT_SHARED_LIBS', 'BLAS_MT_SHARED_LIBS') self.variables.join('BLAS_LAPACK_MT_STATIC_LIBS', 'LAPACK_MT_STATIC_LIBS', 'BLAS_MT_STATIC_LIBS') # add general dependency variables @@ -293,7 +301,9 @@ def _set_scalapack_variables(self): self.variables.nappend('LIBSCALAPACK', self.LIB_EXTRA, position=20) self.variables.nappend('LIBSCALAPACK_MT', self.LIB_EXTRA, position=20) + self.variables.join('SCALAPACK_SHARED_LIBS', 'LIBSCALAPACK') self.variables.join('SCALAPACK_STATIC_LIBS', 'LIBSCALAPACK') + self.variables.join('SCALAPACK_MT_SHARED_LIBS', 'LIBSCALAPACK_MT') self.variables.join('SCALAPACK_MT_STATIC_LIBS', 'LIBSCALAPACK_MT') for root in self.get_software_root(self.SCALAPACK_MODULE_NAME): self.variables.append_exists('SCALAPACK_LIB_DIR', root, self.SCALAPACK_LIB_DIR) diff --git a/easybuild/tools/toolchain/variables.py b/easybuild/tools/toolchain/variables.py index 30a365ad5b..4becb8986b 100644 --- a/easybuild/tools/toolchain/variables.py +++ b/easybuild/tools/toolchain/variables.py @@ -111,8 +111,16 @@ def change(self, separator=None, separator_begin_end=None, prefix=None, prefix_b self.END.PREFIX = prefix_begin_end +class CommaSharedLibs(LibraryList): + """Comma-separated list of shared libraries""" + SEPARATOR = ',' + + PREFIX = 'lib' + SUFFIX = '.so' + + class CommaStaticLibs(LibraryList): - """Comma-separated list""" + """Comma-separated list of static libraries""" SEPARATOR = ',' PREFIX = 'lib' diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 0fa2ff2d56..c02c2b8f42 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1453,6 +1453,8 @@ def test_old_new_iccifort(self): libblas_mt_intel3 += " -Wl,--end-group -Wl,-Bdynamic -liomp5 -lguide -lpthread" # no -lguide + blas_static_libs_intel4 = 'libmkl_intel_lp64.a,libmkl_sequential.a,libmkl_core.a' + blas_shared_libs_intel4 = blas_static_libs_intel4.replace('.a', '.so') libblas_intel4 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_sequential -lmkl_core" libblas_intel4 += " -Wl,--end-group -Wl,-Bdynamic" libblas_mt_intel4 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" @@ -1469,18 +1471,83 @@ def test_old_new_iccifort(self): libscalack_intel4 = "-lmkl_scalapack_lp64 -lmkl_blacs_intelmpi_lp64 -lmkl_intel_lp64 -lmkl_sequential " libscalack_intel4 += "-lmkl_core" - libblas_mt_fosscuda = "-lopenblas -lgfortran -lpthread" + blas_static_libs_fosscuda = "libopenblas.a,libgfortran.a" + blas_shared_libs_fosscuda = blas_static_libs_fosscuda.replace('.a', '.so') + blas_mt_static_libs_fosscuda = blas_static_libs_fosscuda + ",libpthread.a" + blas_mt_shared_libs_fosscuda = blas_mt_static_libs_fosscuda.replace('.a', '.so') + libblas_fosscuda = "-lopenblas -lgfortran" + libblas_mt_fosscuda = libblas_fosscuda + " -lpthread" + + fft_static_libs_fosscuda = "libfftw3.a" + fft_shared_libs_fosscuda = fft_static_libs_fosscuda.replace('.a', '.so') + fft_mt_static_libs_fosscuda = "libfftw3.a,libpthread.a" + fft_mt_shared_libs_fosscuda = fft_mt_static_libs_fosscuda.replace('.a', '.so') + fft_mt_static_libs_fosscuda_omp = "libfftw3_omp.a,libfftw3.a,libpthread.a" + fft_mt_shared_libs_fosscuda_omp = fft_mt_static_libs_fosscuda_omp.replace('.a', '.so') + libfft_fosscuda = "-lfftw3" + libfft_mt_fosscuda = libfft_fosscuda + " -lpthread" + libfft_mt_fosscuda_omp = "-lfftw3_omp " + libfft_fosscuda + " -lpthread" + + lapack_static_libs_fosscuda = "libopenblas.a,libgfortran.a" + lapack_shared_libs_fosscuda = lapack_static_libs_fosscuda.replace('.a', '.so') + lapack_mt_static_libs_fosscuda = lapack_static_libs_fosscuda + ",libpthread.a" + lapack_mt_shared_libs_fosscuda = lapack_mt_static_libs_fosscuda.replace('.a', '.so') + liblapack_fosscuda = "-lopenblas -lgfortran" + liblapack_mt_fosscuda = liblapack_fosscuda + " -lpthread" + libscalack_fosscuda = "-lscalapack -lopenblas -lgfortran" - libfft_mt_fosscuda = "-lfftw3_omp -lfftw3 -lpthread" + libscalack_mt_fosscuda = libscalack_fosscuda + " -lpthread" + scalapack_static_libs_fosscuda = "libscalapack.a,libopenblas.a,libgfortran.a" + scalapack_shared_libs_fosscuda = scalapack_static_libs_fosscuda.replace('.a', '.so') + scalapack_mt_static_libs_fosscuda = "libscalapack.a,libopenblas.a,libgfortran.a,libpthread.a" + scalapack_mt_shared_libs_fosscuda = scalapack_mt_static_libs_fosscuda.replace('.a', '.so') tc = self.get_toolchain('fosscuda', version='2018a') tc.prepare() + self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBBLAS'], libblas_fosscuda) self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_fosscuda) + + self.assertEqual(os.environ['LAPACK_SHARED_LIBS'], lapack_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_STATIC_LIBS'], lapack_static_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_SHARED_LIBS'], lapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_STATIC_LIBS'], lapack_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBLAPACK'], liblapack_fosscuda) + self.assertEqual(os.environ['LIBLAPACK_MT'], liblapack_mt_fosscuda) + + self.assertEqual(os.environ['BLAS_LAPACK_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + + self.assertEqual(os.environ['FFT_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFT_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBFFT'], libfft_fosscuda) + self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda) + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_fosscuda) + self.assertEqual(os.environ['LIBSCALAPACK_MT'], libscalack_mt_fosscuda) + self.assertEqual(os.environ['SCALAPACK_SHARED_LIBS'], scalapack_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_STATIC_LIBS'], scalapack_static_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_SHARED_LIBS'], scalapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_STATIC_LIBS'], scalapack_mt_static_libs_fosscuda) self.modtool.purge() tc = self.get_toolchain('intel', version='2018a') tc.prepare() + self.assertEqual(os.environ.get('BLAS_SHARED_LIBS', "(not set)"), blas_shared_libs_intel4) + self.assertEqual(os.environ.get('BLAS_STATIC_LIBS', "(not set)"), blas_static_libs_intel4) + self.assertEqual(os.environ.get('LAPACK_SHARED_LIBS', "(not set)"), blas_shared_libs_intel4) + self.assertEqual(os.environ.get('LAPACK_STATIC_LIBS', "(not set)"), blas_static_libs_intel4) self.assertEqual(os.environ.get('LIBBLAS', "(not set)"), libblas_intel4) self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_intel4) self.assertEqual(os.environ.get('LIBFFT', "(not set)"), libfft_intel4) @@ -1517,9 +1584,42 @@ def test_old_new_iccifort(self): tc = self.get_toolchain('fosscuda', version='2018a') tc.set_options({'openmp': True}) tc.prepare() + self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBBLAS'], libblas_fosscuda) self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_fosscuda) - self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda) + + self.assertEqual(os.environ['LAPACK_SHARED_LIBS'], lapack_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_STATIC_LIBS'], lapack_static_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_SHARED_LIBS'], lapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_STATIC_LIBS'], lapack_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBLAPACK'], liblapack_fosscuda) + self.assertEqual(os.environ['LIBLAPACK_MT'], liblapack_mt_fosscuda) + + self.assertEqual(os.environ['BLAS_LAPACK_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + + self.assertEqual(os.environ['FFT_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFT_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda_omp) + self.assertEqual(os.environ['FFT_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda_omp) + self.assertEqual(os.environ['FFTW_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda_omp) + self.assertEqual(os.environ['FFTW_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda_omp) + self.assertEqual(os.environ['LIBFFT'], libfft_fosscuda) + self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda_omp) + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_fosscuda) + self.assertEqual(os.environ['LIBSCALAPACK_MT'], libscalack_mt_fosscuda) + self.assertEqual(os.environ['SCALAPACK_SHARED_LIBS'], scalapack_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_STATIC_LIBS'], scalapack_static_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_SHARED_LIBS'], scalapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_STATIC_LIBS'], scalapack_mt_static_libs_fosscuda) def test_standalone_iccifort(self): """Test whether standalone installation of iccifort matches the iccifort toolchain definition.""" From 703813e49e4982901642f05aa4a62ff45483df8c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Aug 2021 19:32:51 +0200 Subject: [PATCH 485/864] only run additional check for find_software_name_for_patch when testing with Python 3 --- test/framework/github.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/framework/github.py b/test/framework/github.py index b280fc5448..423f860a00 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -597,12 +597,13 @@ def test_github_find_patches(self): self.assertEqual(gh.find_software_name_for_patch('test.patch', []), None) - # check behaviour of find_software_name_for_patch when non-UTF8 patch files are present - non_utf8_patch = os.path.join(self.test_prefix, 'problem.patch') - with open(non_utf8_patch, 'wb') as fp: - fp.write(bytes("+ ximage->byte_order=T1_byte_order; /* Set t1lib\xb4s byteorder */\n", 'iso_8859_1')) + # check behaviour of find_software_name_for_patch when non-UTF8 patch files are present (only with Python 3) + if sys.version_info[0] >= 3: + non_utf8_patch = os.path.join(self.test_prefix, 'problem.patch') + with open(non_utf8_patch, 'wb') as fp: + fp.write(bytes("+ ximage->byte_order=T1_byte_order; /* Set t1lib\xb4s byteorder */\n", 'iso_8859_1')) - self.assertEqual(gh.find_software_name_for_patch('test.patch', [self.test_prefix]), None) + self.assertEqual(gh.find_software_name_for_patch('test.patch', [self.test_prefix]), None) def test_github_det_commit_status(self): """Test det_commit_status function.""" From b2662fb1a1d205c23e638e65fd2b1d575db38c38 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 12 Aug 2021 09:42:59 +0200 Subject: [PATCH 486/864] don't hardcode .so, use get_shared_lib_ext function --- easybuild/tools/toolchain/variables.py | 3 ++- test/framework/toolchain.py | 23 +++++++++++++---------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/toolchain/variables.py b/easybuild/tools/toolchain/variables.py index 4becb8986b..6758af24de 100644 --- a/easybuild/tools/toolchain/variables.py +++ b/easybuild/tools/toolchain/variables.py @@ -30,6 +30,7 @@ """ from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.variables import StrList, AbsPathList @@ -116,7 +117,7 @@ class CommaSharedLibs(LibraryList): SEPARATOR = ',' PREFIX = 'lib' - SUFFIX = '.so' + SUFFIX = '.' + get_shared_lib_ext() class CommaStaticLibs(LibraryList): diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index c02c2b8f42..0bf963857c 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -51,6 +51,7 @@ from easybuild.tools.filetools import read_file, symlink, write_file, which from easybuild.tools.py2vs3 import string_type from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.toolchain.mpi import get_mpi_cmd_template from easybuild.tools.toolchain.toolchain import env_vars_external_module from easybuild.tools.toolchain.utilities import get_toolchain, search_toolchain @@ -1448,13 +1449,15 @@ def test_old_new_iccifort(self): self.setup_sandbox_for_intel_fftw(self.test_prefix, imklver='10.2.6.038') self.modtool.prepend_module_path(self.test_prefix) + shlib_ext = get_shared_lib_ext() + # incl. -lguide libblas_mt_intel3 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" libblas_mt_intel3 += " -Wl,--end-group -Wl,-Bdynamic -liomp5 -lguide -lpthread" # no -lguide blas_static_libs_intel4 = 'libmkl_intel_lp64.a,libmkl_sequential.a,libmkl_core.a' - blas_shared_libs_intel4 = blas_static_libs_intel4.replace('.a', '.so') + blas_shared_libs_intel4 = blas_static_libs_intel4.replace('.a', '.' + shlib_ext) libblas_intel4 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_sequential -lmkl_core" libblas_intel4 += " -Wl,--end-group -Wl,-Bdynamic" libblas_mt_intel4 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" @@ -1472,35 +1475,35 @@ def test_old_new_iccifort(self): libscalack_intel4 += "-lmkl_core" blas_static_libs_fosscuda = "libopenblas.a,libgfortran.a" - blas_shared_libs_fosscuda = blas_static_libs_fosscuda.replace('.a', '.so') + blas_shared_libs_fosscuda = blas_static_libs_fosscuda.replace('.a', '.' + shlib_ext) blas_mt_static_libs_fosscuda = blas_static_libs_fosscuda + ",libpthread.a" - blas_mt_shared_libs_fosscuda = blas_mt_static_libs_fosscuda.replace('.a', '.so') + blas_mt_shared_libs_fosscuda = blas_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext) libblas_fosscuda = "-lopenblas -lgfortran" libblas_mt_fosscuda = libblas_fosscuda + " -lpthread" fft_static_libs_fosscuda = "libfftw3.a" - fft_shared_libs_fosscuda = fft_static_libs_fosscuda.replace('.a', '.so') + fft_shared_libs_fosscuda = fft_static_libs_fosscuda.replace('.a', '.' + shlib_ext) fft_mt_static_libs_fosscuda = "libfftw3.a,libpthread.a" - fft_mt_shared_libs_fosscuda = fft_mt_static_libs_fosscuda.replace('.a', '.so') + fft_mt_shared_libs_fosscuda = fft_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext) fft_mt_static_libs_fosscuda_omp = "libfftw3_omp.a,libfftw3.a,libpthread.a" - fft_mt_shared_libs_fosscuda_omp = fft_mt_static_libs_fosscuda_omp.replace('.a', '.so') + fft_mt_shared_libs_fosscuda_omp = fft_mt_static_libs_fosscuda_omp.replace('.a', '.' + shlib_ext) libfft_fosscuda = "-lfftw3" libfft_mt_fosscuda = libfft_fosscuda + " -lpthread" libfft_mt_fosscuda_omp = "-lfftw3_omp " + libfft_fosscuda + " -lpthread" lapack_static_libs_fosscuda = "libopenblas.a,libgfortran.a" - lapack_shared_libs_fosscuda = lapack_static_libs_fosscuda.replace('.a', '.so') + lapack_shared_libs_fosscuda = lapack_static_libs_fosscuda.replace('.a', '.' + shlib_ext) lapack_mt_static_libs_fosscuda = lapack_static_libs_fosscuda + ",libpthread.a" - lapack_mt_shared_libs_fosscuda = lapack_mt_static_libs_fosscuda.replace('.a', '.so') + lapack_mt_shared_libs_fosscuda = lapack_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext) liblapack_fosscuda = "-lopenblas -lgfortran" liblapack_mt_fosscuda = liblapack_fosscuda + " -lpthread" libscalack_fosscuda = "-lscalapack -lopenblas -lgfortran" libscalack_mt_fosscuda = libscalack_fosscuda + " -lpthread" scalapack_static_libs_fosscuda = "libscalapack.a,libopenblas.a,libgfortran.a" - scalapack_shared_libs_fosscuda = scalapack_static_libs_fosscuda.replace('.a', '.so') + scalapack_shared_libs_fosscuda = scalapack_static_libs_fosscuda.replace('.a', '.' + shlib_ext) scalapack_mt_static_libs_fosscuda = "libscalapack.a,libopenblas.a,libgfortran.a,libpthread.a" - scalapack_mt_shared_libs_fosscuda = scalapack_mt_static_libs_fosscuda.replace('.a', '.so') + scalapack_mt_shared_libs_fosscuda = scalapack_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext) tc = self.get_toolchain('fosscuda', version='2018a') tc.prepare() From 9ebdeb96022305e7f878eb20281c28c2223ffe9f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 12 Aug 2021 18:55:10 +0200 Subject: [PATCH 487/864] Avoid checking executable rights of subdirs of temp dirs already checked `run_cmd` is one of the slowest things we can do as it forks the whole process. In the check when setting a temp dir we run a dummy file to see if the folder is actually executable. This means running this check at least twice per test case and another time for each init_config call in the test amounting to e.g. 263 calls in a single test (test_compiler_dependent_optarch) So cache the results and don't repeat the run_cmd if it already succeeded for a parent folder. Likely no change at all in regular mode but really helps for the tests --- easybuild/tools/options.py | 41 ++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 678363ab35..7d6d0f117d 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1756,24 +1756,31 @@ def set_tmpdir(tmpdir=None, raise_error=False): # reset to make sure tempfile picks up new temporary directory to use tempfile.tempdir = None - # test if temporary directory allows to execute files, warn if it doesn't - try: - fd, tmptest_file = tempfile.mkstemp() - os.close(fd) - os.chmod(tmptest_file, 0o700) - if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False, force_in_dry_run=True, trace=False, - stream_output=False): - msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir() - msg += "This can cause problems in the build process, consider using --tmpdir." - if raise_error: - raise EasyBuildError(msg) + # Skip the executable check if it already succeeded for any parent folder + # Especially important for the unit test suite, less so for actual execution + executable_tmp_paths = getattr(set_tmpdir, 'executable_tmp_paths', []) + if not any(current_tmpdir.startswith(path) for path in executable_tmp_paths): + # test if temporary directory allows to execute files, warn if it doesn't + try: + fd, tmptest_file = tempfile.mkstemp() + os.close(fd) + os.chmod(tmptest_file, 0o700) + if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False, force_in_dry_run=True, trace=False, + stream_output=False): + msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir() + msg += "This can cause problems in the build process, consider using --tmpdir." + if raise_error: + raise EasyBuildError(msg) + else: + _log.warning(msg) else: - _log.warning(msg) - else: - _log.debug("Temporary directory %s allows to execute files, good!" % tempfile.gettempdir()) - os.remove(tmptest_file) + _log.debug("Temporary directory %s allows to execute files, good!" % tempfile.gettempdir()) + # Put this folder into the cache + executable_tmp_paths.append(current_tmpdir) + set_tmpdir.executable_tmp_paths = executable_tmp_paths + os.remove(tmptest_file) - except OSError as err: - raise EasyBuildError("Failed to test whether temporary directory allows to execute files: %s", err) + except OSError as err: + raise EasyBuildError("Failed to test whether temporary directory allows to execute files: %s", err) return current_tmpdir From 8e60b7511ea0e93f922027f9122bc8da49f7d5d0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 12 Aug 2021 19:09:26 +0200 Subject: [PATCH 488/864] Reduce number of combinations tested in test_compiler_dependent_optarch The full product of options is not required to cover all cases. Instead run only with varying options per active toolchain and a single case (actually 2) for PGI This cuts down the combinations from 216 to 20 and runtime from ~1m to ~7s --- test/framework/toolchain.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 0bf963857c..11a6c083d0 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -752,17 +752,28 @@ def test_compiler_dependent_optarch(self): intel_options = [('intelflag', 'intelflag'), ('GENERIC', 'xSSE2'), ('', '')] gcc_options = [('gccflag', 'gccflag'), ('march=nocona', 'march=nocona'), ('', '')] gcccore_options = [('gcccoreflag', 'gcccoreflag'), ('GENERIC', 'march=x86-64 -mtune=generic'), ('', '')] - toolchains = [ - ('iccifort', '2018.1.163'), - ('GCC', '6.4.0-2.28'), - ('GCCcore', '6.2.0'), - ('PGI', '16.7-GCC-5.4.0-2.26'), - ] - enabled = [True, False] - test_cases = product(intel_options, gcc_options, gcccore_options, toolchains, enabled) + tc_intel = ('iccifort', '2018.1.163') + tc_gcc = ('GCC', '6.4.0-2.28') + tc_gcccore = ('GCCcore', '6.2.0') + tc_pgi = ('PGI', '16.7-GCC-5.4.0-2.26') + enabled = [True, False] - for intel_flags, gcc_flags, gcccore_flags, (toolchain_name, toolchain_ver), enable in test_cases: + test_cases = [] + for i, (tc, options) in enumerate(zip((tc_intel, tc_gcc, tc_gcccore), + (intel_options, gcc_options, gcccore_options))): + # Vary only the compiler specific option + for opt in options: + new_value = [intel_options[0], gcc_options[0], gcccore_options[0], tc] + new_value[i] = opt + test_cases.append(new_value) + # Add one case for PGI + test_cases.append((intel_options[0], gcc_options[0], gcccore_options[0], tc_pgi)) + + # Run each for enabled and disabled + test_cases = list(product(test_cases, enabled)) + + for (intel_flags, gcc_flags, gcccore_flags, (toolchain_name, toolchain_ver)), enable in test_cases: intel_flags, intel_flags_exp = intel_flags gcc_flags, gcc_flags_exp = gcc_flags From 0653e2d69afed02f4664119502b4be2753bdf80f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 17 Aug 2021 12:00:34 +0200 Subject: [PATCH 489/864] Correctly resolve templates for patches in extensions when uploading to github --- easybuild/tools/github.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index d49cbc2f5e..e703b94cc9 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1013,10 +1013,11 @@ def is_patch_for(patch_name, ec): patches = copy.copy(ec['patches']) - for ext in ec['exts_list']: + for ext in ec.get_ref('exts_list'): if isinstance(ext, (list, tuple)) and len(ext) == 3 and isinstance(ext[2], dict): + templates = {'name': ext[0], 'version': ext[1]} ext_options = ext[2] - patches.extend(ext_options.get('patches', [])) + patches.extend(p % templates for p in ext_options.get('patches', [])) for patch in patches: if isinstance(patch, (tuple, list)): From 5d7623c1a0f6e936d56f1a1126cc782bea412463 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 17 Aug 2021 12:13:22 +0200 Subject: [PATCH 490/864] Extend patch search logic to components --- easybuild/tools/github.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index e703b94cc9..8f96f4f0e6 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -34,6 +34,7 @@ import getpass import glob import functools +import itertools import os import random import re @@ -1013,11 +1014,13 @@ def is_patch_for(patch_name, ec): patches = copy.copy(ec['patches']) - for ext in ec.get_ref('exts_list'): - if isinstance(ext, (list, tuple)) and len(ext) == 3 and isinstance(ext[2], dict): - templates = {'name': ext[0], 'version': ext[1]} - ext_options = ext[2] - patches.extend(p % templates for p in ext_options.get('patches', [])) + with ec.disable_templating(): + for ext in itertools.chain(ec['exts_list'], ec.get('components', [])): + if isinstance(ext, (list, tuple)) and len(ext) == 3 and isinstance(ext[2], dict): + templates = {'name': ext[0], 'version': ext[1]} + ext_options = ext[2] + patches.extend(p[0] % templates if isinstance(p, (tuple, list)) else p % templates + for p in ext_options.get('patches', [])) for patch in patches: if isinstance(patch, (tuple, list)): From d8c463f8d1e9091e1d8b3ea8814b06d28d1f3882 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 17 Aug 2021 16:05:02 +0200 Subject: [PATCH 491/864] Update test to use real EC instances --- test/framework/github.py | 106 +++++++++++++----- .../easyblocks/generic/pythonbundle.py | 4 + 2 files changed, 83 insertions(+), 27 deletions(-) diff --git a/test/framework/github.py b/test/framework/github.py index 423f860a00..e6c4abf4a5 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -33,12 +33,14 @@ import random import re import sys +import textwrap from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from time import gmtime from unittest import TextTestRunner import easybuild.tools.testing from easybuild.base.rest import RestClient +from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.framework.easyconfig.tools import categorize_files_by_type from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, module_classes, update_build_option @@ -803,53 +805,103 @@ def test_github_det_patch_specs(self): """Test for det_patch_specs function.""" patch_paths = [os.path.join(self.test_prefix, p) for p in ['1.patch', '2.patch', '3.patch']] - file_info = {'ecs': [ - {'name': 'A', 'patches': ['1.patch'], 'exts_list': []}, - {'name': 'B', 'patches': [], 'exts_list': []}, - ] - } + file_info = {'ecs': []} + + rawtxt = textwrap.dedent(""" + easyblock = 'ConfigureMake' + name = 'A' + version = '42' + homepage = 'http://foo.com/' + description = '' + toolchain = {"name":"GCC", "version": "4.6.3"} + + patches = ['1.patch'] + """) + file_info['ecs'].append(EasyConfig(None, rawtxt=rawtxt)) + rawtxt = textwrap.dedent(""" + easyblock = 'ConfigureMake' + name = 'B' + version = '42' + homepage = 'http://foo.com/' + description = '' + toolchain = {"name":"GCC", "version": "4.6.3"} + """) + file_info['ecs'].append(EasyConfig(None, rawtxt=rawtxt)) + error_pattern = "Failed to determine software name to which patch file .*/2.patch relates" self.mock_stdout(True) self.assertErrorRegex(EasyBuildError, error_pattern, gh.det_patch_specs, patch_paths, file_info, []) self.mock_stdout(False) - file_info['ecs'].append({'name': 'C', 'patches': [('3.patch', 'subdir'), '2.patch'], 'exts_list': []}) + rawtxt = textwrap.dedent(""" + easyblock = 'ConfigureMake' + name = 'C' + version = '42' + homepage = 'http://foo.com/' + description = '' + toolchain = {"name":"GCC", "version": "4.6.3"} + + patches = [('3.patch', 'subdir'), '2.patch'] + """) + file_info['ecs'].append(EasyConfig(None, rawtxt=rawtxt)) self.mock_stdout(True) res = gh.det_patch_specs(patch_paths, file_info, []) self.mock_stdout(False) - self.assertEqual(len(res), 3) - self.assertEqual(os.path.basename(res[0][0]), '1.patch') - self.assertEqual(res[0][1], 'A') - self.assertEqual(os.path.basename(res[1][0]), '2.patch') - self.assertEqual(res[1][1], 'C') - self.assertEqual(os.path.basename(res[2][0]), '3.patch') - self.assertEqual(res[2][1], 'C') + self.assertEqual([i[0] for i in res], patch_paths) + self.assertEqual([i[1] for i in res], ['A', 'C', 'C']) # check if patches for extensions are found - file_info['ecs'][-1] = { - 'name': 'patched_ext', - 'patches': [], - 'exts_list': [ + rawtxt = textwrap.dedent(""" + easyblock = 'ConfigureMake' + name = 'patched_ext' + version = '42' + homepage = 'http://foo.com/' + description = '' + toolchain = {"name":"GCC", "version": "4.6.3"} + + exts_list = [ 'foo', ('bar', '1.2.3'), ('patched', '4.5.6', { - 'patches': [('2.patch', 1), '3.patch'], + 'patches': [('%(name)s-2.patch', 1), '%(name)s-3.patch'], }), - ], - } + ] + """) + patch_paths[1:3] = [os.path.join(self.test_prefix, p) for p in ['patched-2.patch', 'patched-3.patch']] + file_info['ecs'][-1] = EasyConfig(None, rawtxt=rawtxt) + + self.mock_stdout(True) + res = gh.det_patch_specs(patch_paths, file_info, []) + self.mock_stdout(False) + + self.assertEqual([i[0] for i in res], patch_paths) + self.assertEqual([i[1] for i in res], ['A', 'patched_ext', 'patched_ext']) + + # check if patches for components are found + rawtxt = textwrap.dedent(""" + easyblock = 'PythonBundle' + name = 'patched_bundle' + version = '42' + homepage = 'http://foo.com/' + description = '' + toolchain = {"name":"GCC", "version": "4.6.3"} + + components = [ + ('bar', '1.2.3'), + ('patched', '4.5.6', { + 'patches': [('%(name)s-2.patch', 1), '%(name)s-3.patch'], + }), + ] + """) + file_info['ecs'][-1] = EasyConfig(None, rawtxt=rawtxt) self.mock_stdout(True) res = gh.det_patch_specs(patch_paths, file_info, []) self.mock_stdout(False) - self.assertEqual(len(res), 3) - self.assertEqual(os.path.basename(res[0][0]), '1.patch') - self.assertEqual(res[0][1], 'A') - self.assertEqual(os.path.basename(res[1][0]), '2.patch') - self.assertEqual(res[1][1], 'patched_ext') - self.assertEqual(os.path.basename(res[2][0]), '3.patch') - self.assertEqual(res[2][1], 'patched_ext') + self.assertEqual([i[0] for i in res], patch_paths) + self.assertEqual([i[1] for i in res], ['A', 'patched_bundle', 'patched_bundle']) def test_github_restclient(self): """Test use of RestClient.""" diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py b/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py index 7146b8ecd3..0321602f3f 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py @@ -28,6 +28,7 @@ @author: Miguel Dias Costa (National University of Singapore) """ from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easyconfig import CUSTOM class PythonBundle(EasyBlock): @@ -37,4 +38,7 @@ class PythonBundle(EasyBlock): def extra_options(extra_vars=None): if extra_vars is None: extra_vars = {} + extra_vars.update({ + 'components': [(), "List of components to install: tuples w/ name, version and easyblock to use", CUSTOM], + }) return EasyBlock.extra_options(extra_vars) From 0ffd0c9b2f503bb2a06344d7bae8668716ea6845 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 18 Aug 2021 11:19:05 +0200 Subject: [PATCH 492/864] report use of --ignore-test-failure in success message in output --- easybuild/main.py | 5 ++++- test/framework/toy_build.py | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 8cbff81b41..748c068376 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -527,7 +527,10 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)]) overall_success = correct_builds_cnt == len(ordered_ecs) - success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) + success_msg = "Build succeeded " + if build_option('ignore_test_failure'): + success_msg += "(with --ignore-test-failure) " + success_msg += "for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 51d3f1a977..b0ad961027 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3491,6 +3491,14 @@ def test_toy_build_sanity_check_linked_libs(self): self.test_toy_build(ec_file=test_ec, extra_args=args, force=False, raise_error=True, verbose=False, verify=False) + def test_toy_ignore_test_failure(self): + """Check whether use of --ignore-test-failure is mentioned in build output.""" + args = ['--ignore-test-failure'] + stdout, stderr = self.run_test_toy_build_with_output(extra_args=args, verify=False, testing=False) + + self.assertTrue("Build succeeded (with --ignore-test-failure) for 1 out of 1" in stdout) + self.assertFalse(stderr) + def suite(): """ return all the tests in this file """ From 2b5524a9da98f44b38378b22b35807684f87aad1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 19 Aug 2021 17:47:54 +0200 Subject: [PATCH 493/864] add comments to clarify use of function attribute to cache checked paths in set_tmpdir --- easybuild/tools/options.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 7d6d0f117d..9709fb71ea 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1756,10 +1756,13 @@ def set_tmpdir(tmpdir=None, raise_error=False): # reset to make sure tempfile picks up new temporary directory to use tempfile.tempdir = None + # cache for checked paths, via function attribute + executable_tmp_paths = getattr(set_tmpdir, 'executable_tmp_paths', []) + # Skip the executable check if it already succeeded for any parent folder # Especially important for the unit test suite, less so for actual execution - executable_tmp_paths = getattr(set_tmpdir, 'executable_tmp_paths', []) if not any(current_tmpdir.startswith(path) for path in executable_tmp_paths): + # test if temporary directory allows to execute files, warn if it doesn't try: fd, tmptest_file = tempfile.mkstemp() @@ -1775,9 +1778,13 @@ def set_tmpdir(tmpdir=None, raise_error=False): _log.warning(msg) else: _log.debug("Temporary directory %s allows to execute files, good!" % tempfile.gettempdir()) + # Put this folder into the cache executable_tmp_paths.append(current_tmpdir) + + # set function attribute so we can retrieve cache later set_tmpdir.executable_tmp_paths = executable_tmp_paths + os.remove(tmptest_file) except OSError as err: From 9e11d2ecf5fcce25b7008e93db46e5949901f937 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 20 Aug 2021 14:32:18 +0200 Subject: [PATCH 494/864] add get_cuda_cc_template_value method to EasyConfig class --- easybuild/framework/easyconfig/easyconfig.py | 21 +++++++- test/framework/easyconfig.py | 50 ++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 55948e11a2..6597c00aac 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -59,7 +59,7 @@ from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT from easybuild.framework.easyconfig.parser import DEPRECATED_PARAMETERS, REPLACED_PARAMETERS from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig -from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, template_constant_dict +from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC, template_constant_dict from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN @@ -1803,6 +1803,25 @@ def asdict(self): res[key] = value return res + def get_cuda_cc_template_value(self, key): + """ + Get template value based on --cuda-compute-capabilities EasyBuild configuration option + and cuda_compute_capabilities easyconfig parameter. + Returns user-friendly error message in case neither are defined, + or if an unknown key is used. + """ + if key.startswith('cuda_') and key in [x for (x, _) in TEMPLATE_NAMES_DYNAMIC]: + if key in self.template_values: + return self.template_values[key] + else: + error_msg = "(get_cuda_cc_template_value) Template value '%s' is not defined!\n" + error_msg += "Make sure that either the --cuda-compute-capabilities EasyBuild configuration " + error_msg += "option is set, or that the cuda_compute_capabilities easyconfig parameter is defined." + raise EasyBuildError(error_msg, key) + else: + error_msg = "%s is not a template value baed on --cuda-compute-capabilities/cuda_compute_capabilities" + raise EasyBuildError(error_msg, key) + def det_installversion(version, toolchain_name, toolchain_version, prefix, suffix): """Deprecated 'det_installversion' function, to determine exact install version, based on supplied parameters.""" diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index bd0fd80a3c..d904814d4e 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4483,6 +4483,56 @@ def test_easyconfig_import(self): error_pattern = r"Failed to copy '.*' easyconfig parameter" self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, test_ec) + def test_get_cuda_cc_template_value(self): + """ + Test getting template value based on --cuda-compute-capabilities / cuda_compute_capabilities. + """ + self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.14"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = SYSTEM', + ]) + self.prep() + ec = EasyConfig(self.eb_file) + + error_pattern = "foobar is not a template value baed on --cuda-compute-capabilities/cuda_compute_capabilities" + self.assertErrorRegex(EasyBuildError, error_pattern, ec.get_cuda_cc_template_value, 'foobar') + + error_pattern = r"\(get_cuda_cc_template_value\) Template value '%s' is not defined!\n" + error_pattern += r"Make sure that either the --cuda-compute-capabilities EasyBuild configuration " + error_pattern += "option is set, or that the cuda_compute_capabilities easyconfig parameter is defined." + cuda_template_values = { + 'cuda_compute_capabilities': '6.5,7.0', + 'cuda_cc_space_sep': '6.5 7.0', + 'cuda_cc_semicolon_sep': '6.5;7.0', + 'cuda_sm_comma_sep': 'sm_65,sm_70', + 'cuda_sm_space_sep': 'sm_65 sm_70', + } + for key in cuda_template_values: + self.assertErrorRegex(EasyBuildError, error_pattern % key, ec.get_cuda_cc_template_value, key) + + update_build_option('cuda_compute_capabilities', ['6.5', '7.0']) + ec = EasyConfig(self.eb_file) + + for key in cuda_template_values: + self.assertEqual(ec.get_cuda_cc_template_value(key), cuda_template_values[key]) + + update_build_option('cuda_compute_capabilities', None) + ec = EasyConfig(self.eb_file) + + for key in cuda_template_values: + self.assertErrorRegex(EasyBuildError, error_pattern % key, ec.get_cuda_cc_template_value, key) + + self.contents += "\ncuda_compute_capabilities = ['6.5', '7.0']" + self.prep() + ec = EasyConfig(self.eb_file) + + for key in cuda_template_values: + self.assertEqual(ec.get_cuda_cc_template_value(key), cuda_template_values[key]) + def suite(): """ returns all the testcases in this module """ From 2d993457d5a95c06d6023fec02b649bf1719a4ff Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 20 Aug 2021 15:54:39 +0200 Subject: [PATCH 495/864] fix typo in error message --- easybuild/framework/easyconfig/easyconfig.py | 2 +- test/framework/easyconfig.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 6597c00aac..322c9546e2 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1819,7 +1819,7 @@ def get_cuda_cc_template_value(self, key): error_msg += "option is set, or that the cuda_compute_capabilities easyconfig parameter is defined." raise EasyBuildError(error_msg, key) else: - error_msg = "%s is not a template value baed on --cuda-compute-capabilities/cuda_compute_capabilities" + error_msg = "%s is not a template value based on --cuda-compute-capabilities/cuda_compute_capabilities" raise EasyBuildError(error_msg, key) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index d904814d4e..64cb3ea1ae 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4498,7 +4498,7 @@ def test_get_cuda_cc_template_value(self): self.prep() ec = EasyConfig(self.eb_file) - error_pattern = "foobar is not a template value baed on --cuda-compute-capabilities/cuda_compute_capabilities" + error_pattern = "foobar is not a template value based on --cuda-compute-capabilities/cuda_compute_capabilities" self.assertErrorRegex(EasyBuildError, error_pattern, ec.get_cuda_cc_template_value, 'foobar') error_pattern = r"\(get_cuda_cc_template_value\) Template value '%s' is not defined!\n" From 204bcf168f4e2f0958176182ad343fcafe1bd7bc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 21 Aug 2021 12:08:32 +0200 Subject: [PATCH 496/864] apply @Flamefire's suggested code changes Co-authored-by: Alexander Grund --- easybuild/framework/easyconfig/easyconfig.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 322c9546e2..155dc487f1 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1810,10 +1810,10 @@ def get_cuda_cc_template_value(self, key): Returns user-friendly error message in case neither are defined, or if an unknown key is used. """ - if key.startswith('cuda_') and key in [x for (x, _) in TEMPLATE_NAMES_DYNAMIC]: - if key in self.template_values: + if key.startswith('cuda_') and any(x[0] == key for x in TEMPLATE_NAMES_DYNAMIC): + try: return self.template_values[key] - else: + except KeyError: error_msg = "(get_cuda_cc_template_value) Template value '%s' is not defined!\n" error_msg += "Make sure that either the --cuda-compute-capabilities EasyBuild configuration " error_msg += "option is set, or that the cuda_compute_capabilities easyconfig parameter is defined." From f9db1772f9efa235119b8e505a98fe5009dd090b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 21 Aug 2021 12:10:37 +0200 Subject: [PATCH 497/864] drop '(get_cuda_cc_template_value)' from error message --- easybuild/framework/easyconfig/easyconfig.py | 2 +- test/framework/easyconfig.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 155dc487f1..d0470018e8 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1814,7 +1814,7 @@ def get_cuda_cc_template_value(self, key): try: return self.template_values[key] except KeyError: - error_msg = "(get_cuda_cc_template_value) Template value '%s' is not defined!\n" + error_msg = "Template value '%s' is not defined!\n" error_msg += "Make sure that either the --cuda-compute-capabilities EasyBuild configuration " error_msg += "option is set, or that the cuda_compute_capabilities easyconfig parameter is defined." raise EasyBuildError(error_msg, key) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 64cb3ea1ae..96ce325e6e 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4501,7 +4501,7 @@ def test_get_cuda_cc_template_value(self): error_pattern = "foobar is not a template value based on --cuda-compute-capabilities/cuda_compute_capabilities" self.assertErrorRegex(EasyBuildError, error_pattern, ec.get_cuda_cc_template_value, 'foobar') - error_pattern = r"\(get_cuda_cc_template_value\) Template value '%s' is not defined!\n" + error_pattern = r"Template value '%s' is not defined!\n" error_pattern += r"Make sure that either the --cuda-compute-capabilities EasyBuild configuration " error_pattern += "option is set, or that the cuda_compute_capabilities easyconfig parameter is defined." cuda_template_values = { From 178f546c4731c5635e94de1e3a409630a5a4ed10 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 23 Aug 2021 18:10:55 +0000 Subject: [PATCH 498/864] added bash as an option in the fix_*_shebang_for --- easybuild/framework/easyblock.py | 2 +- easybuild/framework/easyconfig/default.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 93810300b4..d6d8b622f1 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2466,7 +2466,7 @@ def package_step(self): def fix_shebang(self): """Fix shebang lines for specified files.""" - for lang in ['perl', 'python']: + for lang in ['bash', 'perl', 'python']: shebang_regex = re.compile(r'^#![ ]*.*[/ ]%s.*' % lang) fix_shebang_for = self.cfg['fix_%s_shebang_for' % lang] if fix_shebang_for: diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 5279332541..70571c12d5 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -95,6 +95,8 @@ 'easybuild_version': [None, "EasyBuild-version this spec-file was written for", BUILD], 'enhance_sanity_check': [False, "Indicate that additional sanity check commands & paths should enhance " "the existin sanity check, not replace it", BUILD], + 'fix_bash_shebang_for': [None, "List of files for which Bash shebang should be fixed " + "to '#!/usr/bin/env bash' (glob patterns supported)", BUILD], 'fix_perl_shebang_for': [None, "List of files for which Perl shebang should be fixed " "to '#!/usr/bin/env perl' (glob patterns supported)", BUILD], 'fix_python_shebang_for': [None, "List of files for which Python shebang should be fixed " From fe60a29569abe7113270e6ca37e243ae27621ee8 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 23 Aug 2021 18:52:33 +0000 Subject: [PATCH 499/864] refactored test_fix_shebang to reduce code duplication, added tests for bash --- test/framework/toy_build.py | 101 ++++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 46 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 51d3f1a977..698494ec55 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2937,6 +2937,7 @@ def test_fix_shebang(self): # copy of bin/toy to use in fix_python_shebang_for and fix_perl_shebang_for " 'cp -a %(installdir)s/bin/toy %(installdir)s/bin/toy.python',", " 'cp -a %(installdir)s/bin/toy %(installdir)s/bin/toy.perl',", + " 'cp -a %(installdir)s/bin/toy %(installdir)s/bin/toy.sh',", # hardcoded path to bin/python " 'echo \"#!/usr/bin/python\\n# test\" > %(installdir)s/bin/t1.py',", @@ -2973,9 +2974,26 @@ def test_fix_shebang(self): # shebang bash " 'echo \"#!/usr/bin/env bash\\n# test\" > %(installdir)s/bin/b2.sh',", + # tests for bash shebang + # hardcoded path to bin/bash + " 'echo \"#!/bin/bash\\n# test\" > %(installdir)s/bin/t1.sh',", + # hardcoded path to usr/bin/bash + " 'echo \"#!/usr/bin/bash\\n# test\" > %(installdir)s/bin/t2.sh',", + # already OK, should remain the same + " 'echo \"#!/usr/bin/env bash\\n# test\" > %(installdir)s/bin/t3.sh',", + # shebang with space, should strip the space + " 'echo \"#! /usr/bin/env bash\\n# test\" > %(installdir)s/bin/t4.sh',", + # no shebang sh + " 'echo \"# test\" > %(installdir)s/bin/t5.sh',", + # shebang python + " 'echo \"#!/usr/bin/env python\\n# test\" > %(installdir)s/bin/b1.py',", + # shebang perl + " 'echo \"#!/usr/bin/env perl\\n# test\" > %(installdir)s/bin/b1.pl',", + "]", - "fix_python_shebang_for = ['bin/t1.py', 'bin/*.py', 'nosuchdir/*.py', 'bin/toy.python', 'bin/b1.sh']", - "fix_perl_shebang_for = ['bin/*.pl', 'bin/b2.sh', 'bin/toy.perl']", + "fix_python_shebang_for = ['bin/t1.py', 'bin/t*.py', 'nosuchdir/*.py', 'bin/toy.python', 'bin/b1.sh']", + "fix_perl_shebang_for = ['bin/t*.pl', 'bin/b2.sh', 'bin/toy.perl']", + "fix_bash_shebang_for = ['bin/t*.sh', 'bin/b1.py', 'bin/b1.pl', 'bin/toy.sh']", ]) write_file(test_ec, test_ec_txt) self.test_toy_build(ec_file=test_ec, raise_error=True) @@ -2984,36 +3002,32 @@ def test_fix_shebang(self): # bin/toy and bin/toy2 should *not* be patched, since they're binary files toy_txt = read_file(os.path.join(toy_bindir, 'toy'), mode='rb') - for fn in ['toy.perl', 'toy.python']: + for fn in ['toy.sh', 'toy.perl', 'toy.python']: fn_txt = read_file(os.path.join(toy_bindir, fn), mode='rb') # no shebang added self.assertFalse(fn_txt.startswith(b"#!/")) # exact same file as original binary (untouched) self.assertEqual(toy_txt, fn_txt) + regexes = { } # no re.M, this should match at start of file! - py_shebang_regex = re.compile(r'^#!/usr/bin/env python\n# test$') - for pybin in ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py']: - pybin_path = os.path.join(toy_bindir, pybin) - pybin_txt = read_file(pybin_path) - self.assertTrue(py_shebang_regex.match(pybin_txt), - "Pattern '%s' found in %s: %s" % (py_shebang_regex.pattern, pybin_path, pybin_txt)) + regexes['py'] = re.compile(r'^#!/usr/bin/env python\n# test$') + regexes['pl'] = re.compile(r'^#!/usr/bin/env perl\n# test$') + regexes['sh'] = re.compile(r'^#!/usr/bin/env bash\n# test$') - # no re.M, this should match at start of file! - perl_shebang_regex = re.compile(r'^#!/usr/bin/env perl\n# test$') - for perlbin in ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl', 't7.pl']: - perlbin_path = os.path.join(toy_bindir, perlbin) - perlbin_txt = read_file(perlbin_path) - self.assertTrue(perl_shebang_regex.match(perlbin_txt), - "Pattern '%s' found in %s: %s" % (perl_shebang_regex.pattern, perlbin_path, perlbin_txt)) - - # There are 2 bash files which shouldn't be influenced by fix_shebang - bash_shebang_regex = re.compile(r'^#!/usr/bin/env bash\n# test$') - for bashbin in ['b1.sh', 'b2.sh']: - bashbin_path = os.path.join(toy_bindir, bashbin) - bashbin_txt = read_file(bashbin_path) - self.assertTrue(bash_shebang_regex.match(bashbin_txt), - "Pattern '%s' found in %s: %s" % (bash_shebang_regex.pattern, bashbin_path, bashbin_txt)) + + # all scripts should have a shebang that matches their extension + scripts = { } + scripts['py'] = ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py', 'b1.py'] + scripts['pl'] = ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl', 't7.pl', 'b1.pl'] + scripts['sh'] = ['t1.sh', 't2.sh', 't3.sh', 't4.sh', 't5.sh', 'b1.sh', 'b2.sh'] + + for ext in ['sh', 'pl', 'py']: + for script in scripts[ext]: + bin_path = os.path.join(toy_bindir, script) + bin_txt = read_file(bin_path) + self.assertTrue(regexes[ext].match(bin_txt), + "Pattern '%s' found in %s: %s" % (regexes[ext].pattern, bin_path, bin_txt)) # now test with a custom env command extra_args = ['--env-for-shebang=/usr/bin/env -S'] @@ -3023,36 +3037,31 @@ def test_fix_shebang(self): # bin/toy and bin/toy2 should *not* be patched, since they're binary files toy_txt = read_file(os.path.join(toy_bindir, 'toy'), mode='rb') - for fn in ['toy.perl', 'toy.python']: + for fn in ['toy.sh', 'toy.perl', 'toy.python']: fn_txt = read_file(os.path.join(toy_bindir, fn), mode='rb') # no shebang added self.assertFalse(fn_txt.startswith(b"#!/")) # exact same file as original binary (untouched) self.assertEqual(toy_txt, fn_txt) + regexes_S = { } # no re.M, this should match at start of file! - py_shebang_regex = re.compile(r'^#!/usr/bin/env -S python\n# test$') - for pybin in ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py']: - pybin_path = os.path.join(toy_bindir, pybin) - pybin_txt = read_file(pybin_path) - self.assertTrue(py_shebang_regex.match(pybin_txt), - "Pattern '%s' found in %s: %s" % (py_shebang_regex.pattern, pybin_path, pybin_txt)) + regexes_S['py'] = re.compile(r'^#!/usr/bin/env -S python\n# test$') + regexes_S['pl'] = re.compile(r'^#!/usr/bin/env -S perl\n# test$') + regexes_S['sh'] = re.compile(r'^#!/usr/bin/env -S bash\n# test$') + + for ext in ['sh', 'pl', 'py']: + for script in scripts[ext]: + bin_path = os.path.join(toy_bindir, script) + bin_txt = read_file(bin_path) + # the scripts b1.py, b1.pl, b1.sh, b2.sh should keep their original shebang + if script.startswith('b'): + self.assertTrue(regexes[ext].match(bin_txt), + "Pattern '%s' found in %s: %s" % (regexes[ext].pattern, bin_path, bin_txt)) + else: + self.assertTrue(regexes_S[ext].match(bin_txt), + "Pattern '%s' found in %s: %s" % (regexes_S[ext].pattern, bin_path, bin_txt)) - # no re.M, this should match at start of file! - perl_shebang_regex = re.compile(r'^#!/usr/bin/env -S perl\n# test$') - for perlbin in ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl', 't7.pl']: - perlbin_path = os.path.join(toy_bindir, perlbin) - perlbin_txt = read_file(perlbin_path) - self.assertTrue(perl_shebang_regex.match(perlbin_txt), - "Pattern '%s' found in %s: %s" % (perl_shebang_regex.pattern, perlbin_path, perlbin_txt)) - - # There are 2 bash files which shouldn't be influenced by fix_shebang - bash_shebang_regex = re.compile(r'^#!/usr/bin/env bash\n# test$') - for bashbin in ['b1.sh', 'b2.sh']: - bashbin_path = os.path.join(toy_bindir, bashbin) - bashbin_txt = read_file(bashbin_path) - self.assertTrue(bash_shebang_regex.match(bashbin_txt), - "Pattern '%s' found in %s: %s" % (bash_shebang_regex.pattern, bashbin_path, bashbin_txt)) def test_toy_system_toolchain_alias(self): """Test use of 'system' toolchain alias.""" From 080acd34159c0241175484e1f0c92c4813c25aba Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 23 Aug 2021 18:57:02 +0000 Subject: [PATCH 500/864] appeasing hound --- test/framework/toy_build.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 698494ec55..d253b0f66b 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3009,15 +3009,14 @@ def test_fix_shebang(self): # exact same file as original binary (untouched) self.assertEqual(toy_txt, fn_txt) - regexes = { } + regexes = {} # no re.M, this should match at start of file! regexes['py'] = re.compile(r'^#!/usr/bin/env python\n# test$') regexes['pl'] = re.compile(r'^#!/usr/bin/env perl\n# test$') regexes['sh'] = re.compile(r'^#!/usr/bin/env bash\n# test$') - # all scripts should have a shebang that matches their extension - scripts = { } + scripts = {} scripts['py'] = ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py', 'b1.py'] scripts['pl'] = ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl', 't7.pl', 'b1.pl'] scripts['sh'] = ['t1.sh', 't2.sh', 't3.sh', 't4.sh', 't5.sh', 'b1.sh', 'b2.sh'] @@ -3044,7 +3043,7 @@ def test_fix_shebang(self): # exact same file as original binary (untouched) self.assertEqual(toy_txt, fn_txt) - regexes_S = { } + regexes_S = {} # no re.M, this should match at start of file! regexes_S['py'] = re.compile(r'^#!/usr/bin/env -S python\n# test$') regexes_S['pl'] = re.compile(r'^#!/usr/bin/env -S perl\n# test$') From a2cb1bc652bd8af3da7ceedee63c9cb5500ce4cf Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 23 Aug 2021 20:35:37 +0000 Subject: [PATCH 501/864] removed extra blank line --- test/framework/toy_build.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index d253b0f66b..4e55a3e9f0 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3061,7 +3061,6 @@ def test_fix_shebang(self): self.assertTrue(regexes_S[ext].match(bin_txt), "Pattern '%s' found in %s: %s" % (regexes_S[ext].pattern, bin_path, bin_txt)) - def test_toy_system_toolchain_alias(self): """Test use of 'system' toolchain alias.""" toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') From dff7ec589dde3b5a2c169189d7b3eb9db2123a4d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 7 Jul 2021 18:11:31 +0200 Subject: [PATCH 502/864] Filter out duplicate paths added to module files --- easybuild/framework/easyblock.py | 6 +-- easybuild/tools/module_generator.py | 64 ++++++++++++++++++++++++----- test/framework/easyblock.py | 9 +++- 3 files changed, 65 insertions(+), 14 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index d8829d8587..4fdaa99323 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3210,9 +3210,7 @@ def make_module_step(self, fake=False): else: trace_msg("generating module file @ %s" % self.mod_filepath) - txt = self.module_generator.MODULE_SHEBANG - if txt: - txt += '\n' + txt = self.module_generator.prepare_module_creation() if self.modules_header: txt += self.modules_header + '\n' @@ -3226,6 +3224,8 @@ def make_module_step(self, fake=False): txt += self.make_module_extra() txt += self.make_module_footer() + self.module_generator.finalize_module_creation() + hook_txt = run_hook(MODULE_WRITE, self.hooks, args=[self, mod_filepath, txt]) if hook_txt is not None: txt = hook_txt diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 685f158850..8134c8a1f6 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -136,17 +136,21 @@ def __init__(self, application, fake=False): self.fake_mod_path = tempfile.mkdtemp() self.modules_tool = modules_tool() + self.added_paths = None + + def prepare_module_creation(self): + """Prepares creating a module and returns the file header (shebang) if any including the newline""" + if self.added_paths is not None: + raise EasyBuildError('Module creation already in process. Did you forget to finalize the module?') + self.added_paths = set() # Clear all + txt = self.MODULE_SHEBANG + if txt: + txt += '\n' + return txt - def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True): - """ - Generate append-path statements for the given list of paths. - - :param key: environment variable to append paths to - :param paths: list of paths to append - :param allow_abs: allow providing of absolute paths - :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir) - """ - return self.update_paths(key, paths, prepend=False, allow_abs=allow_abs, expand_relpaths=expand_relpaths) + def finalize_module_creation(self): + """Finish creating a module. Must be called when done with the generator""" + self.added_paths = None def create_symlinks(self, mod_symlink_paths, fake=False): """Create moduleclass symlink(s) to actual module file.""" @@ -191,6 +195,43 @@ def get_modules_path(self, fake=False, mod_path_suffix=None): return os.path.join(mod_path, mod_path_suffix) + def _filter_paths(self, paths): + """Filter out already added paths and return the remaining ones""" + if self.added_paths is None: + raise EasyBuildError('Module creation has not been started. Call prepare_module_creation first!') + + # paths can be a string + if isinstance(paths, string_type): + if paths in self.added_paths: + filtered_paths = None + else: + self.added_paths.add(paths) + filtered_paths = paths + else: + # Coerce any iterable/generator into a list + if not isinstance(paths, list): + paths = list(paths) + filtered_paths = [x for x in paths if x not in self.added_paths and not self.added_paths.add(x)] + if filtered_paths != paths: + removed_paths = paths if filtered_paths is None else [x for x in paths if x not in filtered_paths] + self.log.warning("Supressed adding the following path(s) to the module as they were already added: %s", + removed_paths) + return filtered_paths + + def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True): + """ + Generate append-path statements for the given list of paths. + + :param key: environment variable to append paths to + :param paths: list of paths to append + :param allow_abs: allow providing of absolute paths + :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir) + """ + paths = self._filter_paths(paths) + if not paths: + return '' + return self.update_paths(key, paths, prepend=False, allow_abs=allow_abs, expand_relpaths=expand_relpaths) + def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True): """ Generate prepend-path statements for the given list of paths. @@ -200,6 +241,9 @@ def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True): :param allow_abs: allow providing of absolute paths :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir) """ + paths = self._filter_paths(paths) + if not paths: + return '' return self.update_paths(key, paths, prepend=True, allow_abs=allow_abs, expand_relpaths=expand_relpaths) def _modulerc_check_module_version(self, module_version): diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 18bfcf07b0..8310fae89e 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1152,7 +1152,7 @@ def test_make_module_step(self): # purposely use a 'nasty' description, that includes (unbalanced) special chars: [, ], {, } descr = "This {is a}} [fancy]] [[description]]. {{[[TEST}]" modextravars = {'PI': '3.1415', 'FOO': 'bar'} - modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'} + modextrapaths = {'PATH': ('bin', 'pibin'), 'CPATH': 'pi/include'} self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "%s"' % name, @@ -1177,6 +1177,10 @@ def test_make_module_step(self): eb.make_builddir() eb.prepare_step() + # Create a dummy file in bin to test if the duplicate entry of modextrapaths is ignored + os.mkdir(os.path.join(eb.installdir, 'bin')) + write_file(os.path.join(eb.installdir, 'bin', 'dummy_exe'), 'hello') + modpath = os.path.join(eb.make_module_step(), name, version) if get_module_syntax() == 'Lua': modpath += '.lua' @@ -1218,6 +1222,9 @@ def test_make_module_step(self): else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) + # Check for duplicates + num_prepends = len(regex.finditer(txt)) + self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt)) for (name, ver) in [('GCC', '6.4.0-2.28')]: if get_module_syntax() == 'Tcl': From e026a09ad62cf084997cbaec2b85f3f62a1c66cf Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 7 Jul 2021 18:14:29 +0200 Subject: [PATCH 503/864] Use print_warning --- easybuild/tools/module_generator.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 8134c8a1f6..ad3e36d4d6 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -41,7 +41,7 @@ from textwrap import wrap from easybuild.base import fancylogger -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option, get_module_syntax, install_path from easybuild.tools.filetools import convert_name, mkdir, read_file, remove_file, resolve_path, symlink, write_file from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, EnvironmentModulesC, Lmod, modules_tool @@ -214,8 +214,9 @@ def _filter_paths(self, paths): filtered_paths = [x for x in paths if x not in self.added_paths and not self.added_paths.add(x)] if filtered_paths != paths: removed_paths = paths if filtered_paths is None else [x for x in paths if x not in filtered_paths] - self.log.warning("Supressed adding the following path(s) to the module as they were already added: %s", - removed_paths) + print_warning("Supressed adding the following path(s) to the module as they were already added: %s", + removed_paths, + log=self.log) return filtered_paths def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True): From a4dd70e8de57ba2dd1b4c0d0abac66fe631223cb Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 7 Jul 2021 18:22:18 +0200 Subject: [PATCH 504/864] Use a contextmanager instead --- easybuild/framework/easyblock.py | 27 ++++++++++++--------------- easybuild/tools/module_generator.py | 23 +++++++++++++++-------- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 4fdaa99323..6c2d92a657 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3210,21 +3210,18 @@ def make_module_step(self, fake=False): else: trace_msg("generating module file @ %s" % self.mod_filepath) - txt = self.module_generator.prepare_module_creation() - - if self.modules_header: - txt += self.modules_header + '\n' - - txt += self.make_module_description() - txt += self.make_module_group_check() - txt += self.make_module_deppaths() - txt += self.make_module_dep() - txt += self.make_module_extend_modpath() - txt += self.make_module_req() - txt += self.make_module_extra() - txt += self.make_module_footer() - - self.module_generator.finalize_module_creation() + with self.module_generator.start_module_creation() as txt: + if self.modules_header: + txt += self.modules_header + '\n' + + txt += self.make_module_description() + txt += self.make_module_group_check() + txt += self.make_module_deppaths() + txt += self.make_module_dep() + txt += self.make_module_extend_modpath() + txt += self.make_module_req() + txt += self.make_module_extra() + txt += self.make_module_footer() hook_txt = run_hook(MODULE_WRITE, self.hooks, args=[self, mod_filepath, txt]) if hook_txt is not None: diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index ad3e36d4d6..f4b4634a8f 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -37,6 +37,7 @@ import os import re import tempfile +from contextlib import contextmanager from distutils.version import LooseVersion from textwrap import wrap @@ -138,19 +139,25 @@ def __init__(self, application, fake=False): self.modules_tool = modules_tool() self.added_paths = None - def prepare_module_creation(self): - """Prepares creating a module and returns the file header (shebang) if any including the newline""" + @contextmanager + def start_module_creation(self): + """ + Prepares creating a module and returns the file header (shebang) if any including the newline + + Meant to be used in a with statement: + with generator.start_module_creation() as txt: + # Write txt + """ if self.added_paths is not None: - raise EasyBuildError('Module creation already in process. Did you forget to finalize the module?') + raise EasyBuildError('Module creation already in process. You cannot create multiple modules at the same time!') self.added_paths = set() # Clear all txt = self.MODULE_SHEBANG if txt: txt += '\n' - return txt - - def finalize_module_creation(self): - """Finish creating a module. Must be called when done with the generator""" - self.added_paths = None + try: + yield txt + finally: + self.added_paths = None def create_symlinks(self, mod_symlink_paths, fake=False): """Create moduleclass symlink(s) to actual module file.""" From 75a41952f38d21f379466b35ab733fd1bf21e289 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 7 Jul 2021 18:24:02 +0200 Subject: [PATCH 505/864] Add line break --- easybuild/tools/module_generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index f4b4634a8f..0623ddaf6b 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -149,7 +149,8 @@ def start_module_creation(self): # Write txt """ if self.added_paths is not None: - raise EasyBuildError('Module creation already in process. You cannot create multiple modules at the same time!') + raise EasyBuildError('Module creation already in process. ' + 'You cannot create multiple modules at the same time!') self.added_paths = set() # Clear all txt = self.MODULE_SHEBANG if txt: From 88851026173bd1403f48507562d9bd0d15f28893 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Jul 2021 11:13:41 +0200 Subject: [PATCH 506/864] Check paths per key not globally --- easybuild/tools/module_generator.py | 34 ++++++++++++++++------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 0623ddaf6b..12d6ba4d26 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -137,7 +137,7 @@ def __init__(self, application, fake=False): self.fake_mod_path = tempfile.mkdtemp() self.modules_tool = modules_tool() - self.added_paths = None + self.added_paths_per_key = None @contextmanager def start_module_creation(self): @@ -148,17 +148,18 @@ def start_module_creation(self): with generator.start_module_creation() as txt: # Write txt """ - if self.added_paths is not None: + if self.added_paths_per_key is not None: raise EasyBuildError('Module creation already in process. ' 'You cannot create multiple modules at the same time!') - self.added_paths = set() # Clear all + # Mapping of keys/env vars to paths already added + self.added_paths_per_key = dict() txt = self.MODULE_SHEBANG if txt: txt += '\n' try: yield txt finally: - self.added_paths = None + self.added_paths_per_key = None def create_symlinks(self, mod_symlink_paths, fake=False): """Create moduleclass symlink(s) to actual module file.""" @@ -203,27 +204,30 @@ def get_modules_path(self, fake=False, mod_path_suffix=None): return os.path.join(mod_path, mod_path_suffix) - def _filter_paths(self, paths): - """Filter out already added paths and return the remaining ones""" - if self.added_paths is None: - raise EasyBuildError('Module creation has not been started. Call prepare_module_creation first!') + def _filter_paths(self, key, paths): + """Filter out paths already added to key and return the remaining ones""" + if self.added_paths_per_key is None: + # For compatibility this is only a warning for now and we don't filter any paths + print_warning('Module creation has not been started. Call prepare_module_creation first!') + return paths + added_paths = self.added_paths_per_key.setdefault(key, set()) # paths can be a string if isinstance(paths, string_type): - if paths in self.added_paths: + if paths in added_paths: filtered_paths = None else: - self.added_paths.add(paths) + added_paths.add(paths) filtered_paths = paths else: # Coerce any iterable/generator into a list if not isinstance(paths, list): paths = list(paths) - filtered_paths = [x for x in paths if x not in self.added_paths and not self.added_paths.add(x)] + filtered_paths = [x for x in paths if x not in added_paths and not added_paths.add(x)] if filtered_paths != paths: removed_paths = paths if filtered_paths is None else [x for x in paths if x not in filtered_paths] - print_warning("Supressed adding the following path(s) to the module as they were already added: %s", - removed_paths, + print_warning("Supressed adding the following path(s) to $%s of the module as they were already added: %s", + key, removed_paths, log=self.log) return filtered_paths @@ -236,7 +240,7 @@ def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True): :param allow_abs: allow providing of absolute paths :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir) """ - paths = self._filter_paths(paths) + paths = self._filter_paths(key, paths) if not paths: return '' return self.update_paths(key, paths, prepend=False, allow_abs=allow_abs, expand_relpaths=expand_relpaths) @@ -250,7 +254,7 @@ def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True): :param allow_abs: allow providing of absolute paths :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir) """ - paths = self._filter_paths(paths) + paths = self._filter_paths(key, paths) if not paths: return '' return self.update_paths(key, paths, prepend=True, allow_abs=allow_abs, expand_relpaths=expand_relpaths) From eace6817013d006d07aa8ec0a8d79c60ee04f40f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Jul 2021 11:13:49 +0200 Subject: [PATCH 507/864] Fix test --- test/framework/easyblock.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 8310fae89e..9f325a91ab 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1178,7 +1178,7 @@ def test_make_module_step(self): eb.prepare_step() # Create a dummy file in bin to test if the duplicate entry of modextrapaths is ignored - os.mkdir(os.path.join(eb.installdir, 'bin')) + os.makedirs(os.path.join(eb.installdir, 'bin')) write_file(os.path.join(eb.installdir, 'bin', 'dummy_exe'), 'hello') modpath = os.path.join(eb.make_module_step(), name, version) @@ -1214,17 +1214,20 @@ def test_make_module_step(self): self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) - for (key, val) in modextrapaths.items(): - if get_module_syntax() == 'Tcl': - regex = re.compile(r'^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M) - elif get_module_syntax() == 'Lua': - regex = re.compile(r'^prepend_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M) - else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) - self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) - # Check for duplicates - num_prepends = len(regex.finditer(txt)) - self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt)) + for (key, vals) in modextrapaths.items(): + if isinstance(vals, string_type): + vals = [vals] + for val in vals: + if get_module_syntax() == 'Tcl': + regex = re.compile(r'^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M) + elif get_module_syntax() == 'Lua': + regex = re.compile(r'^prepend_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) + # Check for duplicates + num_prepends = len(regex.findall(txt)) + self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt)) for (name, ver) in [('GCC', '6.4.0-2.28')]: if get_module_syntax() == 'Tcl': From cd0a7b3dcc82ba6ea3043fb2b0cf657e59a48684 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 9 Jul 2021 11:28:23 +0200 Subject: [PATCH 508/864] Fix extensions writing to modules --- easybuild/framework/easyblock.py | 3 ++- easybuild/tools/module_generator.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 6c2d92a657..35756b8b51 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2441,7 +2441,8 @@ def extensions_step(self, fetch=False, install=True): if install: try: ext.prerun() - txt = ext.run() + with self.module_generator.start_module_creation(): + txt = ext.run() if txt: self.module_extra_extensions += txt ext.postrun() diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 12d6ba4d26..87c0208024 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -208,7 +208,7 @@ def _filter_paths(self, key, paths): """Filter out paths already added to key and return the remaining ones""" if self.added_paths_per_key is None: # For compatibility this is only a warning for now and we don't filter any paths - print_warning('Module creation has not been started. Call prepare_module_creation first!') + print_warning('Module creation has not been started. Call start_module_creation first!') return paths added_paths = self.added_paths_per_key.setdefault(key, set()) From e7a211c95e08e9952b84f994da586876ff8d9b84 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 16 Jul 2021 12:43:29 +0200 Subject: [PATCH 509/864] Fix adding root path (empty string) --- easybuild/tools/module_generator.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 87c0208024..59444b273e 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -229,6 +229,8 @@ def _filter_paths(self, key, paths): print_warning("Supressed adding the following path(s) to $%s of the module as they were already added: %s", key, removed_paths, log=self.log) + if not filtered_paths: + filtered_paths = None return filtered_paths def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True): @@ -241,7 +243,7 @@ def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True): :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir) """ paths = self._filter_paths(key, paths) - if not paths: + if paths is None: return '' return self.update_paths(key, paths, prepend=False, allow_abs=allow_abs, expand_relpaths=expand_relpaths) @@ -255,7 +257,7 @@ def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True): :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir) """ paths = self._filter_paths(key, paths) - if not paths: + if paths is None: return '' return self.update_paths(key, paths, prepend=True, allow_abs=allow_abs, expand_relpaths=expand_relpaths) From 3048a4971bb349b531c574b1b9bade685bf6f966 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 16 Jul 2021 12:43:38 +0200 Subject: [PATCH 510/864] Update tests to use filtering module generator --- test/framework/easyblock.py | 21 +++++++---- test/framework/module_generator.py | 56 +++++++++++++++++------------- 2 files changed, 46 insertions(+), 31 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 9f325a91ab..7393a35c0d 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -435,7 +435,8 @@ def test_make_module_req(self): # this is not a path that should be picked up os.mkdir(os.path.join(eb.installdir, 'CPATH')) - guess = eb.make_module_req() + with eb.module_generator.start_module_creation(): + guess = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M)) @@ -462,7 +463,8 @@ def test_make_module_req(self): # check that bin is only added to PATH if there are files in there write_file(os.path.join(eb.installdir, 'bin', 'test'), 'test') - guess = eb.make_module_req() + with eb.module_generator.start_module_creation(): + guess = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertTrue(re.search(r"^prepend-path\s+PATH\s+\$root/bin$", guess, re.M)) self.assertFalse(re.search(r"^prepend-path\s+PATH\s+\$root/sbin$", guess, re.M)) @@ -481,7 +483,8 @@ def test_make_module_req(self): self.assertFalse('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))' in guess) # -- With files write_file(os.path.join(eb.installdir, 'lib64', 'libfoo.so'), 'test') - guess = eb.make_module_req() + with eb.module_generator.start_module_creation(): + guess = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertTrue(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib64$", guess, re.M)) elif get_module_syntax() == 'Lua': @@ -490,7 +493,8 @@ def test_make_module_req(self): write_file(os.path.join(eb.installdir, 'lib', 'libfoo.so'), 'test') shutil.rmtree(os.path.join(eb.installdir, 'lib64')) os.symlink('lib', os.path.join(eb.installdir, 'lib64')) - guess = eb.make_module_req() + with eb.module_generator.start_module_creation(): + guess = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertFalse(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib64$", guess, re.M)) elif get_module_syntax() == 'Lua': @@ -509,7 +513,8 @@ def test_make_module_req(self): # check for behavior when a string value is used as dict value by make_module_req_guesses eb.make_module_req_guess = lambda: {'PATH': 'bin'} - txt = eb.make_module_req() + with eb.module_generator.start_module_creation(): + txt = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertTrue(re.match(r"^\nprepend-path\s+PATH\s+\$root/bin\n$", txt, re.M)) elif get_module_syntax() == 'Lua': @@ -520,7 +525,8 @@ def test_make_module_req(self): # check for correct behaviour if empty string is specified as one of the values # prepend-path statements should be included for both the 'bin' subdir and the install root eb.make_module_req_guess = lambda: {'PATH': ['bin', '']} - txt = eb.make_module_req() + with eb.module_generator.start_module_creation(): + txt = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertTrue(re.search(r"\nprepend-path\s+PATH\s+\$root/bin\n", txt, re.M)) self.assertTrue(re.search(r"\nprepend-path\s+PATH\s+\$root\n", txt, re.M)) @@ -535,7 +541,8 @@ def test_make_module_req(self): for path in ['pathA', 'pathB', 'pathC']: os.mkdir(os.path.join(eb.installdir, 'lib', path)) write_file(os.path.join(eb.installdir, 'lib', path, 'libfoo.so'), 'test') - txt = eb.make_module_req() + with eb.module_generator.start_module_creation(): + txt = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertTrue(re.search(r"\nprepend-path\s+LD_LIBRARY_PATH\s+\$root/lib/pathC\n" + r"prepend-path\s+LD_LIBRARY_PATH\s+\$root/lib/pathA\n" + diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 517245e741..4bfa3c3344 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -670,6 +670,10 @@ def test_swap(self): def test_append_paths(self): """Test generating append-paths statements.""" # test append_paths + def append_paths(*args, **kwargs): + """Wrap this into start_module_creation which need to be called prior to append_paths""" + with self.modgen.start_module_creation(): + return self.modgen.append_paths(*args, **kwargs) if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: expected = ''.join([ @@ -678,17 +682,17 @@ def test_append_paths(self): "append-path\tkey\t\t$root\n", ]) paths = ['path1', 'path2', ''] - self.assertEqual(expected, self.modgen.append_paths("key", paths)) + self.assertEqual(expected, append_paths("key", paths)) # 2nd call should still give same result, no side-effects like manipulating passed list 'paths'! - self.assertEqual(expected, self.modgen.append_paths("key", paths)) + self.assertEqual(expected, append_paths("key", paths)) expected = "append-path\tbar\t\t$root/foo\n" - self.assertEqual(expected, self.modgen.append_paths("bar", "foo")) + self.assertEqual(expected, append_paths("bar", "foo")) - res = self.modgen.append_paths("key", ["/abs/path"], allow_abs=True) + res = append_paths("key", ["/abs/path"], allow_abs=True) self.assertEqual("append-path\tkey\t\t/abs/path\n", res) - res = self.modgen.append_paths('key', ['1234@example.com'], expand_relpaths=False) + res = append_paths('key', ['1234@example.com'], expand_relpaths=False) self.assertEqual("append-path\tkey\t\t1234@example.com\n", res) else: @@ -698,22 +702,22 @@ def test_append_paths(self): 'append_path("key", root)\n', ]) paths = ['path1', 'path2', ''] - self.assertEqual(expected, self.modgen.append_paths("key", paths)) + self.assertEqual(expected, append_paths("key", paths)) # 2nd call should still give same result, no side-effects like manipulating passed list 'paths'! - self.assertEqual(expected, self.modgen.append_paths("key", paths)) + self.assertEqual(expected, append_paths("key", paths)) expected = 'append_path("bar", pathJoin(root, "foo"))\n' - self.assertEqual(expected, self.modgen.append_paths("bar", "foo")) + self.assertEqual(expected, append_paths("bar", "foo")) expected = 'append_path("key", "/abs/path")\n' - self.assertEqual(expected, self.modgen.append_paths("key", ["/abs/path"], allow_abs=True)) + self.assertEqual(expected, append_paths("key", ["/abs/path"], allow_abs=True)) - res = self.modgen.append_paths('key', ['1234@example.com'], expand_relpaths=False) + res = append_paths('key', ['1234@example.com'], expand_relpaths=False) self.assertEqual('append_path("key", "1234@example.com")\n', res) self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to update_paths " - "which only expects relative paths." % self.modgen.app.installdir, - self.modgen.append_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir]) + "which only expects relative paths." % self.modgen.app.installdir, + append_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir]) def test_module_extensions(self): """test the extensions() for extensions""" @@ -745,6 +749,10 @@ def test_module_extensions(self): def test_prepend_paths(self): """Test generating prepend-paths statements.""" # test prepend_paths + def prepend_paths(*args, **kwargs): + """Wrap this into start_module_creation which need to be called prior to append_paths""" + with self.modgen.start_module_creation(): + return self.modgen.prepend_paths(*args, **kwargs) if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: expected = ''.join([ @@ -753,17 +761,17 @@ def test_prepend_paths(self): "prepend-path\tkey\t\t$root\n", ]) paths = ['path1', 'path2', ''] - self.assertEqual(expected, self.modgen.prepend_paths("key", paths)) + self.assertEqual(expected, prepend_paths("key", paths)) # 2nd call should still give same result, no side-effects like manipulating passed list 'paths'! - self.assertEqual(expected, self.modgen.prepend_paths("key", paths)) + self.assertEqual(expected, prepend_paths("key", paths)) expected = "prepend-path\tbar\t\t$root/foo\n" - self.assertEqual(expected, self.modgen.prepend_paths("bar", "foo")) + self.assertEqual(expected, prepend_paths("bar", "foo")) - res = self.modgen.prepend_paths("key", ["/abs/path"], allow_abs=True) + res = prepend_paths("key", ["/abs/path"], allow_abs=True) self.assertEqual("prepend-path\tkey\t\t/abs/path\n", res) - res = self.modgen.prepend_paths('key', ['1234@example.com'], expand_relpaths=False) + res = prepend_paths('key', ['1234@example.com'], expand_relpaths=False) self.assertEqual("prepend-path\tkey\t\t1234@example.com\n", res) else: @@ -773,22 +781,22 @@ def test_prepend_paths(self): 'prepend_path("key", root)\n', ]) paths = ['path1', 'path2', ''] - self.assertEqual(expected, self.modgen.prepend_paths("key", paths)) + self.assertEqual(expected, prepend_paths("key", paths)) # 2nd call should still give same result, no side-effects like manipulating passed list 'paths'! - self.assertEqual(expected, self.modgen.prepend_paths("key", paths)) + self.assertEqual(expected, prepend_paths("key", paths)) expected = 'prepend_path("bar", pathJoin(root, "foo"))\n' - self.assertEqual(expected, self.modgen.prepend_paths("bar", "foo")) + self.assertEqual(expected, prepend_paths("bar", "foo")) expected = 'prepend_path("key", "/abs/path")\n' - self.assertEqual(expected, self.modgen.prepend_paths("key", ["/abs/path"], allow_abs=True)) + self.assertEqual(expected, prepend_paths("key", ["/abs/path"], allow_abs=True)) - res = self.modgen.prepend_paths('key', ['1234@example.com'], expand_relpaths=False) + res = prepend_paths('key', ['1234@example.com'], expand_relpaths=False) self.assertEqual('prepend_path("key", "1234@example.com")\n', res) self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to update_paths " - "which only expects relative paths." % self.modgen.app.installdir, - self.modgen.prepend_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir]) + "which only expects relative paths." % self.modgen.app.installdir, + prepend_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir]) def test_det_user_modpath(self): """Test for generic det_user_modpath method.""" From 146773e7b16e02b4b46fdc32b9221ae2da9e56cb Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 16 Jul 2021 12:46:32 +0200 Subject: [PATCH 511/864] Fix formatting --- test/framework/module_generator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 4bfa3c3344..c046180883 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -716,8 +716,8 @@ def append_paths(*args, **kwargs): self.assertEqual('append_path("key", "1234@example.com")\n', res) self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to update_paths " - "which only expects relative paths." % self.modgen.app.installdir, - append_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir]) + "which only expects relative paths." % self.modgen.app.installdir, + append_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir]) def test_module_extensions(self): """test the extensions() for extensions""" @@ -795,8 +795,8 @@ def prepend_paths(*args, **kwargs): self.assertEqual('prepend_path("key", "1234@example.com")\n', res) self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to update_paths " - "which only expects relative paths." % self.modgen.app.installdir, - prepend_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir]) + "which only expects relative paths." % self.modgen.app.installdir, + prepend_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir]) def test_det_user_modpath(self): """Test for generic det_user_modpath method.""" From dda3a48aec2dd6ce813565d3724c1a9cfdb86e66 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 30 Aug 2021 16:46:04 +0200 Subject: [PATCH 512/864] Speedup EasyBlock.set_parallel - Small refactoring to reduce dictionary access to cfg[parallel] option - Cache default parallel value in det_parallelism to avoid systemcalls --- easybuild/framework/easyblock.py | 16 +++++++------ easybuild/tools/systemtools.py | 41 ++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index d8829d8587..22a13e630a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1791,18 +1791,20 @@ def set_parallel(self): """Set 'parallel' easyconfig parameter to determine how many cores can/should be used for parallel builds.""" # set level of parallelism for build par = build_option('parallel') - if self.cfg['parallel'] is not None: + cfg_par = self.cfg['parallel'] + if cfg_par is None: + self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par) + else: if par is None: - par = self.cfg['parallel'] + par = cfg_par self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par) else: - par = min(int(par), int(self.cfg['parallel'])) + par = min(int(par), int(cfg_par)) self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par) - else: - self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par) - self.cfg['parallel'] = det_parallelism(par=par, maxpar=self.cfg['maxparallel']) - self.log.info("Setting parallelism: %s" % self.cfg['parallel']) + par = det_parallelism(par, maxpar=self.cfg['maxparallel']) + self.log.info("Setting parallelism: %s" % par) + self.cfg['parallel'] = par def remove_module_file(self): """Remove module file (if it exists), and check for ghost installation directory (and deal with it).""" diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 108cdb4571..16d471f37c 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -976,31 +976,42 @@ def det_parallelism(par=None, maxpar=None): Determine level of parallelism that should be used. Default: educated guess based on # cores and 'ulimit -u' setting: min(# cores, ((ulimit -u) - 15) // 6) """ - if par is not None: - if not isinstance(par, int): + def get_default_parallelism(): + try: + # Get cache value if any + par = det_parallelism._default_parallelism + except AttributeError: + # No cache -> Calculate value from current system values + par = get_avail_core_count() + # check ulimit -u + out, ec = run_cmd('ulimit -u', force_in_dry_run=True, trace=False, stream_output=False) try: - par = int(par) + if out.startswith("unlimited"): + maxuserproc = 2 ** 32 - 1 + else: + maxuserproc = int(out) except ValueError as err: - raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err) - else: - par = get_avail_core_count() - # check ulimit -u - out, ec = run_cmd('ulimit -u', force_in_dry_run=True, trace=False, stream_output=False) - try: - if out.startswith("unlimited"): - out = 2 ** 32 - 1 - maxuserproc = int(out) + raise EasyBuildError("Failed to determine max user processes (%s, %s): %s", ec, out, err) # assume 6 processes per build thread + 15 overhead - par_guess = int((maxuserproc - 15) // 6) + par_guess = (maxuserproc - 15) // 6 if par_guess < par: par = par_guess _log.info("Limit parallel builds to %s because max user processes is %s" % (par, out)) + # Cache value + det_parallelism._default_parallelism = par + return par + + if par is None: + par = get_default_parallelism() + else: + try: + par = int(par) except ValueError as err: - raise EasyBuildError("Failed to determine max user processes (%s, %s): %s", ec, out, err) + raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err) if maxpar is not None and maxpar < par: _log.info("Limiting parallellism from %s to %s" % (par, maxpar)) - par = min(par, maxpar) + par = maxpar return par From 0f19f1c664160d7411bb49d80687d4c80c5af8c3 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 30 Aug 2021 18:14:59 +0200 Subject: [PATCH 513/864] handle test failure --- test/framework/easyconfig.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 96ce325e6e..48aaeb86a4 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3052,6 +3052,10 @@ def test_template_constant_dict(self): self.assertEqual(res, expected) # mock get_avail_core_count which is used by set_parallel -> det_parallelism + try: + del st.det_parallelism._default_parallelism # Remove cache value + except AttributeError: + pass # Ignore if not present orig_get_avail_core_count = st.get_avail_core_count st.get_avail_core_count = lambda: 42 From 04204a14d44ef2ccc469e8362ec1034b997a3a94 Mon Sep 17 00:00:00 2001 From: Christoph Siegert Date: Tue, 31 Aug 2021 14:53:30 +0200 Subject: [PATCH 514/864] add --easystack to ignore_opts for submit_job() --- easybuild/tools/parallelbuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py index df151aa3a1..0621a29516 100644 --- a/easybuild/tools/parallelbuild.py +++ b/easybuild/tools/parallelbuild.py @@ -127,7 +127,7 @@ def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True): curdir = os.getcwd() # regex pattern for options to ignore (help options can't reach here) - ignore_opts = re.compile('^--robot$|^--job|^--try-.*$') + ignore_opts = re.compile('^--robot$|^--job|^--try-.*$|^--easystack$') # generate_cmd_line returns the options in form --longopt=value opts = [o for o in cmd_line_opts if not ignore_opts.match(o.split('=')[0])] From a99001b2faa24846c4727604ca24a5f6a7e69734 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 31 Aug 2021 17:29:14 +0200 Subject: [PATCH 515/864] trivial style fixes --- easybuild/framework/easyblock.py | 11 +++++------ easybuild/tools/systemtools.py | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 22a13e630a..436ac373e3 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1794,13 +1794,12 @@ def set_parallel(self): cfg_par = self.cfg['parallel'] if cfg_par is None: self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par) + elif par is None: + par = cfg_par + self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par) else: - if par is None: - par = cfg_par - self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par) - else: - par = min(int(par), int(cfg_par)) - self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par) + par = min(int(par), int(cfg_par)) + self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par) par = det_parallelism(par, maxpar=self.cfg['maxparallel']) self.log.info("Setting parallelism: %s" % par) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 16d471f37c..a43338e147 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -996,7 +996,7 @@ def get_default_parallelism(): par_guess = (maxuserproc - 15) // 6 if par_guess < par: par = par_guess - _log.info("Limit parallel builds to %s because max user processes is %s" % (par, out)) + _log.info("Limit parallel builds to %s because max user processes is %s", par, out) # Cache value det_parallelism._default_parallelism = par return par @@ -1010,7 +1010,7 @@ def get_default_parallelism(): raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err) if maxpar is not None and maxpar < par: - _log.info("Limiting parallellism from %s to %s" % (par, maxpar)) + _log.info("Limiting parallellism from %s to %s", par, maxpar) par = maxpar return par From 6a831521af6be42d103acdf157421d6d680fe502 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 1 Sep 2021 15:48:31 +0200 Subject: [PATCH 516/864] trivial code style tweaking in get_source_tarball_from_git --- easybuild/tools/filetools.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 983abdcb76..635e897ee9 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2481,7 +2481,8 @@ def get_source_tarball_from_git(filename, targetdir, git_config): if recursive: clone_cmd.append('--recursive') else: - clone_cmd.append('--no-checkout') # We do that manually below + # checkout is done separately below for specific commits + clone_cmd.append('--no-checkout') clone_cmd.append('%s/%s.git' % (url, repo_name)) @@ -2496,6 +2497,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): checkout_cmd.extend(['&&', 'git', 'submodule', 'update', '--init', '--recursive']) run.run_cmd(' '.join(checkout_cmd), log_all=True, simple=True, regexp=False, path=repo_name) + elif not build_option('extended_dry_run'): # If we wanted to get a tag make sure we actually got a tag and not a branch with the same name # This doesn't make sense in dry-run mode as we don't have anything to check @@ -2507,10 +2509,13 @@ def get_source_tarball_from_git(filename, targetdir, git_config): ' with the same name. You might want to alert the maintainers of %s about that issue.', tag, url, repo_name, repo_name) cmds = [] + if not keep_git_dir: - # Make the repo unshallow, same as git fetch --unshallow in git 1.8.3+ - # The first fetch seemingly does nothing, no idea why. + # make the repo unshallow first; + # this is equivalent with 'git fetch -unshallow' in Git 1.8.3+ + # (first fetch seems to do nothing, unclear why) cmds.append('git fetch --depth=2147483647 && git fetch --depth=2147483647') + cmds.append('git checkout refs/tags/' + tag) # Clean all untracked files, e.g. from left-over submodules cmds.append('git clean --force -d -x') From e1bed5f198bb4aae86ddd845b94e1b6a91f3b5d4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 2 Sep 2021 10:40:11 +0200 Subject: [PATCH 517/864] clarify use of both exts_list and components in is_patch_for --- easybuild/tools/github.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 8f96f4f0e6..a495e8de9a 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1015,12 +1015,13 @@ def is_patch_for(patch_name, ec): patches = copy.copy(ec['patches']) with ec.disable_templating(): - for ext in itertools.chain(ec['exts_list'], ec.get('components', [])): - if isinstance(ext, (list, tuple)) and len(ext) == 3 and isinstance(ext[2], dict): - templates = {'name': ext[0], 'version': ext[1]} - ext_options = ext[2] + # take into account both list of extensions (via exts_list) and components (cfr. Bundle easyblock) + for entry in itertools.chain(ec['exts_list'], ec.get('components', [])): + if isinstance(entry, (list, tuple)) and len(entry) == 3 and isinstance(entry[2], dict): + templates = {'name': entry[0], 'version': entry[1]} + options = entry[2] patches.extend(p[0] % templates if isinstance(p, (tuple, list)) else p % templates - for p in ext_options.get('patches', [])) + for p in options.get('patches', [])) for patch in patches: if isinstance(patch, (tuple, list)): From e51c5f9104206eea87e5575d5bbd97b167deff44 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 2 Sep 2021 10:58:02 +0200 Subject: [PATCH 518/864] pick up $MODULES_CMD to facilitate using Environment Modules 4.x as modules tool (fixes #3815) --- easybuild/tools/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index d8df2b0db2..5e48591be5 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -187,7 +187,6 @@ def __init__(self, mod_paths=None, testing=False): self.cmd = env_cmd_path self.log.debug("Set %s command via environment variable %s: %s", self.NAME, self.COMMAND_ENVIRONMENT, self.cmd) - # check whether paths obtained via $PATH and $LMOD_CMD are different elif cmd_path != env_cmd_path: self.log.debug("Different paths found for %s command '%s' via which/$PATH and $%s: %s vs %s", self.NAME, self.COMMAND, self.COMMAND_ENVIRONMENT, cmd_path, env_cmd_path) @@ -1315,6 +1314,7 @@ class EnvironmentModules(EnvironmentModulesTcl): """Interface to environment modules 4.0+""" NAME = "Environment Modules v4" COMMAND = os.path.join(os.getenv('MODULESHOME', 'MODULESHOME_NOT_DEFINED'), 'libexec', 'modulecmd.tcl') + COMMAND_ENVIRONMENT = 'MODULES_CMD' REQ_VERSION = '4.0.0' MAX_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' From f232500a7f2224b9713078ba3a6c88a3bcdc03d7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 2 Sep 2021 16:58:01 +0200 Subject: [PATCH 519/864] use more sensible branch name for creating easyblocks PR with --new-pr --- easybuild/tools/github.py | 2 ++ test/framework/options.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index d49cbc2f5e..a381034816 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -901,6 +901,8 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ if pr_branch is None: if ec_paths and pr_target_repo == GITHUB_EASYCONFIGS_REPO: label = file_info['ecs'][0].name + re.sub('[.-]', '', file_info['ecs'][0].version) + elif pr_target_repo == GITHUB_EASYBLOCKS_REPO and paths.get('py_files'): + label = os.path.splitext(os.path.basename(paths['py_files'][0]))[0] else: label = ''.join(random.choice(ascii_letters) for _ in range(10)) pr_branch = '%s_new_pr_%s' % (time.strftime("%Y%m%d%H%M%S"), label) diff --git a/test/framework/options.py b/test/framework/options.py index f40829b34b..c9a64bfd0e 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4020,7 +4020,7 @@ def test_new_branch_github(self): regexs = [ r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-easyconfigs.git\.\.\.", r"^== copying files to .*/easybuild-easyconfigs\.\.\.", - r"^== pushing branch '.*' to remote '.*' \(%s\) \[DRY RUN\]" % remote, + r"^== pushing branch '[0-9]{14}_new_pr_toy00' to remote '.*' \(%s\) \[DRY RUN\]" % remote, ] self._assert_regexs(regexs, txt) @@ -4041,7 +4041,7 @@ def test_new_branch_github(self): regexs = [ r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-easyblocks.git\.\.\.", r"^== copying files to .*/easybuild-easyblocks\.\.\.", - r"^== pushing branch '.*' to remote '.*' \(%s\) \[DRY RUN\]" % remote, + r"^== pushing branch '[0-9]{14}_new_pr_toy' to remote '.*' \(%s\) \[DRY RUN\]" % remote, ] self._assert_regexs(regexs, txt) @@ -4068,7 +4068,7 @@ def test_new_branch_github(self): regexs = [ r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-framework.git\.\.\.", r"^== copying files to .*/easybuild-framework\.\.\.", - r"^== pushing branch '.*' to remote '.*' \(%s\) \[DRY RUN\]" % remote, + r"^== pushing branch '[0-9]{14}_new_pr_[A-Za-z]{10}' to remote '.*' \(%s\) \[DRY RUN\]" % remote, ] self._assert_regexs(regexs, txt) From a4cbc73f0158b8f670971a660ae39ab370e23934 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 6 Sep 2021 10:27:14 +0800 Subject: [PATCH 520/864] prepare release notes for EasyBuild v4.4.2 + bump version to 4.4.2 --- RELEASE_NOTES | 36 ++++++++++++++++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 402f9a8e9a..ee11381f72 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,42 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.4.2 (September 7th 2021) +--------------------------- + +update/bugfix release + +- add definition for new toolchain nvpsmpic (NVHPC + ParaStationMPI) (#3736) +- various enhancements, including: + - add per-extension timing in output produced by eb command (#3734) + - include list of missing libraries in warning about missing FFTW libraries in imkl toolchain component (#3776) + - check for recursive symlinks by default before copying a folder (#3784) + - add --filter-ecs options to filter out easyconfigs from set of easyconfigs to install (#3796) + - check type of source_tmpl value for extensions, ensure it's a string value (not a list) (#3799) + - also define $BLAS_SHARED_LIBS & co in build environment (analogous to $BLAS_STATIC_LIBS) (#3800) + - report use of --ignore-test-failure in success message in output (#3806) + - add get_cuda_cc_template_value method to EasyConfig class (#3807) + - add support for fix_bash_shebang_for (#3808) + - pick up $MODULES_CMD to facilitate using Environment Modules 4.x as modules tool (#3816) + - use more sensible branch name for creating easyblocks PR with --new-pr (#3817) +- various bug fixes, including: + - remove Python 2.6 from list of supported Python versions in setup.py (#3767) + - don't add directory that doesn't include any files to $PATH or $LD_LIBRARY_PATH (#3769) + - make logdir writable also when --stop/--fetch is used and --read-only-installdir is enabled (#3771) + - fix forgotten renaming of 'l' to 'char' __init__.py that is created for included Python modules (#3773) + - fix verify_imports by deleting all imported modules before re-importing them one by one (#3780) + - fix ignore_test_failure not set for Extensions (#3782) + - update iompi toolchain to intel-compiler subtoolchain for oneAPI versions (>= iompi 2020.12) (#3785) + - don't parse patch files as easyconfigs when searching for where patch file is used (#3786) + - make sure git clone with a tag argument actually downloads a tag (#3795) + - fix CI by excluding GC3Pie 2.6.7 (which is broken with Python 2) and improve error reporting for option parsing (#3798) + - speed up tests by caching checked paths in set_tmpdir + less test cases for test_compiler_dependent_optarch (#3802) + - correctly resolve templates for patches in extensions when uploading to GitHub (#3805) + - add --easystack to ignore_opts for submit_job() (#3813) +- other changes: + - speed up set_parallel method in EasyBlock class (#3812) + + v4.4.1 (July 6th 2021) ---------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 6e9468b67c..70aa973100 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.4.2.dev0') +VERSION = LooseVersion('4.4.2') UNKNOWN = 'UNKNOWN' From def7bd4b5ed7aa92b0220fd49f91fb558697c0a3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 6 Sep 2021 09:04:00 +0200 Subject: [PATCH 521/864] minor tweaks to v4.4.2 release notes --- RELEASE_NOTES | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index ee11381f72..e622dd198b 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -9,12 +9,12 @@ v4.4.2 (September 7th 2021) update/bugfix release -- add definition for new toolchain nvpsmpic (NVHPC + ParaStationMPI) (#3736) - various enhancements, including: - add per-extension timing in output produced by eb command (#3734) + - add definition for new toolchain nvpsmpic (NVHPC + ParaStationMPI + CUDA) (#3736) - include list of missing libraries in warning about missing FFTW libraries in imkl toolchain component (#3776) - check for recursive symlinks by default before copying a folder (#3784) - - add --filter-ecs options to filter out easyconfigs from set of easyconfigs to install (#3796) + - add --filter-ecs configuration option to filter out easyconfigs from set of easyconfigs to install (#3796) - check type of source_tmpl value for extensions, ensure it's a string value (not a list) (#3799) - also define $BLAS_SHARED_LIBS & co in build environment (analogous to $BLAS_STATIC_LIBS) (#3800) - report use of --ignore-test-failure in success message in output (#3806) @@ -28,15 +28,15 @@ update/bugfix release - make logdir writable also when --stop/--fetch is used and --read-only-installdir is enabled (#3771) - fix forgotten renaming of 'l' to 'char' __init__.py that is created for included Python modules (#3773) - fix verify_imports by deleting all imported modules before re-importing them one by one (#3780) - - fix ignore_test_failure not set for Extensions (#3782) + - fix ignore_test_failure not set for Extension instances (#3782) - update iompi toolchain to intel-compiler subtoolchain for oneAPI versions (>= iompi 2020.12) (#3785) - don't parse patch files as easyconfigs when searching for where patch file is used (#3786) - make sure git clone with a tag argument actually downloads a tag (#3795) - fix CI by excluding GC3Pie 2.6.7 (which is broken with Python 2) and improve error reporting for option parsing (#3798) - - speed up tests by caching checked paths in set_tmpdir + less test cases for test_compiler_dependent_optarch (#3802) - correctly resolve templates for patches in extensions when uploading to GitHub (#3805) - - add --easystack to ignore_opts for submit_job() (#3813) + - add --easystack to ignored options when submitting job (#3813) - other changes: + - speed up tests by caching checked paths in set_tmpdir + less test cases for test_compiler_dependent_optarch (#3802) - speed up set_parallel method in EasyBlock class (#3812) From 3d673307d9863b6aba16ed0f6e8e2c8d9975c16e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 Sep 2021 09:02:35 +0200 Subject: [PATCH 522/864] bump version to 4.4.3dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 70aa973100..2216e1f42d 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.4.2') +VERSION = LooseVersion('4.4.3.dev0') UNKNOWN = 'UNKNOWN' From df39e6e3fb61b728c00bedde9401da9ee87ee747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= Date: Wed, 8 Sep 2021 08:58:06 +0200 Subject: [PATCH 523/864] Add progressbar to installation procedure Utilize [`tqdm`](https://github.com/tqdm/tqdm) to track progress when installing EasyBuilds. This require a few changes to allow `print_msg` and the progressbar to interact. --- easybuild/framework/easyblock.py | 57 ++++++++++++++++++++++---------- easybuild/main.py | 18 +++++++--- easybuild/tools/build_log.py | 6 +++- requirements.txt | 2 ++ 4 files changed, 60 insertions(+), 23 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index ee6d2eae49..43a4286b97 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -212,6 +212,9 @@ def __init__(self, ec): self.postmsg = '' # allow a post message to be set, which can be shown as last output self.current_step = None + # Create empty progress bar + self.progressbar = None + # list of loaded modules self.loaded_modules = [] @@ -300,6 +303,13 @@ def close_log(self): self.log.info("Closing log for application name %s version %s" % (self.name, self.version)) fancylogger.logToFile(self.logfile, enable=False) + def set_progressbar(self, progressbar): + """ + Set progress bar, the progress bar is needed when writing messages so + that the progress counter is always at the bottom + """ + self.progressbar = progressbar + # # DRY RUN UTILITIES # @@ -318,7 +328,7 @@ def dry_run_msg(self, msg, *args): """Print dry run message.""" if args: msg = msg % args - dry_run_msg(msg, silent=self.silent) + dry_run_msg(msg, silent=self.silent, progressbar=self.progressbar) # # FETCH UTILITY FUNCTIONS @@ -1637,7 +1647,8 @@ def skip_extensions(self): self.log.debug("exit code: %s, stdout/err: %s", ec, cmdstdouterr) res.append(ext_inst) else: - print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log) + print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log, + progressbar=self.progressbar) self.ext_instances = res @@ -1741,7 +1752,8 @@ def handle_iterate_opts(self): self.log.debug("Found list for %s: %s", opt, self.iter_opts[opt]) if self.iter_opts: - print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent) + print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent, + progressbar=self.progressbar) self.log.info("Current iteration index: %s", self.iter_idx) # pop first element from all iterative easyconfig parameters as next value to use @@ -1882,7 +1894,8 @@ def check_readiness_step(self): hidden = LooseVersion(self.modules_tool.version) < LooseVersion('7.0.0') self.mod_file_backup = back_up_file(self.mod_filepath, hidden=hidden, strip_fn=strip_fn) - print_msg("backup of existing module file stored at %s" % self.mod_file_backup, log=self.log) + print_msg("backup of existing module file stored at %s" % self.mod_file_backup, log=self.log, + progressbar=self.progressbar) # check if main install needs to be skipped # - if a current module can be found, skip is ok @@ -2418,7 +2431,7 @@ def extensions_step(self, fetch=False, install=True): change_dir(self.orig_workdir) tup = (ext.name, ext.version or '', idx + 1, exts_cnt) - print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent) + print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent, progressbar=self.progressbar) start_time = datetime.now() if self.dry_run: @@ -2450,9 +2463,11 @@ def extensions_step(self, fetch=False, install=True): if not self.dry_run: ext_duration = datetime.now() - start_time if ext_duration.total_seconds() >= 1: - print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent) + print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent, + progressbar=self.progressbar) elif self.logdebug or build_option('trace'): - print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent) + print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent, + progressbar=self.progressbar) # cleanup (unload fake module, remove fake module dir) if fake_mod_data: @@ -3250,7 +3265,7 @@ def make_module_step(self, fake=False): else: diff_msg += 'no differences found' self.log.info(diff_msg) - print_msg(diff_msg, log=self.log) + print_msg(diff_msg, log=self.log, progressbar=self.progressbar) self.invalidate_module_caches(modpath) @@ -3569,7 +3584,8 @@ def run_all_steps(self, run_test_cases): steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt()) - print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent) + print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent, + progressbar=self.progressbar) trace_msg("installation prefix: %s" % self.installdir) ignore_locks = build_option('ignore_locks') @@ -3589,12 +3605,12 @@ def run_all_steps(self, run_test_cases): try: for (step_name, descr, step_methods, skippable) in steps: if self.skip_step(step_name, skippable): - print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent) + print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent, progressbar=self.progressbar) else: if self.dry_run: self.dry_run_msg("%s... [DRY RUN]\n", descr) else: - print_msg("%s..." % descr, log=self.log, silent=self.silent) + print_msg("%s..." % descr, log=self.log, silent=self.silent, progressbar=self.progressbar) self.current_step = step_name start_time = datetime.now() try: @@ -3603,9 +3619,11 @@ def run_all_steps(self, run_test_cases): if not self.dry_run: step_duration = datetime.now() - start_time if step_duration.total_seconds() >= 1: - print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent) + print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent, + progressbar=self.progressbar) elif self.logdebug or build_option('trace'): - print_msg("... (took < 1 sec)", log=self.log, silent=self.silent) + print_msg("... (took < 1 sec)", log=self.log, silent=self.silent, + progressbar=self.progressbar) except StopException: pass @@ -3631,7 +3649,7 @@ def print_dry_run_note(loc, silent=True): dry_run_msg(msg, silent=silent) -def build_and_install_one(ecdict, init_env): +def build_and_install_one(ecdict, init_env, progressbar=None): """ Build the software :param ecdict: dictionary contaning parsed easyconfig + metadata @@ -3649,7 +3667,7 @@ def build_and_install_one(ecdict, init_env): if dry_run: dry_run_msg('', silent=silent) - print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent) + print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent, progressbar=progressbar) if dry_run: # print note on interpreting dry run output (argument is reference to location of dry run messages) @@ -3674,6 +3692,7 @@ def build_and_install_one(ecdict, init_env): try: app_class = get_easyblock_class(easyblock, name=name) app = app_class(ecdict['ec']) + app.set_progressbar(progressbar) _log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock)) except EasyBuildError as err: print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg), @@ -3856,7 +3875,8 @@ def ensure_writable_log_dir(log_dir): application_log = app.logfile req_time = time2str(end_timestamp - start_timestamp) - print_msg("%s: Installation %s %s (took %s)" % (summary, ended, succ, req_time), log=_log, silent=silent) + print_msg("%s: Installation %s %s (took %s)" % (summary, ended, succ, req_time), log=_log, silent=silent, + progressbar=progressbar) # check for errors if run.errors_found_in_log > 0: @@ -3864,7 +3884,7 @@ def ensure_writable_log_dir(log_dir): "build logs, please verify the build.", run.errors_found_in_log) if app.postmsg: - print_msg("\nWARNING: %s\n" % app.postmsg, log=_log, silent=silent) + print_msg("\nWARNING: %s\n" % app.postmsg, log=_log, silent=silent, progressbar=progressbar) if dry_run: # print note on interpreting dry run output (argument is reference to location of dry run messages) @@ -3878,7 +3898,8 @@ def ensure_writable_log_dir(log_dir): if application_log: # there may be multiple log files, or the file name may be different due to zipping logs = glob.glob('%s*' % application_log) - print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent) + print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent, + progressbar=progressbar) del app diff --git a/easybuild/main.py b/easybuild/main.py index 748c068376..1da4362597 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -39,6 +39,7 @@ import os import stat import sys +import tqdm import traceback # IMPORTANT this has to be the first easybuild import as it customises the logging @@ -98,7 +99,7 @@ def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing= return [(ec_file, generated)] -def build_and_install_software(ecs, init_session_state, exit_on_failure=True): +def build_and_install_software(ecs, init_session_state, exit_on_failure=True, progress=None): """ Build and install software for all provided parsed easyconfig files. @@ -113,9 +114,11 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): res = [] for ec in ecs: + if progress: + progress.set_description("Installing %s" % ec['short_mod_name']) ec_res = {} try: - (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env) + (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progressbar=progress) ec_res['log_file'] = app_log if not ec_res['success']: ec_res['err'] = EasyBuildError(err) @@ -153,6 +156,8 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): raise EasyBuildError(test_msg) res.append((ec, ec_res)) + if progress: + progress.update() return res @@ -520,8 +525,13 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) - - ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) + # Create progressbar around software to install + progress_bar = tqdm.tqdm(total=len(ordered_ecs), desc="EasyBuild", + leave=False, unit='EB') + ecs_with_res = build_and_install_software( + ordered_ecs, init_session_state, exit_on_failure=exit_on_failure, + progress=progress_bar) + progress_bar.close() else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 2cf97c5f2d..2242a78dea 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -258,6 +258,7 @@ def print_msg(msg, *args, **kwargs): prefix = kwargs.pop('prefix', True) newline = kwargs.pop('newline', True) stderr = kwargs.pop('stderr', False) + pbar = kwargs.pop('progressbar', None) if kwargs: raise EasyBuildError("Unknown named arguments passed to print_msg: %s", kwargs) @@ -272,6 +273,8 @@ def print_msg(msg, *args, **kwargs): if stderr: sys.stderr.write(msg) + elif pbar: + pbar.write(msg, end='') else: sys.stdout.write(msg) @@ -304,6 +307,7 @@ def dry_run_msg(msg, *args, **kwargs): msg = msg % args silent = kwargs.pop('silent', False) + pbar = kwargs.pop('progressbar', None) if kwargs: raise EasyBuildError("Unknown named arguments passed to dry_run_msg: %s", kwargs) @@ -311,7 +315,7 @@ def dry_run_msg(msg, *args, **kwargs): if dry_run_var is not None: msg = dry_run_var[0].sub(dry_run_var[1], msg) - print_msg(msg, silent=silent, prefix=False) + print_msg(msg, silent=silent, prefix=False, progressbar=pbar) def dry_run_warning(msg, *args, **kwargs): diff --git a/requirements.txt b/requirements.txt index e63085eb51..77d393c398 100644 --- a/requirements.txt +++ b/requirements.txt @@ -62,3 +62,5 @@ archspec; python_version >= '2.7' # cryptography 3.4.0 no longer supports Python 2.7 cryptography==3.3.2; python_version == '2.7' cryptography; python_version >= '3.5' + +tqdm; python_version >= '2.7' From 671e7faceda523c0a7511268bb363c15cb572d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= Date: Wed, 8 Sep 2021 09:06:58 +0200 Subject: [PATCH 524/864] Moved initialization of easyblock progressbar Moved initialization outside try block and added logging for easier future debugging --- easybuild/framework/easyblock.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 43a4286b97..9bdd64ed68 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3692,12 +3692,15 @@ def build_and_install_one(ecdict, init_env, progressbar=None): try: app_class = get_easyblock_class(easyblock, name=name) app = app_class(ecdict['ec']) - app.set_progressbar(progressbar) _log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock)) except EasyBuildError as err: print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg), silent=silent) + # Setup progressbar + if progressbar: + app.set_progressbar(progressbar) + _log.info("Updated progressbar instance for easyblock %s" % easyblock) # application settings stop = build_option('stop') if stop is not None: From 9f5fff0b41c4027a3dea6d03ed1b079ecc8d3155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= Date: Thu, 9 Sep 2021 08:52:42 +0200 Subject: [PATCH 525/864] Moved to `rich` library and more fine grained ticks --- easybuild/framework/easyblock.py | 64 ++++++++++++++++---------------- easybuild/main.py | 29 +++++++++------ easybuild/tools/build_log.py | 6 +-- requirements.txt | 2 +- 4 files changed, 53 insertions(+), 48 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 9bdd64ed68..8f4c4a713b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -214,6 +214,7 @@ def __init__(self, ec): # Create empty progress bar self.progressbar = None + self.pbar_task = None # list of loaded modules self.loaded_modules = [] @@ -303,12 +304,20 @@ def close_log(self): self.log.info("Closing log for application name %s version %s" % (self.name, self.version)) fancylogger.logToFile(self.logfile, enable=False) - def set_progressbar(self, progressbar): + def set_progressbar(self, progressbar, task_id): """ Set progress bar, the progress bar is needed when writing messages so that the progress counter is always at the bottom """ self.progressbar = progressbar + self.pbar_task = task_id + + def advance_progress(self, tick=1.0): + """ + Advance the progress bar forward with `tick` + """ + if self.progressbar and self.pbar_task is not None: + self.progressbar.advance(self.pbar_task, tick) # # DRY RUN UTILITIES @@ -328,7 +337,7 @@ def dry_run_msg(self, msg, *args): """Print dry run message.""" if args: msg = msg % args - dry_run_msg(msg, silent=self.silent, progressbar=self.progressbar) + dry_run_msg(msg, silent=self.silent) # # FETCH UTILITY FUNCTIONS @@ -1647,8 +1656,7 @@ def skip_extensions(self): self.log.debug("exit code: %s, stdout/err: %s", ec, cmdstdouterr) res.append(ext_inst) else: - print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log, - progressbar=self.progressbar) + print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log) self.ext_instances = res @@ -1752,8 +1760,7 @@ def handle_iterate_opts(self): self.log.debug("Found list for %s: %s", opt, self.iter_opts[opt]) if self.iter_opts: - print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent, - progressbar=self.progressbar) + print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent) self.log.info("Current iteration index: %s", self.iter_idx) # pop first element from all iterative easyconfig parameters as next value to use @@ -1894,8 +1901,7 @@ def check_readiness_step(self): hidden = LooseVersion(self.modules_tool.version) < LooseVersion('7.0.0') self.mod_file_backup = back_up_file(self.mod_filepath, hidden=hidden, strip_fn=strip_fn) - print_msg("backup of existing module file stored at %s" % self.mod_file_backup, log=self.log, - progressbar=self.progressbar) + print_msg("backup of existing module file stored at %s" % self.mod_file_backup, log=self.log) # check if main install needs to be skipped # - if a current module can be found, skip is ok @@ -2431,7 +2437,7 @@ def extensions_step(self, fetch=False, install=True): change_dir(self.orig_workdir) tup = (ext.name, ext.version or '', idx + 1, exts_cnt) - print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent, progressbar=self.progressbar) + print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent) start_time = datetime.now() if self.dry_run: @@ -2463,11 +2469,9 @@ def extensions_step(self, fetch=False, install=True): if not self.dry_run: ext_duration = datetime.now() - start_time if ext_duration.total_seconds() >= 1: - print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent, - progressbar=self.progressbar) + print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent) elif self.logdebug or build_option('trace'): - print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent, - progressbar=self.progressbar) + print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent) # cleanup (unload fake module, remove fake module dir) if fake_mod_data: @@ -3265,7 +3269,7 @@ def make_module_step(self, fake=False): else: diff_msg += 'no differences found' self.log.info(diff_msg) - print_msg(diff_msg, log=self.log, progressbar=self.progressbar) + print_msg(diff_msg, log=self.log) self.invalidate_module_caches(modpath) @@ -3583,9 +3587,10 @@ def run_all_steps(self, run_test_cases): return True steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt()) + # Calculate progress bar tick + tick = 1.0 / float(len(steps)) - print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent, - progressbar=self.progressbar) + print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent) trace_msg("installation prefix: %s" % self.installdir) ignore_locks = build_option('ignore_locks') @@ -3605,12 +3610,12 @@ def run_all_steps(self, run_test_cases): try: for (step_name, descr, step_methods, skippable) in steps: if self.skip_step(step_name, skippable): - print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent, progressbar=self.progressbar) + print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent) else: if self.dry_run: self.dry_run_msg("%s... [DRY RUN]\n", descr) else: - print_msg("%s..." % descr, log=self.log, silent=self.silent, progressbar=self.progressbar) + print_msg("%s..." % descr, log=self.log, silent=self.silent) self.current_step = step_name start_time = datetime.now() try: @@ -3619,11 +3624,10 @@ def run_all_steps(self, run_test_cases): if not self.dry_run: step_duration = datetime.now() - start_time if step_duration.total_seconds() >= 1: - print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent, - progressbar=self.progressbar) + print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent) elif self.logdebug or build_option('trace'): - print_msg("... (took < 1 sec)", log=self.log, silent=self.silent, - progressbar=self.progressbar) + print_msg("... (took < 1 sec)", log=self.log, silent=self.silent) + self.advance_progress(tick) except StopException: pass @@ -3649,7 +3653,7 @@ def print_dry_run_note(loc, silent=True): dry_run_msg(msg, silent=silent) -def build_and_install_one(ecdict, init_env, progressbar=None): +def build_and_install_one(ecdict, init_env, progressbar=None, task_id=None): """ Build the software :param ecdict: dictionary contaning parsed easyconfig + metadata @@ -3667,7 +3671,7 @@ def build_and_install_one(ecdict, init_env, progressbar=None): if dry_run: dry_run_msg('', silent=silent) - print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent, progressbar=progressbar) + print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent) if dry_run: # print note on interpreting dry run output (argument is reference to location of dry run messages) @@ -3698,8 +3702,8 @@ def build_and_install_one(ecdict, init_env, progressbar=None): silent=silent) # Setup progressbar - if progressbar: - app.set_progressbar(progressbar) + if progressbar and task_id is not None: + app.set_progressbar(progressbar, task_id) _log.info("Updated progressbar instance for easyblock %s" % easyblock) # application settings stop = build_option('stop') @@ -3878,8 +3882,7 @@ def ensure_writable_log_dir(log_dir): application_log = app.logfile req_time = time2str(end_timestamp - start_timestamp) - print_msg("%s: Installation %s %s (took %s)" % (summary, ended, succ, req_time), log=_log, silent=silent, - progressbar=progressbar) + print_msg("%s: Installation %s %s (took %s)" % (summary, ended, succ, req_time), log=_log, silent=silent) # check for errors if run.errors_found_in_log > 0: @@ -3887,7 +3890,7 @@ def ensure_writable_log_dir(log_dir): "build logs, please verify the build.", run.errors_found_in_log) if app.postmsg: - print_msg("\nWARNING: %s\n" % app.postmsg, log=_log, silent=silent, progressbar=progressbar) + print_msg("\nWARNING: %s\n" % app.postmsg, log=_log, silent=silent) if dry_run: # print note on interpreting dry run output (argument is reference to location of dry run messages) @@ -3901,8 +3904,7 @@ def ensure_writable_log_dir(log_dir): if application_log: # there may be multiple log files, or the file name may be different due to zipping logs = glob.glob('%s*' % application_log) - print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent, - progressbar=progressbar) + print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent) del app diff --git a/easybuild/main.py b/easybuild/main.py index 1da4362597..987819fb0d 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -39,7 +39,6 @@ import os import stat import sys -import tqdm import traceback # IMPORTANT this has to be the first easybuild import as it customises the logging @@ -74,6 +73,7 @@ from easybuild.tools.parallelbuild import submit_jobs from easybuild.tools.repository.repository import init_repository from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state +from rich.progress import Progress, TextColumn, BarColumn, TimeElapsedColumn _log = None @@ -112,13 +112,17 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, pr # e.g. via easyconfig.handle_allowed_system_deps init_env = copy.deepcopy(os.environ) + # Initialize progress bar with overall installation task + if progress: + task_id = progress.add_task("", total=len(ecs)) res = [] for ec in ecs: if progress: - progress.set_description("Installing %s" % ec['short_mod_name']) + progress.update(task_id, description=ec['short_mod_name']) ec_res = {} try: - (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progressbar=progress) + (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progressbar=progress, + task_id=task_id) ec_res['log_file'] = app_log if not ec_res['success']: ec_res['err'] = EasyBuildError(err) @@ -156,8 +160,6 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, pr raise EasyBuildError(test_msg) res.append((ec, ec_res)) - if progress: - progress.update() return res @@ -526,12 +528,17 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) # Create progressbar around software to install - progress_bar = tqdm.tqdm(total=len(ordered_ecs), desc="EasyBuild", - leave=False, unit='EB') - ecs_with_res = build_and_install_software( - ordered_ecs, init_session_state, exit_on_failure=exit_on_failure, - progress=progress_bar) - progress_bar.close() + progress_bar = Progress( + TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"), + BarColumn(), + "[progress.percentage]{task.percentage:>3.1f}%", + "•", + TimeElapsedColumn() + ) + with progress_bar: + ecs_with_res = build_and_install_software( + ordered_ecs, init_session_state, exit_on_failure=exit_on_failure, + progress=progress_bar) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 2242a78dea..2cf97c5f2d 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -258,7 +258,6 @@ def print_msg(msg, *args, **kwargs): prefix = kwargs.pop('prefix', True) newline = kwargs.pop('newline', True) stderr = kwargs.pop('stderr', False) - pbar = kwargs.pop('progressbar', None) if kwargs: raise EasyBuildError("Unknown named arguments passed to print_msg: %s", kwargs) @@ -273,8 +272,6 @@ def print_msg(msg, *args, **kwargs): if stderr: sys.stderr.write(msg) - elif pbar: - pbar.write(msg, end='') else: sys.stdout.write(msg) @@ -307,7 +304,6 @@ def dry_run_msg(msg, *args, **kwargs): msg = msg % args silent = kwargs.pop('silent', False) - pbar = kwargs.pop('progressbar', None) if kwargs: raise EasyBuildError("Unknown named arguments passed to dry_run_msg: %s", kwargs) @@ -315,7 +311,7 @@ def dry_run_msg(msg, *args, **kwargs): if dry_run_var is not None: msg = dry_run_var[0].sub(dry_run_var[1], msg) - print_msg(msg, silent=silent, prefix=False, progressbar=pbar) + print_msg(msg, silent=silent, prefix=False) def dry_run_warning(msg, *args, **kwargs): diff --git a/requirements.txt b/requirements.txt index 77d393c398..3f82c41b26 100644 --- a/requirements.txt +++ b/requirements.txt @@ -63,4 +63,4 @@ archspec; python_version >= '2.7' cryptography==3.3.2; python_version == '2.7' cryptography; python_version >= '3.5' -tqdm; python_version >= '2.7' +rich; python_version >= '2.7' From 9b0a12d99aad17a6a093fb671e6c3affcc7dbd10 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Sep 2021 14:05:43 +0200 Subject: [PATCH 526/864] use Rich as optional dependency for showing progress bar --- easybuild/framework/easyblock.py | 21 +++++++------ easybuild/main.py | 54 +++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 8f4c4a713b..58bed6afde 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -213,7 +213,7 @@ def __init__(self, ec): self.current_step = None # Create empty progress bar - self.progressbar = None + self.progress_bar = None self.pbar_task = None # list of loaded modules @@ -304,20 +304,20 @@ def close_log(self): self.log.info("Closing log for application name %s version %s" % (self.name, self.version)) fancylogger.logToFile(self.logfile, enable=False) - def set_progressbar(self, progressbar, task_id): + def set_progress_bar(self, progress_bar, task_id): """ Set progress bar, the progress bar is needed when writing messages so that the progress counter is always at the bottom """ - self.progressbar = progressbar + self.progress_bar = progress_bar self.pbar_task = task_id def advance_progress(self, tick=1.0): """ Advance the progress bar forward with `tick` """ - if self.progressbar and self.pbar_task is not None: - self.progressbar.advance(self.pbar_task, tick) + if self.progress_bar and self.pbar_task is not None: + self.progress_bar.advance(self.pbar_task, tick) # # DRY RUN UTILITIES @@ -3653,7 +3653,7 @@ def print_dry_run_note(loc, silent=True): dry_run_msg(msg, silent=silent) -def build_and_install_one(ecdict, init_env, progressbar=None, task_id=None): +def build_and_install_one(ecdict, init_env, progress_bar=None, task_id=None): """ Build the software :param ecdict: dictionary contaning parsed easyconfig + metadata @@ -3701,10 +3701,11 @@ def build_and_install_one(ecdict, init_env, progressbar=None, task_id=None): print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg), silent=silent) - # Setup progressbar - if progressbar and task_id is not None: - app.set_progressbar(progressbar, task_id) - _log.info("Updated progressbar instance for easyblock %s" % easyblock) + # Setup progress bar + if progress_bar and task_id is not None: + app.set_progress_bar(progress_bar, task_id) + _log.info("Updated progress bar instance for easyblock %s", easyblock) + # application settings stop = build_option('stop') if stop is not None: diff --git a/easybuild/main.py b/easybuild/main.py index 987819fb0d..1a5e427917 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -73,7 +73,13 @@ from easybuild.tools.parallelbuild import submit_jobs from easybuild.tools.repository.repository import init_repository from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state -from rich.progress import Progress, TextColumn, BarColumn, TimeElapsedColumn + +try: + from rich.progress import Progress, TextColumn, BarColumn, TimeElapsedColumn + HAVE_RICH = True +except ImportError: + HAVE_RICH = False + _log = None @@ -99,13 +105,14 @@ def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing= return [(ec_file, generated)] -def build_and_install_software(ecs, init_session_state, exit_on_failure=True, progress=None): +def build_and_install_software(ecs, init_session_state, exit_on_failure=True, progress_bar=None): """ Build and install software for all provided parsed easyconfig files. :param ecs: easyconfig files to install software with :param init_session_state: initial session state, to use in test reports :param exit_on_failure: whether or not to exit on installation failure + :param progress_bar: ProgressBar instance to use to report progress """ # obtain a copy of the starting environment so each build can start afresh # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since @@ -113,15 +120,20 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, pr init_env = copy.deepcopy(os.environ) # Initialize progress bar with overall installation task - if progress: - task_id = progress.add_task("", total=len(ecs)) + if progress_bar: + task_id = progress_bar.add_task("", total=len(ecs)) + else: + task_id = None + res = [] for ec in ecs: - if progress: - progress.update(task_id, description=ec['short_mod_name']) + + if progress_bar: + progress_bar.update(task_id, description=ec['short_mod_name']) + ec_res = {} try: - (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progressbar=progress, + (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progress_bar=progress_bar, task_id=task_id) ec_res['log_file'] = app_log if not ec_res['success']: @@ -527,18 +539,22 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) - # Create progressbar around software to install - progress_bar = Progress( - TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"), - BarColumn(), - "[progress.percentage]{task.percentage:>3.1f}%", - "•", - TimeElapsedColumn() - ) - with progress_bar: - ecs_with_res = build_and_install_software( - ordered_ecs, init_session_state, exit_on_failure=exit_on_failure, - progress=progress_bar) + + if HAVE_RICH: + # Create progressbar around software to install + progress_bar = Progress( + TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"), + BarColumn(), + "[progress.percentage]{task.percentage:>3.1f}%", + "•", + TimeElapsedColumn() + ) + with progress_bar: + ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, + exit_on_failure=exit_on_failure, + progress_bar=progress_bar) + else: + ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] From 33f8e546ae43e933ca27434b6aa0c03aff157324 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Sep 2021 14:27:12 +0200 Subject: [PATCH 527/864] add create_progress_bar function in new easybuild.tools.output module --- easybuild/main.py | 29 ++++----------- easybuild/tools/output.py | 75 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 22 deletions(-) create mode 100644 easybuild/tools/output.py diff --git a/easybuild/main.py b/easybuild/main.py index 1a5e427917..9c2243a50c 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -68,18 +68,13 @@ from easybuild.tools.hooks import START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import set_up_configuration, use_color +from easybuild.tools.output import create_progress_bar from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs from easybuild.tools.package.utilities import check_pkg_support from easybuild.tools.parallelbuild import submit_jobs from easybuild.tools.repository.repository import init_repository from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state -try: - from rich.progress import Progress, TextColumn, BarColumn, TimeElapsedColumn - HAVE_RICH = True -except ImportError: - HAVE_RICH = False - _log = None @@ -112,7 +107,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, pr :param ecs: easyconfig files to install software with :param init_session_state: initial session state, to use in test reports :param exit_on_failure: whether or not to exit on installation failure - :param progress_bar: ProgressBar instance to use to report progress + :param progress_bar: progress bar to use to report progress """ # obtain a copy of the starting environment so each build can start afresh # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since @@ -540,21 +535,11 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) - if HAVE_RICH: - # Create progressbar around software to install - progress_bar = Progress( - TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"), - BarColumn(), - "[progress.percentage]{task.percentage:>3.1f}%", - "•", - TimeElapsedColumn() - ) - with progress_bar: - ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, - exit_on_failure=exit_on_failure, - progress_bar=progress_bar) - else: - ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) + progress_bar = create_progress_bar() + with progress_bar: + ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, + exit_on_failure=exit_on_failure, + progress_bar=progress_bar) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py new file mode 100644 index 0000000000..f20f027d31 --- /dev/null +++ b/easybuild/tools/output.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# # +# Copyright 2021-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Tools for controlling output to terminal produced by EasyBuild. + +:author: Kenneth Hoste (Ghent University) +:author: Jørgen Nordmoen (University of Oslo) +""" +try: + from rich.progress import Progress, TextColumn, BarColumn, TimeElapsedColumn + HAVE_RICH = True +except ImportError: + HAVE_RICH = False + + +class DummyProgress(object): + """Shim for Rich's Progress class.""" + + # __enter__ and __exit__ must be implemented to allow use as context manager + def __enter__(self, *args, **kwargs): + pass + + def __exit__(self, *args, **kwargs): + pass + + # dummy implementations for methods supported by rich.progress.Progress class + def add_task(self, *args, **kwargs): + pass + + def update(self, *args, **kwargs): + pass + + +def create_progress_bar(): + """ + Create progress bar to display overall progress. + + Returns rich.progress.Progress instance if the Rich Python package is available, + or a shim DummyProgress instance otherwise. + """ + if HAVE_RICH: + progress_bar = Progress( + TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"), + BarColumn(), + "[progress.percentage]{task.percentage:>3.1f}%", + "•", + TimeElapsedColumn() + ) + else: + progress_bar = DummyProgress() + + return progress_bar From d779a618d3fe70afc0b7f68c277bd4f55be36862 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Sep 2021 14:33:20 +0200 Subject: [PATCH 528/864] fix requirements.txt: Rich only supports Python 3.6+ --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3f82c41b26..591fc502f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -63,4 +63,5 @@ archspec; python_version >= '2.7' cryptography==3.3.2; python_version == '2.7' cryptography; python_version >= '3.5' -rich; python_version >= '2.7' +# rich is only supported for Python 3.6+ +rich; python_version >= '3.6' From 771f95d9108401582bea4a52399f8ac84b9a7515 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Sep 2021 16:44:10 +0200 Subject: [PATCH 529/864] use transient progress bar, so it disappears when installation is complete --- easybuild/tools/output.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index f20f027d31..359e7746d2 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -67,7 +67,8 @@ def create_progress_bar(): BarColumn(), "[progress.percentage]{task.percentage:>3.1f}%", "•", - TimeElapsedColumn() + TimeElapsedColumn(), + transient=True, ) else: progress_bar = DummyProgress() From 33df74b6de0e2fe400c588dfba37dfecd928e980 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Sep 2021 17:05:24 +0200 Subject: [PATCH 530/864] use random spinner in progress bar --- easybuild/tools/output.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 359e7746d2..831e0674bb 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -29,8 +29,10 @@ :author: Kenneth Hoste (Ghent University) :author: Jørgen Nordmoen (University of Oslo) """ +import random + try: - from rich.progress import Progress, TextColumn, BarColumn, TimeElapsedColumn + from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn HAVE_RICH = True except ImportError: HAVE_RICH = False @@ -62,11 +64,15 @@ def create_progress_bar(): or a shim DummyProgress instance otherwise. """ if HAVE_RICH: + + # pick random spinner, from a selected subset of available spinner (see 'python3 -m rich.spinner') + spinner = random.choice(('aesthetic', 'arc', 'bounce', 'dots', 'line', 'monkey', 'point', 'simpleDots')) + progress_bar = Progress( TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"), BarColumn(), "[progress.percentage]{task.percentage:>3.1f}%", - "•", + SpinnerColumn(spinner), TimeElapsedColumn(), transient=True, ) From f714ea6132362500e3ac6f09f422e8cfbb829b10 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Sep 2021 17:45:54 +0200 Subject: [PATCH 531/864] add configuration option to allow disabling progress bar --- easybuild/tools/config.py | 1 + easybuild/tools/options.py | 7 ++++--- easybuild/tools/output.py | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 8f660331f5..94b02d4424 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -292,6 +292,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'map_toolchains', 'modules_tool_version_check', 'pre_create_installdir', + 'show_progress_bar', ], WARN: [ 'check_ebroot_env_vars', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index b1aa1dd3cb..64948af45c 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -406,6 +406,8 @@ def override_options(self): 'force-download': ("Force re-downloading of sources and/or patches, " "even if they are available already in source path", 'choice', 'store_or_None', DEFAULT_FORCE_DOWNLOAD, FORCE_DOWNLOAD_CHOICES), + 'generate-devel-module': ("Generate a develop module file, implies --force if disabled", + None, 'store_true', True), 'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None), 'group-writable-installdir': ("Enable group write permissions on installation directory after installation", None, 'store_true', False), @@ -468,13 +470,12 @@ def override_options(self): None, 'store_true', False), 'set-default-module': ("Set the generated module as default", None, 'store_true', False), 'set-gid-bit': ("Set group ID bit on newly created directories", None, 'store_true', False), + 'show-progress-bar': ("Show progress bar in terminal output", None, 'store_true', True), 'silence-deprecation-warnings': ("Silence specified deprecation warnings", 'strlist', 'extend', None), - 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), 'skip-extensions': ("Skip installation of extensions", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), 'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False), - 'generate-devel-module': ("Generate a develop module file, implies --force if disabled", - None, 'store_true', True), + 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), 'sysroot': ("Location root directory of system, prefix for standard paths like /usr/lib and /usr/include", None, 'store', None), 'trace': ("Provide more information in output to stdout on progress", None, 'store_true', False, 'T'), diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 831e0674bb..35c6d962a3 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -31,6 +31,8 @@ """ import random +from easybuild.tools.config import build_option + try: from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn HAVE_RICH = True @@ -63,7 +65,7 @@ def create_progress_bar(): Returns rich.progress.Progress instance if the Rich Python package is available, or a shim DummyProgress instance otherwise. """ - if HAVE_RICH: + if HAVE_RICH and build_option('show_progress_bar'): # pick random spinner, from a selected subset of available spinner (see 'python3 -m rich.spinner') spinner = random.choice(('aesthetic', 'arc', 'bounce', 'dots', 'line', 'monkey', 'point', 'simpleDots')) From 37e6877942f46277285c51e3706b1fd7369e8e3f Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 12 Sep 2021 08:32:27 +0100 Subject: [PATCH 532/864] expand progress bar to full screen width --- easybuild/tools/output.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 35c6d962a3..7c936a7261 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -72,11 +72,12 @@ def create_progress_bar(): progress_bar = Progress( TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"), - BarColumn(), + BarColumn(bar_width=None), "[progress.percentage]{task.percentage:>3.1f}%", SpinnerColumn(spinner), TimeElapsedColumn(), transient=True, + expand=True, ) else: progress_bar = DummyProgress() From 681e53f227fe2a839c906b811bec77e83b940fb0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 12 Sep 2021 10:07:31 +0200 Subject: [PATCH 533/864] reorder progress bar components --- easybuild/tools/output.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 7c936a7261..52ee836b50 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -71,10 +71,10 @@ def create_progress_bar(): spinner = random.choice(('aesthetic', 'arc', 'bounce', 'dots', 'line', 'monkey', 'point', 'simpleDots')) progress_bar = Progress( - TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"), - BarColumn(bar_width=None), - "[progress.percentage]{task.percentage:>3.1f}%", SpinnerColumn(spinner), + "[progress.percentage]{task.percentage:>3.1f}%", + TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total} done)"), + BarColumn(bar_width=None), TimeElapsedColumn(), transient=True, expand=True, From 2db1befd6b45981db1fb33a9720294db7b1352d3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 12 Sep 2021 12:20:23 +0200 Subject: [PATCH 534/864] add support for checking required/optional EasyBuild dependencies via 'eb --check-eb-deps' --- easybuild/main.py | 5 ++ easybuild/tools/modules.py | 13 +++- easybuild/tools/options.py | 2 + easybuild/tools/systemtools.py | 124 ++++++++++++++++++++++++++++++++- test/framework/options.py | 22 ++++++ 5 files changed, 162 insertions(+), 4 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 9c2243a50c..84b02dc186 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -73,6 +73,7 @@ from easybuild.tools.package.utilities import check_pkg_support from easybuild.tools.parallelbuild import submit_jobs from easybuild.tools.repository.repository import init_repository +from easybuild.tools.systemtools import check_easybuild_deps from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state @@ -259,6 +260,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename, terse=options.terse) + if options.check_eb_deps: + print(check_easybuild_deps(modtool)) + # GitHub options that warrant a silent cleanup & exit if options.check_github: check_github() @@ -297,6 +301,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ options.add_pr_labels, + options.check_eb_deps, options.check_github, options.create_index, options.install_github_token, diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 5e48591be5..0860b810d8 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -207,6 +207,15 @@ def __init__(self, mod_paths=None, testing=False): self.set_and_check_version() self.supports_depends_on = False + def __str__(self): + """String representation of this ModulesTool instance.""" + res = self.NAME + if self.version: + res += ' ' + self.version + else: + res += ' (unknown version)' + return res + def buildstats(self): """Return tuple with data to be included in buildstats""" return (self.NAME, self.cmd, self.version) @@ -1177,7 +1186,7 @@ def update(self): class EnvironmentModulesC(ModulesTool): """Interface to (C) environment modules (modulecmd).""" - NAME = "Environment Modules v3" + NAME = "Environment Modules" COMMAND = "modulecmd" REQ_VERSION = '3.2.10' MAX_VERSION = '3.99' @@ -1312,7 +1321,7 @@ def remove_module_path(self, path, set_mod_paths=True): class EnvironmentModules(EnvironmentModulesTcl): """Interface to environment modules 4.0+""" - NAME = "Environment Modules v4" + NAME = "Environment Modules" COMMAND = os.path.join(os.getenv('MODULESHOME', 'MODULESHOME_NOT_DEFINED'), 'libexec', 'modulecmd.tcl') COMMAND_ENVIRONMENT = 'MODULES_CMD' REQ_VERSION = '4.0.0' diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 64948af45c..759e8d8e42 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -617,6 +617,8 @@ def informative_options(self): 'avail-hooks': ("Show list of known hooks", None, 'store_true', False), 'avail-toolchain-opts': ("Show options for toolchain", 'str', 'store', None), 'check-conflicts': ("Check for version conflicts in dependency graphs", None, 'store_true', False), + 'check-eb-deps': ("Check presence and version of (required and optional) EasyBuild dependencies", + None, 'store_true', False), 'dep-graph': ("Create dependency graph", None, 'store', None, {'metavar': 'depgraph.'}), 'dump-env-script': ("Dump source script to set up build environment based on toolchain/dependencies", None, 'store_true', False), diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index a43338e147..bf96fb7745 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -34,6 +34,7 @@ import grp # @UnresolvedImport import os import platform +import pkg_resources import pwd import re import struct @@ -152,6 +153,36 @@ RPM = 'rpm' DPKG = 'dpkg' +SYSTEM_TOOLS = ('7z', 'bunzip2', DPKG, 'gunzip', 'make', 'patch', RPM, 'sed', 'tar', 'unxz', 'unzip') + +OPT_DEPS = { + 'archspec': "determining name of CPU microarchitecture", + 'autopep8': "auto-formatting for dumped easyconfigs", + 'GC3Pie': "backend for --job", + 'GitPython': "GitHub integration + using Git repository as easyconfigs archive", + 'graphviz-python': "rendering dependency graph with Graphviz: --dep-graph", + 'keyring': "storing GitHub token", + 'pep8': "fallback for code style checking: --check-style, --check-contrib", + 'pycodestyle': "code style checking: --check-style, --check-contrib", + 'pysvn': "using SVN repository as easyconfigs archive", + 'python-graph-core': "creating dependency graph: --dep-graph", + 'python-graph-dot': "saving dependency graph as dot file: --dep-graph", + 'python-hglib': "using Mercurial repository as easyconfigs archive", + 'requests': "fallback library for downloading files", + 'Rich': "eb command rich terminal output", + 'PyYAML': "easystack files and .yeb easyconfig format", +} + +OPT_DEP_PKG_NAMES = { + 'GC3Pie': 'gc3libs', + 'GitPython': 'git', + 'graphviz-python': 'gv', + 'python-graph-core': 'pygraph.classes.digraph', + 'python-graph-dot': 'pygraph.readwrite.dot', + 'python-hglib': 'hglib', + 'PyYAML': 'yaml', +} + class SystemToolsException(Exception): """raised when systemtools fails""" @@ -722,14 +753,14 @@ def check_os_dependency(dep): return found -def get_tool_version(tool, version_option='--version'): +def get_tool_version(tool, version_option='--version', ignore_ec=False): """ Get output of running version option for specific command line tool. Output is returned as a single-line string (newlines are replaced by '; '). """ out, ec = run_cmd(' '.join([tool, version_option]), simple=False, log_ok=False, force_in_dry_run=True, trace=False, stream_output=False) - if ec: + if not ignore_ec and ec: _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, out)) return UNKNOWN else: @@ -1103,3 +1134,92 @@ def pick_dep_version(dep_version): raise EasyBuildError("Unknown value type for version: %s (%s), should be string value", typ, dep_version) return result + + +def check_easybuild_deps(modtool): + """ + Check presence and version of required and optional EasyBuild dependencies, and report back to terminal. + """ + version_regex = re.compile(r'\s(?P[0-9][0-9.]+[a-z]*)') + + def extract_version(tool): + """Helper function to extract (only) version for specific command line tool.""" + out = get_tool_version(tool, ignore_ec=True) + res = version_regex.search(out) + if res: + version = res.group('version') + else: + version = "UNKNOWN version" + + return version + + python_version = extract_version(sys.executable) + + opt_dep_versions = {} + for key in OPT_DEPS: + + pkg = OPT_DEP_PKG_NAMES.get(key, key.lower()) + + try: + mod = __import__(pkg) + except ImportError: + mod = None + + if mod: + try: + dep_version = pkg_resources.get_distribution(pkg).version + except pkg_resources.DistributionNotFound: + try: + dep_version = pkg_resources.get_distribution(key).version + except pkg_resources.DistributionNotFound: + if hasattr(mod, '__version__'): + dep_version = mod.__version__ + else: + dep_version = '(unknown version)' + else: + dep_version = '(NOT AVAILABLE)' + + opt_dep_versions[key] = dep_version + + lines = [ + '', + "Required dependencies:", + "----------------------", + '', + "* Python %s" % python_version, + "* %s (modules tool)" % modtool, + '', + "Optional dependencies:", + "----------------------", + '', + ] + for pkg in sorted(opt_dep_versions, key=lambda x: x.lower()): + line = "* %s %s" % (pkg, opt_dep_versions[pkg]) + line = line.ljust(40) + " [%s]" % OPT_DEPS[pkg] + lines.append(line) + + lines.extend([ + '', + "System tools:", + "-------------", + '', + ]) + + tools = list(SYSTEM_TOOLS) + ['Slurm'] + cmds = {'Slurm': 'sbatch'} + + for tool in sorted(tools, key=lambda x: x.lower()): + line = "* %s " % tool + cmd = cmds.get(tool, tool) + if which(cmd): + version = extract_version(cmd) + if version.startswith('UNKNOWN'): + line += "(available, %s)" % version + else: + line += version + else: + line += "(NOT AVAILABLE)" + + lines.append(line) + + return '\n'.join(lines) diff --git a/test/framework/options.py b/test/framework/options.py index c9a64bfd0e..de338e4045 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5845,6 +5845,28 @@ def test_show_system_info(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + def test_check_eb_deps(self): + """Test for --check-eb-deps.""" + txt, _ = self._run_mock_eb(['--check-eb-deps'], raise_error=True) + patterns = [ + r"^Required dependencies:", + r"^\* Python [23][0-9.]+$", + r"^\* [A-Za-z ]+ [0-9.]+ \(modules tool\)$", + r"^Optional dependencies:", + r"^\* archspec ([0-9.]+|\(NOT AVAILABLE\))+\s+\[determining name of CPU microarchitecture\]$", + r"^\* GitPython ([0-9.]+|\(NOT AVAILABLE\))+\s+\[GitHub integration .*\]$", + r"^\* Rich ([0-9.]+|\(NOT AVAILABLE\))+\s+\[eb command rich terminal output\]$", + r"^System tools:", + r"^\* make ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", + r"^\* patch ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", + r"^\* sed ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", + r"^\* Slurm ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", + ] + + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + def test_tmp_logdir(self): """Test use of --tmp-logdir.""" From c97661a0efb2107cbd60b9c186dfd5cf35c5c8ce Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 12 Sep 2021 14:19:45 +0200 Subject: [PATCH 535/864] avoid making setuptools a required dependency by only using pkg_resources if it's available --- easybuild/tools/systemtools.py | 49 ++++++++++++++++++++++++++-------- test/framework/options.py | 8 +++--- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index bf96fb7745..f8a115ba0f 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -34,7 +34,6 @@ import grp # @UnresolvedImport import os import platform -import pkg_resources import pwd import re import struct @@ -43,6 +42,14 @@ from ctypes.util import find_library from socket import gethostname +# pkg_resources is provided by the setuptools Python package, +# which we really want to keep as an *optional* dependency +try: + import pkg_resources + HAVE_PKG_RESOURCES = True +except ImportError: + HAVE_PKG_RESOURCES = False + try: # only needed on macOS, may not be available on Linux import ctypes.macholib.dyld @@ -162,6 +169,7 @@ 'GitPython': "GitHub integration + using Git repository as easyconfigs archive", 'graphviz-python': "rendering dependency graph with Graphviz: --dep-graph", 'keyring': "storing GitHub token", + 'pbs-python': "using Torque as --job backend", 'pep8': "fallback for code style checking: --check-style, --check-contrib", 'pycodestyle': "code style checking: --check-style, --check-contrib", 'pysvn': "using SVN repository as easyconfigs archive", @@ -171,12 +179,14 @@ 'requests': "fallback library for downloading files", 'Rich': "eb command rich terminal output", 'PyYAML': "easystack files and .yeb easyconfig format", + 'setuptools': "obtaining information on Python packages via pkg_resources module", } OPT_DEP_PKG_NAMES = { 'GC3Pie': 'gc3libs', 'GitPython': 'git', 'graphviz-python': 'gv', + 'pbs-python': 'pbs', 'python-graph-core': 'pygraph.classes.digraph', 'python-graph-dot': 'pygraph.readwrite.dot', 'python-hglib': 'hglib', @@ -1136,6 +1146,30 @@ def pick_dep_version(dep_version): return result +def det_pypkg_version(pkg_name, imported_pkg, import_name=None): + """Determine version of a Python package.""" + + version = None + + if HAVE_PKG_RESOURCES: + if import_name: + try: + version = pkg_resources.get_distribution(import_name).version + except pkg_resources.DistributionNotFound as err: + _log.debug("%s Python package not found: %s", import_name, err) + + if version is None: + try: + version = pkg_resources.get_distribution(pkg_name).version + except pkg_resources.DistributionNotFound as err: + _log.debug("%s Python package not found: %s", pkg_name, err) + + if version is None and hasattr(imported_pkg, '__version__'): + version = imported_pkg.__version__ + + return version + + def check_easybuild_deps(modtool): """ Check presence and version of required and optional EasyBuild dependencies, and report back to terminal. @@ -1166,16 +1200,9 @@ def extract_version(tool): mod = None if mod: - try: - dep_version = pkg_resources.get_distribution(pkg).version - except pkg_resources.DistributionNotFound: - try: - dep_version = pkg_resources.get_distribution(key).version - except pkg_resources.DistributionNotFound: - if hasattr(mod, '__version__'): - dep_version = mod.__version__ - else: - dep_version = '(unknown version)' + dep_version = det_pypkg_version(key, mod, import_name=pkg) + if dep_version is None: + dep_version = '(unknown version)' else: dep_version = '(NOT AVAILABLE)' diff --git a/test/framework/options.py b/test/framework/options.py index de338e4045..b3e60904f0 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5848,14 +5848,16 @@ def test_show_system_info(self): def test_check_eb_deps(self): """Test for --check-eb-deps.""" txt, _ = self._run_mock_eb(['--check-eb-deps'], raise_error=True) + opt_dep_version_pattern = r'([0-9.]+|\(NOT AVAILABLE\)|\(unknown version\))' patterns = [ r"^Required dependencies:", r"^\* Python [23][0-9.]+$", r"^\* [A-Za-z ]+ [0-9.]+ \(modules tool\)$", r"^Optional dependencies:", - r"^\* archspec ([0-9.]+|\(NOT AVAILABLE\))+\s+\[determining name of CPU microarchitecture\]$", - r"^\* GitPython ([0-9.]+|\(NOT AVAILABLE\))+\s+\[GitHub integration .*\]$", - r"^\* Rich ([0-9.]+|\(NOT AVAILABLE\))+\s+\[eb command rich terminal output\]$", + r"^\* archspec %s\s+\[determining name of CPU microarchitecture\]$" % opt_dep_version_pattern, + r"^\* GitPython %s\s+\[GitHub integration .*\]$" % opt_dep_version_pattern, + r"^\* Rich %s\s+\[eb command rich terminal output\]$" % opt_dep_version_pattern, + r"^\* setuptools %s\s+\[obtaining information on Python packages .*\]$" % opt_dep_version_pattern, r"^System tools:", r"^\* make ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", r"^\* patch ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", From 331820fa9edbe6500b2b589f0666fca5c089b6b4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 12 Sep 2021 15:45:03 +0200 Subject: [PATCH 536/864] add print_checks function in easybuild.tools.output and leverage it to produce rich output for --check-eb-deps --- easybuild/main.py | 4 +- easybuild/tools/output.py | 71 ++++++++++++++++++++++++++++ easybuild/tools/systemtools.py | 86 ++++++++++++++++++---------------- test/framework/options.py | 31 ++++++------ 4 files changed, 136 insertions(+), 56 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 84b02dc186..1e5792fd0e 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -68,7 +68,7 @@ from easybuild.tools.hooks import START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import set_up_configuration, use_color -from easybuild.tools.output import create_progress_bar +from easybuild.tools.output import create_progress_bar, print_checks from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs from easybuild.tools.package.utilities import check_pkg_support from easybuild.tools.parallelbuild import submit_jobs @@ -261,7 +261,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): terse=options.terse) if options.check_eb_deps: - print(check_easybuild_deps(modtool)) + print_checks(check_easybuild_deps(modtool)) # GitHub options that warrant a silent cleanup & exit if options.check_github: diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 35c6d962a3..e5d72119d0 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -32,8 +32,11 @@ import random from easybuild.tools.config import build_option +from easybuild.tools.py2vs3 import OrderedDict try: + from rich.console import Console + from rich.table import Table from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn HAVE_RICH = True except ImportError: @@ -82,3 +85,71 @@ def create_progress_bar(): progress_bar = DummyProgress() return progress_bar + + +def print_checks(checks_data): + """Print overview of checks that were made.""" + + col_titles = checks_data.pop('col_titles', ('name', 'info', 'description')) + + col2_label = col_titles[1] + + if HAVE_RICH: + console = Console() + # don't use console.print, which causes SyntaxError in Python 2 + console_print = getattr(console, 'print') + console_print('') + + for section in checks_data: + section_checks = checks_data[section] + + if HAVE_RICH: + table = Table(title=section) + table.add_column(col_titles[0]) + table.add_column(col_titles[1]) + # only add 3rd column if there's any information to include in it + if any(x[1] for x in section_checks.values()): + table.add_column(col_titles[2]) + else: + lines = [ + '', + section + ':', + '-' * (len(section) + 1), + '', + ] + + if isinstance(section_checks, OrderedDict): + check_names = section_checks.keys() + else: + check_names = sorted(section_checks, key=lambda x: x.lower()) + + if HAVE_RICH: + for check_name in check_names: + (info, descr) = checks_data[section][check_name] + if info is None: + info = ':yellow_circle: [yellow]%s?!' % col2_label + elif info is False: + info = ':cross_mark: [red]not found' + else: + info = ':white_heavy_check_mark: [green]%s' % info + if descr: + table.add_row(check_name.rstrip(':'), info, descr) + else: + table.add_row(check_name.rstrip(':'), info) + else: + for check_name in check_names: + (info, descr) = checks_data[section][check_name] + if info is None: + info = '(found, UNKNOWN %s)' % col2_label + elif info is False: + info = '(NOT FOUND)' + line = "* %s %s" % (check_name, info) + if descr: + line = line.ljust(40) + '[%s]' % descr + lines.append(line) + lines.append('') + + if HAVE_RICH: + console_print(table) + else: + print('\n'.join(lines)) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index f8a115ba0f..2c7052f4ef 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -59,7 +59,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import is_readable, read_file, which -from easybuild.tools.py2vs3 import string_type +from easybuild.tools.py2vs3 import OrderedDict, string_type from easybuild.tools.run import run_cmd @@ -160,7 +160,24 @@ RPM = 'rpm' DPKG = 'dpkg' -SYSTEM_TOOLS = ('7z', 'bunzip2', DPKG, 'gunzip', 'make', 'patch', RPM, 'sed', 'tar', 'unxz', 'unzip') +SYSTEM_TOOLS = { + '7z': "extracting sources (.iso)", + 'bunzip2': "decompressing sources (.bz2, .tbz, .tbz2, ...)", + DPKG: "checking OS dependencies (Debian, Ubuntu, ...)", + 'gunzip': "decompressing source files (.gz, .tgz, ...)", + 'make': "build tool", + 'patch': "applying patch files", + RPM: "checking OS dependencies (CentOS, RHEL, OpenSuSE, SLES, ...)", + 'sed': "runtime patching", + 'Slurm': "backend for --job (sbatch command)", + 'tar': "unpacking source files (.tar)", + 'unxz': "decompressing source files (.xz, .txz)", + 'unzip': "decompressing files (.zip)", +} + +SYSTEM_TOOL_CMDS = { + 'Slurm': 'sbatch', +} OPT_DEPS = { 'archspec': "determining name of CPU microarchitecture", @@ -1176,6 +1193,8 @@ def check_easybuild_deps(modtool): """ version_regex = re.compile(r'\s(?P[0-9][0-9.]+[a-z]*)') + checks_data = OrderedDict() + def extract_version(tool): """Helper function to extract (only) version for specific command line tool.""" out = get_tool_version(tool, ignore_ec=True) @@ -1201,52 +1220,39 @@ def extract_version(tool): if mod: dep_version = det_pypkg_version(key, mod, import_name=pkg) - if dep_version is None: - dep_version = '(unknown version)' else: - dep_version = '(NOT AVAILABLE)' + dep_version = False opt_dep_versions[key] = dep_version - lines = [ - '', - "Required dependencies:", - "----------------------", - '', - "* Python %s" % python_version, - "* %s (modules tool)" % modtool, - '', - "Optional dependencies:", - "----------------------", - '', - ] - for pkg in sorted(opt_dep_versions, key=lambda x: x.lower()): - line = "* %s %s" % (pkg, opt_dep_versions[pkg]) - line = line.ljust(40) + " [%s]" % OPT_DEPS[pkg] - lines.append(line) - - lines.extend([ - '', - "System tools:", - "-------------", - '', - ]) - - tools = list(SYSTEM_TOOLS) + ['Slurm'] - cmds = {'Slurm': 'sbatch'} - - for tool in sorted(tools, key=lambda x: x.lower()): - line = "* %s " % tool - cmd = cmds.get(tool, tool) + checks_data['col_titles'] = ('name', 'version', 'used for') + + req_deps_key = "Required dependencies" + checks_data[req_deps_key] = OrderedDict() + checks_data[req_deps_key]['Python'] = (python_version, None) + checks_data[req_deps_key]['modules tool:'] = (str(modtool), None) + + opt_deps_key = "Optional dependencies" + checks_data[opt_deps_key] = {} + + for pkg in opt_dep_versions: + checks_data[opt_deps_key][pkg] = (opt_dep_versions[pkg], OPT_DEPS[pkg]) + + sys_tools_key = "System tools" + checks_data[sys_tools_key] = {} + + for tool in SYSTEM_TOOLS: + tool_info = None + cmd = SYSTEM_TOOL_CMDS.get(tool, tool) if which(cmd): version = extract_version(cmd) if version.startswith('UNKNOWN'): - line += "(available, %s)" % version + tool_info = None else: - line += version + tool_info = version else: - line += "(NOT AVAILABLE)" + tool_info = False - lines.append(line) + checks_data[sys_tools_key][tool] = (tool_info, None) - return '\n'.join(lines) + return checks_data diff --git a/test/framework/options.py b/test/framework/options.py index b3e60904f0..2dda9e9a1c 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5848,21 +5848,24 @@ def test_show_system_info(self): def test_check_eb_deps(self): """Test for --check-eb-deps.""" txt, _ = self._run_mock_eb(['--check-eb-deps'], raise_error=True) - opt_dep_version_pattern = r'([0-9.]+|\(NOT AVAILABLE\)|\(unknown version\))' + + # keep in mind that these patterns should match with both normal output and Rich output! + opt_dep_info_pattern = r'([0-9.]+|\(NOT FOUND\)|not found|\(unknown version\))' + tool_info_pattern = r'([0-9.]+|\(NOT FOUND\)|not found|\(found, UNKNOWN version\)|version\?\!)' patterns = [ - r"^Required dependencies:", - r"^\* Python [23][0-9.]+$", - r"^\* [A-Za-z ]+ [0-9.]+ \(modules tool\)$", - r"^Optional dependencies:", - r"^\* archspec %s\s+\[determining name of CPU microarchitecture\]$" % opt_dep_version_pattern, - r"^\* GitPython %s\s+\[GitHub integration .*\]$" % opt_dep_version_pattern, - r"^\* Rich %s\s+\[eb command rich terminal output\]$" % opt_dep_version_pattern, - r"^\* setuptools %s\s+\[obtaining information on Python packages .*\]$" % opt_dep_version_pattern, - r"^System tools:", - r"^\* make ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", - r"^\* patch ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", - r"^\* sed ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", - r"^\* Slurm ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", + r"Required dependencies", + r"Python.* [23][0-9.]+", + r"modules tool.* [A-Za-z0-9.\s-]+", + r"Optional dependencies", + r"archspec.* %s.*determining name" % opt_dep_info_pattern, + r"GitPython.* %s.*GitHub integration" % opt_dep_info_pattern, + r"Rich.* %s.*eb command rich terminal output" % opt_dep_info_pattern, + r"setuptools.* %s.*information on Python packages" % opt_dep_info_pattern, + r"System tools", + r"make.* %s" % tool_info_pattern, + r"patch.* %s" % tool_info_pattern, + r"sed.* %s" % tool_info_pattern, + r"Slurm.* %s" % tool_info_pattern, ] for pattern in patterns: From 93c3277f3e0d9c9b2715a9472c950aa8a8aff2ae Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Sep 2021 09:04:47 +0200 Subject: [PATCH 537/864] silence the Hound on accessing console.print method via getattr --- easybuild/tools/output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index e5d72119d0..bd629f3bd3 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -97,7 +97,7 @@ def print_checks(checks_data): if HAVE_RICH: console = Console() # don't use console.print, which causes SyntaxError in Python 2 - console_print = getattr(console, 'print') + console_print = getattr(console, 'print') # noqa: B009 console_print('') for section in checks_data: From 79cc83b96d758eb4f6ef623b492d8fcd28a5655f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Sep 2021 09:50:41 +0200 Subject: [PATCH 538/864] ensure that path configuration options have absolute path values --- easybuild/tools/options.py | 32 ++++++++++++++++++++++++++++++++ test/framework/options.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 64948af45c..47804a28ab 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1047,8 +1047,40 @@ def _postprocess_checks(self): self.log.info("Checks on configuration options passed") + def _ensure_abs_path(self, opt_name): + """Ensure that path value for specified configuration option is an absolute path.""" + + def _ensure_abs_path(opt_name, path): + """Helper function to make path value for a configuration option an absolute path.""" + if os.path.isabs(path): + abs_path = path + else: + abs_path = os.path.abspath(path) + self.log.info("Relative path value for '%s' configuration option resolved to absolute path: %s", + path, abs_path) + return abs_path + + opt_val = getattr(self.options, opt_name) + if opt_val: + if isinstance(opt_val, string_type): + setattr(self.options, opt_name, _ensure_abs_path(opt_name, opt_val)) + elif isinstance(opt_val, list): + abs_paths = [_ensure_abs_path(opt_name, p) for p in opt_val] + setattr(self.options, opt_name, abs_paths) + else: + error_msg = "Don't know how to ensure absolute path(s) for '%s' configuration option (value type: %s)" + raise EasyBuildError(error_msg, opt_name, type(opt_val)) + def _postprocess_config(self): """Postprocessing of configuration options""" + + # resolve relative paths for configuration options that specify a location + path_opt_names = ('buildpath', 'containerpath', 'git_working_dirs_path', 'installpath', + 'installpath_modules', 'installpath_software', 'prefix', 'packagepath', + 'repositorypath', 'robot_paths', 'sourcepath') + for opt_name in path_opt_names: + self._ensure_abs_path(opt_name) + if self.options.prefix is not None: # prefix applies to all paths, and repository has to be reinitialised to take new repositorypath in account # in the legacy-style configuration, repository is initialised in configuration file itself diff --git a/test/framework/options.py b/test/framework/options.py index c9a64bfd0e..045e868790 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6233,6 +6233,42 @@ def test_accept_eula_for(self): self.eb_main(args, do_build=True, raise_error=True) self.assertTrue(os.path.exists(toy_modfile)) + def test_config_abs_path(self): + """Test ensuring of absolute path values for path configuration options.""" + + test_topdir = os.path.join(self.test_prefix, 'test_topdir') + test_subdir = os.path.join(test_topdir, 'test_middle_dir', 'test_subdir') + mkdir(test_subdir, parents=True) + change_dir(test_subdir) + + # a relative path specified in a configuration file is positively weird, but fine :) + cfgfile = os.path.join(self.test_prefix, 'test.cfg') + cfgtxt = '\n'.join([ + "[config]", + "containerpath = ..", + ]) + write_file(cfgfile, cfgtxt) + + os.environ['EASYBUILD_INSTALLPATH'] = '../..' + + args = [ + '--configfiles=%s' % cfgfile, + '--prefix=..', + '--sourcepath=.', + '--show-config', + ] + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) + + patterns = [ + r"^containerpath\s+\(F\) = .*/test_topdir/test_middle_dir$", + r"^installpath\s+\(E\) = .*/test_topdir$", + r"^prefix\s+\(C\) = .*/test_topdir/test_middle_dir$", + r"^sourcepath\s+\(C\) = .*/test_topdir/test_middle_dir/test_subdir$", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (pattern, txt)) + # end-to-end testing of unknown filename def test_easystack_wrong_read(self): """Test for --easystack when wrong name is provided""" From 3252979f29b1b92a7223f5163cef02c7c03fa564 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Sep 2021 12:16:03 +0200 Subject: [PATCH 539/864] handle repositorypath as special case when ensuring absolute path values --- easybuild/tools/options.py | 38 ++++++++++++++++++++++++-------------- test/framework/config.py | 2 +- test/framework/options.py | 2 ++ 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 47804a28ab..8d7b3af441 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1047,25 +1047,25 @@ def _postprocess_checks(self): self.log.info("Checks on configuration options passed") + def get_cfg_opt_abs_path(self, opt_name, path): + """Get path value of configuration option as absolute path.""" + if os.path.isabs(path): + abs_path = path + else: + abs_path = os.path.abspath(path) + self.log.info("Relative path value for '%s' configuration option resolved to absolute path: %s", + path, abs_path) + return abs_path + def _ensure_abs_path(self, opt_name): """Ensure that path value for specified configuration option is an absolute path.""" - def _ensure_abs_path(opt_name, path): - """Helper function to make path value for a configuration option an absolute path.""" - if os.path.isabs(path): - abs_path = path - else: - abs_path = os.path.abspath(path) - self.log.info("Relative path value for '%s' configuration option resolved to absolute path: %s", - path, abs_path) - return abs_path - opt_val = getattr(self.options, opt_name) if opt_val: if isinstance(opt_val, string_type): - setattr(self.options, opt_name, _ensure_abs_path(opt_name, opt_val)) + setattr(self.options, opt_name, self.get_cfg_opt_abs_path(opt_name, opt_val)) elif isinstance(opt_val, list): - abs_paths = [_ensure_abs_path(opt_name, p) for p in opt_val] + abs_paths = [self.get_cfg_opt_abs_path(opt_name, p) for p in opt_val] setattr(self.options, opt_name, abs_paths) else: error_msg = "Don't know how to ensure absolute path(s) for '%s' configuration option (value type: %s)" @@ -1075,9 +1075,19 @@ def _postprocess_config(self): """Postprocessing of configuration options""" # resolve relative paths for configuration options that specify a location - path_opt_names = ('buildpath', 'containerpath', 'git_working_dirs_path', 'installpath', + path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath', 'installpath_modules', 'installpath_software', 'prefix', 'packagepath', - 'repositorypath', 'robot_paths', 'sourcepath') + 'robot_paths', 'sourcepath'] + + # repositorypath is a special case: only first part is a path; + # 2nd (optional) part is a relative subdir and should not be resolved to an absolute path! + repositorypath = self.options.repositorypath + if isinstance(repositorypath, (list, tuple)) and len(repositorypath) == 2: + abs_path = self.get_cfg_opt_abs_path('repositorypath', repositorypath[0]) + self.options.repositorypath = (abs_path, repositorypath[1]) + else: + path_opt_names.append('repositorypath') + for opt_name in path_opt_names: self._ensure_abs_path(opt_name) diff --git a/test/framework/config.py b/test/framework/config.py index cb13d348a5..0c4489a412 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -302,7 +302,7 @@ def test_generaloption_config_file(self): self.assertEqual(install_path('mod'), installpath_modules), # via config file self.assertEqual(source_paths(), [testpath2]) # via command line self.assertEqual(build_path(), testpath1) # via config file - self.assertEqual(get_repositorypath(), [os.path.join(topdir, 'ebfiles_repo'), 'somesubdir']) # via config file + self.assertEqual(get_repositorypath(), (os.path.join(topdir, 'ebfiles_repo'), 'somesubdir')) # via config file # hardcoded first entry self.assertEqual(options.robot_paths[0], '/tmp/foo') diff --git a/test/framework/options.py b/test/framework/options.py index 045e868790..a8bc6fe985 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6246,6 +6246,7 @@ def test_config_abs_path(self): cfgtxt = '\n'.join([ "[config]", "containerpath = ..", + "repositorypath = /apps/easyconfigs_archive, somesubdir", ]) write_file(cfgfile, cfgtxt) @@ -6263,6 +6264,7 @@ def test_config_abs_path(self): r"^containerpath\s+\(F\) = .*/test_topdir/test_middle_dir$", r"^installpath\s+\(E\) = .*/test_topdir$", r"^prefix\s+\(C\) = .*/test_topdir/test_middle_dir$", + r"^repositorypath\s+\(F\) = \('/apps/easyconfigs_archive', ' somesubdir'\)$", r"^sourcepath\s+\(C\) = .*/test_topdir/test_middle_dir/test_subdir$", ] for pattern in patterns: From 25470adfb03bd407732055be8b74d2e31b899964 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Sep 2021 16:40:37 +0200 Subject: [PATCH 540/864] collapse OPT_DEPS and OPT_DEP_PKG_NAMES to EASYBUILD_OPTIONAL_DEPENDENCIES --- easybuild/tools/systemtools.py | 57 ++++++++++++++-------------------- 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 2c7052f4ef..52c6ee63c2 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -179,35 +179,24 @@ 'Slurm': 'sbatch', } -OPT_DEPS = { - 'archspec': "determining name of CPU microarchitecture", - 'autopep8': "auto-formatting for dumped easyconfigs", - 'GC3Pie': "backend for --job", - 'GitPython': "GitHub integration + using Git repository as easyconfigs archive", - 'graphviz-python': "rendering dependency graph with Graphviz: --dep-graph", - 'keyring': "storing GitHub token", - 'pbs-python': "using Torque as --job backend", - 'pep8': "fallback for code style checking: --check-style, --check-contrib", - 'pycodestyle': "code style checking: --check-style, --check-contrib", - 'pysvn': "using SVN repository as easyconfigs archive", - 'python-graph-core': "creating dependency graph: --dep-graph", - 'python-graph-dot': "saving dependency graph as dot file: --dep-graph", - 'python-hglib': "using Mercurial repository as easyconfigs archive", - 'requests': "fallback library for downloading files", - 'Rich': "eb command rich terminal output", - 'PyYAML': "easystack files and .yeb easyconfig format", - 'setuptools': "obtaining information on Python packages via pkg_resources module", -} - -OPT_DEP_PKG_NAMES = { - 'GC3Pie': 'gc3libs', - 'GitPython': 'git', - 'graphviz-python': 'gv', - 'pbs-python': 'pbs', - 'python-graph-core': 'pygraph.classes.digraph', - 'python-graph-dot': 'pygraph.readwrite.dot', - 'python-hglib': 'hglib', - 'PyYAML': 'yaml', +EASYBUILD_OPTIONAL_DEPENDENCIES = { + 'archspec': (None, "determining name of CPU microarchitecture"), + 'autopep8': (None, "auto-formatting for dumped easyconfigs"), + 'GC3Pie': ('gc3libs', "backend for --job"), + 'GitPython': ('git', "GitHub integration + using Git repository as easyconfigs archive"), + 'graphviz-python': ('gv', "rendering dependency graph with Graphviz: --dep-graph"), + 'keyring': (None, "storing GitHub token"), + 'pbs-python': ('pbs', "using Torque as --job backend"), + 'pep8': (None, "fallback for code style checking: --check-style, --check-contrib"), + 'pycodestyle': (None, "code style checking: --check-style, --check-contrib"), + 'pysvn': (None, "using SVN repository as easyconfigs archive"), + 'python-graph-core': ('pygraph.classes.digraph', "creating dependency graph: --dep-graph"), + 'python-graph-dot': ('pygraph.readwrite.dot', "saving dependency graph as dot file: --dep-graph"), + 'python-hglib': ('hglib', "using Mercurial repository as easyconfigs archive"), + 'requests': (None, "fallback library for downloading files"), + 'Rich': (None, "eb command rich terminal output"), + 'PyYAML': ('yaml', "easystack files and .yeb easyconfig format"), + 'setuptools': ('pkg_resources', "obtaining information on Python packages via pkg_resources module"), } @@ -1209,9 +1198,11 @@ def extract_version(tool): python_version = extract_version(sys.executable) opt_dep_versions = {} - for key in OPT_DEPS: + for key in EASYBUILD_OPTIONAL_DEPENDENCIES: - pkg = OPT_DEP_PKG_NAMES.get(key, key.lower()) + pkg = EASYBUILD_OPTIONAL_DEPENDENCIES[key][0] + if pkg is None: + pkg = key.lower() try: mod = __import__(pkg) @@ -1235,8 +1226,8 @@ def extract_version(tool): opt_deps_key = "Optional dependencies" checks_data[opt_deps_key] = {} - for pkg in opt_dep_versions: - checks_data[opt_deps_key][pkg] = (opt_dep_versions[pkg], OPT_DEPS[pkg]) + for key in opt_dep_versions: + checks_data[opt_deps_key][key] = (opt_dep_versions[key], EASYBUILD_OPTIONAL_DEPENDENCIES[key][1]) sys_tools_key = "System tools" checks_data[sys_tools_key] = {} From dc40a2b7f671ffbcaa2824728142cfc544e59cbe Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Sep 2021 17:13:18 +0200 Subject: [PATCH 541/864] fix log message in get_cfg_opt_abs_path --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 8d7b3af441..3dc979b149 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1054,7 +1054,7 @@ def get_cfg_opt_abs_path(self, opt_name, path): else: abs_path = os.path.abspath(path) self.log.info("Relative path value for '%s' configuration option resolved to absolute path: %s", - path, abs_path) + opt_name, abs_path) return abs_path def _ensure_abs_path(self, opt_name): From 5b87003ea590fdcd2d570d6e3a911345fb3048de Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Sep 2021 17:40:19 +0200 Subject: [PATCH 542/864] also ensure absolute paths for 'robot' configuration option + enhance test_config_abs_path to also check on robot and robot-paths configuration options --- easybuild/tools/options.py | 4 ++-- test/framework/options.py | 32 ++++++++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 3dc979b149..a0592bc2fb 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1077,7 +1077,7 @@ def _postprocess_config(self): # resolve relative paths for configuration options that specify a location path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath', 'installpath_modules', 'installpath_software', 'prefix', 'packagepath', - 'robot_paths', 'sourcepath'] + 'robot', 'robot_paths', 'sourcepath'] # repositorypath is a special case: only first part is a path; # 2nd (optional) part is a relative subdir and should not be resolved to an absolute path! @@ -1133,7 +1133,7 @@ def _postprocess_config(self): # paths specified to --robot have preference over --robot-paths # keep both values in sync if robot is enabled, which implies enabling dependency resolver - self.options.robot_paths = [os.path.abspath(path) for path in self.options.robot + self.options.robot_paths] + self.options.robot_paths = self.options.robot + self.options.robot_paths self.options.robot = self.options.robot_paths # Update the search_paths (if any) to absolute paths diff --git a/test/framework/options.py b/test/framework/options.py index a8bc6fe985..0425402049 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6250,7 +6250,10 @@ def test_config_abs_path(self): ]) write_file(cfgfile, cfgtxt) + # relative paths in environment variables is also weird, + # but OK for the sake of testing... os.environ['EASYBUILD_INSTALLPATH'] = '../..' + os.environ['EASYBUILD_ROBOT_PATHS'] = '../..' args = [ '--configfiles=%s' % cfgfile, @@ -6258,19 +6261,40 @@ def test_config_abs_path(self): '--sourcepath=.', '--show-config', ] + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) patterns = [ - r"^containerpath\s+\(F\) = .*/test_topdir/test_middle_dir$", - r"^installpath\s+\(E\) = .*/test_topdir$", - r"^prefix\s+\(C\) = .*/test_topdir/test_middle_dir$", + r"^containerpath\s+\(F\) = /.*/test_topdir/test_middle_dir$", + r"^installpath\s+\(E\) = /.*/test_topdir$", + r"^prefix\s+\(C\) = /.*/test_topdir/test_middle_dir$", r"^repositorypath\s+\(F\) = \('/apps/easyconfigs_archive', ' somesubdir'\)$", - r"^sourcepath\s+\(C\) = .*/test_topdir/test_middle_dir/test_subdir$", + r"^sourcepath\s+\(C\) = /.*/test_topdir/test_middle_dir/test_subdir$", + r"^robot-paths\s+\(E\) = /.*/test_topdir$", ] for pattern in patterns: regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (pattern, txt)) + # if --robot is also used, that wins and $EASYBUILD_ROBOT_PATHS doesn't matter anymore + change_dir(test_subdir) + args.append('--robot=..:.') + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) + + patterns.pop(-1) + robot_value_pattern = ', '.join([ + r'/.*/test_topdir/test_middle_dir', # via --robot (first path) + r'/.*/test_topdir/test_middle_dir/test_subdir', # via --robot (second path) + r'/.*/test_topdir', # via $EASYBUILD_ROBOT_PATHS + ]) + patterns.extend([ + r"^robot-paths\s+\(C\) = %s$" % robot_value_pattern, + r"^robot\s+\(C\) = %s$" % robot_value_pattern, + ]) + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (pattern, txt)) + # end-to-end testing of unknown filename def test_easystack_wrong_read(self): """Test for --easystack when wrong name is provided""" From 74175a78cd6754c9420c8d18329c119bed2c772b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Sep 2021 18:37:08 +0200 Subject: [PATCH 543/864] fix comment in test_config_abs_path --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 0425402049..c547f2328d 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6276,7 +6276,7 @@ def test_config_abs_path(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (pattern, txt)) - # if --robot is also used, that wins and $EASYBUILD_ROBOT_PATHS doesn't matter anymore + # paths specified via --robot have precedence over those specified via $EASYBUILD_ROBOT_PATHS change_dir(test_subdir) args.append('--robot=..:.') txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) From 593105e058c29d68719ad7f65ace4428beddf2eb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Sep 2021 19:37:39 +0200 Subject: [PATCH 544/864] add --output-style configuration option, which can be used to disable use of Rich or type of any colored output --- easybuild/tools/config.py | 32 ++++++++++++++++++++++++++++++++ easybuild/tools/options.py | 6 +++++- easybuild/tools/output.py | 20 ++++++++++++-------- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 94b02d4424..fc583e7c4f 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -48,6 +48,12 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.py2vs3 import ascii_letters, create_base_metaclass, string_type +try: + import rich + HAVE_RICH = True +except ImportError: + HAVE_RICH = False + _log = fancylogger.getLogger('config', fname=False) @@ -137,6 +143,13 @@ LOCAL_VAR_NAMING_CHECKS = [LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG, LOCAL_VAR_NAMING_CHECK_WARN] +OUTPUT_STYLE_AUTO = 'auto' +OUTPUT_STYLE_BASIC = 'basic' +OUTPUT_STYLE_NO_COLOR = 'no_color' +OUTPUT_STYLE_RICH = 'rich' +OUTPUT_STYLES = (OUTPUT_STYLE_AUTO, OUTPUT_STYLE_BASIC, OUTPUT_STYLE_NO_COLOR, OUTPUT_STYLE_RICH) + + class Singleton(ABCMeta): """Serves as metaclass for classes that should implement the Singleton pattern. @@ -342,6 +355,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_WAIT_ON_LOCK_INTERVAL: [ 'wait_on_lock_interval', ], + OUTPUT_STYLE_AUTO: [ + 'output_style', + ], } # build option that do not have a perfectly matching command line option BUILD_OPTIONS_OTHER = { @@ -688,6 +704,22 @@ def get_module_syntax(): return ConfigurationVariables()['module_syntax'] +def get_output_style(): + """Return output style to use.""" + output_style = build_option('output_style') + + if output_style == OUTPUT_STYLE_AUTO: + if HAVE_RICH: + output_style = OUTPUT_STYLE_RICH + else: + output_style = OUTPUT_STYLE_BASIC + + if output_style == OUTPUT_STYLE_RICH and not HAVE_RICH: + raise EasyBuildError("Can't use '%s' output style, Rich Python package is not available!", OUTPUT_STYLE_RICH) + + return output_style + + def log_file_format(return_directory=False, ec=None, date=None, timestamp=None): """ Return the format for the logfile or the directory diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 759e8d8e42..0d4cec6787 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -69,7 +69,8 @@ from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS -from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS, WARN +from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS +from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path from easybuild.tools.configobj import ConfigObj, ConfigObjError from easybuild.tools.docs import FORMAT_TXT, FORMAT_RST @@ -448,6 +449,9 @@ def override_options(self): 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_TXT, FORMAT_RST]), + 'output-style': ("Control output style; auto implies using Rich if available to produce rich output, " + "with fallback to basic colored output", + 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), 'parallel': ("Specify (maximum) level of parallellism used during build procedure", 'int', 'store', None), 'pre-create-installdir': ("Create installation directory before submitting build jobs", diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 5b9c9ba3dd..fb9ad176c4 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -31,16 +31,15 @@ """ import random -from easybuild.tools.config import build_option +from easybuild.tools.config import OUTPUT_STYLE_RICH, build_option, get_output_style from easybuild.tools.py2vs3 import OrderedDict try: from rich.console import Console from rich.table import Table from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn - HAVE_RICH = True except ImportError: - HAVE_RICH = False + pass class DummyProgress(object): @@ -61,6 +60,11 @@ def update(self, *args, **kwargs): pass +def use_rich(): + """Return whether or not to use Rich to produce rich output.""" + return get_output_style() == OUTPUT_STYLE_RICH + + def create_progress_bar(): """ Create progress bar to display overall progress. @@ -68,7 +72,7 @@ def create_progress_bar(): Returns rich.progress.Progress instance if the Rich Python package is available, or a shim DummyProgress instance otherwise. """ - if HAVE_RICH and build_option('show_progress_bar'): + if use_rich() and build_option('show_progress_bar'): # pick random spinner, from a selected subset of available spinner (see 'python3 -m rich.spinner') spinner = random.choice(('aesthetic', 'arc', 'bounce', 'dots', 'line', 'monkey', 'point', 'simpleDots')) @@ -95,7 +99,7 @@ def print_checks(checks_data): col2_label = col_titles[1] - if HAVE_RICH: + if use_rich(): console = Console() # don't use console.print, which causes SyntaxError in Python 2 console_print = getattr(console, 'print') # noqa: B009 @@ -104,7 +108,7 @@ def print_checks(checks_data): for section in checks_data: section_checks = checks_data[section] - if HAVE_RICH: + if use_rich(): table = Table(title=section) table.add_column(col_titles[0]) table.add_column(col_titles[1]) @@ -124,7 +128,7 @@ def print_checks(checks_data): else: check_names = sorted(section_checks, key=lambda x: x.lower()) - if HAVE_RICH: + if use_rich(): for check_name in check_names: (info, descr) = checks_data[section][check_name] if info is None: @@ -150,7 +154,7 @@ def print_checks(checks_data): lines.append(line) lines.append('') - if HAVE_RICH: + if use_rich(): console_print(table) else: print('\n'.join(lines)) From f538359a8ce653f14e534562639de1c6ebfacf2d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Sep 2021 21:25:09 +0200 Subject: [PATCH 545/864] silence the Hound on unused import of rich in tools.config --- easybuild/tools/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index fc583e7c4f..18902ae799 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -49,7 +49,7 @@ from easybuild.tools.py2vs3 import ascii_letters, create_base_metaclass, string_type try: - import rich + import rich # noqa HAVE_RICH = True except ImportError: HAVE_RICH = False From 476a826c6f18ffbe610fd4c45f7fb782580a450e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 14 Sep 2021 09:07:05 +0200 Subject: [PATCH 546/864] add tests for function provided by easybuild.tools.output --- test/framework/output.py | 110 +++++++++++++++++++++++++++++++++++++++ test/framework/suite.py | 3 +- 2 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 test/framework/output.py diff --git a/test/framework/output.py b/test/framework/output.py new file mode 100644 index 0000000000..b69a90cd35 --- /dev/null +++ b/test/framework/output.py @@ -0,0 +1,110 @@ +# # +# Copyright 2021-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Unit tests for functionality in easybuild.tools.output + +@author: Kenneth Hoste (Ghent University) +""" +import sys +from unittest import TextTestRunner +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered + +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option, get_output_style, update_build_option +from easybuild.tools.output import DummyProgress, create_progress_bar, use_rich + +try: + import rich.progress + HAVE_RICH = True +except ImportError: + HAVE_RICH = False + + +class OutputTest(EnhancedTestCase): + """Tests for functions controlling terminal output.""" + + def test_create_progress_bar(self): + """Test create_progress_bar function.""" + + progress_bar = create_progress_bar() + if HAVE_RICH: + progress_bar_class = rich.progress.ProgressBar + else: + progress_bar_class = DummyProgress + self.assertTrue(isinstance(progress_bar, progress_bar_class)) + + update_build_option('output_style', 'basic') + self.assertTrue(isinstance(progress_bar, DummyProgress)) + + update_build_option('output_style', 'rich') + self.assertTrue(isinstance(progress_bar, progress_bar_class)) + + update_build_option('show_progress_bar', False) + self.assertTrue(isinstance(progress_bar, DummyProgress)) + + def test_get_output_style(self): + """Test get_output_style function.""" + + self.assertEqual(build_option('output_style'), 'auto') + + for style in (None, 'auto'): + if style: + update_build_option('output_style', style) + + if HAVE_RICH: + self.assertEqual(get_output_style(), 'rich') + else: + self.assertEqual(get_output_style(), 'basic') + + test_styles = ['basic', 'no_color'] + if HAVE_RICH: + test_styles.append('rich') + + for style in test_styles: + update_build_option('output_style', style) + self.assertEqual(get_output_style(), style) + + if not HAVE_RICH: + update_build_option('output_style', 'rich') + error_pattern = "Can't use 'rich' output style, Rich Python package is not available!" + self.assertErrorRegex(EasyBuildError, error_pattern, get_output_style) + + def test_use_rich(self): + """Test use_rich function.""" + try: + import rich # noqa + self.assertTrue(use_rich()) + except ImportError: + self.assertFalse(use_rich()) + + +def suite(): + """ returns all the testcases in this module """ + return TestLoaderFiltered().loadTestsFromTestCase(OutputTest, sys.argv[1:]) + + +if __name__ == '__main__': + res = TextTestRunner(verbosity=1).run(suite()) + sys.exit(len(res.failures)) diff --git a/test/framework/suite.py b/test/framework/suite.py index 1633bba103..80bce4983f 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -66,6 +66,7 @@ import test.framework.modules as m import test.framework.modulestool as mt import test.framework.options as o +import test.framework.output as ou import test.framework.parallelbuild as p import test.framework.package as pkg import test.framework.repository as r @@ -120,7 +121,7 @@ # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, lic, f_c, - tw, p, i, pkg, d, env, et, y, st, h, ct, lib, u, es] + tw, p, i, pkg, d, env, et, y, st, h, ct, lib, u, es, ou] SUITE = unittest.TestSuite([x.suite() for x in tests]) res = unittest.TextTestRunner().run(SUITE) From 4c9f72dc0eea2e31cc9ae62384292bc9d6a2e822 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 14 Sep 2021 09:49:23 +0200 Subject: [PATCH 547/864] fix test_create_progress_bar --- test/framework/output.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/test/framework/output.py b/test/framework/output.py index b69a90cd35..174f8164d2 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -48,20 +48,27 @@ class OutputTest(EnhancedTestCase): def test_create_progress_bar(self): """Test create_progress_bar function.""" - progress_bar = create_progress_bar() if HAVE_RICH: - progress_bar_class = rich.progress.ProgressBar + expected_progress_bar_class = rich.progress.Progress else: - progress_bar_class = DummyProgress - self.assertTrue(isinstance(progress_bar, progress_bar_class)) + expected_progress_bar_class = DummyProgress + + progress_bar = create_progress_bar() + error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class) + self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg) update_build_option('output_style', 'basic') + progress_bar = create_progress_bar() self.assertTrue(isinstance(progress_bar, DummyProgress)) - update_build_option('output_style', 'rich') - self.assertTrue(isinstance(progress_bar, progress_bar_class)) + if HAVE_RICH: + update_build_option('output_style', 'rich') + progress_bar = create_progress_bar() + error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class) + self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg) update_build_option('show_progress_bar', False) + progress_bar = create_progress_bar() self.assertTrue(isinstance(progress_bar, DummyProgress)) def test_get_output_style(self): From 6934dd84c5d1de61851bf9cd73d2791a206ea90a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 14 Sep 2021 10:37:23 +0200 Subject: [PATCH 548/864] handle ensuring of absolute paths in 'robot' configuration value separately, due to special treatment needed for --robot argument --- easybuild/tools/options.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index a0592bc2fb..06ce7f6013 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1074,10 +1074,12 @@ def _ensure_abs_path(self, opt_name): def _postprocess_config(self): """Postprocessing of configuration options""" - # resolve relative paths for configuration options that specify a location + # resolve relative paths for configuration options that specify a location; + # ensuring absolute paths for 'robot' is handled separately below, + # because we need to be careful with the argument pass to --robot path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath', 'installpath_modules', 'installpath_software', 'prefix', 'packagepath', - 'robot', 'robot_paths', 'sourcepath'] + 'robot_paths', 'sourcepath'] # repositorypath is a special case: only first part is a path; # 2nd (optional) part is a relative subdir and should not be resolved to an absolute path! @@ -1133,7 +1135,7 @@ def _postprocess_config(self): # paths specified to --robot have preference over --robot-paths # keep both values in sync if robot is enabled, which implies enabling dependency resolver - self.options.robot_paths = self.options.robot + self.options.robot_paths + self.options.robot_paths = [os.path.abspath(p) for p in self.options.robot + self.options.robot_paths] self.options.robot = self.options.robot_paths # Update the search_paths (if any) to absolute paths From 88034abbd152d283edd51edf8af0b28152242a94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= Date: Wed, 15 Sep 2021 08:57:05 +0200 Subject: [PATCH 549/864] Explicitly check if a file exist when copying Previous behavior would simply skip copying a file if it did not exist and report copy success. This commit changes that behavior to error out early if the file does not exist and explicitly warn if trying to copy something that is neither a file nor a symbolic link. --- easybuild/tools/filetools.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 0e4afabe1a..99096c34fc 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2270,6 +2270,8 @@ def copy_file(path, target_path, force_in_dry_run=False): :param target_path: path to copy the file to :param force_in_dry_run: force copying of file during dry run """ + if not os.path.exists(path): + raise EasyBuildError("could not copy '%s' it does not exist!" % path) if not force_in_dry_run and build_option('extended_dry_run'): dry_run_msg("copied file %s to %s" % (path, target_path)) else: @@ -2285,13 +2287,16 @@ def copy_file(path, target_path, force_in_dry_run=False): _log.info("Copied contents of file %s to %s", path, target_path) else: mkdir(os.path.dirname(target_path), parents=True) - if os.path.exists(path): + if os.path.isfile(path) and not os.path.islink(path): shutil.copy2(path, target_path) + _log.info("%s copied to %s", path, target_path) elif os.path.islink(path): # special care for copying broken symlinks link_target = os.readlink(path) symlink(link_target, target_path) - _log.info("%s copied to %s", path, target_path) + _log.info("created symlink from %s to %s", path, target_path) + else: + _log.warn("ignoring '%s' since it is neither a file nor a symlink" % path) except (IOError, OSError, shutil.Error) as err: raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err) From a7dde5fcfd98defd59325a53bb295a3a527ec79e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= Date: Wed, 15 Sep 2021 09:20:21 +0200 Subject: [PATCH 550/864] Do not check if path is a file Previous intention is simply to let the error bubble up if the copy tries to affect a directory. --- easybuild/tools/filetools.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 99096c34fc..89579b188a 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2287,16 +2287,14 @@ def copy_file(path, target_path, force_in_dry_run=False): _log.info("Copied contents of file %s to %s", path, target_path) else: mkdir(os.path.dirname(target_path), parents=True) - if os.path.isfile(path) and not os.path.islink(path): - shutil.copy2(path, target_path) - _log.info("%s copied to %s", path, target_path) - elif os.path.islink(path): + if os.path.islink(path): # special care for copying broken symlinks link_target = os.readlink(path) symlink(link_target, target_path) _log.info("created symlink from %s to %s", path, target_path) else: - _log.warn("ignoring '%s' since it is neither a file nor a symlink" % path) + shutil.copy2(path, target_path) + _log.info("%s copied to %s", path, target_path) except (IOError, OSError, shutil.Error) as err: raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err) From 94e9876038530c89dcd3ad6b81287ee15cb0264d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 15 Sep 2021 09:59:41 +0200 Subject: [PATCH 551/864] only call os.path.abspath on paths passed to robot paths, robot_paths is already handled --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 06ce7f6013..614ba78f97 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1135,7 +1135,7 @@ def _postprocess_config(self): # paths specified to --robot have preference over --robot-paths # keep both values in sync if robot is enabled, which implies enabling dependency resolver - self.options.robot_paths = [os.path.abspath(p) for p in self.options.robot + self.options.robot_paths] + self.options.robot_paths = [os.path.abspath(p) for p in self.options.robot] + self.options.robot_paths self.options.robot = self.options.robot_paths # Update the search_paths (if any) to absolute paths From a9745d99c91da5898666038872229076257ca7e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= Date: Wed, 15 Sep 2021 10:37:42 +0200 Subject: [PATCH 552/864] Added check if path is a broken symlink --- easybuild/tools/filetools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 89579b188a..66b2249b3a 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2270,7 +2270,8 @@ def copy_file(path, target_path, force_in_dry_run=False): :param target_path: path to copy the file to :param force_in_dry_run: force copying of file during dry run """ - if not os.path.exists(path): + # NOTE: 'exists' will return False if 'path' is a broken symlink + if not os.path.exists(path) and not os.path.islink(path): raise EasyBuildError("could not copy '%s' it does not exist!" % path) if not force_in_dry_run and build_option('extended_dry_run'): dry_run_msg("copied file %s to %s" % (path, target_path)) From 467c29c6a1d2ff1c6162c43176d6c8b1bc83fc76 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 14 Sep 2021 21:06:11 +0200 Subject: [PATCH 553/864] disable progress bars when running the tests to avoid messing up test suite output --- test/framework/output.py | 3 +++ test/framework/utilities.py | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/test/framework/output.py b/test/framework/output.py index 174f8164d2..be5d4d5046 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -48,6 +48,9 @@ class OutputTest(EnhancedTestCase): def test_create_progress_bar(self): """Test create_progress_bar function.""" + # restore default (was disabled in EnhancedTestCase.setUp to avoid messing up test output) + update_build_option('show_progress_bar', True) + if HAVE_RICH: expected_progress_bar_class = rich.progress.Progress else: diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 4a82aaf6a8..e6188991b4 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -48,7 +48,7 @@ from easybuild.framework.easyblock import EasyBlock from easybuild.main import main from easybuild.tools import config -from easybuild.tools.config import GENERAL_CLASS, Singleton, module_classes +from easybuild.tools.config import GENERAL_CLASS, Singleton, module_classes, update_build_option from easybuild.tools.configobj import ConfigObj from easybuild.tools.environment import modify_env from easybuild.tools.filetools import copy_dir, mkdir, read_file, which @@ -137,6 +137,11 @@ def setUp(self): init_config() + # disable progress bars when running the tests, + # since it messes up with test suite progress output when test installations are performed + os.environ['EASYBUILD_DISABLE_SHOW_PROGRESS_BAR'] = '1' + update_build_option('show_progress_bar', False) + import easybuild # try to import easybuild.easyblocks(.generic) packages # it's OK if it fails here, but important to import first before fiddling with sys.path From 90c24677edde832e82e8e06c9d0a88299e14540c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= Date: Thu, 16 Sep 2021 08:01:39 +0200 Subject: [PATCH 554/864] Added test for copy of non-existing file --- easybuild/tools/filetools.py | 2 +- test/framework/filetools.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 66b2249b3a..2cd50d2791 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2272,7 +2272,7 @@ def copy_file(path, target_path, force_in_dry_run=False): """ # NOTE: 'exists' will return False if 'path' is a broken symlink if not os.path.exists(path) and not os.path.islink(path): - raise EasyBuildError("could not copy '%s' it does not exist!" % path) + raise EasyBuildError("Could not copy '%s' it does not exist!" % path) if not force_in_dry_run and build_option('extended_dry_run'): dry_run_msg("copied file %s to %s" % (path, target_path)) else: diff --git a/test/framework/filetools.py b/test/framework/filetools.py index f3eed568f8..55684fa56e 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1661,6 +1661,10 @@ def test_copy_file(self): self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path)) self.assertEqual(txt, '') + # Test that a non-existing file raises an exception + src, target = os.path.join(self.test_prefix, 'this_file_does_not_exist'), os.path.join(self.test_prefix, 'toy') + self.assertErrorRegex(EasyBuildError, "Could not copy *", ft.copy_file, src, target) + def test_copy_files(self): """Test copy_files function.""" test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') From d5ccae9252369d6d5b3bb8e2d66b33512e3f3199 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 17 Sep 2021 08:39:37 +0200 Subject: [PATCH 555/864] fix comment --- test/framework/utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index e6188991b4..14248243d3 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -138,7 +138,7 @@ def setUp(self): init_config() # disable progress bars when running the tests, - # since it messes up with test suite progress output when test installations are performed + # since it messes with test suite progress output when test installations are performed os.environ['EASYBUILD_DISABLE_SHOW_PROGRESS_BAR'] = '1' update_build_option('show_progress_bar', False) From cf12d627a4fb6d6a883636a4baa94b192e3e2e9d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 17 Sep 2021 10:47:13 +0200 Subject: [PATCH 556/864] take into account changed error raised by Python 3.9.7 when copying directory via shutil.copyfile --- test/framework/filetools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index f3eed568f8..722710430f 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1610,7 +1610,9 @@ def test_copy_file(self): # clean error when trying to copy a directory with copy_file src, target = os.path.dirname(to_copy), os.path.join(self.test_prefix, 'toy') - self.assertErrorRegex(EasyBuildError, "Failed to copy file.*Is a directory", ft.copy_file, src, target) + # error message was changed in Python 3.9.7 to "FileNotFoundError: Directory does not exist" + error_pattern = "Failed to copy file.*(Is a directory|Directory does not exist)" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.copy_file, src, target) # test overwriting of existing file owned by someone else, # which should make copy_file use shutil.copyfile rather than shutil.copy2 From c951010574a82030eaa9d129739f5b937697945d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= Date: Fri, 17 Sep 2021 12:03:32 +0200 Subject: [PATCH 557/864] Do not raise exception in dry run mode --- easybuild/tools/filetools.py | 8 +++++--- test/framework/filetools.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 2cd50d2791..9d589bd90a 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2272,9 +2272,11 @@ def copy_file(path, target_path, force_in_dry_run=False): """ # NOTE: 'exists' will return False if 'path' is a broken symlink if not os.path.exists(path) and not os.path.islink(path): - raise EasyBuildError("Could not copy '%s' it does not exist!" % path) - if not force_in_dry_run and build_option('extended_dry_run'): - dry_run_msg("copied file %s to %s" % (path, target_path)) + if force_in_dry_run: + raise EasyBuildError("Could not copy '%s' it does not exist!", path) + else: + _log.debug("Ignoring non-existing file in 'copy_file' because of dry run mode: %s", path) + dry_run_msg("copied file %s to %s" % (path, target_path)) else: try: target_exists = os.path.exists(target_path) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 55684fa56e..9af1f8a1a7 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1662,8 +1662,18 @@ def test_copy_file(self): self.assertEqual(txt, '') # Test that a non-existing file raises an exception + update_build_option('extended_dry_run', False) src, target = os.path.join(self.test_prefix, 'this_file_does_not_exist'), os.path.join(self.test_prefix, 'toy') self.assertErrorRegex(EasyBuildError, "Could not copy *", ft.copy_file, src, target) + # Test that copying a non-existing file in 'dry_run' mode does noting + update_build_option('extended_dry_run', True) + self.mock_stdout(True) + ft.copy_file(src, target, force_in_dry_run=False) + txt = self.get_stdout() + self.mock_stdout(False) + self.assertTrue(re.search("^copied file %s to %s" % (src, target), txt)) + # However, if we add 'force_in_dry_run=True' it should throw an exception + self.assertErrorRegex(EasyBuildError, "Could not copy *", ft.copy_file, src, target, force_in_dry_run=True) def test_copy_files(self): """Test copy_files function.""" From f0ab45899917aecb1749269008d09cf2ab0024d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= Date: Fri, 17 Sep 2021 12:08:50 +0200 Subject: [PATCH 558/864] Added import of 'update_build_options' --- test/framework/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 9af1f8a1a7..9bea250b75 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -45,7 +45,7 @@ from easybuild.tools import run import easybuild.tools.filetools as ft from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import IGNORE, ERROR +from easybuild.tools.config import IGNORE, ERROR, update_build_option from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import std_urllib From 9313e40eb97faccd17c3360731bc1e5dc092c9c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= Date: Fri, 17 Sep 2021 13:54:26 +0200 Subject: [PATCH 559/864] Fixed so that copy_file adheres to dry run --- easybuild/tools/filetools.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 9d589bd90a..723ac5d6e1 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2270,13 +2270,12 @@ def copy_file(path, target_path, force_in_dry_run=False): :param target_path: path to copy the file to :param force_in_dry_run: force copying of file during dry run """ - # NOTE: 'exists' will return False if 'path' is a broken symlink - if not os.path.exists(path) and not os.path.islink(path): - if force_in_dry_run: - raise EasyBuildError("Could not copy '%s' it does not exist!", path) - else: - _log.debug("Ignoring non-existing file in 'copy_file' because of dry run mode: %s", path) - dry_run_msg("copied file %s to %s" % (path, target_path)) + if not force_in_dry_run and build_option('extended_dry_run'): + # If in dry run mode, do not copy any files, just lie about it + dry_run_msg("copied file %s to %s" % (path, target_path)) + elif not os.path.exists(path) and not os.path.islink(path): + # NOTE: 'exists' will return False if 'path' is a broken symlink + raise EasyBuildError("Could not copy '%s' it does not exist!", path) else: try: target_exists = os.path.exists(target_path) From fdd13a4b8f7c131d98b12a636a6dfa8e1893a4ac Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 17 Sep 2021 16:08:23 +0200 Subject: [PATCH 560/864] Restore edited comment --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a325d5c868..2b6febcfb7 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3959,7 +3959,7 @@ def reproduce_build(app, reprod_dir_root): except NotImplementedError as err: _log.warning("Unable to dump easyconfig instance to %s: %s", reprod_spec, err) - # also archive all the relevant easyblocks + # also archive all the relevant easyblocks (including any used by extensions) copy_easyblocks_for_reprod([app], reprod_dir) # if there is a hook file we should also archive it From 76471d9d34fa0bb0b0e85b9a9663cf12cf3e0c6c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 21 Sep 2021 10:44:13 +0200 Subject: [PATCH 561/864] use separate progress bars for overall progress, installation steps, downloading sources --- easybuild/framework/easyblock.py | 29 +---- easybuild/main.py | 28 ++--- easybuild/tools/filetools.py | 40 ++++++- easybuild/tools/output.py | 194 +++++++++++++++++++++++++++---- 4 files changed, 223 insertions(+), 68 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 58bed6afde..a719073333 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -89,6 +89,7 @@ from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX from easybuild.tools.modules import Lmod, curr_module_paths, invalidate_module_caches_for, get_software_root from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name +from easybuild.tools.output import PROGRESS_BAR_EASYCONFIG, start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.package.utilities import package from easybuild.tools.py2vs3 import extract_method_name, string_type from easybuild.tools.repository.repository import init_repository @@ -304,21 +305,6 @@ def close_log(self): self.log.info("Closing log for application name %s version %s" % (self.name, self.version)) fancylogger.logToFile(self.logfile, enable=False) - def set_progress_bar(self, progress_bar, task_id): - """ - Set progress bar, the progress bar is needed when writing messages so - that the progress counter is always at the bottom - """ - self.progress_bar = progress_bar - self.pbar_task = task_id - - def advance_progress(self, tick=1.0): - """ - Advance the progress bar forward with `tick` - """ - if self.progress_bar and self.pbar_task is not None: - self.progress_bar.advance(self.pbar_task, tick) - # # DRY RUN UTILITIES # @@ -3587,8 +3573,8 @@ def run_all_steps(self, run_test_cases): return True steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt()) - # Calculate progress bar tick - tick = 1.0 / float(len(steps)) + + start_progress_bar(PROGRESS_BAR_EASYCONFIG, len(steps), label=self.full_mod_name) print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent) trace_msg("installation prefix: %s" % self.installdir) @@ -3627,7 +3613,7 @@ def run_all_steps(self, run_test_cases): print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent) elif self.logdebug or build_option('trace'): print_msg("... (took < 1 sec)", log=self.log, silent=self.silent) - self.advance_progress(tick) + update_progress_bar(PROGRESS_BAR_EASYCONFIG, progress_size=1) except StopException: pass @@ -3653,7 +3639,7 @@ def print_dry_run_note(loc, silent=True): dry_run_msg(msg, silent=silent) -def build_and_install_one(ecdict, init_env, progress_bar=None, task_id=None): +def build_and_install_one(ecdict, init_env): """ Build the software :param ecdict: dictionary contaning parsed easyconfig + metadata @@ -3701,11 +3687,6 @@ def build_and_install_one(ecdict, init_env, progress_bar=None, task_id=None): print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg), silent=silent) - # Setup progress bar - if progress_bar and task_id is not None: - app.set_progress_bar(progress_bar, task_id) - _log.info("Updated progress bar instance for easyblock %s", easyblock) - # application settings stop = build_option('stop') if stop is not None: diff --git a/easybuild/main.py b/easybuild/main.py index 1e5792fd0e..59d7c60c20 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -68,7 +68,8 @@ from easybuild.tools.hooks import START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import set_up_configuration, use_color -from easybuild.tools.output import create_progress_bar, print_checks +from easybuild.tools.output import PROGRESS_BAR_OVERALL, print_checks, rich_live_cm +from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs from easybuild.tools.package.utilities import check_pkg_support from easybuild.tools.parallelbuild import submit_jobs @@ -101,36 +102,27 @@ def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing= return [(ec_file, generated)] -def build_and_install_software(ecs, init_session_state, exit_on_failure=True, progress_bar=None): +def build_and_install_software(ecs, init_session_state, exit_on_failure=True): """ Build and install software for all provided parsed easyconfig files. :param ecs: easyconfig files to install software with :param init_session_state: initial session state, to use in test reports :param exit_on_failure: whether or not to exit on installation failure - :param progress_bar: progress bar to use to report progress """ # obtain a copy of the starting environment so each build can start afresh # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since # e.g. via easyconfig.handle_allowed_system_deps init_env = copy.deepcopy(os.environ) - # Initialize progress bar with overall installation task - if progress_bar: - task_id = progress_bar.add_task("", total=len(ecs)) - else: - task_id = None + start_progress_bar(PROGRESS_BAR_OVERALL, size=len(ecs)) res = [] for ec in ecs: - if progress_bar: - progress_bar.update(task_id, description=ec['short_mod_name']) - ec_res = {} try: - (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progress_bar=progress_bar, - task_id=task_id) + (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env) ec_res['log_file'] = app_log if not ec_res['success']: ec_res['err'] = EasyBuildError(err) @@ -169,6 +161,10 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, pr res.append((ec, ec_res)) + update_progress_bar(PROGRESS_BAR_OVERALL, progress_size=1) + + stop_progress_bar(PROGRESS_BAR_OVERALL, visible=True) + return res @@ -540,11 +536,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) - progress_bar = create_progress_bar() - with progress_bar: + with rich_live_cm(): ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, - exit_on_failure=exit_on_failure, - progress_bar=progress_bar) + exit_on_failure=exit_on_failure) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 0e4afabe1a..125975bdfd 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -54,6 +54,7 @@ import tempfile import time import zlib +from functools import partial from easybuild.base import fancylogger from easybuild.tools import run @@ -61,6 +62,7 @@ from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN from easybuild.tools.config import build_option, install_path +from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD, start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.py2vs3 import HTMLParser, std_urllib, string_type from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars @@ -215,7 +217,8 @@ def read_file(path, log_error=True, mode='r'): return txt -def write_file(path, data, append=False, forced=False, backup=False, always_overwrite=True, verbose=False): +def write_file(path, data, append=False, forced=False, backup=False, always_overwrite=True, verbose=False, + show_progress=False, size=None): """ Write given contents to file at given path; overwrites current file contents without backup by default! @@ -227,6 +230,8 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over :param backup: back up existing file before overwriting or modifying it :param always_overwrite: don't require --force to overwrite an existing file :param verbose: be verbose, i.e. inform where backup file was created + :param show_progress: show progress bar while writing file + :param size: size (in bytes) of data to write (used for progress bar) """ # early exit in 'dry run' mode if not forced and build_option('extended_dry_run'): @@ -256,15 +261,30 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over if sys.version_info[0] >= 3 and (isinstance(data, bytes) or data_is_file_obj): mode += 'b' + # don't bother showing a progress bar for small files + if size and size < 1024: + _log.info("Not showing progress bar for downloading small file (size %s)", size) + show_progress = False + + if show_progress: + start_progress_bar(PROGRESS_BAR_DOWNLOAD, size, label=os.path.basename(path)) + # note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block try: mkdir(os.path.dirname(path), parents=True) with open_file(path, mode) as fh: if data_is_file_obj: - # if a file-like object was provided, use copyfileobj (which reads the file in chunks) - shutil.copyfileobj(data, fh) + # if a file-like object was provided, read file in 1MB chunks + for chunk in iter(partial(data.read, 1024 ** 2), b''): + fh.write(chunk) + if show_progress: + update_progress_bar(PROGRESS_BAR_DOWNLOAD, progress_size=len(chunk)) else: fh.write(data) + + if show_progress: + stop_progress_bar(PROGRESS_BAR_DOWNLOAD) + except IOError as err: raise EasyBuildError("Failed to write to %s: %s", path, err) @@ -752,11 +772,21 @@ def download_file(filename, url, path, forced=False): while not downloaded and attempt_cnt < max_attempts: attempt_cnt += 1 try: + size = None if used_urllib is std_urllib: # urllib2 (Python 2) / urllib.request (Python 3) does the right thing for http proxy setups, # urllib does not! url_fd = std_urllib.urlopen(url_req, timeout=timeout) status_code = url_fd.getcode() + http_header = url_fd.info() + len_key = 'Content-Length' + if len_key in http_header: + size = http_header[len_key] + try: + size = int(size) + except (ValueError, TypeError) as err: + _log.warning("Failed to interpret size '%s' as integer value: %s", size, err) + size = None else: response = requests.get(url, headers=headers, stream=True, timeout=timeout) status_code = response.status_code @@ -768,8 +798,8 @@ def download_file(filename, url, path, forced=False): # to ensure the data is read in chunks (which prevents problems in Python 3.9+); # cfr. https://github.com/easybuilders/easybuild-framework/issues/3455 # and https://bugs.python.org/issue42853 - write_file(path, url_fd, forced=forced, backup=True) - _log.info("Downloaded file %s from url %s to %s" % (filename, url, path)) + write_file(path, url_fd, forced=forced, backup=True, show_progress=True, size=size) + _log.info("Downloaded file %s from url %s to %s", filename, url, path) downloaded = True url_fd.close() except used_urllib.HTTPError as err: diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index fb9ad176c4..81cd31db05 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -29,21 +29,35 @@ :author: Kenneth Hoste (Ghent University) :author: Jørgen Nordmoen (University of Oslo) """ +import functools import random +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import OUTPUT_STYLE_RICH, build_option, get_output_style from easybuild.tools.py2vs3 import OrderedDict try: - from rich.console import Console + from rich.console import Console, RenderGroup + from rich.live import Live from rich.table import Table from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn + from rich.progress import DownloadColumn, FileSizeColumn, TransferSpeedColumn, TimeRemainingColumn except ImportError: pass -class DummyProgress(object): - """Shim for Rich's Progress class.""" +PROGRESS_BAR_DOWNLOAD = 'download' +PROGRESS_BAR_EASYCONFIG = 'easyconfig' +PROGRESS_BAR_OVERALL = 'overall' + +_progress_bar_cache = {} + + +class DummyRich(object): + """ + Dummy shim for Rich classes. + Used in case Rich is not available, or when EasyBuild is not configured to use rich output style. + """ # __enter__ and __exit__ must be implemented to allow use as context manager def __enter__(self, *args, **kwargs): @@ -61,37 +75,173 @@ def update(self, *args, **kwargs): def use_rich(): - """Return whether or not to use Rich to produce rich output.""" + """ + Return whether or not to use Rich to produce rich output. + """ return get_output_style() == OUTPUT_STYLE_RICH -def create_progress_bar(): +def rich_live_cm(): """ - Create progress bar to display overall progress. - - Returns rich.progress.Progress instance if the Rich Python package is available, - or a shim DummyProgress instance otherwise. + Return Live instance to use as context manager. """ if use_rich() and build_option('show_progress_bar'): - - # pick random spinner, from a selected subset of available spinner (see 'python3 -m rich.spinner') - spinner = random.choice(('aesthetic', 'arc', 'bounce', 'dots', 'line', 'monkey', 'point', 'simpleDots')) - - progress_bar = Progress( - SpinnerColumn(spinner), - "[progress.percentage]{task.percentage:>3.1f}%", - TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total} done)"), - BarColumn(bar_width=None), - TimeElapsedColumn(), - transient=True, - expand=True, + overall_pbar = overall_progress_bar() + easyconfig_pbar = easyconfig_progress_bar() + download_pbar = download_progress_bar() + download_pbar_bis = download_progress_bar_unknown_size() + pbar_group = RenderGroup( + download_pbar, + download_pbar_bis, + easyconfig_pbar, + overall_pbar ) + live = Live(pbar_group) else: - progress_bar = DummyProgress() + live = DummyRich() + + return live + + +def progress_bar_cache(func): + """ + Function decorator to cache created progress bars for easy retrieval. + """ + @functools.wraps(func) + def new_func(): + if hasattr(func, 'cached'): + progress_bar = func.cached + elif use_rich() and build_option('show_progress_bar'): + progress_bar = func() + else: + progress_bar = DummyRich() + + func.cached = progress_bar + return func.cached + + return new_func + + +@progress_bar_cache +def overall_progress_bar(): + """ + Get progress bar to display overall progress. + """ + progress_bar = Progress( + TimeElapsedColumn(), + TextColumn("{task.description}({task.completed} out of {task.total} easyconfigs done)"), + BarColumn(bar_width=None), + expand=True, + ) + + return progress_bar + + +@progress_bar_cache +def easyconfig_progress_bar(): + """ + Get progress bar to display progress for installing a single easyconfig file. + """ + progress_bar = Progress( + TextColumn("[bold blue]{task.description} ({task.completed} out of {task.total} steps done)"), + BarColumn(), + TimeElapsedColumn(), + ) + + return progress_bar + + +@progress_bar_cache +def download_progress_bar(): + """ + Get progress bar to show progress for downloading a file of known size. + """ + progress_bar = Progress( + TextColumn('[bold yellow]Downloading {task.description}'), + BarColumn(), + DownloadColumn(), + TransferSpeedColumn(), + TimeRemainingColumn(), + ) return progress_bar +@progress_bar_cache +def download_progress_bar_unknown_size(): + """ + Get progress bar to show progress for downloading a file of unknown size. + """ + progress_bar = Progress( + TextColumn('[bold yellow]Downloading {task.description}'), + FileSizeColumn(), + TransferSpeedColumn(), + ) + + return progress_bar + + +def get_progress_bar(bar_type, size=None): + """ + Get progress bar of given type. + """ + progress_bar_types = { + PROGRESS_BAR_DOWNLOAD: download_progress_bar, + PROGRESS_BAR_EASYCONFIG: easyconfig_progress_bar, + PROGRESS_BAR_OVERALL: overall_progress_bar, + } + + if bar_type == PROGRESS_BAR_DOWNLOAD and not size: + pbar = download_progress_bar_unknown_size() + elif bar_type in progress_bar_types: + pbar = progress_bar_types[bar_type]() + else: + raise EasyBuildError("Unknown progress bar type: %s", bar_type) + + return pbar + + +def start_progress_bar(bar_type, size, label=None): + """ + Start progress bar of given type. + + :param label: label for progress bar + :param size: total target size of progress bar + """ + pbar = get_progress_bar(bar_type, size=size) + task_id = pbar.add_task('') + _progress_bar_cache[bar_type] = (pbar, task_id) + if size: + pbar.update(task_id, total=size) + if label: + pbar.update(task_id, description=label) + + +def update_progress_bar(bar_type, label=None, progress_size=None): + """ + Update progress bar of given type, add progress of given size. + + :param bar_type: type of progress bar + :param label: label for progress bar + :param progress_size: size of progress made + """ + (pbar, task_id) = _progress_bar_cache[bar_type] + if label: + pbar.update(task_id, description=label) + if progress_size: + pbar.update(task_id, advance=progress_size) + + +def stop_progress_bar(bar_type, visible=False): + """ + Stop progress bar of given type. + """ + (pbar, task_id) = _progress_bar_cache[bar_type] + pbar.stop_task(task_id) + if not visible: + pbar.update(task_id, visible=False) + + def print_checks(checks_data): """Print overview of checks that were made.""" From b0b64f7f03e9e918b073f991914c9c306e4254c7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 21 Sep 2021 11:19:30 +0200 Subject: [PATCH 562/864] set progress bar label to 'done' for finished installations --- easybuild/framework/easyblock.py | 14 +++++++++++--- easybuild/tools/output.py | 5 ++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a719073333..766085ece0 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3574,7 +3574,11 @@ def run_all_steps(self, run_test_cases): steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt()) - start_progress_bar(PROGRESS_BAR_EASYCONFIG, len(steps), label=self.full_mod_name) + progress_label_tmpl = "%s (%d out of %d steps done)" + + n_steps = len(steps) + progress_label = progress_label_tmpl % (self.full_mod_name, 0, n_steps) + start_progress_bar(PROGRESS_BAR_EASYCONFIG, n_steps, label=progress_label) print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent) trace_msg("installation prefix: %s" % self.installdir) @@ -3594,7 +3598,7 @@ def run_all_steps(self, run_test_cases): create_lock(lock_name) try: - for (step_name, descr, step_methods, skippable) in steps: + for step_id, (step_name, descr, step_methods, skippable) in enumerate(steps): if self.skip_step(step_name, skippable): print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent) else: @@ -3613,7 +3617,9 @@ def run_all_steps(self, run_test_cases): print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent) elif self.logdebug or build_option('trace'): print_msg("... (took < 1 sec)", log=self.log, silent=self.silent) - update_progress_bar(PROGRESS_BAR_EASYCONFIG, progress_size=1) + + progress_label = progress_label_tmpl % (self.full_mod_name, step_id, n_steps) + update_progress_bar(PROGRESS_BAR_EASYCONFIG, progress_size=1, label=progress_label) except StopException: pass @@ -3621,6 +3627,8 @@ def run_all_steps(self, run_test_cases): if not ignore_locks: remove_lock(lock_name) + update_progress_bar(PROGRESS_BAR_EASYCONFIG, label="%s done!" % self.full_mod_name) + # return True for successfull build (or stopped build) return True diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 81cd31db05..8743441dd4 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -30,7 +30,6 @@ :author: Jørgen Nordmoen (University of Oslo) """ import functools -import random from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import OUTPUT_STYLE_RICH, build_option, get_output_style @@ -40,7 +39,7 @@ from rich.console import Console, RenderGroup from rich.live import Live from rich.table import Table - from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn + from rich.progress import BarColumn, Progress, TextColumn, TimeElapsedColumn from rich.progress import DownloadColumn, FileSizeColumn, TransferSpeedColumn, TimeRemainingColumn except ImportError: pass @@ -143,7 +142,7 @@ def easyconfig_progress_bar(): Get progress bar to display progress for installing a single easyconfig file. """ progress_bar = Progress( - TextColumn("[bold blue]{task.description} ({task.completed} out of {task.total} steps done)"), + TextColumn("[bold blue]{task.description}"), BarColumn(), TimeElapsedColumn(), ) From 42398de3ac10284683f0626e67b09fe1dc38c0a8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 21 Sep 2021 11:20:21 +0200 Subject: [PATCH 563/864] change default progress size for update_progress_bar function --- easybuild/framework/easyblock.py | 2 +- easybuild/tools/output.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 766085ece0..7e886e748d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3619,7 +3619,7 @@ def run_all_steps(self, run_test_cases): print_msg("... (took < 1 sec)", log=self.log, silent=self.silent) progress_label = progress_label_tmpl % (self.full_mod_name, step_id, n_steps) - update_progress_bar(PROGRESS_BAR_EASYCONFIG, progress_size=1, label=progress_label) + update_progress_bar(PROGRESS_BAR_EASYCONFIG, label=progress_label) except StopException: pass diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 8743441dd4..10457b1b93 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -216,13 +216,13 @@ def start_progress_bar(bar_type, size, label=None): pbar.update(task_id, description=label) -def update_progress_bar(bar_type, label=None, progress_size=None): +def update_progress_bar(bar_type, label=None, progress_size=1): """ Update progress bar of given type, add progress of given size. :param bar_type: type of progress bar :param label: label for progress bar - :param progress_size: size of progress made + :param progress_size: amount of progress made """ (pbar, task_id) = _progress_bar_cache[bar_type] if label: From 3808f298c6a5a7c7d8b1ffa09874ab4e42ba9217 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 21 Sep 2021 11:21:37 +0200 Subject: [PATCH 564/864] fix unused import --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 7e886e748d..e33c9e796c 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -89,7 +89,7 @@ from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX from easybuild.tools.modules import Lmod, curr_module_paths, invalidate_module_caches_for, get_software_root from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name -from easybuild.tools.output import PROGRESS_BAR_EASYCONFIG, start_progress_bar, stop_progress_bar, update_progress_bar +from easybuild.tools.output import PROGRESS_BAR_EASYCONFIG, start_progress_bar, update_progress_bar from easybuild.tools.package.utilities import package from easybuild.tools.py2vs3 import extract_method_name, string_type from easybuild.tools.repository.repository import init_repository From f8be1c5a99c0381f0437d5b7aab00f1df635f69e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Sep 2021 16:11:22 +0200 Subject: [PATCH 565/864] fix tests for overall_progress_bar --- easybuild/tools/output.py | 4 ++-- test/framework/output.py | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 10457b1b93..e99301c8bb 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -107,8 +107,8 @@ def progress_bar_cache(func): Function decorator to cache created progress bars for easy retrieval. """ @functools.wraps(func) - def new_func(): - if hasattr(func, 'cached'): + def new_func(ignore_cache=False): + if hasattr(func, 'cached') and not ignore_cache: progress_bar = func.cached elif use_rich() and build_option('show_progress_bar'): progress_bar = func() diff --git a/test/framework/output.py b/test/framework/output.py index 174f8164d2..28ec84ab3e 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -33,7 +33,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_output_style, update_build_option -from easybuild.tools.output import DummyProgress, create_progress_bar, use_rich +from easybuild.tools.output import DummyRich, overall_progress_bar, use_rich try: import rich.progress @@ -45,31 +45,31 @@ class OutputTest(EnhancedTestCase): """Tests for functions controlling terminal output.""" - def test_create_progress_bar(self): - """Test create_progress_bar function.""" + def test_overall_progress_bar(self): + """Test overall_progress_bar function.""" if HAVE_RICH: expected_progress_bar_class = rich.progress.Progress else: - expected_progress_bar_class = DummyProgress + expected_progress_bar_class = DummyRich - progress_bar = create_progress_bar() + progress_bar = overall_progress_bar() error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class) self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg) update_build_option('output_style', 'basic') - progress_bar = create_progress_bar() - self.assertTrue(isinstance(progress_bar, DummyProgress)) + progress_bar = overall_progress_bar(ignore_cache=True) + self.assertTrue(isinstance(progress_bar, DummyRich)) if HAVE_RICH: update_build_option('output_style', 'rich') - progress_bar = create_progress_bar() + progress_bar = overall_progress_bar(ignore_cache=True) error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class) self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg) update_build_option('show_progress_bar', False) - progress_bar = create_progress_bar() - self.assertTrue(isinstance(progress_bar, DummyProgress)) + progress_bar = overall_progress_bar(ignore_cache=True) + self.assertTrue(isinstance(progress_bar, DummyRich)) def test_get_output_style(self): """Test get_output_style function.""" From d5950ecbc60bd116b6ba6618925318d6f242b4d1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Sep 2021 17:25:52 +0200 Subject: [PATCH 566/864] add dummy implementation of stop_task to DummyRich --- easybuild/tools/output.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index e99301c8bb..dca4e8fe0c 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -69,6 +69,9 @@ def __exit__(self, *args, **kwargs): def add_task(self, *args, **kwargs): pass + def stop_task(self, *args, **kwargs): + pass + def update(self, *args, **kwargs): pass From 4080cbba9f77cec2ae0019fabede8eefe06dc589 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 26 Sep 2021 10:20:08 +0200 Subject: [PATCH 567/864] add dummy implementation to __rich_console__ method to DummyRich --- easybuild/tools/output.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index dca4e8fe0c..1e31613183 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -75,6 +75,10 @@ def stop_task(self, *args, **kwargs): def update(self, *args, **kwargs): pass + # internal Rich methods + def __rich_console__(self, *args, **kwargs): + pass + def use_rich(): """ From ff9e9723d5e89e44654656bf762ceaf30d83a161 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 27 Sep 2021 15:35:53 +0200 Subject: [PATCH 568/864] Ensure newer location of CUDA stubs is taken into account by rpath filter --- easybuild/tools/toolchain/toolchain.py | 8 +++++--- test/framework/toolchain.py | 8 ++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 782d6634b3..45229ba722 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -960,9 +960,11 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None # always include filter for 'stubs' library directory, # cfr. https://github.com/easybuilders/easybuild-framework/issues/2683 - lib_stubs_pattern = '.*/lib(64)?/stubs/?' - if lib_stubs_pattern not in rpath_filter_dirs: - rpath_filter_dirs.append(lib_stubs_pattern) + # (since CUDA 11.something the stubs are in $EBROOTCUDA/stubs/lib64) + lib_stubs_patterns = ['.*/lib(64)?/stubs/?', '.*/stubs/lib(64)?/?'] + for lib_stubs_pattern in lib_stubs_patterns: + if lib_stubs_pattern not in rpath_filter_dirs: + rpath_filter_dirs.append(lib_stubs_pattern) # directory where all wrappers will be placed wrappers_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 11a6c083d0..403f6774ca 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -2437,12 +2437,16 @@ def test_toolchain_prepare_rpath(self): # check whether 'stubs' library directory are correctly filtered out paths = [ 'prefix/software/CUDA/1.2.3/lib/stubs/', # should be filtered (no -rpath) + 'prefix/software/CUDA/1.2.3/stubs/lib/', # should be filtered (no -rpath) 'tmp/foo/', 'prefix/software/stubs/1.2.3/lib', # should NOT be filtered 'prefix/software/CUDA/1.2.3/lib/stubs', # should be filtered (no -rpath) + 'prefix/software/CUDA/1.2.3/stubs/lib', # should be filtered (no -rpath) 'prefix/software/CUDA/1.2.3/lib64/stubs/', # should be filtered (no -rpath) + 'prefix/software/CUDA/1.2.3/stubs/lib64/', # should be filtered (no -rpath) 'prefix/software/foobar/4.5/notreallystubs', # should NOT be filtered 'prefix/software/CUDA/1.2.3/lib64/stubs', # should be filtered (no -rpath) + 'prefix/software/CUDA/1.2.3/stubs/lib64', # should be filtered (no -rpath) 'prefix/software/zlib/1.2.11/lib', 'prefix/software/bleh/0/lib/stubs', # should be filtered (no -rpath) 'prefix/software/foobar/4.5/stubsbutnotreally', # should NOT be filtered @@ -2465,12 +2469,16 @@ def test_toolchain_prepare_rpath(self): '-Wl,-rpath=%s/prefix/software/foobar/4.5/stubsbutnotreally' % self.test_prefix, '%(user)s.c', '-L%s/prefix/software/CUDA/1.2.3/lib/stubs/' % self.test_prefix, + '-L%s/prefix/software/CUDA/1.2.3/stubs/lib/' % self.test_prefix, '-L%s/tmp/foo/' % self.test_prefix, '-L%s/prefix/software/stubs/1.2.3/lib' % self.test_prefix, '-L%s/prefix/software/CUDA/1.2.3/lib/stubs' % self.test_prefix, + '-L%s/prefix/software/CUDA/1.2.3/stubs/lib' % self.test_prefix, '-L%s/prefix/software/CUDA/1.2.3/lib64/stubs/' % self.test_prefix, + '-L%s/prefix/software/CUDA/1.2.3/stubs/lib64/' % self.test_prefix, '-L%s/prefix/software/foobar/4.5/notreallystubs' % self.test_prefix, '-L%s/prefix/software/CUDA/1.2.3/lib64/stubs' % self.test_prefix, + '-L%s/prefix/software/CUDA/1.2.3/stubs/lib64' % self.test_prefix, '-L%s/prefix/software/zlib/1.2.11/lib' % self.test_prefix, '-L%s/prefix/software/bleh/0/lib/stubs' % self.test_prefix, '-L%s/prefix/software/foobar/4.5/stubsbutnotreally' % self.test_prefix, From b01f1414625b7d07c93c47aa5294ef75a3035685 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Tue, 28 Sep 2021 11:38:25 +0100 Subject: [PATCH 569/864] GPU Info --- easybuild/tools/options.py | 13 ++++++++++++- easybuild/tools/systemtools.py | 27 +++++++++++++++++++++++++++ easybuild/tools/testing.py | 15 +++++++++++++-- 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 8ed73b66d5..5d313b8a13 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -99,7 +99,7 @@ from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME from easybuild.tools.repository.repository import avail_repositories from easybuild.tools.systemtools import UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family -from easybuild.tools.systemtools import get_cpu_features, get_system_info +from easybuild.tools.systemtools import get_cpu_features, get_gpu_info, get_system_info from easybuild.tools.version import this_is_easybuild @@ -1292,6 +1292,7 @@ def show_system_info(self): """Show system information.""" system_info = get_system_info() cpu_features = get_cpu_features() + gpu_info = get_gpu_info() cpu_arch_name = system_info['cpu_arch_name'] lines = [ "System information (%s):" % system_info['hostname'], @@ -1324,6 +1325,16 @@ def show_system_info(self): " -> Python version: %s" % sys.version.split(' ')[0], ]) + if gpu_info: + lines.extend([ + '', + "* GPU:", + ]) + for vendor in gpu_info: + lines.append(" -> %s" % vendor) + for gpu, num in gpu_info[vendor].items(): + lines.append(" -> %s: %s" % (gpu, num)) + return '\n'.join(lines) def show_config(self): diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 52c6ee63c2..e177d62583 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -574,6 +574,33 @@ def get_cpu_features(): return cpu_feat +def get_gpu_info(): + """ + Get the GPU info + """ + gpu_info = {} + os_type = get_os_type() + + if os_type == LINUX: + try: + cmd = "nvidia-smi --query-gpu=gpu_name,driver_version --format=csv,noheader" + _log.debug("Trying to determine NVIDIA GPU info on Linux via cmd '%s'", cmd) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) + if ec == 0: + for line in out.strip().split('\n'): + if 'NVIDIA' in gpu_info and line in gpu_info['NVIDIA']: + gpu_info['NVIDIA'][line] += 1 + else: + gpu_info['NVIDIA'] = {} + gpu_info['NVIDIA'][line] = 1 + except Exception: + _log.debug("No NVIDIA GPUs detected") + else: + _log.debug("Only know how to get GPU info on Linux") + + return gpu_info + + def get_kernel_name(): """NO LONGER SUPPORTED: use get_os_type() instead""" _log.nosupport("get_kernel_name() is replaced by get_os_type()", '2.0') diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 5d765b6b82..f9e4d0886c 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -50,7 +50,7 @@ from easybuild.tools.jenkins import aggregate_xml_in_dirs from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel from easybuild.tools.robot import resolve_dependencies -from easybuild.tools.systemtools import UNKNOWN, get_system_info +from easybuild.tools.systemtools import UNKNOWN, get_gpu_info, get_system_info from easybuild.tools.version import FRAMEWORK_VERSION, EASYBLOCKS_VERSION @@ -295,11 +295,22 @@ def post_pr_test_report(pr_nrs, repo_type, test_report, msg, init_session_state, if system_info['cpu_arch_name'] != UNKNOWN: system_info['cpu_model'] += " (%s)" % system_info['cpu_arch_name'] + # add GPU info, if known + gpu_info = get_gpu_info() + if gpu_info: + gpu_str = " gpu: " + for vendor in gpu_info: + for gpu, num in gpu_info[vendor].items(): + gpu_str += "%s %s x %s; " % (vendor, num, gpu) + else: + gpu_str = "" + os_info = '%(hostname)s - %(os_type)s %(os_name)s %(os_version)s' % system_info - short_system_info = "%(os_info)s, %(cpu_arch)s, %(cpu_model)s, Python %(pyver)s" % { + short_system_info = "%(os_info)s, %(cpu_arch)s, %(cpu_model)s%(gpu)s, Python %(pyver)s" % { 'os_info': os_info, 'cpu_arch': system_info['cpu_arch'], 'cpu_model': system_info['cpu_model'], + 'gpu': gpu_str, 'pyver': system_info['python_version'].split(' ')[0], } From 2e344200c1671bdbdf2c9e79815416186462f683 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Tue, 28 Sep 2021 17:12:42 +0100 Subject: [PATCH 570/864] improve code, from review --- easybuild/tools/systemtools.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index e177d62583..284aa5c5ee 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -588,15 +588,16 @@ def get_gpu_info(): out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) if ec == 0: for line in out.strip().split('\n'): - if 'NVIDIA' in gpu_info and line in gpu_info['NVIDIA']: - gpu_info['NVIDIA'][line] += 1 - else: - gpu_info['NVIDIA'] = {} - gpu_info['NVIDIA'][line] = 1 - except Exception: - _log.debug("No NVIDIA GPUs detected") + nvidia_gpu_info = gpu_info.setdefault('NVIDIA', {}) + nvidia_gpu_info.setdefault(line, 0) + nvidia_gpu_info[line] += 1 + else: + _log.debug("None zero exit (%s) from nvidia-smi: %s" % (ec, out)) + except Exception as err: + _log.debug("Exception was raised when running nvidia-smi: %s", err) + _log.info("No NVIDIA GPUs detected") else: - _log.debug("Only know how to get GPU info on Linux") + _log.info("Only know how to get GPU info on Linux, assuming no GPUs are present") return gpu_info From dc0ae01804d20cf76b123341ef36c3db57c2918c Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Tue, 28 Sep 2021 17:46:00 +0100 Subject: [PATCH 571/864] improve PR test report message formatting --- easybuild/tools/testing.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index f9e4d0886c..f9edc8d865 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -297,13 +297,11 @@ def post_pr_test_report(pr_nrs, repo_type, test_report, msg, init_session_state, # add GPU info, if known gpu_info = get_gpu_info() + gpu_str = "" if gpu_info: - gpu_str = " gpu: " for vendor in gpu_info: for gpu, num in gpu_info[vendor].items(): - gpu_str += "%s %s x %s; " % (vendor, num, gpu) - else: - gpu_str = "" + gpu_str += ", %s %s x %s" % (vendor, num, gpu) os_info = '%(hostname)s - %(os_type)s %(os_name)s %(os_version)s' % system_info short_system_info = "%(os_info)s, %(cpu_arch)s, %(cpu_model)s%(gpu)s, Python %(pyver)s" % { From db8a157b893b551dfa4275274e0cf9a42e505d24 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 29 Sep 2021 06:09:48 +0100 Subject: [PATCH 572/864] Improve info Co-authored-by: SebastianAchilles --- easybuild/tools/testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index f9edc8d865..8f2e4d3529 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -301,7 +301,7 @@ def post_pr_test_report(pr_nrs, repo_type, test_report, msg, init_session_state, if gpu_info: for vendor in gpu_info: for gpu, num in gpu_info[vendor].items(): - gpu_str += ", %s %s x %s" % (vendor, num, gpu) + gpu_str += ", %s x %s %s" % (num, vendor, gpu) os_info = '%(hostname)s - %(os_type)s %(os_name)s %(os_version)s' % system_info short_system_info = "%(os_info)s, %(cpu_arch)s, %(cpu_model)s%(gpu)s, Python %(pyver)s" % { From e92a2acf0ea8edb1b39dd208495b542adaa834b6 Mon Sep 17 00:00:00 2001 From: Christoph Siegert Date: Wed, 29 Sep 2021 14:29:34 +0200 Subject: [PATCH 573/864] replace which by command -v to avoid dependency Same rational as on line 65, see #2976 --- eb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb b/eb index 53a345f08f..880759f62c 100755 --- a/eb +++ b/eb @@ -104,7 +104,7 @@ if [ -z $PYTHON ]; then echo "(EasyBuild requires Python 2.${REQ_MIN_PY2VER}+ or 3.${REQ_MIN_PY3VER}+)" >&2 exit 1 else - verbose "Selected Python command: $python_cmd (`which $python_cmd`)" + verbose "Selected Python command: $python_cmd (`command -v $python_cmd`)" fi # enable optimization, unless $PYTHONOPTIMIZE is defined (use "export PYTHONOPTIMIZE=0" to disable optimization) From 9f3a957507e6aae68fbfa7164535e000fcf8f5d3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 29 Sep 2021 21:05:05 +0200 Subject: [PATCH 574/864] tweak GPU part of output produced by --show-system-info --- easybuild/tools/options.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 5d313b8a13..e4fc7661a4 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1318,11 +1318,6 @@ def show_system_info(self): " -> speed: %s" % system_info['cpu_speed'], " -> cores: %s" % system_info['core_count'], " -> features: %s" % ','.join(cpu_features), - '', - "* software:", - " -> glibc version: %s" % system_info['glibc_version'], - " -> Python binary: %s" % sys.executable, - " -> Python version: %s" % sys.version.split(' ')[0], ]) if gpu_info: @@ -1333,7 +1328,15 @@ def show_system_info(self): for vendor in gpu_info: lines.append(" -> %s" % vendor) for gpu, num in gpu_info[vendor].items(): - lines.append(" -> %s: %s" % (gpu, num)) + lines.append(" -> %sx %s" % (num, gpu)) + + lines.extend([ + '', + "* software:", + " -> glibc version: %s" % system_info['glibc_version'], + " -> Python binary: %s" % sys.executable, + " -> Python version: %s" % sys.version.split(' ')[0], + ]) return '\n'.join(lines) From ab90fa194f07276709201d46b547db5c879cb419 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 29 Sep 2021 21:09:49 +0200 Subject: [PATCH 575/864] use lazy logging in get_gpu_info --- easybuild/tools/systemtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 284aa5c5ee..6783b49eaa 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -592,7 +592,7 @@ def get_gpu_info(): nvidia_gpu_info.setdefault(line, 0) nvidia_gpu_info[line] += 1 else: - _log.debug("None zero exit (%s) from nvidia-smi: %s" % (ec, out)) + _log.debug("None zero exit (%s) from nvidia-smi: %s", ec, out) except Exception as err: _log.debug("Exception was raised when running nvidia-smi: %s", err) _log.info("No NVIDIA GPUs detected") From 80381ad82b7747d9ae5a0d8be59593784c926da5 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 30 Sep 2021 08:52:14 +0200 Subject: [PATCH 576/864] Fix copy_file so it doesn't fail if the target_path is an existing dir. --- easybuild/tools/filetools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 723ac5d6e1..788e554dc9 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2277,6 +2277,8 @@ def copy_file(path, target_path, force_in_dry_run=False): # NOTE: 'exists' will return False if 'path' is a broken symlink raise EasyBuildError("Could not copy '%s' it does not exist!", path) else: + if os.path.isdir(target_path): + target_path = os.path.join(target_path, os.path.basename(path)) try: target_exists = os.path.exists(target_path) if target_exists and os.path.samefile(path, target_path): From e04a838e806eaa02e1819f5361534b2138eeaf74 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 30 Sep 2021 09:03:39 +0200 Subject: [PATCH 577/864] Add test for copy_file that it shouldn't fail if it copies a new file into a partially occupied target_dir --- test/framework/filetools.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index ee9eb2ab48..10328f006e 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1608,6 +1608,13 @@ def test_copy_file(self): self.assertTrue(os.path.exists(target_path)) self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path)) + # Make sure it doesn't fail if target_path is a dir with some other file(s) in it + to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb') + target_path = self.test_prefix + ft.copy_file(to_copy, target_path) + self.assertTrue(os.path.exists(os.path.join(target_path, os.path.basename(to_copy)))) + self.assertTrue(ft.read_file(to_copy) == ft.read_file(os.path.join(target_path, os.path.basename(to_copy)))) + # clean error when trying to copy a directory with copy_file src, target = os.path.dirname(to_copy), os.path.join(self.test_prefix, 'toy') # error message was changed in Python 3.9.7 to "FileNotFoundError: Directory does not exist" From 44035e78cb32a13c88c2d45a9fc51c70af8a8c4c Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 30 Sep 2021 09:23:26 +0200 Subject: [PATCH 578/864] Adjust copy_file test to test the actual problem from issue https://github.com/easybuilders/easybuild-framework/issues/3854 I.e., copying a symlink into a dir --- test/framework/filetools.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 10328f006e..89631d07c0 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1608,11 +1608,12 @@ def test_copy_file(self): self.assertTrue(os.path.exists(target_path)) self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path)) - # Make sure it doesn't fail if target_path is a dir with some other file(s) in it - to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb') + # Make sure it doesn't fail if path is a symlink and target_path is a dir + to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-link-0.0.eb') target_path = self.test_prefix ft.copy_file(to_copy, target_path) self.assertTrue(os.path.exists(os.path.join(target_path, os.path.basename(to_copy)))) + self.assertTrue(os.path.islink(os.path.join(target_path, os.path.basename(to_copy)))) self.assertTrue(ft.read_file(to_copy) == ft.read_file(os.path.join(target_path, os.path.basename(to_copy)))) # clean error when trying to copy a directory with copy_file From 3b341973926117f6f5de631c338a1ae98d500b00 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 30 Sep 2021 09:24:41 +0200 Subject: [PATCH 579/864] Add the symlink file for the new copy_file test --- test/framework/easyconfigs/test_ecs/t/toy/toy-link-0.0.eb | 1 + 1 file changed, 1 insertion(+) create mode 120000 test/framework/easyconfigs/test_ecs/t/toy/toy-link-0.0.eb diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-link-0.0.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-link-0.0.eb new file mode 120000 index 0000000000..72dfcf609f --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-link-0.0.eb @@ -0,0 +1 @@ +toy-0.0.eb \ No newline at end of file From 79013ed08e02f53ce79c177a4994272c3978fd79 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 30 Sep 2021 10:34:37 +0200 Subject: [PATCH 580/864] Only change target_path in copy_file if the source path is a symlink Change test to correctly test for this situation. --- easybuild/tools/filetools.py | 9 +++++---- test/framework/filetools.py | 14 ++++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 788e554dc9..4dd40bd4f4 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2277,8 +2277,6 @@ def copy_file(path, target_path, force_in_dry_run=False): # NOTE: 'exists' will return False if 'path' is a broken symlink raise EasyBuildError("Could not copy '%s' it does not exist!", path) else: - if os.path.isdir(target_path): - target_path = os.path.join(target_path, os.path.basename(path)) try: target_exists = os.path.exists(target_path) if target_exists and os.path.samefile(path, target_path): @@ -2292,10 +2290,13 @@ def copy_file(path, target_path, force_in_dry_run=False): else: mkdir(os.path.dirname(target_path), parents=True) if os.path.islink(path): + if os.path.isdir(target_path): + target_path = os.path.join(target_path, os.path.basename(path)) + _log.info("target_path changed to %s", target_path) # special care for copying broken symlinks link_target = os.readlink(path) - symlink(link_target, target_path) - _log.info("created symlink from %s to %s", path, target_path) + symlink(link_target, target_path, use_abspath_source=False) + _log.info("created symlink %s to %s", link_target, target_path) else: shutil.copy2(path, target_path) _log.info("%s copied to %s", path, target_path) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 89631d07c0..4b25de75cf 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1609,12 +1609,14 @@ def test_copy_file(self): self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path)) # Make sure it doesn't fail if path is a symlink and target_path is a dir - to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-link-0.0.eb') - target_path = self.test_prefix - ft.copy_file(to_copy, target_path) - self.assertTrue(os.path.exists(os.path.join(target_path, os.path.basename(to_copy)))) - self.assertTrue(os.path.islink(os.path.join(target_path, os.path.basename(to_copy)))) - self.assertTrue(ft.read_file(to_copy) == ft.read_file(os.path.join(target_path, os.path.basename(to_copy)))) + link_to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-link-0.0.eb') + base_dir = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy') + link_to_copy = 'toy-link-0.0.eb' + dir_target_path = self.test_prefix + ft.copy_file(os.path.join(base_dir, link_to_copy), dir_target_path) + self.assertTrue(os.path.islink(os.path.join(dir_target_path, link_to_copy))) + self.assertTrue(os.readlink(os.path.join(dir_target_path, link_to_copy)) == os.readlink(os.path.join(base_dir, link_to_copy))) + os.remove(os.path.join(dir_target_path, link_to_copy)) # clean error when trying to copy a directory with copy_file src, target = os.path.dirname(to_copy), os.path.join(self.test_prefix, 'toy') From c273b105df0c28be0845daed390cc14a7e7d709f Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 30 Sep 2021 10:44:30 +0200 Subject: [PATCH 581/864] Forgot to update the check of the number of .eb files in the test_ecs tree after adding the test-link-0.0.eb --- test/framework/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 4b25de75cf..c9fee0b707 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2203,7 +2203,7 @@ def test_index_functions(self): # test with specified path with and without trailing '/'s for path in [test_ecs, test_ecs + '/', test_ecs + '//']: index = ft.create_index(path) - self.assertEqual(len(index), 89) + self.assertEqual(len(index), 90) expected = [ os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), From d5ee6f40ac7f643919918fae24263f119bdf5a65 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 30 Sep 2021 10:47:35 +0200 Subject: [PATCH 582/864] only create symlink to test copy_file with locally in test_copy_file --- .../test_ecs/t/toy/toy-link-0.0.eb | 1 - test/framework/filetools.py | 33 ++++++++++--------- 2 files changed, 17 insertions(+), 17 deletions(-) delete mode 120000 test/framework/easyconfigs/test_ecs/t/toy/toy-link-0.0.eb diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-link-0.0.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-link-0.0.eb deleted file mode 120000 index 72dfcf609f..0000000000 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-link-0.0.eb +++ /dev/null @@ -1 +0,0 @@ -toy-0.0.eb \ No newline at end of file diff --git a/test/framework/filetools.py b/test/framework/filetools.py index c9fee0b707..2f8052bc03 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1602,24 +1602,25 @@ def test_apply_patch(self): def test_copy_file(self): """Test copy_file function.""" testdir = os.path.dirname(os.path.abspath(__file__)) - to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') target_path = os.path.join(self.test_prefix, 'toy.eb') - ft.copy_file(to_copy, target_path) + ft.copy_file(toy_ec, target_path) self.assertTrue(os.path.exists(target_path)) - self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path)) + self.assertTrue(ft.read_file(toy_ec) == ft.read_file(target_path)) # Make sure it doesn't fail if path is a symlink and target_path is a dir - link_to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-link-0.0.eb') - base_dir = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy') - link_to_copy = 'toy-link-0.0.eb' - dir_target_path = self.test_prefix - ft.copy_file(os.path.join(base_dir, link_to_copy), dir_target_path) - self.assertTrue(os.path.islink(os.path.join(dir_target_path, link_to_copy))) - self.assertTrue(os.readlink(os.path.join(dir_target_path, link_to_copy)) == os.readlink(os.path.join(base_dir, link_to_copy))) - os.remove(os.path.join(dir_target_path, link_to_copy)) + toy_link_fn = 'toy-link-0.0.eb' + toy_link = os.path.join(self.test_prefix, toy_link_fn) + ft.symlink(toy_ec, toy_link) + dir_target_path = os.path.join(self.test_prefix, 'subdir') + ft.mkdir(dir_target_path) + ft.copy_file(toy_link, dir_target_path) + self.assertTrue(os.path.islink(os.path.join(dir_target_path, toy_link_fn))) + self.assertEqual(os.readlink(os.path.join(dir_target_path, toy_link_fn)), os.readlink(toy_link)) + os.remove(os.path.join(dir_target_path, toy_link)) # clean error when trying to copy a directory with copy_file - src, target = os.path.dirname(to_copy), os.path.join(self.test_prefix, 'toy') + src, target = os.path.dirname(toy_ec), os.path.join(self.test_prefix, 'toy') # error message was changed in Python 3.9.7 to "FileNotFoundError: Directory does not exist" error_pattern = "Failed to copy file.*(Is a directory|Directory does not exist)" self.assertErrorRegex(EasyBuildError, error_pattern, ft.copy_file, src, target) @@ -1656,7 +1657,7 @@ def test_copy_file(self): os.remove(target_path) self.mock_stdout(True) - ft.copy_file(to_copy, target_path) + ft.copy_file(toy_ec, target_path) txt = self.get_stdout() self.mock_stdout(False) @@ -1665,12 +1666,12 @@ def test_copy_file(self): # forced copy, even in dry run mode self.mock_stdout(True) - ft.copy_file(to_copy, target_path, force_in_dry_run=True) + ft.copy_file(toy_ec, target_path, force_in_dry_run=True) txt = self.get_stdout() self.mock_stdout(False) self.assertTrue(os.path.exists(target_path)) - self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path)) + self.assertTrue(ft.read_file(toy_ec) == ft.read_file(target_path)) self.assertEqual(txt, '') # Test that a non-existing file raises an exception @@ -2203,7 +2204,7 @@ def test_index_functions(self): # test with specified path with and without trailing '/'s for path in [test_ecs, test_ecs + '/', test_ecs + '//']: index = ft.create_index(path) - self.assertEqual(len(index), 90) + self.assertEqual(len(index), 89) expected = [ os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), From ea161c15f4e24ef3401ffb1bffceb639491b73dd Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 30 Sep 2021 10:48:43 +0200 Subject: [PATCH 583/864] Fix long line --- test/framework/filetools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index c9fee0b707..91f037fd75 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1615,7 +1615,9 @@ def test_copy_file(self): dir_target_path = self.test_prefix ft.copy_file(os.path.join(base_dir, link_to_copy), dir_target_path) self.assertTrue(os.path.islink(os.path.join(dir_target_path, link_to_copy))) - self.assertTrue(os.readlink(os.path.join(dir_target_path, link_to_copy)) == os.readlink(os.path.join(base_dir, link_to_copy))) + link_source = os.readlink(os.path.join(base_dir, link_to_copy)) + link_target = os.readlink(os.path.join(dir_target_path, link_to_copy)) + self.assertTrue(link_target == link_source) os.remove(os.path.join(dir_target_path, link_to_copy)) # clean error when trying to copy a directory with copy_file From 0cdf0c5fd86bad44aea7184fbfb6543d6d379b06 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Oct 2021 08:40:56 +0200 Subject: [PATCH 584/864] extend test for use_rich --- test/framework/output.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test/framework/output.py b/test/framework/output.py index 9fbe3ec5d9..32da43ebcc 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -103,12 +103,20 @@ def test_get_output_style(self): def test_use_rich(self): """Test use_rich function.""" - try: - import rich # noqa + + self.assertEqual(build_option('output_style'), 'auto') + + if HAVE_RICH: + self.assertTrue(use_rich()) + + update_build_option('output_style', 'rich') self.assertTrue(use_rich()) - except ImportError: + else: self.assertFalse(use_rich()) + update_build_option('output_style', 'basic') + self.assertFalse(use_rich()) + def suite(): """ returns all the testcases in this module """ From a7a5910651032e064e2cc54bb499234591a578c5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Oct 2021 08:46:56 +0200 Subject: [PATCH 585/864] always ignore cache when testing overall_progress_bar function --- test/framework/output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/output.py b/test/framework/output.py index 32da43ebcc..ad0280179d 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -56,7 +56,7 @@ def test_overall_progress_bar(self): else: expected_progress_bar_class = DummyRich - progress_bar = overall_progress_bar() + progress_bar = overall_progress_bar(ignore_cache=True) error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class) self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg) From 0dead5fe36e043699eaf8ffc90b5f093e0adb836 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Oct 2021 10:15:35 +0200 Subject: [PATCH 586/864] show progress bar for extensions --- easybuild/framework/easyblock.py | 12 +++++++++- easybuild/tools/output.py | 40 ++++++++++++++++++++++++-------- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e33c9e796c..57551c04c6 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -89,7 +89,8 @@ from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX from easybuild.tools.modules import Lmod, curr_module_paths, invalidate_module_caches_for, get_software_root from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name -from easybuild.tools.output import PROGRESS_BAR_EASYCONFIG, start_progress_bar, update_progress_bar +from easybuild.tools.output import PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS +from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.package.utilities import package from easybuild.tools.py2vs3 import extract_method_name, string_type from easybuild.tools.repository.repository import init_repository @@ -2415,6 +2416,9 @@ def extensions_step(self, fetch=False, install=True): self.skip_extensions() exts_cnt = len(self.ext_instances) + + start_progress_bar(PROGRESS_BAR_EXTENSIONS, exts_cnt) + for idx, ext in enumerate(self.ext_instances): self.log.debug("Starting extension %s" % ext.name) @@ -2422,8 +2426,12 @@ def extensions_step(self, fetch=False, install=True): # always go back to original work dir to avoid running stuff from a dir that no longer exists change_dir(self.orig_workdir) + progress_label = "Installing '%s' extension" % ext.name + update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label) + tup = (ext.name, ext.version or '', idx + 1, exts_cnt) print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent) + start_time = datetime.now() if self.dry_run: @@ -2459,6 +2467,8 @@ def extensions_step(self, fetch=False, install=True): elif self.logdebug or build_option('trace'): print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent) + stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False) + # cleanup (unload fake module, remove fake module dir) if fake_mod_data: self.clean_up_fake_module(fake_mod_data) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 1e31613183..dd895af1c0 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -46,6 +46,7 @@ PROGRESS_BAR_DOWNLOAD = 'download' +PROGRESS_BAR_EXTENSIONS = 'extensions' PROGRESS_BAR_EASYCONFIG = 'easyconfig' PROGRESS_BAR_OVERALL = 'overall' @@ -87,20 +88,24 @@ def use_rich(): return get_output_style() == OUTPUT_STYLE_RICH +def show_progress_bars(): + """ + Return whether or not to show progress bars. + """ + return use_rich() and build_option('show_progress_bar') + + def rich_live_cm(): """ Return Live instance to use as context manager. """ - if use_rich() and build_option('show_progress_bar'): - overall_pbar = overall_progress_bar() - easyconfig_pbar = easyconfig_progress_bar() - download_pbar = download_progress_bar() - download_pbar_bis = download_progress_bar_unknown_size() + if show_progress_bars(): pbar_group = RenderGroup( - download_pbar, - download_pbar_bis, - easyconfig_pbar, - overall_pbar + download_progress_bar(), + download_progress_bar_unknown_size(), + extensions_progress_bar(), + easyconfig_progress_bar(), + overall_progress_bar(), ) live = Live(pbar_group) else: @@ -149,7 +154,7 @@ def easyconfig_progress_bar(): Get progress bar to display progress for installing a single easyconfig file. """ progress_bar = Progress( - TextColumn("[bold blue]{task.description}"), + TextColumn("[bold green]{task.description}"), BarColumn(), TimeElapsedColumn(), ) @@ -187,12 +192,27 @@ def download_progress_bar_unknown_size(): return progress_bar +@progress_bar_cache +def extensions_progress_bar(): + """ + Get progress bar to show progress for installing extensions. + """ + progress_bar = Progress( + TextColumn("[bold blue]{task.description} ({task.completed}/{task.total})"), + BarColumn(), + TimeElapsedColumn(), + ) + + return progress_bar + + def get_progress_bar(bar_type, size=None): """ Get progress bar of given type. """ progress_bar_types = { PROGRESS_BAR_DOWNLOAD: download_progress_bar, + PROGRESS_BAR_EXTENSIONS: extensions_progress_bar, PROGRESS_BAR_EASYCONFIG: easyconfig_progress_bar, PROGRESS_BAR_OVERALL: overall_progress_bar, } From b956e8b040ea33e0791054a22668173ecd9d0890 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Oct 2021 11:15:52 +0200 Subject: [PATCH 587/864] show separate progress bar to report progress on fetching of sources/patches + only show download progress for files >= 10MB --- easybuild/framework/easyblock.py | 27 ++++++++++++++++++++++++- easybuild/tools/filetools.py | 12 +++++------ easybuild/tools/output.py | 34 ++++++++++++++++++++++++-------- 3 files changed, 58 insertions(+), 15 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 57551c04c6..3b2994f2c2 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -89,7 +89,7 @@ from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX from easybuild.tools.modules import Lmod, curr_module_paths, invalidate_module_caches_for, get_software_root from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name -from easybuild.tools.output import PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS +from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ALL, PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.package.utilities import package from easybuild.tools.py2vs3 import extract_method_name, string_type @@ -690,6 +690,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No """ srcpaths = source_paths() + update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename) + # should we download or just try and find it? if re.match(r"^(https?|ftp)://", filename): # URL detected, so let's try and download it @@ -1921,6 +1923,27 @@ def fetch_step(self, skip_checksums=False): raise EasyBuildError("EasyBuild-version %s is newer than the currently running one. Aborting!", easybuild_version) + # count number of files to download: sources + patches (incl. extensions) + # FIXME: to make this count fully correct, we need the Extension instances first, + # which requires significant refactoring in fetch_extension_sources + # (also needed to fix https://github.com/easybuilders/easybuild-framework/issues/3849) + cnt = len(self.cfg['sources']) + len(self.cfg['patches']) + if self.cfg['exts_list']: + for ext in self.cfg['exts_list']: + if isinstance(ext, tuple) and len(ext) >= 3: + ext_opts = ext[2] + if 'source' in ext_opts: + cnt += 1 + elif 'sources' in ext_opts: + cnt += len(ext_opts['sources']) + else: + # assume there's always one source file; + # for extensions using PythonPackage, no 'source' or 'sources' may be specified + cnt += 1 + cnt += len(ext_opts.get('patches', [])) + + start_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, cnt) + if self.dry_run: self.dry_run_msg("Available download URLs for sources/patches:") @@ -2003,6 +2026,8 @@ def fetch_step(self, skip_checksums=False): else: self.log.info("Skipped installation dirs check per user request") + stop_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL) + def checksum_step(self): """Verify checksum of sources and patches, if a checksum is available.""" for fil in self.src + self.patches: diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index f26cc73385..ebeb9a9618 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -62,7 +62,7 @@ from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN from easybuild.tools.config import build_option, install_path -from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD, start_progress_bar, stop_progress_bar, update_progress_bar +from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.py2vs3 import HTMLParser, std_urllib, string_type from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars @@ -261,13 +261,13 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over if sys.version_info[0] >= 3 and (isinstance(data, bytes) or data_is_file_obj): mode += 'b' - # don't bother showing a progress bar for small files - if size and size < 1024: + # don't bother showing a progress bar for small files (< 10MB) + if size and size < 10 * (1024 ** 2): _log.info("Not showing progress bar for downloading small file (size %s)", size) show_progress = False if show_progress: - start_progress_bar(PROGRESS_BAR_DOWNLOAD, size, label=os.path.basename(path)) + start_progress_bar(PROGRESS_BAR_DOWNLOAD_ONE, size, label=os.path.basename(path)) # note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block try: @@ -278,12 +278,12 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over for chunk in iter(partial(data.read, 1024 ** 2), b''): fh.write(chunk) if show_progress: - update_progress_bar(PROGRESS_BAR_DOWNLOAD, progress_size=len(chunk)) + update_progress_bar(PROGRESS_BAR_DOWNLOAD_ONE, progress_size=len(chunk)) else: fh.write(data) if show_progress: - stop_progress_bar(PROGRESS_BAR_DOWNLOAD) + stop_progress_bar(PROGRESS_BAR_DOWNLOAD_ONE) except IOError as err: raise EasyBuildError("Failed to write to %s: %s", path, err) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index dd895af1c0..225385fc4b 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -45,7 +45,8 @@ pass -PROGRESS_BAR_DOWNLOAD = 'download' +PROGRESS_BAR_DOWNLOAD_ALL = 'download_all' +PROGRESS_BAR_DOWNLOAD_ONE = 'download_one' PROGRESS_BAR_EXTENSIONS = 'extensions' PROGRESS_BAR_EASYCONFIG = 'easyconfig' PROGRESS_BAR_OVERALL = 'overall' @@ -101,8 +102,9 @@ def rich_live_cm(): """ if show_progress_bars(): pbar_group = RenderGroup( - download_progress_bar(), - download_progress_bar_unknown_size(), + download_one_progress_bar(), + download_one_progress_bar_unknown_size(), + download_all_progress_bar(), extensions_progress_bar(), easyconfig_progress_bar(), overall_progress_bar(), @@ -163,7 +165,22 @@ def easyconfig_progress_bar(): @progress_bar_cache -def download_progress_bar(): +def download_all_progress_bar(): + """ + Get progress bar to show progress on downloading of all source files. + """ + progress_bar = Progress( + TextColumn("[bold blue]Fetching files: {task.percentage:>3.0f}% ({task.completed}/{task.total})"), + BarColumn(), + TimeElapsedColumn(), + TextColumn("({task.description})"), + ) + + return progress_bar + + +@progress_bar_cache +def download_one_progress_bar(): """ Get progress bar to show progress for downloading a file of known size. """ @@ -179,7 +196,7 @@ def download_progress_bar(): @progress_bar_cache -def download_progress_bar_unknown_size(): +def download_one_progress_bar_unknown_size(): """ Get progress bar to show progress for downloading a file of unknown size. """ @@ -211,14 +228,15 @@ def get_progress_bar(bar_type, size=None): Get progress bar of given type. """ progress_bar_types = { - PROGRESS_BAR_DOWNLOAD: download_progress_bar, + PROGRESS_BAR_DOWNLOAD_ALL: download_all_progress_bar, + PROGRESS_BAR_DOWNLOAD_ONE: download_one_progress_bar, PROGRESS_BAR_EXTENSIONS: extensions_progress_bar, PROGRESS_BAR_EASYCONFIG: easyconfig_progress_bar, PROGRESS_BAR_OVERALL: overall_progress_bar, } - if bar_type == PROGRESS_BAR_DOWNLOAD and not size: - pbar = download_progress_bar_unknown_size() + if bar_type == PROGRESS_BAR_DOWNLOAD_ONE and not size: + pbar = download_one_progress_bar_unknown_size() elif bar_type in progress_bar_types: pbar = progress_bar_types[bar_type]() else: From 8d5bf09abf61c8542a0ed4d70843dfc09e720a0b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Oct 2021 11:24:13 +0200 Subject: [PATCH 588/864] don't show progress bar if there's only a single task --- easybuild/tools/output.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 225385fc4b..7d235a41f9 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -255,8 +255,13 @@ def start_progress_bar(bar_type, size, label=None): pbar = get_progress_bar(bar_type, size=size) task_id = pbar.add_task('') _progress_bar_cache[bar_type] = (pbar, task_id) - if size: + + # don't bother showing progress bar if there's only 1 item to make progress on + if size == 1: + pbar.update(task_id, visible=False) + elif size: pbar.update(task_id, total=size) + if label: pbar.update(task_id, description=label) From a9a9d23a78d31ab20392ce9720102a3c75dfb95d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Oct 2021 11:40:51 +0200 Subject: [PATCH 589/864] take into account --stop, --fetch, --module-only when determining number of steps that will be run --- easybuild/framework/easyblock.py | 18 +++++++++++------- easybuild/tools/output.py | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 3b2994f2c2..b8d22fb106 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3496,6 +3496,7 @@ def run_step(self, step, step_methods): run_hook(step, self.hooks, post_step_hook=True, args=[self]) if self.cfg['stop'] == step: + update_progress_bar(PROGRESS_BAR_EASYCONFIG) self.log.info("Stopping after %s step.", step) raise StopException(step) @@ -3609,11 +3610,15 @@ def run_all_steps(self, run_test_cases): steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt()) - progress_label_tmpl = "%s (%d out of %d steps done)" + # figure out how many steps will actually be run (not be skipped) + step_cnt = 0 + for (step_name, _, _, skippable) in steps: + if not self.skip_step(step_name, skippable): + step_cnt += 1 + if self.cfg['stop'] == step_name: + break - n_steps = len(steps) - progress_label = progress_label_tmpl % (self.full_mod_name, 0, n_steps) - start_progress_bar(PROGRESS_BAR_EASYCONFIG, n_steps, label=progress_label) + start_progress_bar(PROGRESS_BAR_EASYCONFIG, step_cnt, label=self.full_mod_name) print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent) trace_msg("installation prefix: %s" % self.installdir) @@ -3653,8 +3658,7 @@ def run_all_steps(self, run_test_cases): elif self.logdebug or build_option('trace'): print_msg("... (took < 1 sec)", log=self.log, silent=self.silent) - progress_label = progress_label_tmpl % (self.full_mod_name, step_id, n_steps) - update_progress_bar(PROGRESS_BAR_EASYCONFIG, label=progress_label) + update_progress_bar(PROGRESS_BAR_EASYCONFIG) except StopException: pass @@ -3662,7 +3666,7 @@ def run_all_steps(self, run_test_cases): if not ignore_locks: remove_lock(lock_name) - update_progress_bar(PROGRESS_BAR_EASYCONFIG, label="%s done!" % self.full_mod_name) + update_progress_bar(PROGRESS_BAR_EASYCONFIG, label="%s done!" % self.full_mod_name, progress_size=0) # return True for successfull build (or stopped build) return True diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 7d235a41f9..bb87b63bff 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -156,7 +156,7 @@ def easyconfig_progress_bar(): Get progress bar to display progress for installing a single easyconfig file. """ progress_bar = Progress( - TextColumn("[bold green]{task.description}"), + TextColumn("[bold green]{task.description} ({task.completed} out of {task.total} steps done)"), BarColumn(), TimeElapsedColumn(), ) From 9e2e0b0daaf6b311a823427b3b19132562643718 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Oct 2021 12:06:28 +0200 Subject: [PATCH 590/864] also determine file size when downloading file via requests module --- easybuild/tools/filetools.py | 33 ++++++++++++++++++++++----------- test/framework/filetools.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index ebeb9a9618..52ac6ac748 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -721,6 +721,22 @@ def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_header return urlpat_headers +def det_file_size(http_header): + """ + Determine size of file from provided HTTP header info (without downloading it). + """ + res = None + len_key = 'Content-Length' + if len_key in http_header: + size = http_header[len_key] + try: + res = int(size) + except (ValueError, TypeError) as err: + _log.warning("Failed to interpret size '%s' as integer value: %s", size, err) + + return res + + def download_file(filename, url, path, forced=False): """Download a file from the given URL, to the specified path.""" @@ -772,28 +788,23 @@ def download_file(filename, url, path, forced=False): while not downloaded and attempt_cnt < max_attempts: attempt_cnt += 1 try: - size = None if used_urllib is std_urllib: # urllib2 (Python 2) / urllib.request (Python 3) does the right thing for http proxy setups, # urllib does not! url_fd = std_urllib.urlopen(url_req, timeout=timeout) status_code = url_fd.getcode() - http_header = url_fd.info() - len_key = 'Content-Length' - if len_key in http_header: - size = http_header[len_key] - try: - size = int(size) - except (ValueError, TypeError) as err: - _log.warning("Failed to interpret size '%s' as integer value: %s", size, err) - size = None + size = det_file_size(url_fd.info()) else: response = requests.get(url, headers=headers, stream=True, timeout=timeout) status_code = response.status_code response.raise_for_status() + size = det_file_size(response.headers) url_fd = response.raw url_fd.decode_content = True - _log.debug('response code for given url %s: %s' % (url, status_code)) + + _log.debug("HTTP response code for given url %s: %s", url, status_code) + _log.info("File size for %s: %s", url, size) + # note: we pass the file object to write_file rather than reading the file first, # to ensure the data is read in chunks (which prevents problems in Python 3.9+); # cfr. https://github.com/easybuilders/easybuild-framework/issues/3455 diff --git a/test/framework/filetools.py b/test/framework/filetools.py index ee9eb2ab48..fdaa934165 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -376,6 +376,34 @@ def test_normalize_path(self): self.assertEqual(ft.normalize_path('/././foo//bar/././baz/'), '/foo/bar/baz') self.assertEqual(ft.normalize_path('//././foo//bar/././baz/'), '//foo/bar/baz') + def test_det_file_size(self): + """Test det_file_size function.""" + + self.assertEqual(ft.det_file_size({'Content-Length': '12345'}), 12345) + + # missing content length, or invalid value + self.assertEqual(ft.det_file_size({}), None) + self.assertEqual(ft.det_file_size({'Content-Length': 'foo'}), None) + + test_url = 'https://github.com/easybuilders/easybuild-framework/raw/develop/' + test_url += 'test/framework/sandbox/sources/toy/toy-0.0.tar.gz' + expected_size = 273 + + # also try with actual HTTP header + try: + with std_urllib.urlopen(test_url) as fh: + self.assertEqual(ft.det_file_size(fh.info()), expected_size) + + # also try using requests, which is used as a fallback in download_file + try: + import requests + with requests.get(test_url) as res: + self.assertEqual(ft.det_file_size(res.headers), expected_size) + except ImportError: + pass + except std_urllib.URLError: + print("Skipping online test for det_file_size (working offline)") + def test_download_file(self): """Test download_file function.""" fn = 'toy-0.0.tar.gz' From 5350e5967c82ad17344f5b0ffb1e705eb68f5a2e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Oct 2021 13:52:00 +0200 Subject: [PATCH 591/864] fix broken test_toy_multi_deps by re-disabling showing of progress bars --- test/framework/toy_build.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 265c83f22a..6c047c9fe6 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2902,13 +2902,16 @@ def check_toy_load(depends_on=False): modify_env(os.environ, self.orig_environ, verbose=False) self.modtool.use(test_mod_path) + # disable showing of progress bars (again), doesn't make sense when running tests + os.environ['EASYBUILD_DISABLE_SHOW_PROGRESS_BAR'] = '1' + write_file(test_ec, test_ec_txt) # also check behaviour when using 'depends_on' rather than 'load' statements (requires Lmod 7.6.1 or newer) if self.modtool.supports_depends_on: remove_file(toy_mod_file) - self.test_toy_build(ec_file=test_ec, extra_args=['--module-depends-on']) + self.test_toy_build(ec_file=test_ec, extra_args=['--module-depends-on'], raise_error=True) toy_mod_txt = read_file(toy_mod_file) From a4215be02acf4be66d925078e6005f80cc3772cc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Oct 2021 14:38:50 +0200 Subject: [PATCH 592/864] add test for toy build with showing of progress bars enabled --- test/framework/toy_build.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 6c047c9fe6..2cbba1cebf 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -57,6 +57,7 @@ from easybuild.tools.module_generator import ModuleGeneratorTcl from easybuild.tools.modules import Lmod from easybuild.tools.py2vs3 import reload, string_type +from easybuild.tools.output import show_progress_bars from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.version import VERSION as EASYBUILD_VERSION @@ -3523,6 +3524,19 @@ def test_toy_ignore_test_failure(self): self.assertTrue("Build succeeded (with --ignore-test-failure) for 1 out of 1" in stdout) self.assertFalse(stderr) + def test_toy_build_with_progress_bars(self): + """Test installation with showing of progress bars enabled.""" + + # don't disable showing of progress bars, to ensure we catch any problems + # that are only triggered when progress bars are being shown... + del os.environ['EASYBUILD_DISABLE_SHOW_PROGRESS_BAR'] + + stdout, _ = self.run_test_toy_build_with_output() + + if show_progress_bars(): + regex = re.compile(r"^toy/0.0 done! \(17 out of 17 steps done\) ━+ [0-9:]+", re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def suite(): """ return all the tests in this file """ From c61e8487e66103d9e0e90891b11306f4230067bc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Oct 2021 14:39:02 +0200 Subject: [PATCH 593/864] also test show_progress_bars() function in output tests --- test/framework/output.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/framework/output.py b/test/framework/output.py index ad0280179d..d27be8aa0f 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -33,7 +33,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_output_style, update_build_option -from easybuild.tools.output import DummyRich, overall_progress_bar, use_rich +from easybuild.tools.output import DummyRich, overall_progress_bar, show_progress_bars, use_rich try: import rich.progress @@ -101,21 +101,28 @@ def test_get_output_style(self): error_pattern = "Can't use 'rich' output style, Rich Python package is not available!" self.assertErrorRegex(EasyBuildError, error_pattern, get_output_style) - def test_use_rich(self): - """Test use_rich function.""" + def test_use_rich_show_progress_bars(self): + """Test use_rich and show_progress_bar functions.""" + + # restore default configuration to show progress bars (disabled to avoid mangled test output) + update_build_option('show_progress_bar', True) self.assertEqual(build_option('output_style'), 'auto') if HAVE_RICH: self.assertTrue(use_rich()) + self.assertTrue(show_progress_bars()) update_build_option('output_style', 'rich') self.assertTrue(use_rich()) + self.assertTrue(show_progress_bars()) else: self.assertFalse(use_rich()) + self.assertFalse(show_progress_bars()) update_build_option('output_style', 'basic') self.assertFalse(use_rich()) + self.assertFalse(show_progress_bars()) def suite(): From 4e09da4f6e41c8a7a31a6fb3dc907dabeecf92ab Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Oct 2021 14:41:44 +0200 Subject: [PATCH 594/864] remove unused step_id variable in EasyBlock.run_all_steps --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index b8d22fb106..5d47884789 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3638,7 +3638,7 @@ def run_all_steps(self, run_test_cases): create_lock(lock_name) try: - for step_id, (step_name, descr, step_methods, skippable) in enumerate(steps): + for step_name, descr, step_methods, skippable in steps: if self.skip_step(step_name, skippable): print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent) else: From 56660a229a091d96f4c3b57db2af0ebe4e9c0a3a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Oct 2021 19:53:45 +0200 Subject: [PATCH 595/864] fix test_det_file_size for Python 2.7 (result of urllib2.urlopen can't be used as context manager) --- test/framework/filetools.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index fdaa934165..0b868d9278 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -391,14 +391,16 @@ def test_det_file_size(self): # also try with actual HTTP header try: - with std_urllib.urlopen(test_url) as fh: - self.assertEqual(ft.det_file_size(fh.info()), expected_size) + fh = std_urllib.urlopen(test_url) + self.assertEqual(ft.det_file_size(fh.info()), expected_size) + fh.close() # also try using requests, which is used as a fallback in download_file try: import requests - with requests.get(test_url) as res: - self.assertEqual(ft.det_file_size(res.headers), expected_size) + res = requests.get(test_url) + self.assertEqual(ft.det_file_size(res.headers), expected_size) + res.close() except ImportError: pass except std_urllib.URLError: From ab103630c29e1b046a54ef00f9d71eec3c9b9439 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Oct 2021 22:51:50 +0200 Subject: [PATCH 596/864] remove test_toy_build_with_progress_bars, fails in CI due to 'err: object None is not renderable' --- test/framework/toy_build.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 2cbba1cebf..6c047c9fe6 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -57,7 +57,6 @@ from easybuild.tools.module_generator import ModuleGeneratorTcl from easybuild.tools.modules import Lmod from easybuild.tools.py2vs3 import reload, string_type -from easybuild.tools.output import show_progress_bars from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.version import VERSION as EASYBUILD_VERSION @@ -3524,19 +3523,6 @@ def test_toy_ignore_test_failure(self): self.assertTrue("Build succeeded (with --ignore-test-failure) for 1 out of 1" in stdout) self.assertFalse(stderr) - def test_toy_build_with_progress_bars(self): - """Test installation with showing of progress bars enabled.""" - - # don't disable showing of progress bars, to ensure we catch any problems - # that are only triggered when progress bars are being shown... - del os.environ['EASYBUILD_DISABLE_SHOW_PROGRESS_BAR'] - - stdout, _ = self.run_test_toy_build_with_output() - - if show_progress_bars(): - regex = re.compile(r"^toy/0.0 done! \(17 out of 17 steps done\) ━+ [0-9:]+", re.M) - self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) - def suite(): """ return all the tests in this file """ From 4337df1dad790dbb7a84e353d29e8dc7b687a971 Mon Sep 17 00:00:00 2001 From: Joris Rommelse Date: Thu, 14 Oct 2021 18:18:27 +0200 Subject: [PATCH 597/864] Added support for --insecure-download option. Similar to --no-check-certificate for wget or --insecure for curl. --- easybuild/framework/easyblock.py | 17 +++++++++++------ easybuild/tools/config.py | 1 + easybuild/tools/filetools.py | 12 +++++++++--- easybuild/tools/options.py | 2 ++ 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 58bed6afde..e37d96a63d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -400,8 +400,10 @@ def fetch_source(self, source, checksum=None, extension=False): # check if the sources can be located force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES] + insecure_download = build_option('insecure_download') path = self.obtain_file(filename, extension=extension, download_filename=download_filename, - force_download=force_download, urls=source_urls, git_config=git_config) + force_download=force_download, insecure_download=insecure_download, + urls=source_urls, git_config=git_config) if path is None: raise EasyBuildError('No file found for source %s', filename) @@ -487,7 +489,8 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None): patch_file = patch_spec force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES] - path = self.obtain_file(patch_file, extension=extension, force_download=force_download) + insecure_download = build_option('insecure_download') + path = self.obtain_file(patch_file, extension=extension, force_download=force_download, insecure_download=insecure_download) if path: self.log.debug('File %s found for patch %s' % (path, patch_spec)) patchspec = { @@ -527,6 +530,7 @@ def fetch_extension_sources(self, skip_checksums=False): self.dry_run_msg("\nList of sources/patches for extensions:") force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES] + insecure_download = build_option('insecure_download') for ext in exts_list: if (isinstance(ext, list) or isinstance(ext, tuple)) and ext: @@ -620,7 +624,7 @@ def fetch_extension_sources(self, skip_checksums=False): raise EasyBuildError(error_msg, type(src_fn).__name__, src_fn) src_path = self.obtain_file(src_fn, extension=True, urls=source_urls, - force_download=force_download) + force_download=force_download, insecure_download=insecure_download) if src_path: ext_src.update({'src': src_path}) else: @@ -689,7 +693,7 @@ def fetch_extension_sources(self, skip_checksums=False): return exts_sources def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False, - git_config=None): + insecure_download=False, git_config=None): """ Locate the file with the given name - searches in different subdirectories of source path @@ -699,6 +703,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No :param urls: list of source URLs where this file may be available :param download_filename: filename with which the file should be downloaded, and then renamed to :param force_download: always try to download file, even if it's already available in source path + :param insecure_download: don't check the server certificate against the available certificate authorities :param git_config: dictionary to define how to download a git repository """ srcpaths = source_paths() @@ -728,7 +733,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No self.log.info("Found file %s at %s, no need to download it", filename, filepath) return fullpath - if download_file(filename, url, fullpath): + if download_file(filename, url, fullpath, insecure=insecure_download): return fullpath except IOError as err: @@ -855,7 +860,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No self.log.debug("Trying to download file %s from %s to %s ..." % (filename, fullurl, targetpath)) downloaded = False try: - if download_file(filename, fullurl, targetpath): + if download_file(filename, fullurl, targetpath, insecure=insecure_download): downloaded = True except IOError as err: diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 18902ae799..7b669764f4 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -199,6 +199,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'hide_toolchains', 'http_header_fields_urlpat', 'force_download', + 'insecure_download', 'from_pr', 'git_working_dirs_path', 'github_user', diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 4dd40bd4f4..a84696321a 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -50,6 +50,7 @@ import shutil import signal import stat +import ssl import sys import tempfile import time @@ -701,7 +702,7 @@ def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_header return urlpat_headers -def download_file(filename, url, path, forced=False): +def download_file(filename, url, path, forced=False, insecure=False): """Download a file from the given URL, to the specified path.""" _log.debug("Trying to download %s from %s to %s", filename, url, path) @@ -752,13 +753,18 @@ def download_file(filename, url, path, forced=False): while not downloaded and attempt_cnt < max_attempts: attempt_cnt += 1 try: + if insecure: + print_warning("Not checking server certificates while downloading %s from %s." % (filename, url)) if used_urllib is std_urllib: # urllib2 (Python 2) / urllib.request (Python 3) does the right thing for http proxy setups, # urllib does not! - url_fd = std_urllib.urlopen(url_req, timeout=timeout) + if insecure: + url_fd = std_urllib.urlopen(url_req, timeout=timeout, context=ssl._create_unverified_context()) + else: + url_fd = std_urllib.urlopen(url_req, timeout=timeout) status_code = url_fd.getcode() else: - response = requests.get(url, headers=headers, stream=True, timeout=timeout) + response = requests.get(url, headers=headers, stream=True, timeout=timeout, verify=( not insecure )) status_code = response.status_code response.raise_for_status() url_fd = response.raw diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index e4fc7661a4..8055221d87 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -407,6 +407,8 @@ def override_options(self): 'force-download': ("Force re-downloading of sources and/or patches, " "even if they are available already in source path", 'choice', 'store_or_None', DEFAULT_FORCE_DOWNLOAD, FORCE_DOWNLOAD_CHOICES), + 'insecure-download': ("Don't check the server certificate against the available certificate authorities.", + None, 'store_true', False), 'generate-devel-module': ("Generate a develop module file, implies --force if disabled", None, 'store_true', True), 'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None), From f3020f75a6e1d0510618d7a7fc3b87911562137a Mon Sep 17 00:00:00 2001 From: Joris Rommelse Date: Thu, 14 Oct 2021 21:03:09 +0200 Subject: [PATCH 598/864] Resolved 3 minor issues that were flagged by the hound bot in pull request 3859. --- easybuild/framework/easyblock.py | 3 ++- easybuild/tools/filetools.py | 2 +- easybuild/tools/options.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e37d96a63d..672b928ee3 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -490,7 +490,8 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None): force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES] insecure_download = build_option('insecure_download') - path = self.obtain_file(patch_file, extension=extension, force_download=force_download, insecure_download=insecure_download) + path = self.obtain_file(patch_file, extension=extension, force_download=force_download, + insecure_download=insecure_download) if path: self.log.debug('File %s found for patch %s' % (path, patch_spec)) patchspec = { diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index a84696321a..9ba9675f70 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -764,7 +764,7 @@ def download_file(filename, url, path, forced=False, insecure=False): url_fd = std_urllib.urlopen(url_req, timeout=timeout) status_code = url_fd.getcode() else: - response = requests.get(url, headers=headers, stream=True, timeout=timeout, verify=( not insecure )) + response = requests.get(url, headers=headers, stream=True, timeout=timeout, verify=(not insecure)) status_code = response.status_code response.raise_for_status() url_fd = response.raw diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 8055221d87..4c2f7091d9 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -408,7 +408,7 @@ def override_options(self): "even if they are available already in source path", 'choice', 'store_or_None', DEFAULT_FORCE_DOWNLOAD, FORCE_DOWNLOAD_CHOICES), 'insecure-download': ("Don't check the server certificate against the available certificate authorities.", - None, 'store_true', False), + None, 'store_true', False), 'generate-devel-module': ("Generate a develop module file, implies --force if disabled", None, 'store_true', True), 'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None), From eb2e71db8fbdb52a5b1d73c536fe403ec0991df3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 14 Oct 2021 21:08:25 +0200 Subject: [PATCH 599/864] add back spinner, mention step name in easyconfig progress bar, stop/hide easyconfig progress bar when done --- easybuild/framework/easyblock.py | 7 ++++--- easybuild/tools/output.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 5d47884789..775d43b5d2 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3618,7 +3618,7 @@ def run_all_steps(self, run_test_cases): if self.cfg['stop'] == step_name: break - start_progress_bar(PROGRESS_BAR_EASYCONFIG, step_cnt, label=self.full_mod_name) + start_progress_bar(PROGRESS_BAR_EASYCONFIG, step_cnt, label="Installing %s" % self.full_mod_name) print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent) trace_msg("installation prefix: %s" % self.installdir) @@ -3658,7 +3658,8 @@ def run_all_steps(self, run_test_cases): elif self.logdebug or build_option('trace'): print_msg("... (took < 1 sec)", log=self.log, silent=self.silent) - update_progress_bar(PROGRESS_BAR_EASYCONFIG) + progress_label = "Installing %s: %s" % (self.full_mod_name, descr) + update_progress_bar(PROGRESS_BAR_EASYCONFIG, label=progress_label) except StopException: pass @@ -3666,7 +3667,7 @@ def run_all_steps(self, run_test_cases): if not ignore_locks: remove_lock(lock_name) - update_progress_bar(PROGRESS_BAR_EASYCONFIG, label="%s done!" % self.full_mod_name, progress_size=0) + stop_progress_bar(PROGRESS_BAR_EASYCONFIG) # return True for successfull build (or stopped build) return True diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index bb87b63bff..e838f1fd52 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -39,7 +39,7 @@ from rich.console import Console, RenderGroup from rich.live import Live from rich.table import Table - from rich.progress import BarColumn, Progress, TextColumn, TimeElapsedColumn + from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn from rich.progress import DownloadColumn, FileSizeColumn, TransferSpeedColumn, TimeRemainingColumn except ImportError: pass @@ -156,6 +156,7 @@ def easyconfig_progress_bar(): Get progress bar to display progress for installing a single easyconfig file. """ progress_bar = Progress( + SpinnerColumn('line'), TextColumn("[bold green]{task.description} ({task.completed} out of {task.total} steps done)"), BarColumn(), TimeElapsedColumn(), From 230aaeab0afda61c62a1f79abc21ea162286d679 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 14 Oct 2021 21:10:08 +0200 Subject: [PATCH 600/864] stop showing overall progress bar when done --- easybuild/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 59d7c60c20..cf514d619d 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -161,9 +161,9 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): res.append((ec, ec_res)) - update_progress_bar(PROGRESS_BAR_OVERALL, progress_size=1) + update_progress_bar(PROGRESS_BAR_OVERALL) - stop_progress_bar(PROGRESS_BAR_OVERALL, visible=True) + stop_progress_bar(PROGRESS_BAR_OVERALL) return res From 13fe3467367913bd386e9807c40c132427b69bd6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 14 Oct 2021 21:13:53 +0200 Subject: [PATCH 601/864] don't show progress bars in dry run mode --- easybuild/tools/output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index e838f1fd52..f9f0cef9b6 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -93,7 +93,7 @@ def show_progress_bars(): """ Return whether or not to show progress bars. """ - return use_rich() and build_option('show_progress_bar') + return use_rich() and build_option('show_progress_bar') and not build_option('extended_dry_run') def rich_live_cm(): From c791b0b0884d58070d44aa8705ebc17220653642 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 14 Oct 2021 22:18:19 +0200 Subject: [PATCH 602/864] use nicer spinner in easyconfig progress bar --- easybuild/tools/output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index f9f0cef9b6..3652ea9682 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -156,7 +156,7 @@ def easyconfig_progress_bar(): Get progress bar to display progress for installing a single easyconfig file. """ progress_bar = Progress( - SpinnerColumn('line'), + SpinnerColumn('point'), TextColumn("[bold green]{task.description} ({task.completed} out of {task.total} steps done)"), BarColumn(), TimeElapsedColumn(), From c84b82dcfbeb23ca7f0762577b99b7b01a7dacfa Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 16 Oct 2021 14:11:46 +0200 Subject: [PATCH 603/864] don't expand overall progress bar, to be consistent with other progress bars which aren't extended either --- easybuild/tools/output.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 3652ea9682..ddac5bc718 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -144,7 +144,6 @@ def overall_progress_bar(): TimeElapsedColumn(), TextColumn("{task.description}({task.completed} out of {task.total} easyconfigs done)"), BarColumn(bar_width=None), - expand=True, ) return progress_bar From 5065be65c39e9b2e139bac672cde25f25156d547 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 16 Oct 2021 14:45:21 +0200 Subject: [PATCH 604/864] add EasyConfig.count_files method, and leverage it in EasyBlock.fetch_step for download progrss counter --- easybuild/framework/easyblock.py | 21 +------- easybuild/framework/easyconfig/easyconfig.py | 22 ++++++++ test/framework/easyconfig.py | 53 ++++++++++++++++++++ 3 files changed, 76 insertions(+), 20 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 775d43b5d2..dc5eeb41a0 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1923,26 +1923,7 @@ def fetch_step(self, skip_checksums=False): raise EasyBuildError("EasyBuild-version %s is newer than the currently running one. Aborting!", easybuild_version) - # count number of files to download: sources + patches (incl. extensions) - # FIXME: to make this count fully correct, we need the Extension instances first, - # which requires significant refactoring in fetch_extension_sources - # (also needed to fix https://github.com/easybuilders/easybuild-framework/issues/3849) - cnt = len(self.cfg['sources']) + len(self.cfg['patches']) - if self.cfg['exts_list']: - for ext in self.cfg['exts_list']: - if isinstance(ext, tuple) and len(ext) >= 3: - ext_opts = ext[2] - if 'source' in ext_opts: - cnt += 1 - elif 'sources' in ext_opts: - cnt += len(ext_opts['sources']) - else: - # assume there's always one source file; - # for extensions using PythonPackage, no 'source' or 'sources' may be specified - cnt += 1 - cnt += len(ext_opts.get('patches', [])) - - start_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, cnt) + start_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, self.cfg.count_files()) if self.dry_run: diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 8bc89606ce..0cbcb11ef3 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -773,6 +773,28 @@ def remove_false_versions(deps): # indicate that this is a parsed easyconfig self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"] + def count_files(self): + """ + Determine number of files (sources + patches) required for this easyconfig. + """ + cnt = len(self['sources']) + len(self['patches']) + + for ext in self['exts_list']: + if isinstance(ext, tuple) and len(ext) >= 3: + ext_opts = ext[2] + # check for 'sources' first, since that's also considered first by EasyBlock.fetch_extension_sources + if 'sources' in ext_opts: + cnt += len(ext_opts['sources']) + elif 'source_tmpl' in ext_opts: + cnt += 1 + else: + # assume there's always one source file; + # for extensions using PythonPackage, no 'source' or 'sources' may be specified + cnt += 1 + cnt += len(ext_opts.get('patches', [])) + + return cnt + def local_var_naming(self, local_var_naming_check): """Deal with local variables that do not follow the recommended naming scheme (if any).""" diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 48aaeb86a4..7491c109e0 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4537,6 +4537,59 @@ def test_get_cuda_cc_template_value(self): for key in cuda_template_values: self.assertEqual(ec.get_cuda_cc_template_value(key), cuda_template_values[key]) + def test_count_files(self): + """Tests for EasyConfig.count_files method.""" + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + + foss = os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb') + toy = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') + toy_exts = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + + # no sources or patches for toolchain => 0 + foss_ec = EasyConfig(foss) + self.assertEqual(foss_ec['sources'], []) + self.assertEqual(foss_ec['patches'], []) + self.assertEqual(foss_ec.count_files(), 0) + # 1 source + 2 patches => 3 + toy_ec = EasyConfig(toy) + self.assertEqual(len(toy_ec['sources']), 1) + self.assertEqual(len(toy_ec['patches']), 2) + self.assertEqual(toy_ec['exts_list'], []) + self.assertEqual(toy_ec.count_files(), 3) + # 1 source + 1 patch + # 4 extensions + # * ls: no sources/patches (only name is specified) + # * bar: 1 source (implied, using default source_tmpl) + 2 patches + # * barbar: 1 source (implied, using default source_tmpl) + # * toy: 1 source (implied, using default source_tmpl) + # => 7 files in total + toy_exts_ec = EasyConfig(toy_exts) + self.assertEqual(len(toy_exts_ec['sources']), 1) + self.assertEqual(len(toy_exts_ec['patches']), 1) + self.assertEqual(len(toy_exts_ec['exts_list']), 4) + self.assertEqual(toy_exts_ec.count_files(), 7) + + test_ec = os.path.join(self.test_prefix, 'test.eb') + copy_file(toy_exts, test_ec) + # add a couple of additional extensions to verify correct file count + test_ec_extra = '\n'.join([ + 'exts_list += [', + ' ("test-ext-one", "0.0", {', + ' "sources": ["test-ext-one-0.0-part1.tgz", "test-ext-one-0.0-part2.zip"],', + # if both 'sources' and 'source_tmpl' are specified, 'source_tmpl' is ignored, + # see EasyBlock.fetch_extension_sources, so it should be too when counting files + ' "source_tmpl": "test-ext-one-%(version)s.tar.gz",', + ' }),', + ' ("test-ext-two", "0.0", {', + ' "source_tmpl": "test-ext-two-0.0-part1.tgz",', + ' "patches": ["test-ext-two.patch"],', + ' }),', + ']', + ]) + write_file(test_ec, test_ec_extra, append=True) + test_ec = EasyConfig(test_ec) + self.assertEqual(test_ec.count_files(), 11) + def suite(): """ returns all the testcases in this module """ From 988e813dc7cd774e9b5ba6b738e7d4cb8e2c6830 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 16 Oct 2021 15:29:37 +0200 Subject: [PATCH 605/864] tweak test_module_only_extensions to catch bug reported in https://github.com/easybuilders/easybuild-framework/issues/3849 --- test/framework/toy_build.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 265c83f22a..8f8af2b139 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1746,6 +1746,16 @@ def test_module_only_extensions(self): # remove module file so we can try --module-only remove_file(toy_mod) + # make sure that sources for extensions can't be found, + # they should not be needed when using --module-only + # (cfr. https://github.com/easybuilders/easybuild-framework/issues/3849) + del os.environ['EASYBUILD_SOURCEPATH'] + + # first try normal --module-only, should work fine + self.eb_main([test_ec, '--module-only'], do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_mod)) + remove_file(toy_mod) + # rename file required for barbar extension, so we can check whether sanity check catches it libbarbar = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'lib', 'libbarbar.a') move_file(libbarbar, libbarbar + '.foobar') From b7edf1d2f804aa82a5e4dd587dd4fa4e60295e95 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 16 Oct 2021 16:10:51 +0200 Subject: [PATCH 606/864] refactor EasyBlock to decouple collecting of information on extension source/patch files from downloading them (fixes #3849) includes deprecating EasyBlock.fetch_extension_sources and replacing it with EasyBlock.collect_exts_file_info --- easybuild/framework/easyblock.py | 120 ++++++++++++++----------------- easybuild/tools/filetools.py | 38 ++++++++++ test/framework/easyblock.py | 83 ++++++++++++++++++++- test/framework/easyconfig.py | 2 +- test/framework/filetools.py | 17 +++++ 5 files changed, 192 insertions(+), 68 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 58bed6afde..5a3a57409a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -72,9 +72,10 @@ from easybuild.tools.config import install_path, log_path, package_path, source_paths from easybuild.tools.environment import restore_env, sanitize_env from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256 -from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, convert_name -from easybuild.tools.filetools import compute_checksum, copy_file, check_lock, create_lock, derive_alt_pypi_url -from easybuild.tools.filetools import diff_files, dir_contains_files, download_file, encode_class_name, extract_file +from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, create_patch_info +from easybuild.tools.filetools import convert_name, compute_checksum, copy_file, check_lock, create_lock +from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file +from easybuild.tools.filetools import encode_class_name, extract_file from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink @@ -372,7 +373,7 @@ def fetch_source(self, source, checksum=None, extension=False): :param source: source to be found (single dictionary in 'sources' list, or filename) :param checksum: checksum corresponding to source - :param extension: flag if being called from fetch_extension_sources() + :param extension: flag if being called from collect_exts_file_info() """ filename, download_filename, extract_cmd, source_urls, git_config = None, None, None, None, None @@ -461,52 +462,19 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None): patches = [] for index, patch_spec in enumerate(patch_specs): - # check if the patches can be located - copy_file = False - suff = None - level = None - if isinstance(patch_spec, (list, tuple)): - if not len(patch_spec) == 2: - raise EasyBuildError("Unknown patch specification '%s', only 2-element lists/tuples are supported!", - str(patch_spec)) - patch_file = patch_spec[0] - - # this *must* be of typ int, nothing else - # no 'isinstance(..., int)', since that would make True/False also acceptable - if isinstance(patch_spec[1], int): - level = patch_spec[1] - elif isinstance(patch_spec[1], string_type): - # non-patch files are assumed to be files to copy - if not patch_spec[0].endswith('.patch'): - copy_file = True - suff = patch_spec[1] - else: - raise EasyBuildError("Wrong patch spec '%s', only int/string are supported as 2nd element", - str(patch_spec)) - else: - patch_file = patch_spec + patch_info = create_patch_info(patch_spec) force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES] - path = self.obtain_file(patch_file, extension=extension, force_download=force_download) + path = self.obtain_file(patch_info['name'], extension=extension, force_download=force_download) if path: - self.log.debug('File %s found for patch %s' % (path, patch_spec)) - patchspec = { - 'name': patch_file, - 'path': path, - 'checksum': self.get_checksum_for(checksums, index=index), - } - if suff: - if copy_file: - patchspec['copy'] = suff - else: - patchspec['sourcepath'] = suff - if level is not None: - patchspec['level'] = level + self.log.debug('File %s found for patch %s', path, patch_spec) + patch_info['path'] = path + patch_info['checksum'] = self.get_checksum_for(checksums, index=index) if extension: - patches.append(patchspec) + patches.append(patch_info) else: - self.patches.append(patchspec) + self.patches.append(patch_info) else: raise EasyBuildError('No file found for patch %s', patch_spec) @@ -514,25 +482,38 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None): self.log.info("Fetched extension patches: %s", patches) return patches else: - self.log.info("Added patches: %s" % self.patches) + self.log.info("Added patches: %s", self.patches) def fetch_extension_sources(self, skip_checksums=False): """ - Find source file for extensions. + Fetch source and patch files for extensions (DEPRECATED, use collect_exts_file_info instead). + """ + depr_msg = "EasyBlock.fetch_extension_sources is deprecated, use EasyBlock.collect_exts_file_info instead" + self.log.deprecated(depr_msg, '5.0') + return self.collect_exts_file_info(fetch_files=True, verify_checksums=not skip_checksums) + + def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): + """ + Collect information on source and patch files for extensions. + + :param fetch_files: whether or not to fetch files (if False, path to files will be missing from info) + :param verify_checksums: whether or not to verify checksums + :return: list of dict values, one per extension, with information on source/patch files. """ exts_sources = [] exts_list = self.cfg.get_ref('exts_list') + if verify_checksums and not fetch_files: + raise EasyBuildError("Can't verify checksums for extension files if they are not being fetched") + if self.dry_run: self.dry_run_msg("\nList of sources/patches for extensions:") force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES] for ext in exts_list: - if (isinstance(ext, list) or isinstance(ext, tuple)) and ext: - + if isinstance(ext, (list, tuple)) and ext: # expected format: (name, version, options (dict)) - ext_name = ext[0] if len(ext) == 1: exts_sources.append({'name': ext_name}) @@ -602,10 +583,10 @@ def fetch_extension_sources(self, skip_checksums=False): if 'source_urls' not in source: source['source_urls'] = source_urls - src = self.fetch_source(source, checksums, extension=True) - - # copy 'path' entry to 'src' for use with extensions - ext_src.update({'src': src['path']}) + if fetch_files: + src = self.fetch_source(source, checksums, extension=True) + # copy 'path' entry to 'src' for use with extensions + ext_src.update({'src': src['path']}) else: # use default template for name of source file if none is specified @@ -619,15 +600,16 @@ def fetch_extension_sources(self, skip_checksums=False): error_msg = "source_tmpl value must be a string! (found value of type '%s'): %s" raise EasyBuildError(error_msg, type(src_fn).__name__, src_fn) - src_path = self.obtain_file(src_fn, extension=True, urls=source_urls, - force_download=force_download) - if src_path: - ext_src.update({'src': src_path}) - else: - raise EasyBuildError("Source for extension %s not found.", ext) + if fetch_files: + src_path = self.obtain_file(src_fn, extension=True, urls=source_urls, + force_download=force_download) + if src_path: + ext_src.update({'src': src_path}) + else: + raise EasyBuildError("Source for extension %s not found.", ext) # verify checksum for extension sources - if 'src' in ext_src and not skip_checksums: + if verify_checksums and 'src' in ext_src: src_path = ext_src['src'] src_fn = os.path.basename(src_path) @@ -647,12 +629,17 @@ def fetch_extension_sources(self, skip_checksums=False): raise EasyBuildError('Checksum verification for extension source %s failed', src_fn) # locate extension patches (if any), and verify checksums - ext_patches = self.fetch_patches(patch_specs=ext_options.get('patches', []), extension=True) + ext_patches = ext_options.get('patches', []) + if fetch_files: + ext_patches = self.fetch_patches(patch_specs=ext_patches, extension=True) + else: + ext_patches = [create_patch_info(p) for p in ext_patches] + if ext_patches: self.log.debug('Found patches for extension %s: %s', ext_name, ext_patches) ext_src.update({'patches': ext_patches}) - if not skip_checksums: + if verify_checksums: for patch in ext_patches: patch = patch['path'] # report both MD5 and SHA256 checksums, @@ -1995,7 +1982,7 @@ def fetch_step(self, skip_checksums=False): # fetch extensions if self.cfg.get_ref('exts_list'): - self.exts = self.fetch_extension_sources(skip_checksums=skip_checksums) + self.exts = self.collect_exts_file_info(fetch_files=True, verify_checksums=not skip_checksums) # create parent dirs in install and modules path already # this is required when building in parallel @@ -2311,8 +2298,11 @@ def init_ext_instances(self): self.ext_instances = [] exts_classmap = self.cfg['exts_classmap'] + # self.exts may already be populated at this point through collect_exts_file_info; + # if it's not, we do it lightweight here, by skipping fetching of the files; + # information on location of source/patch files will be lacking in that case (but that should be fine) if exts_list and not self.exts: - self.exts = self.fetch_extension_sources() + self.exts = self.collect_exts_file_info(fetch_files=False, verify_checksums=False) # obtain name and module path for default extention class exts_defaultclass = self.cfg['exts_defaultclass'] @@ -2410,7 +2400,7 @@ def extensions_step(self, fetch=False, install=True): self.prepare_for_extensions() if fetch: - self.exts = self.fetch_extension_sources() + self.exts = self.collect_exts_file_info(fetch_files=True) self.exts_all = self.exts[:] # retain a copy of all extensions, regardless of filtering/skipping diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 723ac5d6e1..b6a4bc623a 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1409,6 +1409,44 @@ def guess_patch_level(patched_files, parent_dir): return patch_level +def create_patch_info(patch_spec): + """ + Create info dictionary from specified patch spec. + """ + if isinstance(patch_spec, (list, tuple)): + if not len(patch_spec) == 2: + error_msg = "Unknown patch specification '%s', only 2-element lists/tuples are supported!" + raise EasyBuildError(error_msg, str(patch_spec)) + + patch_info = {'name': patch_spec[0]} + + patch_arg = patch_spec[1] + # patch level *must* be of type int, nothing else (not True/False!) + # note that 'isinstance(..., int)' returns True for True/False values... + if isinstance(patch_arg, int) and not isinstance(patch_arg, bool): + patch_info['level'] = patch_arg + + # string value as patch argument can be either path where patch should be applied, + # or path to where a non-patch file should be copied + elif isinstance(patch_arg, string_type): + if patch_spec[0].endswith('.patch'): + patch_info['sourcepath'] = patch_arg + # non-patch files are assumed to be files to copy + else: + patch_info['copy'] = patch_arg + else: + raise EasyBuildError("Wrong patch spec '%s', only int/string are supported as 2nd element", + str(patch_spec)) + + elif isinstance(patch_spec, string_type): + patch_info = {'name': patch_spec} + else: + error_msg = "Wrong patch spec, should be string of 2-tuple with patch name + argument: %s" + raise EasyBuildError(error_msg, patch_spec) + + return patch_info + + def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=False, use_git=False): """ Apply a patch to source code in directory dest diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 18bfcf07b0..241e533690 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1617,6 +1617,65 @@ def test_fallback_source_url(self): self.assertTrue(verify_checksum(expected_path, eb.cfg['checksums'][0])) + def test_collect_exts_file_info(self): + """Test collect_exts_file_info method.""" + testdir = os.path.abspath(os.path.dirname(__file__)) + toy_sources = os.path.join(testdir, 'sandbox', 'sources', 'toy') + toy_ext_sources = os.path.join(toy_sources, 'extensions') + toy_ec_file = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec = process_easyconfig(toy_ec_file)[0] + toy_eb = EasyBlock(toy_ec['ec']) + + exts_file_info = toy_eb.collect_exts_file_info() + + self.assertTrue(isinstance(exts_file_info, list)) + self.assertEqual(len(exts_file_info), 4) + + self.assertEqual(exts_file_info[0], {'name': 'ls'}) + + self.assertEqual(exts_file_info[1]['name'], 'bar') + self.assertEqual(exts_file_info[1]['src'], os.path.join(toy_ext_sources, 'bar-0.0.tar.gz')) + bar_patch1 = 'bar-0.0_fix-silly-typo-in-printf-statement.patch' + self.assertEqual(exts_file_info[1]['patches'][0]['name'], bar_patch1) + self.assertEqual(exts_file_info[1]['patches'][0]['path'], os.path.join(toy_ext_sources, bar_patch1)) + bar_patch2 = 'bar-0.0_fix-very-silly-typo-in-printf-statement.patch' + self.assertEqual(exts_file_info[1]['patches'][1]['name'], bar_patch2) + self.assertEqual(exts_file_info[1]['patches'][1]['path'], os.path.join(toy_ext_sources, bar_patch2)) + + self.assertEqual(exts_file_info[2]['name'], 'barbar') + self.assertEqual(exts_file_info[2]['src'], os.path.join(toy_ext_sources, 'barbar-0.0.tar.gz')) + self.assertFalse('patches' in exts_file_info[2]) + + self.assertEqual(exts_file_info[3]['name'], 'toy') + self.assertEqual(exts_file_info[3]['src'], os.path.join(toy_sources, 'toy-0.0.tar.gz')) + self.assertFalse('patches' in exts_file_info[3]) + + # location of files is missing when fetch_files is set to False + exts_file_info = toy_eb.collect_exts_file_info(fetch_files=False, verify_checksums=False) + + self.assertTrue(isinstance(exts_file_info, list)) + self.assertEqual(len(exts_file_info), 4) + + self.assertEqual(exts_file_info[0], {'name': 'ls'}) + + self.assertEqual(exts_file_info[1]['name'], 'bar') + self.assertFalse('src' in exts_file_info[1]) + self.assertEqual(exts_file_info[1]['patches'][0]['name'], bar_patch1) + self.assertFalse('path' in exts_file_info[1]['patches'][0]) + self.assertEqual(exts_file_info[1]['patches'][1]['name'], bar_patch2) + self.assertFalse('path' in exts_file_info[1]['patches'][1]) + + self.assertEqual(exts_file_info[2]['name'], 'barbar') + self.assertFalse('src' in exts_file_info[2]) + self.assertFalse('patches' in exts_file_info[2]) + + self.assertEqual(exts_file_info[3]['name'], 'toy') + self.assertFalse('src' in exts_file_info[3]) + self.assertFalse('patches' in exts_file_info[3]) + + error_msg = "Can't verify checksums for extension files if they are not being fetched" + self.assertErrorRegex(EasyBuildError, error_msg, toy_eb.collect_exts_file_info, fetch_files=False) + def test_obtain_file_extension(self): """Test use of obtain_file method on an extension.""" @@ -2062,9 +2121,16 @@ def test_checksum_step(self): error_msg = "Checksum verification for .*/toy-0.0.tar.gz using .* failed" self.assertErrorRegex(EasyBuildError, error_msg, eb.checksum_step) - # also check verification of checksums for extensions, which is part of fetch_extension_sources + # also check verification of checksums for extensions, which is part of collect_exts_file_info error_msg = "Checksum verification for extension source bar-0.0.tar.gz failed" + self.assertErrorRegex(EasyBuildError, error_msg, eb.collect_exts_file_info) + + # also check with deprecated fetch_extension_sources method + self.allow_deprecated_behaviour() + self.mock_stderr(True) self.assertErrorRegex(EasyBuildError, error_msg, eb.fetch_extension_sources) + self.mock_stderr(False) + self.disallow_deprecated_behaviour() # if --ignore-checksums is enabled, faulty checksums are reported but otherwise ignored (no error) build_options = { @@ -2084,7 +2150,7 @@ def test_checksum_step(self): self.mock_stderr(True) self.mock_stdout(True) - eb.fetch_extension_sources() + eb.collect_exts_file_info() stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stderr(False) @@ -2092,6 +2158,19 @@ def test_checksum_step(self): self.assertEqual(stdout, '') self.assertEqual(stderr.strip(), "WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz") + # also check with deprecated fetch_extension_sources method + self.allow_deprecated_behaviour() + self.mock_stderr(True) + self.mock_stdout(True) + eb.fetch_extension_sources() + stderr = self.get_stderr() + stdout = self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stdout, '') + self.assertTrue(stderr.strip().endswith("WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz")) + self.disallow_deprecated_behaviour() + def test_check_checksums(self): """Test for check_checksums_for and check_checksums methods.""" testdir = os.path.abspath(os.path.dirname(__file__)) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 48aaeb86a4..ab11c94c50 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -474,7 +474,7 @@ def test_exts_list(self): self.prep() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) - exts_sources = eb.fetch_extension_sources() + exts_sources = eb.collect_exts_file_info() self.assertEqual(len(exts_sources), 2) self.assertEqual(exts_sources[0]['name'], 'ext1') diff --git a/test/framework/filetools.py b/test/framework/filetools.py index ee9eb2ab48..fafd1e54d3 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1510,6 +1510,23 @@ def test_derive_alt_pypi_url(self): url = 'https://pypi.python.org/packages/source/n/nosuchpackageonpypiever/nosuchpackageonpypiever-0.0.0.tar.gz' self.assertEqual(ft.derive_alt_pypi_url(url), None) + def test_create_patch_info(self): + """Test create_patch_info function.""" + + self.assertEqual(ft.create_patch_info('foo.patch'), {'name': 'foo.patch'}) + self.assertEqual(ft.create_patch_info('foo.txt'), {'name': 'foo.txt'}) + self.assertEqual(ft.create_patch_info(('foo.patch', 1)), {'name': 'foo.patch', 'level': 1}) + self.assertEqual(ft.create_patch_info(('foo.patch', 'subdir')), {'name': 'foo.patch', 'sourcepath': 'subdir'}) + self.assertEqual(ft.create_patch_info(('foo.txt', 'subdir')), {'name': 'foo.txt', 'copy': 'subdir'}) + + # faulty input + error_msg = "Wrong patch spec" + self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, None) + self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, {'name': 'foo.patch'}) + self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, ('foo.patch', [1, 2])) + error_msg = "Unknown patch specification" + self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, ('foo.patch', 1, 'subdir')) + def test_apply_patch(self): """ Test apply_patch """ testdir = os.path.dirname(os.path.abspath(__file__)) From 1fa4c08202a2d565f8462ddad2c381b3c969de8a Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Sun, 17 Oct 2021 01:30:29 +0000 Subject: [PATCH 607/864] Make toolchain logic aware of imkl-FFTW module When this module is detected use it to obtain the cluster (cdft) MPI FFTW3 interfaces, but obtain the non-MPI FFTW3 interfaces from the main MKL libraries from the imkl binary distribution, which have provided them since MKL 10.2 --- easybuild/toolchains/fft/intelfftw.py | 6 ++ test/framework/modules/impi/2021.4.0 | 42 +++++++++ .../modules/intel-compilers/2021.4.0 | 41 +++++++++ test/framework/modules/intel/2021b | 36 ++++++++ test/framework/toolchain.py | 85 ++++++++++++++++++- 5 files changed, 206 insertions(+), 4 deletions(-) create mode 100644 test/framework/modules/impi/2021.4.0 create mode 100644 test/framework/modules/intel-compilers/2021.4.0 create mode 100644 test/framework/modules/intel/2021b diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py index b748fd5e7e..6181903c4d 100644 --- a/easybuild/toolchains/fft/intelfftw.py +++ b/easybuild/toolchains/fft/intelfftw.py @@ -95,6 +95,12 @@ def _set_fftw_variables(self): # so make sure libraries are there before FFT_LIB is set imklroot = get_software_root(self.FFT_MODULE_NAME[0]) fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR] + imklfftwroot = get_software_root('imkl-FFTW') + if imklfftwroot: + # only get cluster_interface_lib from seperate module imkl-FFTW, rest via libmkl_gf/libmkl_intel + fft_lib_dirs += [os.path.join(imklfftwroot, 'lib')] + fftw_libs.remove(interface_lib) + fftw_mt_libs.remove(interface_lib) def fftw_lib_exists(libname): """Helper function to check whether FFTW library with specified name exists.""" diff --git a/test/framework/modules/impi/2021.4.0 b/test/framework/modules/impi/2021.4.0 new file mode 100644 index 0000000000..a8003ced0e --- /dev/null +++ b/test/framework/modules/impi/2021.4.0 @@ -0,0 +1,42 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +Intel MPI Library, compatible with MPICH ABI + + +More information +================ + - Homepage: https://software.intel.com/content/www/us/en/develop/tools/mpi-library.html + } +} + +module-whatis {Description: Intel MPI Library, compatible with MPICH ABI} +module-whatis {Homepage: https://software.intel.com/content/www/us/en/develop/tools/mpi-library.html} +module-whatis {URL: https://software.intel.com/content/www/us/en/develop/tools/mpi-library.html} + +set root /tmp/impi/2021.4.0 + +conflict impi + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/mpi/2021.4.0/include +prepend-path FI_PROVIDER_PATH $root/mpi/2021.4.0/libfabric/lib/prov +prepend-path LD_LIBRARY_PATH $root/mpi/2021.4.0/lib +prepend-path LD_LIBRARY_PATH $root/mpi/2021.4.0/lib/release +prepend-path LD_LIBRARY_PATH $root/mpi/2021.4.0/libfabric/lib +prepend-path LIBRARY_PATH $root/mpi/2021.4.0/lib +prepend-path LIBRARY_PATH $root/mpi/2021.4.0/lib/release +prepend-path LIBRARY_PATH $root/mpi/2021.4.0/libfabric/lib +prepend-path MANPATH $root/mpi/2021.4.0/man +prepend-path PATH $root/mpi/2021.4.0/bin +prepend-path PATH $root/mpi/2021.4.0/libfabric/bin +setenv EBROOTIMPI "$root" +setenv EBVERSIONIMPI "2021.4.0" +setenv EBDEVELIMPI "$root/easybuild/impi-2021.4.0-easybuild-devel" + +setenv I_MPI_ROOT "$root/mpi/2021.4.0" +setenv UCX_TLS "all" +# Built with EasyBuild version 4.5.0dev diff --git a/test/framework/modules/intel-compilers/2021.4.0 b/test/framework/modules/intel-compilers/2021.4.0 new file mode 100644 index 0000000000..b9e93096d1 --- /dev/null +++ b/test/framework/modules/intel-compilers/2021.4.0 @@ -0,0 +1,41 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +Intel C, C++ & Fortran compilers (classic and oneAPI) + + +More information +================ + - Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html + } +} + +module-whatis {Description: Intel C, C++ & Fortran compilers (classic and oneAPI)} +module-whatis {Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html} +module-whatis {URL: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html} + +set root /tmp/intel-compilers/2021.4.0 + +conflict intel-compilers + +prepend-path CPATH $root/tbb/2021.4.0/include +prepend-path LD_LIBRARY_PATH $root/compiler/2021.4.0/linux/lib +prepend-path LD_LIBRARY_PATH $root/compiler/2021.4.0/linux/lib/x64 +prepend-path LD_LIBRARY_PATH $root/compiler/2021.4.0/linux/compiler/lib/intel64_lin +prepend-path LD_LIBRARY_PATH $root/tbb/2021.4.0/lib/intel64/gcc4.8 +prepend-path LIBRARY_PATH $root/compiler/2021.4.0/linux/lib +prepend-path LIBRARY_PATH $root/compiler/2021.4.0/linux/lib/x64 +prepend-path LIBRARY_PATH $root/compiler/2021.4.0/linux/compiler/lib/intel64_lin +prepend-path LIBRARY_PATH $root/tbb/2021.4.0/lib/intel64/gcc4.8 +prepend-path OCL_ICD_FILENAMES $root/compiler/2021.4.0/linux/lib/x64/libintelocl.so +prepend-path PATH $root/compiler/2021.4.0/linux/bin +prepend-path PATH $root/compiler/2021.4.0/linux/bin/intel64 +prepend-path TBBROOT $root/tbb/2021.4.0 +setenv EBROOTINTELMINCOMPILERS "$root" +setenv EBVERSIONINTELMINCOMPILERS "2021.4.0" +setenv EBDEVELINTELMINCOMPILERS "$root/easybuild/Core-intel-compilers-2021.4.0-easybuild-devel" + +# Built with EasyBuild version 4.5.0dev diff --git a/test/framework/modules/intel/2021b b/test/framework/modules/intel/2021b new file mode 100644 index 0000000000..5695e2e0b3 --- /dev/null +++ b/test/framework/modules/intel/2021b @@ -0,0 +1,36 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and Fortran compilers, Intel MPI & Intel MKL. - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/ + } +} + +module-whatis {Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and Fortran compilers, Intel MPI & Intel MKL. - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/} + +set root /tmp/intel/2021b + +conflict intel + +if { ![is-loaded intel-compilers/2021.4.0] } { + module load intel-compilers/2021.4.0 +} + +if { ![is-loaded impi/2021.4.0] } { + module load impi/2021.4.0 +} + +if { ![is-loaded imkl/2021.4.0] } { + module load imkl/2021.4.0 +} + +if { ![is-loaded imkl-FFTW/2021.4.0] } { + module load imkl-FFTW/2021.4.0 +} + + +setenv EBROOTINTEL "$root" +setenv EBVERSIONINTEL "2021b" +setenv EBDEVELINTEL "$root/easybuild/intel-2021b-easybuild-devel" + + +# built with EasyBuild version 4.5.0dev diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 403f6774ca..8bad3a7a8d 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1079,6 +1079,63 @@ def test_fft_env_vars_intel(self): libfft_mt += '-Wl,-Bdynamic -liomp5 -lpthread' self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt) + self.setup_sandbox_for_intel_fftw(self.test_prefix, imklver='2021.4.0') + tc = self.get_toolchain('intel', version='2021b') + tc.prepare() + + fft_static_libs = 'libmkl_intel_lp64.a,libmkl_sequential.a,libmkl_core.a' + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs) + + fft_static_libs_mt = 'libmkl_intel_lp64.a,libmkl_intel_thread.a,libmkl_core.a,' + fft_static_libs_mt += 'libiomp5.a,libpthread.a' + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS_MT'), fft_static_libs_mt) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS_MT'), fft_static_libs_mt) + + libfft = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_sequential -lmkl_core " + libfft += "-Wl,--end-group -Wl,-Bdynamic" + self.assertEqual(tc.get_variable('LIBFFT'), libfft) + + libfft_mt = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core " + libfft_mt += "-Wl,--end-group -Wl,-Bdynamic -liomp5 -lpthread" + self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt) + + tc = self.get_toolchain('intel', version='2021b') + tc.set_options({'openmp': True}) + tc.prepare() + + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs) + + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS_MT'), fft_static_libs_mt) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS_MT'), fft_static_libs_mt) + + self.assertEqual(tc.get_variable('LIBFFT'), libfft) + self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt) + + tc = self.get_toolchain('intel', version='2021b') + tc.set_options({'usempi': True}) + tc.prepare() + + fft_static_libs = 'libfftw3x_cdft_lp64.a,libmkl_cdft_core.a,libmkl_blacs_intelmpi_lp64.a,' + fft_static_libs += 'libmkl_intel_lp64.a,libmkl_sequential.a,libmkl_core.a' + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs) + + fft_static_libs_mt = 'libfftw3x_cdft_lp64.a,libmkl_cdft_core.a,libmkl_blacs_intelmpi_lp64.a,' + fft_static_libs_mt += 'libmkl_intel_lp64.a,libmkl_intel_thread.a,libmkl_core.a,libiomp5.a,libpthread.a' + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS_MT'), fft_static_libs_mt) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS_MT'), fft_static_libs_mt) + + libfft = '-Wl,-Bstatic -Wl,--start-group -lfftw3x_cdft_lp64 -lmkl_cdft_core ' + libfft += '-lmkl_blacs_intelmpi_lp64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -Wl,--end-group -Wl,-Bdynamic' + self.assertEqual(tc.get_variable('LIBFFT'), libfft) + + libfft_mt = '-Wl,-Bstatic -Wl,--start-group -lfftw3x_cdft_lp64 -lmkl_cdft_core ' + libfft_mt += '-lmkl_blacs_intelmpi_lp64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -Wl,--end-group ' + libfft_mt += '-Wl,-Bdynamic -liomp5 -lpthread' + self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt) + def test_fosscuda(self): """Test whether fosscuda is handled properly.""" tc = self.get_toolchain("fosscuda", version="2018a") @@ -1115,7 +1172,9 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'): # create dummy imkl module and put required lib*.a files in place imkl_module_path = os.path.join(moddir, 'imkl', imklver) + imkl_fftw_module_path = os.path.join(moddir, 'imkl-FFTW', imklver) imkl_dir = os.path.join(self.test_prefix, 'software', 'imkl', imklver) + imkl_fftw_dir = os.path.join(self.test_prefix, 'software', 'imkl-FFTW', imklver) imkl_mod_txt = '\n'.join([ "#%Module", @@ -1124,17 +1183,35 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'): ]) write_file(imkl_module_path, imkl_mod_txt) - fftw_libs = ['fftw3xc_intel', 'fftw3xc_pgi', 'mkl_cdft_core', 'mkl_blacs_intelmpi_lp64'] - fftw_libs += ['mkl_intel_lp64', 'mkl_sequential', 'mkl_core', 'mkl_intel_ilp64'] + imkl_fftw_mod_txt = '\n'.join([ + "#%Module", + "setenv EBROOTIMKLMINFFTW %s" % imkl_fftw_dir, + "setenv EBVERSIONIMKLMINFFTW %s" % imklver, + ]) + write_file(imkl_fftw_module_path, imkl_fftw_mod_txt) + + mkl_libs = ['mkl_cdft_core', 'mkl_blacs_intelmpi_lp64'] + mkl_libs += ['mkl_intel_lp64', 'mkl_sequential', 'mkl_core', 'mkl_intel_ilp64'] + fftw_libs = ['fftw3xc_intel', 'fftw3xc_pgi'] if LooseVersion(imklver) >= LooseVersion('11'): fftw_libs.extend(['fftw3x_cdft_ilp64', 'fftw3x_cdft_lp64']) else: fftw_libs.append('fftw3x_cdft') - for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64', 'lib/em64t']: + if LooseVersion(imklver) >= LooseVersion('2021.4.0'): + subdir = 'mkl/%s/lib/intel64' % imklver os.makedirs(os.path.join(imkl_dir, subdir)) - for fftlib in fftw_libs: + for fftlib in mkl_libs: write_file(os.path.join(imkl_dir, subdir, 'lib%s.a' % fftlib), 'foo') + subdir = 'lib' + os.makedirs(os.path.join(imkl_fftw_dir, subdir)) + for fftlib in fftw_libs: + write_file(os.path.join(imkl_fftw_dir, subdir, 'lib%s.a' % fftlib), 'foo') + else: + for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64', 'lib/em64t']: + os.makedirs(os.path.join(imkl_dir, subdir)) + for fftlib in mkl_libs + fftw_libs: + write_file(os.path.join(imkl_dir, subdir, 'lib%s.a' % fftlib), 'foo') def test_intel_toolchain(self): """Test for intel toolchain.""" From e15e748bdeb520b4ed5a7fa84826e5679b97f566 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Sun, 17 Oct 2021 02:10:26 +0000 Subject: [PATCH 608/864] Add explicit test modules for imkl, imkl-FFTW/2021.4.0 Without these the test_avail check fails when checking dependencies for intel/2021b. --- test/framework/modules/imkl-FFTW/2021.4.0 | 31 +++++++++++++++++++ test/framework/modules/imkl/2021.4.0 | 37 +++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 test/framework/modules/imkl-FFTW/2021.4.0 create mode 100644 test/framework/modules/imkl/2021.4.0 diff --git a/test/framework/modules/imkl-FFTW/2021.4.0 b/test/framework/modules/imkl-FFTW/2021.4.0 new file mode 100644 index 0000000000..955bf68727 --- /dev/null +++ b/test/framework/modules/imkl-FFTW/2021.4.0 @@ -0,0 +1,31 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +FFTW interfaces using Intel oneAPI Math Kernel Library + + +More information +================ + - Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html + } +} + +module-whatis {Description: FFTW interfaces using Intel oneAPI Math Kernel Library} +module-whatis {Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html} +module-whatis {URL: https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html} + +set root /tmp/imkl-FFTW/2021.4.0 + +conflict imkl-FFTW + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +setenv EBROOTIMKLMINFFTW "$root" +setenv EBVERSIONIMKLMINFFTW "2021.4.0" +setenv EBDEVELIMKLMINFFTW "$root/easybuild/imkl-FFTW-2021.4.0-easybuild-devel" + +# Built with EasyBuild version 4.5.0dev diff --git a/test/framework/modules/imkl/2021.4.0 b/test/framework/modules/imkl/2021.4.0 new file mode 100644 index 0000000000..f188251b48 --- /dev/null +++ b/test/framework/modules/imkl/2021.4.0 @@ -0,0 +1,37 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +Intel oneAPI Math Kernel Library + + +More information +================ + - Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html + } +} + +module-whatis {Description: Intel oneAPI Math Kernel Library} +module-whatis {Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html} +module-whatis {URL: https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html} + +set root /tmp/eb-bI0pBy/eb-DmuEpJ/eb-leoYDw/eb-UtJJqp/tmp8P3FOY + +conflict imkl + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/mkl/2021.4.0/include +prepend-path CPATH $root/mkl/2021.4.0/include/fftw +prepend-path LD_LIBRARY_PATH $root/compiler/2021.4.0/linux/compiler/lib/intel64_lin +prepend-path LD_LIBRARY_PATH $root/mkl/2021.4.0/lib/intel64 +prepend-path LIBRARY_PATH $root/compiler/2021.4.0/linux/compiler/lib/intel64_lin +prepend-path LIBRARY_PATH $root/mkl/2021.4.0/lib/intel64 +setenv EBROOTIMKL "$root" +setenv EBVERSIONIMKL "2021.4.0" +setenv EBDEVELIMKL "$root/easybuild/Core-imkl-2021.4.0-easybuild-devel" + +setenv MKL_EXAMPLES "$root/mkl/2021.4.0/examples" +setenv MKLROOT "$root/mkl/2021.4.0" +# Built with EasyBuild version 4.5.0dev From fcca35487b42914f6ef0a697bee4966b9fcafc5c Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Sun, 17 Oct 2021 02:27:36 +0000 Subject: [PATCH 609/864] TEST_MODULES_COUNT now at 87 with 5 new ones --- test/framework/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index b56c08bb2f..9c834477ee 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -54,7 +54,7 @@ # number of modules included for testing purposes -TEST_MODULES_COUNT = 82 +TEST_MODULES_COUNT = 87 class ModulesTest(EnhancedTestCase): From 7a635124d20f68c7d1eabe346108b2ad6e86e8f2 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Sun, 17 Oct 2021 11:46:56 +0200 Subject: [PATCH 610/864] Make sure the contrib/hooks tree is included in the distribution. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index e95b620a65..4b2d936ec6 100644 --- a/setup.py +++ b/setup.py @@ -101,6 +101,7 @@ def find_rel_test(): data_files=[ ('easybuild/scripts', glob.glob('easybuild/scripts/*')), ('etc', glob.glob('etc/*')), + ('contrib/hooks', glob.glob('contrib/hooks/*')), ], long_description=read('README.rst'), classifiers=[ From af5a22245e3a0f48dcadf106c0e1391c9ec69557 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Sun, 17 Oct 2021 17:25:04 +0000 Subject: [PATCH 611/864] Move imkl_fftw logic under if statement and else remove the module --- test/framework/toolchain.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 8bad3a7a8d..14b26cfbf3 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1172,9 +1172,7 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'): # create dummy imkl module and put required lib*.a files in place imkl_module_path = os.path.join(moddir, 'imkl', imklver) - imkl_fftw_module_path = os.path.join(moddir, 'imkl-FFTW', imklver) imkl_dir = os.path.join(self.test_prefix, 'software', 'imkl', imklver) - imkl_fftw_dir = os.path.join(self.test_prefix, 'software', 'imkl-FFTW', imklver) imkl_mod_txt = '\n'.join([ "#%Module", @@ -1183,13 +1181,6 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'): ]) write_file(imkl_module_path, imkl_mod_txt) - imkl_fftw_mod_txt = '\n'.join([ - "#%Module", - "setenv EBROOTIMKLMINFFTW %s" % imkl_fftw_dir, - "setenv EBVERSIONIMKLMINFFTW %s" % imklver, - ]) - write_file(imkl_fftw_module_path, imkl_fftw_mod_txt) - mkl_libs = ['mkl_cdft_core', 'mkl_blacs_intelmpi_lp64'] mkl_libs += ['mkl_intel_lp64', 'mkl_sequential', 'mkl_core', 'mkl_intel_ilp64'] fftw_libs = ['fftw3xc_intel', 'fftw3xc_pgi'] @@ -1199,6 +1190,15 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'): fftw_libs.append('fftw3x_cdft') if LooseVersion(imklver) >= LooseVersion('2021.4.0'): + imkl_fftw_module_path = os.path.join(moddir, 'imkl-FFTW', imklver) + imkl_fftw_dir = os.path.join(self.test_prefix, 'software', 'imkl-FFTW', imklver) + imkl_fftw_mod_txt = '\n'.join([ + "#%Module", + "setenv EBROOTIMKLMINFFTW %s" % imkl_fftw_dir, + "setenv EBVERSIONIMKLMINFFTW %s" % imklver, + ]) + write_file(imkl_fftw_module_path, imkl_fftw_mod_txt) + subdir = 'mkl/%s/lib/intel64' % imklver os.makedirs(os.path.join(imkl_dir, subdir)) for fftlib in mkl_libs: @@ -1208,6 +1208,7 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'): for fftlib in fftw_libs: write_file(os.path.join(imkl_fftw_dir, subdir, 'lib%s.a' % fftlib), 'foo') else: + self.modtool.unload(['imkl-FFTW']) for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64', 'lib/em64t']: os.makedirs(os.path.join(imkl_dir, subdir)) for fftlib in mkl_libs + fftw_libs: From 5d840ce339a1f664057fd2d396bac08c7723be62 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Sun, 17 Oct 2021 18:38:17 +0000 Subject: [PATCH 612/864] Purge modules before testing with different intel module. Otherwise auto-swap functionality doesn't work so well with environment modules (it does work with Lmod) --- test/framework/toolchain.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 14b26cfbf3..8139b274d5 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1020,6 +1020,7 @@ def test_fft_env_vars_foss(self): def test_fft_env_vars_intel(self): """Test setting of $FFT* environment variables using intel toolchain.""" + self.modtool.purge() self.setup_sandbox_for_intel_fftw(self.test_prefix) self.modtool.prepend_module_path(self.test_prefix) @@ -1079,6 +1080,7 @@ def test_fft_env_vars_intel(self): libfft_mt += '-Wl,-Bdynamic -liomp5 -lpthread' self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt) + self.modtool.purge() self.setup_sandbox_for_intel_fftw(self.test_prefix, imklver='2021.4.0') tc = self.get_toolchain('intel', version='2021b') tc.prepare() @@ -1208,7 +1210,6 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'): for fftlib in fftw_libs: write_file(os.path.join(imkl_fftw_dir, subdir, 'lib%s.a' % fftlib), 'foo') else: - self.modtool.unload(['imkl-FFTW']) for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64', 'lib/em64t']: os.makedirs(os.path.join(imkl_dir, subdir)) for fftlib in mkl_libs + fftw_libs: From 7a4d8ce161808869d9935b59c9dc42ab19477ba4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Oct 2021 11:26:54 +0200 Subject: [PATCH 613/864] fix alphabetical ordering of filetools import statements in easyblock.py --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 5a3a57409a..a4d83d3f9d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -72,8 +72,8 @@ from easybuild.tools.config import install_path, log_path, package_path, source_paths from easybuild.tools.environment import restore_env, sanitize_env from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256 -from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, create_patch_info -from easybuild.tools.filetools import convert_name, compute_checksum, copy_file, check_lock, create_lock +from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock +from easybuild.tools.filetools import compute_checksum, convert_name, copy_file, create_lock, create_patch_info from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file from easybuild.tools.filetools import encode_class_name, extract_file from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url From 28d107476bb702692d95fa1c3de264559cc0fbfa Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Oct 2021 14:14:42 +0200 Subject: [PATCH 614/864] fix step description in easyconfig progress bar --- easybuild/framework/easyblock.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 989fb34ec2..e29f9cda3b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3613,6 +3613,9 @@ def run_all_steps(self, run_test_cases): if self.skip_step(step_name, skippable): print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent) else: + progress_label = "Installing %s: %s" % (self.full_mod_name, descr) + update_progress_bar(PROGRESS_BAR_EASYCONFIG, label=progress_label, progress_size=0) + if self.dry_run: self.dry_run_msg("%s... [DRY RUN]\n", descr) else: @@ -3629,8 +3632,7 @@ def run_all_steps(self, run_test_cases): elif self.logdebug or build_option('trace'): print_msg("... (took < 1 sec)", log=self.log, silent=self.silent) - progress_label = "Installing %s: %s" % (self.full_mod_name, descr) - update_progress_bar(PROGRESS_BAR_EASYCONFIG, label=progress_label) + update_progress_bar(PROGRESS_BAR_EASYCONFIG) except StopException: pass From 106d9a9ebba0b844239a1b57dc49162c45bb16f2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Oct 2021 14:21:34 +0200 Subject: [PATCH 615/864] always stop easyconfig progress bar, also when installation failed (or when lock could not be created) --- easybuild/framework/easyblock.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e29f9cda3b..426306a3fa 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3596,19 +3596,19 @@ def run_all_steps(self, run_test_cases): ignore_locks = build_option('ignore_locks') - if ignore_locks: - self.log.info("Ignoring locks...") - else: - lock_name = self.installdir.replace('/', '_') + try: + if ignore_locks: + self.log.info("Ignoring locks...") + else: + lock_name = self.installdir.replace('/', '_') - # check if lock already exists; - # either aborts with an error or waits until it disappears (depends on --wait-on-lock) - check_lock(lock_name) + # check if lock already exists; + # either aborts with an error or waits until it disappears (depends on --wait-on-lock) + check_lock(lock_name) - # create lock to avoid that another installation running in parallel messes things up - create_lock(lock_name) + # create lock to avoid that another installation running in parallel messes things up + create_lock(lock_name) - try: for step_name, descr, step_methods, skippable in steps: if self.skip_step(step_name, skippable): print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent) @@ -3640,7 +3640,7 @@ def run_all_steps(self, run_test_cases): if not ignore_locks: remove_lock(lock_name) - stop_progress_bar(PROGRESS_BAR_EASYCONFIG) + stop_progress_bar(PROGRESS_BAR_EASYCONFIG) # return True for successfull build (or stopped build) return True From 04f16288666d97b0bd5a0b679b866e0e91b0fc4c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Oct 2021 14:42:56 +0200 Subject: [PATCH 616/864] use Group rather than RenderGroup, since latter is deprecated in Rich --- easybuild/tools/output.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index ddac5bc718..9b66060f76 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -36,7 +36,7 @@ from easybuild.tools.py2vs3 import OrderedDict try: - from rich.console import Console, RenderGroup + from rich.console import Console, Group from rich.live import Live from rich.table import Table from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn @@ -101,7 +101,7 @@ def rich_live_cm(): Return Live instance to use as context manager. """ if show_progress_bars(): - pbar_group = RenderGroup( + pbar_group = Group( download_one_progress_bar(), download_one_progress_bar_unknown_size(), download_all_progress_bar(), From 447c96f3c54e521d19bde301b88d56317ef838ca Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Oct 2021 15:02:26 +0200 Subject: [PATCH 617/864] tweak overall progress bar: show result for easyconfigs that are already handled (ok vs failed) --- easybuild/main.py | 11 ++++++++--- easybuild/tools/output.py | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index cf514d619d..05b4f2a1ad 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -68,7 +68,7 @@ from easybuild.tools.hooks import START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import set_up_configuration, use_color -from easybuild.tools.output import PROGRESS_BAR_OVERALL, print_checks, rich_live_cm +from easybuild.tools.output import COLOR_GREEN, COLOR_RED, PROGRESS_BAR_OVERALL, colorize, print_checks, rich_live_cm from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs from easybuild.tools.package.utilities import check_pkg_support @@ -118,19 +118,24 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): start_progress_bar(PROGRESS_BAR_OVERALL, size=len(ecs)) res = [] + ec_results = [] for ec in ecs: ec_res = {} try: (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env) ec_res['log_file'] = app_log - if not ec_res['success']: + if ec_res['success']: + ec_results.append(ec['full_mod_name'] + ' (' + colorize('OK', COLOR_GREEN) + ')') + else: ec_res['err'] = EasyBuildError(err) + ec_results.append(ec['full_mod_name'] + ' (' + colorize('FAILED', COLOR_RED) + ')') except Exception as err: # purposely catch all exceptions ec_res['success'] = False ec_res['err'] = err ec_res['traceback'] = traceback.format_exc() + ec_results.append(ec['full_mod_name'] + ' (' + colorize('FAILED', COLOR_RED) + ')') # keep track of success/total count if ec_res['success']: @@ -161,7 +166,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): res.append((ec, ec_res)) - update_progress_bar(PROGRESS_BAR_OVERALL) + update_progress_bar(PROGRESS_BAR_OVERALL, label=': ' + ', '.join(ec_results)) stop_progress_bar(PROGRESS_BAR_OVERALL) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 9b66060f76..ef24dc4ea6 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -45,6 +45,18 @@ pass +COLOR_GREEN = 'green' +COLOR_RED = 'red' +COLOR_YELLOW = 'yellow' + +# map known colors to ANSII color codes +KNOWN_COLORS = { + COLOR_GREEN: '\033[0;32m', + COLOR_RED: '\033[0;31m', + COLOR_YELLOW: '\033[1;33m', +} +COLOR_END = '\033[0m' + PROGRESS_BAR_DOWNLOAD_ALL = 'download_all' PROGRESS_BAR_DOWNLOAD_ONE = 'download_one' PROGRESS_BAR_EXTENSIONS = 'extensions' @@ -54,6 +66,21 @@ _progress_bar_cache = {} +def colorize(txt, color): + """ + Colorize given text, with specified color. + """ + if color in KNOWN_COLORS: + if use_rich(): + coltxt = '[bold %s]%s[/bold %s]' % (color, txt, color) + else: + coltxt = KNOWN_COLORS[color] + txt + COLOR_END + else: + raise EasyBuildError("Unknown color: %s", color) + + return coltxt + + class DummyRich(object): """ Dummy shim for Rich classes. @@ -142,8 +169,7 @@ def overall_progress_bar(): """ progress_bar = Progress( TimeElapsedColumn(), - TextColumn("{task.description}({task.completed} out of {task.total} easyconfigs done)"), - BarColumn(bar_width=None), + TextColumn("{task.completed} out of {task.total} easyconfigs done{task.description}"), ) return progress_bar @@ -155,10 +181,11 @@ def easyconfig_progress_bar(): Get progress bar to display progress for installing a single easyconfig file. """ progress_bar = Progress( - SpinnerColumn('point'), + SpinnerColumn('point', speed=0.2), TextColumn("[bold green]{task.description} ({task.completed} out of {task.total} steps done)"), BarColumn(), TimeElapsedColumn(), + refresh_per_second=1, ) return progress_bar From 5d1c0177cd124743a1798487571271d31e2388d9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Oct 2021 15:18:12 +0200 Subject: [PATCH 618/864] add tests for colorize function --- test/framework/output.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/framework/output.py b/test/framework/output.py index d27be8aa0f..f35b281a66 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -33,7 +33,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_output_style, update_build_option -from easybuild.tools.output import DummyRich, overall_progress_bar, show_progress_bars, use_rich +from easybuild.tools.output import DummyRich, colorize, overall_progress_bar, show_progress_bars, use_rich try: import rich.progress @@ -124,6 +124,20 @@ def test_use_rich_show_progress_bars(self): self.assertFalse(use_rich()) self.assertFalse(show_progress_bars()) + def test_colorize(self): + """ + Test colorize function + """ + if HAVE_RICH: + for color in ('green', 'red', 'yellow'): + self.assertEqual(colorize('test', color), '[bold %s]test[/bold %s]' % (color, color)) + else: + self.assertEqual(colorize('test', 'green'), '\x1b[0;32mtest\x1b[0m') + self.assertEqual(colorize('test', 'red'), '\x1b[0;31mtest\x1b[0m') + self.assertEqual(colorize('test', 'yellow'), '\x1b[1;33mtest\x1b[0m') + + self.assertErrorRegex(EasyBuildError, "Unknown color: nosuchcolor", colorize, 'test', 'nosuchcolor') + def suite(): """ returns all the testcases in this module """ From 99afb2ad658b31ffa95e5ea5179ee5bda02722dc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Oct 2021 16:34:33 +0200 Subject: [PATCH 619/864] show processed easyconfigs in reverse order (since status bar doesn't line wrap if it becomes too long), show failed easyconfigs first, mention number of failed easyconfigs so far --- easybuild/main.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 05b4f2a1ad..baab1f1db3 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -119,23 +119,32 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): res = [] ec_results = [] + failed_cnt = 0 + + def collect_result(mod_name, success): + """ + Keep track of failed easyconfig + """ + for ec in ecs: ec_res = {} try: (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env) ec_res['log_file'] = app_log - if ec_res['success']: - ec_results.append(ec['full_mod_name'] + ' (' + colorize('OK', COLOR_GREEN) + ')') - else: + if not ec_res['success']: ec_res['err'] = EasyBuildError(err) - ec_results.append(ec['full_mod_name'] + ' (' + colorize('FAILED', COLOR_RED) + ')') except Exception as err: # purposely catch all exceptions ec_res['success'] = False ec_res['err'] = err ec_res['traceback'] = traceback.format_exc() + + if ec_res['success']: + ec_results.append(ec['full_mod_name'] + ' (' + colorize('OK', COLOR_GREEN) + ')') + else: ec_results.append(ec['full_mod_name'] + ' (' + colorize('FAILED', COLOR_RED) + ')') + failed_cnt += 1 # keep track of success/total count if ec_res['success']: @@ -166,7 +175,16 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): res.append((ec, ec_res)) - update_progress_bar(PROGRESS_BAR_OVERALL, label=': ' + ', '.join(ec_results)) + if failed_cnt: + # if installations failed: indicate th + status_label = '(%s): ' % colorize('%s failed!' % failed_cnt, COLOR_RED) + failed_ecs = [x for x in ec_results[::-1] if 'FAILED' in x] + ok_ecs = [x for x in ec_results[::-1] if x not in failed_ecs] + status_label += ', '.join(failed_ecs + ok_ecs) + else: + status_label = ': ' + ', '.join(ec_results[::-1]) + + update_progress_bar(PROGRESS_BAR_OVERALL, label=status_label) stop_progress_bar(PROGRESS_BAR_OVERALL) From d440c118e021c577821e49f2e0f3db11e8827e4e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Oct 2021 16:43:00 +0200 Subject: [PATCH 620/864] add extra space to status bar in case of failed builds --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index baab1f1db3..13ed75e941 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -177,7 +177,7 @@ def collect_result(mod_name, success): if failed_cnt: # if installations failed: indicate th - status_label = '(%s): ' % colorize('%s failed!' % failed_cnt, COLOR_RED) + status_label = ' (%s): ' % colorize('%s failed!' % failed_cnt, COLOR_RED) failed_ecs = [x for x in ec_results[::-1] if 'FAILED' in x] ok_ecs = [x for x in ec_results[::-1] if x not in failed_ecs] status_label += ', '.join(failed_ecs + ok_ecs) From 0f6fb795b345f4a46e56d370ace9e0cc2c93ffaf Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Oct 2021 16:46:16 +0200 Subject: [PATCH 621/864] rename overall_progress_bar to status_bar --- easybuild/main.py | 8 ++++---- easybuild/tools/output.py | 8 ++++---- test/framework/output.py | 14 +++++++------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 13ed75e941..ddc3cce156 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -68,7 +68,7 @@ from easybuild.tools.hooks import START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import set_up_configuration, use_color -from easybuild.tools.output import COLOR_GREEN, COLOR_RED, PROGRESS_BAR_OVERALL, colorize, print_checks, rich_live_cm +from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs from easybuild.tools.package.utilities import check_pkg_support @@ -115,7 +115,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): # e.g. via easyconfig.handle_allowed_system_deps init_env = copy.deepcopy(os.environ) - start_progress_bar(PROGRESS_BAR_OVERALL, size=len(ecs)) + start_progress_bar(STATUS_BAR, size=len(ecs)) res = [] ec_results = [] @@ -184,9 +184,9 @@ def collect_result(mod_name, success): else: status_label = ': ' + ', '.join(ec_results[::-1]) - update_progress_bar(PROGRESS_BAR_OVERALL, label=status_label) + update_progress_bar(STATUS_BAR, label=status_label) - stop_progress_bar(PROGRESS_BAR_OVERALL) + stop_progress_bar(STATUS_BAR) return res diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index ef24dc4ea6..542f7e3fd0 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -61,7 +61,7 @@ PROGRESS_BAR_DOWNLOAD_ONE = 'download_one' PROGRESS_BAR_EXTENSIONS = 'extensions' PROGRESS_BAR_EASYCONFIG = 'easyconfig' -PROGRESS_BAR_OVERALL = 'overall' +STATUS_BAR = 'status' _progress_bar_cache = {} @@ -134,7 +134,7 @@ def rich_live_cm(): download_all_progress_bar(), extensions_progress_bar(), easyconfig_progress_bar(), - overall_progress_bar(), + status_bar(), ) live = Live(pbar_group) else: @@ -163,7 +163,7 @@ def new_func(ignore_cache=False): @progress_bar_cache -def overall_progress_bar(): +def status_bar(): """ Get progress bar to display overall progress. """ @@ -259,7 +259,7 @@ def get_progress_bar(bar_type, size=None): PROGRESS_BAR_DOWNLOAD_ONE: download_one_progress_bar, PROGRESS_BAR_EXTENSIONS: extensions_progress_bar, PROGRESS_BAR_EASYCONFIG: easyconfig_progress_bar, - PROGRESS_BAR_OVERALL: overall_progress_bar, + STATUS_BAR: status_bar, } if bar_type == PROGRESS_BAR_DOWNLOAD_ONE and not size: diff --git a/test/framework/output.py b/test/framework/output.py index f35b281a66..00c46185cf 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -33,7 +33,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_output_style, update_build_option -from easybuild.tools.output import DummyRich, colorize, overall_progress_bar, show_progress_bars, use_rich +from easybuild.tools.output import DummyRich, colorize, status_bar, show_progress_bars, use_rich try: import rich.progress @@ -45,8 +45,8 @@ class OutputTest(EnhancedTestCase): """Tests for functions controlling terminal output.""" - def test_overall_progress_bar(self): - """Test overall_progress_bar function.""" + def test_status_bar(self): + """Test status_bar function.""" # restore default (was disabled in EnhancedTestCase.setUp to avoid messing up test output) update_build_option('show_progress_bar', True) @@ -56,22 +56,22 @@ def test_overall_progress_bar(self): else: expected_progress_bar_class = DummyRich - progress_bar = overall_progress_bar(ignore_cache=True) + progress_bar = status_bar(ignore_cache=True) error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class) self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg) update_build_option('output_style', 'basic') - progress_bar = overall_progress_bar(ignore_cache=True) + progress_bar = status_bar(ignore_cache=True) self.assertTrue(isinstance(progress_bar, DummyRich)) if HAVE_RICH: update_build_option('output_style', 'rich') - progress_bar = overall_progress_bar(ignore_cache=True) + progress_bar = status_bar(ignore_cache=True) error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class) self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg) update_build_option('show_progress_bar', False) - progress_bar = overall_progress_bar(ignore_cache=True) + progress_bar = status_bar(ignore_cache=True) self.assertTrue(isinstance(progress_bar, DummyRich)) def test_get_output_style(self): From 8984aa55ae8d99eab6c4306bbe01d43593bf0565 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Oct 2021 17:30:31 +0200 Subject: [PATCH 622/864] avoid disappearing of time elapsed in status bar by specifying minimal width --- easybuild/tools/output.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 542f7e3fd0..23270fecc4 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -38,9 +38,9 @@ try: from rich.console import Console, Group from rich.live import Live - from rich.table import Table from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn from rich.progress import DownloadColumn, FileSizeColumn, TransferSpeedColumn, TimeRemainingColumn + from rich.table import Column, Table except ImportError: pass @@ -168,7 +168,7 @@ def status_bar(): Get progress bar to display overall progress. """ progress_bar = Progress( - TimeElapsedColumn(), + TimeElapsedColumn(Column(min_width=7, no_wrap=True)), TextColumn("{task.completed} out of {task.total} easyconfigs done{task.description}"), ) From 619f70ee63a0c5cfffaa1460ca945b1e4b0c0c2b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Oct 2021 19:55:11 +0200 Subject: [PATCH 623/864] remove unused helper function in build_and_install_software --- easybuild/main.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index ddc3cce156..b41df476c7 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -121,11 +121,6 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): ec_results = [] failed_cnt = 0 - def collect_result(mod_name, success): - """ - Keep track of failed easyconfig - """ - for ec in ecs: ec_res = {} From 563cbf4b0dad5387ed70f22aeb9557a064d779bd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Oct 2021 19:56:17 +0200 Subject: [PATCH 624/864] fix import order in tests for tools/output.py --- test/framework/output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/output.py b/test/framework/output.py index 00c46185cf..6cacc80fd6 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -33,7 +33,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_output_style, update_build_option -from easybuild.tools.output import DummyRich, colorize, status_bar, show_progress_bars, use_rich +from easybuild.tools.output import DummyRich, colorize, show_progress_bars, status_bar, use_rich try: import rich.progress From 55481e1a94a7b47363a519e4804e6348b7cc4aa3 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 21 Oct 2021 16:56:34 +0800 Subject: [PATCH 625/864] don't sort alphabetically the result of find_related_easyconfigs --- easybuild/framework/easyconfig/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index ff6882f33a..b8cd49f404 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -483,7 +483,7 @@ def find_related_easyconfigs(path, ec): else: _log.debug("No related easyconfigs in potential paths using '%s'" % regex) - return sorted(res) + return res def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False, max_ecs=None, filter_ecs=None): From 950e24f30e397cba0a363fd3153b9a338d8ea0e6 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 21 Oct 2021 17:58:06 +0800 Subject: [PATCH 626/864] fix order of easyconfigs in test of find_related_easyconfigs after removing sort --- test/framework/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index bf207bf8f4..5a8ec46868 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -2826,7 +2826,7 @@ def test_find_related_easyconfigs(self): ec['toolchain'] = {'name': 'gompi', 'version': '1.5.16'} ec['versionsuffix'] = '-foobar' res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] - self.assertEqual(res, ['toy-0.0-gompi-2018a-test.eb', 'toy-0.0-gompi-2018a.eb']) + self.assertEqual(res, ['toy-0.0-gompi-2018a.eb', 'toy-0.0-gompi-2018a-test.eb']) # restore original versionsuffix => matching versionsuffix wins over matching toolchain (name) ec['versionsuffix'] = '-deps' From 6053c0cb1d768d0e5970e1cf1ebf21dafa2d9e13 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 14:36:44 +0200 Subject: [PATCH 627/864] also update extensions progress bar when installing extensions in parallel --- easybuild/framework/easyblock.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e3b34f5c2b..a2383b6a0d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1733,6 +1733,21 @@ def install_extensions_parallel(self, install=True): exts_cnt = len(all_ext_names) exts_queue = self.ext_instances[:] + start_progress_bar(PROGRESS_BAR_EXTENSIONS, exts_cnt) + + def update_exts_progress_bar(running_exts, progress_size): + """Helper function to update extensions progress bar.""" + running_exts_cnt = len(running_exts) + if running_exts_cnt > 1: + progress_label = "Installing %d extensions: " % running_exts_cnt + elif running_exts_cnt == 1: + progress_label = "Installing extension " + else: + progress_label = "Not installing extensions (yet)" + + progress_label += ' '.join(e.name for e in running_exts) + update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label, progress_size=progress_size) + iter_id = 0 while exts_queue or running_exts: @@ -1750,6 +1765,7 @@ def install_extensions_parallel(self, install=True): ext.postrun() running_exts.remove(ext) installed_ext_names.append(ext.name) + update_exts_progress_bar(running_exts, 1) else: self.log.debug("Installation of %s is still running...", ext.name) @@ -1811,7 +1827,10 @@ def install_extensions_parallel(self, install=True): ext.prerun() ext.run(asynchronous=True) running_exts.append(ext) - self.log.debug("Started installation of extension %s in the background...", ext.name) + self.log.info("Started installation of extension %s in the background...", ext.name) + update_exts_progress_bar(running_exts, 0) + + stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False) # # MISCELLANEOUS UTILITY FUNCTIONS From 3fe1f688b3903433c36ed538a18fdf7a693dc2e9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 14:59:01 +0200 Subject: [PATCH 628/864] mark support for installing extensions in parallel as experimental + add --parallel-extensions-install configuration option to opt-in to it --- easybuild/framework/easyblock.py | 3 ++- easybuild/tools/config.py | 1 + easybuild/tools/options.py | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a2383b6a0d..dbc31f1ca6 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1646,7 +1646,8 @@ def install_extensions(self, install=True, parallel=False): """ self.log.debug("List of loaded modules: %s", self.modules_tool.list()) - if parallel: + if build_option('parallel_extensions_install') and parallel: + self.log.experimental("installing extensions in parallel") self.install_extensions_parallel(install=install) else: self.install_extensions_sequential(install=install) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 18902ae799..32642ac224 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -269,6 +269,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'module_extensions', 'module_only', 'package', + 'parallel_extensions_install', 'read_only_installdir', 'remove_ghost_install_dirs', 'rebuild', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index e4fc7661a4..7ef8e7232e 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -454,6 +454,8 @@ def override_options(self): 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), 'parallel': ("Specify (maximum) level of parallellism used during build procedure", 'int', 'store', None), + 'parallel-extensions-install': ("Install list of extensions in parallel (if supported)", + None, 'store_true', False), 'pre-create-installdir': ("Create installation directory before submitting build jobs", None, 'store_true', True), 'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"), From 2e5c6ab8db7440b56a063cf3672280205c96a691 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 15:14:24 +0200 Subject: [PATCH 629/864] start extensions progress bar a bit earlier, also mention preparatory steps (like creating of Extension instances) --- easybuild/framework/easyblock.py | 17 +++++++++-------- easybuild/tools/output.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index dbc31f1ca6..636c0d30a6 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1661,7 +1661,6 @@ def install_extensions_sequential(self, install=True): self.log.info("Installing extensions sequentially...") exts_cnt = len(self.ext_instances) - start_progress_bar(PROGRESS_BAR_EXTENSIONS, exts_cnt) for idx, ext in enumerate(self.ext_instances): @@ -1670,7 +1669,7 @@ def install_extensions_sequential(self, install=True): # always go back to original work dir to avoid running stuff from a dir that no longer exists change_dir(self.orig_workdir) - progress_label = "Installing '%s' extension" % ext.name + progress_label = "Installing '%s' extension (%s/%s)" % (ext.name, idx + 1, exts_cnt) update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label) tup = (ext.name, ext.version or '', idx + 1, exts_cnt) @@ -1711,8 +1710,6 @@ def install_extensions_sequential(self, install=True): elif self.logdebug or build_option('trace'): print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent) - stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False) - def install_extensions_parallel(self, install=True): """ Install extensions in parallel. @@ -1734,8 +1731,6 @@ def install_extensions_parallel(self, install=True): exts_cnt = len(all_ext_names) exts_queue = self.ext_instances[:] - start_progress_bar(PROGRESS_BAR_EXTENSIONS, exts_cnt) - def update_exts_progress_bar(running_exts, progress_size): """Helper function to update extensions progress bar.""" running_exts_cnt = len(running_exts) @@ -1747,6 +1742,7 @@ def update_exts_progress_bar(running_exts, progress_size): progress_label = "Not installing extensions (yet)" progress_label += ' '.join(e.name for e in running_exts) + progress_label += "(%d/%d done)" % (len(installed_ext_names), exts_cnt) update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label, progress_size=progress_size) iter_id = 0 @@ -1831,8 +1827,6 @@ def update_exts_progress_bar(running_exts, progress_size): self.log.info("Started installation of extension %s in the background...", ext.name) update_exts_progress_bar(running_exts, 0) - stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False) - # # MISCELLANEOUS UTILITY FUNCTIONS # @@ -2587,9 +2581,12 @@ def extensions_step(self, fetch=False, install=True): fake_mod_data = self.load_fake_module(purge=True, extra_modules=build_dep_mods) + start_progress_bar(PROGRESS_BAR_EXTENSIONS, len(self.cfg['exts_list'])) + self.prepare_for_extensions() if fetch: + update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="fetching extension sources/patches", progress_size=0) self.exts = self.collect_exts_file_info(fetch_files=True) self.exts_all = self.exts[:] # retain a copy of all extensions, regardless of filtering/skipping @@ -2603,9 +2600,11 @@ def extensions_step(self, fetch=False, install=True): self.clean_up_fake_module(fake_mod_data) raise EasyBuildError("ERROR: No default extension class set for %s", self.name) + update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="creating Extension instances", progress_size=0) self.init_ext_instances() if self.skip: + update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="skipping installed extensions", progress_size=0) self.skip_extensions() self.install_extensions(install=install) @@ -2614,6 +2613,8 @@ def extensions_step(self, fetch=False, install=True): if fake_mod_data: self.clean_up_fake_module(fake_mod_data) + stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False) + def package_step(self): """Package installed software (e.g., into an RPM), if requested, using selected package tool.""" diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 23270fecc4..ff8250b7b2 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -242,7 +242,7 @@ def extensions_progress_bar(): Get progress bar to show progress for installing extensions. """ progress_bar = Progress( - TextColumn("[bold blue]{task.description} ({task.completed}/{task.total})"), + TextColumn("[bold blue]{task.description}"), BarColumn(), TimeElapsedColumn(), ) From 3748d9c444a290c3b7649821376a34c504b10854 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 15:26:00 +0200 Subject: [PATCH 630/864] add and use update_exts_progress_bar method to EasyBlock --- easybuild/framework/easyblock.py | 36 +++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 636c0d30a6..39b06987dc 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1669,8 +1669,8 @@ def install_extensions_sequential(self, install=True): # always go back to original work dir to avoid running stuff from a dir that no longer exists change_dir(self.orig_workdir) - progress_label = "Installing '%s' extension (%s/%s)" % (ext.name, idx + 1, exts_cnt) - update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label) + progress_info = "Installing '%s' extension (%s/%s)" % (ext.name, idx + 1, exts_cnt) + self.update_exts_progress_bar(progress_info) tup = (ext.name, ext.version or '', idx + 1, exts_cnt) print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent, log=self.log) @@ -1710,6 +1710,8 @@ def install_extensions_sequential(self, install=True): elif self.logdebug or build_option('trace'): print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent) + self.update_exts_progress_bar(progress_info, progress_size=1) + def install_extensions_parallel(self, install=True): """ Install extensions in parallel. @@ -1731,19 +1733,19 @@ def install_extensions_parallel(self, install=True): exts_cnt = len(all_ext_names) exts_queue = self.ext_instances[:] - def update_exts_progress_bar(running_exts, progress_size): + def update_exts_progress_bar_helper(running_exts, progress_size): """Helper function to update extensions progress bar.""" running_exts_cnt = len(running_exts) if running_exts_cnt > 1: - progress_label = "Installing %d extensions: " % running_exts_cnt + progress_info = "Installing %d extensions: " % running_exts_cnt elif running_exts_cnt == 1: - progress_label = "Installing extension " + progress_info = "Installing extension " else: - progress_label = "Not installing extensions (yet)" + progress_info = "Not installing extensions (yet)" - progress_label += ' '.join(e.name for e in running_exts) - progress_label += "(%d/%d done)" % (len(installed_ext_names), exts_cnt) - update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label, progress_size=progress_size) + progress_info += ' '.join(e.name for e in running_exts) + progress_info += "(%d/%d done)" % (len(installed_ext_names), exts_cnt) + self.update_exts_progress_bar(progress_info, progress_size=progress_size) iter_id = 0 while exts_queue or running_exts: @@ -1762,7 +1764,7 @@ def update_exts_progress_bar(running_exts, progress_size): ext.postrun() running_exts.remove(ext) installed_ext_names.append(ext.name) - update_exts_progress_bar(running_exts, 1) + update_exts_progress_bar_helper(running_exts, 1) else: self.log.debug("Installation of %s is still running...", ext.name) @@ -1825,7 +1827,7 @@ def update_exts_progress_bar(running_exts, progress_size): ext.run(asynchronous=True) running_exts.append(ext) self.log.info("Started installation of extension %s in the background...", ext.name) - update_exts_progress_bar(running_exts, 0) + update_exts_progress_bar_helper(running_exts, 0) # # MISCELLANEOUS UTILITY FUNCTIONS @@ -2560,6 +2562,12 @@ def init_ext_instances(self): self.ext_instances.append(inst) + def update_exts_progress_bar(self, info, progress_size=0): + """ + Update extensions progress bar with specified info and amount of progress made + """ + update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=info, progress_size=progress_size) + def extensions_step(self, fetch=False, install=True): """ After make install, run this. @@ -2586,7 +2594,7 @@ def extensions_step(self, fetch=False, install=True): self.prepare_for_extensions() if fetch: - update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="fetching extension sources/patches", progress_size=0) + self.update_exts_progress_bar("fetching extension sources/patches") self.exts = self.collect_exts_file_info(fetch_files=True) self.exts_all = self.exts[:] # retain a copy of all extensions, regardless of filtering/skipping @@ -2600,11 +2608,11 @@ def extensions_step(self, fetch=False, install=True): self.clean_up_fake_module(fake_mod_data) raise EasyBuildError("ERROR: No default extension class set for %s", self.name) - update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="creating Extension instances", progress_size=0) + self.update_exts_progress_bar("creating internal datastructures") self.init_ext_instances() if self.skip: - update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="skipping installed extensions", progress_size=0) + self.update_exts_progress_bar("skipping install extensions") self.skip_extensions() self.install_extensions(install=install) From 45a2627a382df986bc28d2330fb6a8734e04f0ac Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 16:57:38 +0200 Subject: [PATCH 631/864] fix formatting for extension progress bar when installing extensions in parallel --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 39b06987dc..e0fca04973 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1743,8 +1743,8 @@ def update_exts_progress_bar_helper(running_exts, progress_size): else: progress_info = "Not installing extensions (yet)" - progress_info += ' '.join(e.name for e in running_exts) - progress_info += "(%d/%d done)" % (len(installed_ext_names), exts_cnt) + progress_info += ', '.join(e.name for e in running_exts) + progress_info += " (%d/%d done)" % (len(installed_ext_names), exts_cnt) self.update_exts_progress_bar(progress_info, progress_size=progress_size) iter_id = 0 From e47fead521fec0f87a07d8229c0a7c19e0f05e4d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 17:54:46 +0200 Subject: [PATCH 632/864] add check_async_cmd function to facilitate checking on asynchronously running commands --- easybuild/tools/run.py | 36 +++++++++++++++++++++++++++++++ test/framework/run.py | 49 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index ce04900204..aa13a14f77 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -248,6 +248,42 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True regexp=regexp, stream_output=stream_output, trace=trace) +def check_async_cmd(proc, cmd, owd, start_time, cmd_log, output_read_size=1024, output=''): + """ + Check status of command that was started asynchronously. + + :param proc: subprocess.Popen instance representing asynchronous command + :param cmd: command being run + :param owd: original working directory + :param start_time: start time of command (datetime instance) + :param cmd_log: log file to print command output to + :param output_read_size: number of bytes to read from output + :param output: already collected output for this command + + :result: dict value with result of the check (boolean 'done', 'exit_code', 'output') + """ + # use small read size, to avoid waiting for a long time until sufficient output is produced + add_out = get_output_from_process(proc, read_size=output_read_size) + _log.debug("Additional output from asynchronous command '%s': %s" % (cmd, add_out)) + output += add_out + + exit_code = proc.poll() + if exit_code is None: + _log.debug("Asynchronous command '%s' still running..." % cmd) + done = False + else: + _log.debug("Asynchronous command '%s' completed!", cmd) + output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output, simple=False) + done = True + + res = { + 'done': done, + 'exit_code': exit_code, + 'output': output, + } + return res + + def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False, simple=False, regexp=True, stream_output=None, trace=True, output=''): """ diff --git a/test/framework/run.py b/test/framework/run.py index 368a4b14ed..1f48129f2f 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -47,7 +47,7 @@ import easybuild.tools.utilities from easybuild.tools.build_log import EasyBuildError, init_logging, stop_logging from easybuild.tools.filetools import adjust_permissions, read_file, write_file -from easybuild.tools.run import check_log_for_errors, complete_cmd, get_output_from_process +from easybuild.tools.run import check_async_cmd, check_log_for_errors, complete_cmd, get_output_from_process from easybuild.tools.run import parse_log_for_error, run_cmd, run_cmd_qa from easybuild.tools.config import ERROR, IGNORE, WARN @@ -575,7 +575,8 @@ def test_run_cmd_async(self): os.environ['TEST'] = 'test123' - cmd_info = run_cmd("sleep 2; echo $TEST", asynchronous=True) + test_cmd = "echo 'sleeping...'; sleep 2; echo $TEST" + cmd_info = run_cmd(test_cmd, asynchronous=True) proc = cmd_info[0] # change value of $TEST to check that command is completed with correct environment @@ -585,18 +586,41 @@ def test_run_cmd_async(self): ec = proc.poll() self.assertEqual(ec, None) + # wait until command is done while ec is None: time.sleep(1) ec = proc.poll() out, ec = complete_cmd(*cmd_info, simple=False) self.assertEqual(ec, 0) - self.assertEqual(out, 'test123\n') + self.assertEqual(out, 'sleeping...\ntest123\n') + + # also test use of check_async_cmd function + os.environ['TEST'] = 'test123' + cmd_info = run_cmd(test_cmd, asynchronous=True) + + # first check, only read first 12 output characters + # (otherwise we'll be waiting until command is completed) + res = check_async_cmd(*cmd_info, output_read_size=12) + self.assertEqual(res, {'done': False, 'exit_code': None, 'output': 'sleeping...\n'}) + + # 2nd check with default output size (1024) gets full output + res = check_async_cmd(*cmd_info, output=res['output']) + self.assertEqual(res, {'done': True, 'exit_code': 0, 'output': 'sleeping...\ntest123\n'}) # also test with a command that produces a lot of output, # since that tends to lock up things unless we frequently grab some output... - cmd = "echo start; for i in $(seq 1 50); do sleep 0.1; for j in $(seq 1000); do echo foo; done; done; echo done" - cmd_info = run_cmd(cmd, asynchronous=True) + verbose_test_cmd = ';'.join([ + "echo start", + "for i in $(seq 1 50)", + "do sleep 0.1", + "for j in $(seq 1000)", + "do echo foo", + "done", + "done", + "echo done", + ]) + cmd_info = run_cmd(verbose_test_cmd, asynchronous=True) proc = cmd_info[0] output = '' @@ -613,6 +637,21 @@ def test_run_cmd_async(self): self.assertTrue(out.startswith('start\n')) self.assertTrue(out.endswith('\ndone\n')) + # also test use of check_async_cmd on verbose test command + cmd_info = run_cmd(verbose_test_cmd, asynchronous=True) + res = check_async_cmd(*cmd_info) + self.assertEqual(res['done'], False) + self.assertEqual(res['exit_code'], None) + self.assertTrue(res['output'].startswith('start\n')) + self.assertFalse(res['output'].endswith('\ndone\n')) + # keep checking until command is complete + while not res['done']: + res = check_async_cmd(*cmd_info, output=res['output']) + self.assertEqual(res['done'], True) + self.assertEqual(res['exit_code'], 0) + self.assertTrue(res['output'].startswith('start\n')) + self.assertTrue(res['output'].endswith('\ndone\n')) + def test_check_log_for_errors(self): fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') os.close(fd) From 471ff3ec649a832feec9acf5f4251bc41f83ff74 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 21 Oct 2021 18:24:56 +0200 Subject: [PATCH 633/864] Fix IntelCompilers libpaths to actually point to the compiler libs --- easybuild/toolchains/compiler/intel_compilers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/toolchains/compiler/intel_compilers.py b/easybuild/toolchains/compiler/intel_compilers.py index 98b4ea5b7f..8bc4bdc763 100644 --- a/easybuild/toolchains/compiler/intel_compilers.py +++ b/easybuild/toolchains/compiler/intel_compilers.py @@ -47,11 +47,11 @@ def _set_compiler_vars(self): Compiler._set_compiler_vars(self) root = self.get_software_root(self.COMPILER_MODULE_NAME)[0] + version = self.get_software_version(self.COMPILER_MODULE_NAME)[0] + libbase = os.path.join('compiler', version, 'linux') libpaths = [ - 'lib', - os.path.join('lib', 'x64'), - os.path.join('compiler', 'lib', 'intel64_lin'), + os.path.join(libbase, 'compiler', 'lib', 'intel64'), ] self.variables.append_subdirs("LDFLAGS", root, subdirs=libpaths) From 0825d425d028bfaa230534b9aadbf1a8ef22ca62 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 18:27:58 +0200 Subject: [PATCH 634/864] don't read any output in check_async_cmd if output_read_size is set to 0 --- easybuild/tools/run.py | 7 ++++--- test/framework/run.py | 7 +++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index aa13a14f77..5340c76b76 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -263,9 +263,10 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, output_read_size=1024, :result: dict value with result of the check (boolean 'done', 'exit_code', 'output') """ # use small read size, to avoid waiting for a long time until sufficient output is produced - add_out = get_output_from_process(proc, read_size=output_read_size) - _log.debug("Additional output from asynchronous command '%s': %s" % (cmd, add_out)) - output += add_out + if output_read_size: + add_out = get_output_from_process(proc, read_size=output_read_size) + _log.debug("Additional output from asynchronous command '%s': %s" % (cmd, add_out)) + output += add_out exit_code = proc.poll() if exit_code is None: diff --git a/test/framework/run.py b/test/framework/run.py index 1f48129f2f..6a348e3eb2 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -639,6 +639,13 @@ def test_run_cmd_async(self): # also test use of check_async_cmd on verbose test command cmd_info = run_cmd(verbose_test_cmd, asynchronous=True) + + # with output_read_size set to 0, no output is read yet, only status of command is checked + res = check_async_cmd(*cmd_info, output_read_size=0) + self.assertEqual(res['done'], False) + self.assertEqual(res['exit_code'], None) + self.assertEqual(res['output'], '') + res = check_async_cmd(*cmd_info) self.assertEqual(res['done'], False) self.assertEqual(res['exit_code'], None) From 3ea4d147795f2cb5b5c5b668444bfbd019cf5523 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 19:27:55 +0200 Subject: [PATCH 635/864] verify that output_read_size passed to check_async_cmd is a positive integer value --- easybuild/tools/run.py | 2 ++ test/framework/run.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 5340c76b76..e70e6702ce 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -264,6 +264,8 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, output_read_size=1024, """ # use small read size, to avoid waiting for a long time until sufficient output is produced if output_read_size: + if not isinstance(output_read_size, int) or output_read_size < 0: + raise EasyBuildError("Number of output bytes to read should be a positive integer value") add_out = get_output_from_process(proc, read_size=output_read_size) _log.debug("Additional output from asynchronous command '%s': %s" % (cmd, add_out)) output += add_out diff --git a/test/framework/run.py b/test/framework/run.py index 6a348e3eb2..d84fbf2711 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -640,6 +640,10 @@ def test_run_cmd_async(self): # also test use of check_async_cmd on verbose test command cmd_info = run_cmd(verbose_test_cmd, asynchronous=True) + error_pattern = "Number of output bytes to read should be a positive integer value" + self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size=-1) + self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size='foo') + # with output_read_size set to 0, no output is read yet, only status of command is checked res = check_async_cmd(*cmd_info, output_read_size=0) self.assertEqual(res['done'], False) From 78faeabafa37aef7fc9b8a3c5a66ad1a388e6ae9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 19:55:25 +0200 Subject: [PATCH 636/864] update extensions progress bar with more detailed info when creating Extension instances + checking for extensions to skip --- easybuild/framework/easyblock.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e0fca04973..b22322544c 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1616,14 +1616,18 @@ def skip_extensions(self): - use this to detect existing extensions and to remove them from self.ext_instances - based on initial R version """ + self.update_exts_progress_bar("skipping installed extensions") + # obtaining untemplated reference value is required here to support legacy string templates like name/version exts_filter = self.cfg.get_ref('exts_filter') if not exts_filter or len(exts_filter) == 0: raise EasyBuildError("Skipping of extensions, but no exts_filter set in easyconfig") + exts_cnt = len(self.ext_instances) + res = [] - for ext_inst in self.ext_instances: + for idx, ext_inst in enumerate(self.ext_instances): cmd, stdin = resolve_exts_filter_template(exts_filter, ext_inst) (cmdstdouterr, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False) self.log.info("exts_filter result %s %s", cmdstdouterr, ec) @@ -1634,6 +1638,8 @@ def skip_extensions(self): else: print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log) + self.update_exts_progress_bar("skipping installed extensions (%d/%d checked)" % (idx + 1, exts_cnt)) + self.ext_instances = res def install_extensions(self, install=True, parallel=False): @@ -2477,6 +2483,8 @@ def init_ext_instances(self): """ exts_list = self.cfg.get_ref('exts_list') + self.update_exts_progress_bar("creating internal datastructures for extensions") + # early exit if there are no extensions if not exts_list: return @@ -2500,7 +2508,9 @@ def init_ext_instances(self): error_msg = "Improper default extension class specification, should be string: %s (%s)" raise EasyBuildError(error_msg, exts_defaultclass, type(exts_defaultclass)) - for ext in self.exts: + exts_cnt = len(self.exts) + + for idx, ext in enumerate(self.exts): ext_name = ext['name'] self.log.debug("Creating class instance for extension %s...", ext_name) @@ -2561,6 +2571,9 @@ def init_ext_instances(self): self.log.debug("Installing extension %s with class %s (from %s)", ext_name, class_name, mod_path) self.ext_instances.append(inst) + pbar_label = "creating internal datastructures for extensions " + pbar_label += "(%d/%d done)" % (idx + 1, exts_cnt) + self.update_exts_progress_bar(pbar_label) def update_exts_progress_bar(self, info, progress_size=0): """ @@ -2608,11 +2621,9 @@ def extensions_step(self, fetch=False, install=True): self.clean_up_fake_module(fake_mod_data) raise EasyBuildError("ERROR: No default extension class set for %s", self.name) - self.update_exts_progress_bar("creating internal datastructures") self.init_ext_instances() if self.skip: - self.update_exts_progress_bar("skipping install extensions") self.skip_extensions() self.install_extensions(install=install) From 6b431687fb75101b444ddb30c96f9354aa844810 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 20:26:41 +0200 Subject: [PATCH 637/864] also mention zero as valid value for output_read_size in check_async_cmd --- easybuild/tools/run.py | 2 +- test/framework/run.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index e70e6702ce..dbfcf4d2d4 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -265,7 +265,7 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, output_read_size=1024, # use small read size, to avoid waiting for a long time until sufficient output is produced if output_read_size: if not isinstance(output_read_size, int) or output_read_size < 0: - raise EasyBuildError("Number of output bytes to read should be a positive integer value") + raise EasyBuildError("Number of output bytes to read should be a positive integer value (or zero)") add_out = get_output_from_process(proc, read_size=output_read_size) _log.debug("Additional output from asynchronous command '%s': %s" % (cmd, add_out)) output += add_out diff --git a/test/framework/run.py b/test/framework/run.py index d84fbf2711..d421d72cc1 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -640,7 +640,7 @@ def test_run_cmd_async(self): # also test use of check_async_cmd on verbose test command cmd_info = run_cmd(verbose_test_cmd, asynchronous=True) - error_pattern = "Number of output bytes to read should be a positive integer value" + error_pattern = r"Number of output bytes to read should be a positive integer value \(or zero\)" self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size=-1) self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size='foo') From e9ca5f7e35d3fde6b02c948ab0f233e6d38e43fd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 20:28:35 +0200 Subject: [PATCH 638/864] disable trace output when calling complete_cmd in check_async_cmd --- easybuild/tools/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index dbfcf4d2d4..2713b4b007 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -276,7 +276,7 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, output_read_size=1024, done = False else: _log.debug("Asynchronous command '%s' completed!", cmd) - output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output, simple=False) + output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output, simple=False, trace=False) done = True res = { From 6c19701ac2777bc94bda33a6134d8291ae5b2b67 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 20:39:48 +0200 Subject: [PATCH 639/864] add support to check_async_cmd for not failing when command exited with an error --- easybuild/tools/run.py | 5 +++-- test/framework/run.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 2713b4b007..69cebc57a9 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -248,7 +248,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True regexp=regexp, stream_output=stream_output, trace=trace) -def check_async_cmd(proc, cmd, owd, start_time, cmd_log, output_read_size=1024, output=''): +def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, output_read_size=1024, output=''): """ Check status of command that was started asynchronously. @@ -276,7 +276,8 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, output_read_size=1024, done = False else: _log.debug("Asynchronous command '%s' completed!", cmd) - output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output, simple=False, trace=False) + output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output, + simple=False, trace=False, log_ok=fail_on_error, log_all=fail_on_error) done = True res = { diff --git a/test/framework/run.py b/test/framework/run.py index d421d72cc1..cfcaac2805 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -608,6 +608,16 @@ def test_run_cmd_async(self): res = check_async_cmd(*cmd_info, output=res['output']) self.assertEqual(res, {'done': True, 'exit_code': 0, 'output': 'sleeping...\ntest123\n'}) + # check asynchronous running of failing command + error_test_cmd = "sleep 2; echo 'FAIL!' >&2; exit 123" + cmd_info = run_cmd(error_test_cmd, asynchronous=True) + error_pattern = 'cmd ".*" exited with exit code 123' + self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info) + + cmd_info = run_cmd(error_test_cmd, asynchronous=True) + res = check_async_cmd(*cmd_info, fail_on_error=False) + self.assertEqual(res, {'done': True, 'exit_code': 123, 'output': "FAIL!\n"}) + # also test with a command that produces a lot of output, # since that tends to lock up things unless we frequently grab some output... verbose_test_cmd = ';'.join([ From d64ecae91335be8a7e0556755067da6addf6f69a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 22:19:30 +0200 Subject: [PATCH 640/864] make update_progress_bar a bit more robust by just doing nothing if the corresponding progress bar was not started (and making stopping of a non-started progress bar fatal) --- easybuild/tools/output.py | 24 ++++++++++++++---------- test/framework/output.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 23270fecc4..193a6c05ef 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -295,27 +295,31 @@ def start_progress_bar(bar_type, size, label=None): def update_progress_bar(bar_type, label=None, progress_size=1): """ - Update progress bar of given type, add progress of given size. + Update progress bar of given type (if it was started), add progress of given size. :param bar_type: type of progress bar :param label: label for progress bar :param progress_size: amount of progress made """ - (pbar, task_id) = _progress_bar_cache[bar_type] - if label: - pbar.update(task_id, description=label) - if progress_size: - pbar.update(task_id, advance=progress_size) + if bar_type in _progress_bar_cache: + (pbar, task_id) = _progress_bar_cache[bar_type] + if label: + pbar.update(task_id, description=label) + if progress_size: + pbar.update(task_id, advance=progress_size) def stop_progress_bar(bar_type, visible=False): """ Stop progress bar of given type. """ - (pbar, task_id) = _progress_bar_cache[bar_type] - pbar.stop_task(task_id) - if not visible: - pbar.update(task_id, visible=False) + if bar_type in _progress_bar_cache: + (pbar, task_id) = _progress_bar_cache[bar_type] + pbar.stop_task(task_id) + if not visible: + pbar.update(task_id, visible=False) + else: + raise EasyBuildError("Failed to stop %s progress bar, since it was never started?!", bar_type) def print_checks(checks_data): diff --git a/test/framework/output.py b/test/framework/output.py index 6cacc80fd6..b3ff7e558a 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -33,7 +33,8 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_output_style, update_build_option -from easybuild.tools.output import DummyRich, colorize, show_progress_bars, status_bar, use_rich +from easybuild.tools.output import PROGRESS_BAR_EXTENSIONS, DummyRich, colorize, get_progress_bar, show_progress_bars +from easybuild.tools.output import start_progress_bar, status_bar, stop_progress_bar, update_progress_bar, use_rich try: import rich.progress @@ -138,6 +139,34 @@ def test_colorize(self): self.assertErrorRegex(EasyBuildError, "Unknown color: nosuchcolor", colorize, 'test', 'nosuchcolor') + def test_get_start_update_stop_progress_bar(self): + """ + Test starting/updating/stopping of progress bars. + """ + # restore default configuration to show progress bars (disabled to avoid mangled test output) + update_build_option('show_progress_bar', True) + + # stopping a progress bar that never was started results in an error + error_pattern = "Failed to stop extensions progress bar, since it was never started" + self.assertErrorRegex(EasyBuildError, error_pattern, stop_progress_bar, PROGRESS_BAR_EXTENSIONS) + + # updating a progress bar that never was started is silently ignored on purpose + update_progress_bar(PROGRESS_BAR_EXTENSIONS) + update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="foo") + update_progress_bar(PROGRESS_BAR_EXTENSIONS, progress_size=100) + + # also test normal cycle: start, update, stop + start_progress_bar(PROGRESS_BAR_EXTENSIONS, 100) + update_progress_bar(PROGRESS_BAR_EXTENSIONS) # single step progress + update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="test123", progress_size=5) + stop_progress_bar(PROGRESS_BAR_EXTENSIONS) + + pbar = get_progress_bar(PROGRESS_BAR_EXTENSIONS) + if HAVE_RICH: + self.assertTrue(isinstance(pbar, rich.progress.Progress)) + else: + self.assertTrue(isinstance(pbar, DummyRich)) + def suite(): """ returns all the testcases in this module """ From 6e0097ffb7f5e1ae1f0b2060e5857e7a3936d778 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Fri, 22 Oct 2021 09:48:26 +0800 Subject: [PATCH 641/864] reverse sort potential_paths in find_related_easyconfigs and fix order of easyconfigs in respective tests --- easybuild/framework/easyconfig/tools.py | 2 +- test/framework/easyconfig.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index a08b016bd4..595481d74a 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -457,7 +457,7 @@ def find_related_easyconfigs(path, ec): toolchain_pattern = '' potential_paths = [glob.glob(ec_path) for ec_path in create_paths(path, name, '*')] - potential_paths = sum(potential_paths, []) # flatten + potential_paths = sorted(sum(potential_paths, []), reverse=True) # flatten _log.debug("found these potential paths: %s" % potential_paths) parsed_version = LooseVersion(version).version diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 5a8ec46868..c62b767069 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -2807,12 +2807,12 @@ def test_find_related_easyconfigs(self): # tweak version to 4.6.1, GCC/4.6.x easyconfigs are found as closest match ec['version'] = '4.6.1' res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] - self.assertEqual(res, ['GCC-4.6.3.eb', 'GCC-4.6.4.eb']) + self.assertEqual(res, ['GCC-4.6.4.eb', 'GCC-4.6.3.eb']) # tweak version to 4.5.0, GCC/4.x easyconfigs are found as closest match ec['version'] = '4.5.0' res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] - expected = ['GCC-4.6.3.eb', 'GCC-4.6.4.eb', 'GCC-4.8.2.eb', 'GCC-4.8.3.eb', 'GCC-4.9.2.eb'] + expected = ['GCC-4.9.2.eb', 'GCC-4.8.3.eb', 'GCC-4.8.2.eb', 'GCC-4.6.4.eb', 'GCC-4.6.3.eb'] self.assertEqual(res, expected) ec_file = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-deps.eb') From 8946355c4019f23f1a1f456132a79ee495458312 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Fri, 22 Oct 2021 12:23:54 +0800 Subject: [PATCH 642/864] also test --review-pr-max and --review-pr-filter in test_github_review_pr --- test/framework/options.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 7f0eb39d40..4e78999742 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3786,6 +3786,36 @@ def test_github_review_pr(self): self.mock_stderr(False) self.assertTrue("This PR should be labelled with 'update'" in txt) + # test --review-pr-max + self.mock_stdout(True) + self.mock_stderr(True) + args = [ + '--color=never', + '--github-user=%s' % GITHUB_TEST_ACCOUNT, + '--review-pr=5365', + '--review-pr-max=1', + ] + self.eb_main(args, raise_error=True, testing=True) + txt = self.get_stdout() + self.mock_stdout(False) + self.mock_stderr(False) + self.assertTrue("2016.04" not in txt) + + # test --review-pr-filter + self.mock_stdout(True) + self.mock_stderr(True) + args = [ + '--color=never', + '--github-user=%s' % GITHUB_TEST_ACCOUNT, + '--review-pr=5365', + '--review-pr-filter=2016a', + ] + self.eb_main(args, raise_error=True, testing=True) + txt = self.get_stdout() + self.mock_stdout(False) + self.mock_stderr(False) + self.assertTrue("2016.04" not in txt) + def test_set_tmpdir(self): """Test set_tmpdir config function.""" self.purge_environment() From c2c7557e6d86810e0734ac3ae2ef936bdb698177 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 22 Oct 2021 08:25:18 +0200 Subject: [PATCH 643/864] fix remarks for check_async_cmd --- easybuild/tools/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 69cebc57a9..322a9fddf5 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -257,6 +257,7 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, out :param owd: original working directory :param start_time: start time of command (datetime instance) :param cmd_log: log file to print command output to + :param fail_on_error: raise EasyBuildError when command exited with an error :param output_read_size: number of bytes to read from output :param output: already collected output for this command @@ -277,7 +278,7 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, out else: _log.debug("Asynchronous command '%s' completed!", cmd) output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output, - simple=False, trace=False, log_ok=fail_on_error, log_all=fail_on_error) + simple=False, trace=False, log_ok=fail_on_error) done = True res = { From e6a82ca571856c9a2e9f5cbf1dfbf6fc153bdf4a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 22 Oct 2021 09:14:56 +0200 Subject: [PATCH 644/864] clear progress bar cache first in test_get_start_update_stop_progress_bar --- test/framework/output.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/framework/output.py b/test/framework/output.py index b3ff7e558a..a34c3d2de1 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -31,6 +31,7 @@ from unittest import TextTestRunner from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered +import easybuild.tools.output from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_output_style, update_build_option from easybuild.tools.output import PROGRESS_BAR_EXTENSIONS, DummyRich, colorize, get_progress_bar, show_progress_bars @@ -143,6 +144,9 @@ def test_get_start_update_stop_progress_bar(self): """ Test starting/updating/stopping of progress bars. """ + # clear progress bar cache first, this test assumes we start with a clean slate + easybuild.tools.output._progress_bar_cache.clear() + # restore default configuration to show progress bars (disabled to avoid mangled test output) update_build_option('show_progress_bar', True) From a7a970aed65caee67195494d385316dce43f6505 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 22 Oct 2021 09:37:04 +0200 Subject: [PATCH 645/864] add constant for progress bar types + support use of ignore_cache in get_progress_bar --- easybuild/tools/output.py | 25 ++++++++++++++----------- test/framework/output.py | 24 +++++++++++++++++------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 193a6c05ef..e408962338 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -250,22 +250,15 @@ def extensions_progress_bar(): return progress_bar -def get_progress_bar(bar_type, size=None): +def get_progress_bar(bar_type, ignore_cache=False, size=None): """ Get progress bar of given type. """ - progress_bar_types = { - PROGRESS_BAR_DOWNLOAD_ALL: download_all_progress_bar, - PROGRESS_BAR_DOWNLOAD_ONE: download_one_progress_bar, - PROGRESS_BAR_EXTENSIONS: extensions_progress_bar, - PROGRESS_BAR_EASYCONFIG: easyconfig_progress_bar, - STATUS_BAR: status_bar, - } if bar_type == PROGRESS_BAR_DOWNLOAD_ONE and not size: - pbar = download_one_progress_bar_unknown_size() - elif bar_type in progress_bar_types: - pbar = progress_bar_types[bar_type]() + pbar = download_one_progress_bar_unknown_size(ignore_cache=ignore_cache) + elif bar_type in PROGRESS_BAR_TYPES: + pbar = PROGRESS_BAR_TYPES[bar_type](ignore_cache=ignore_cache) else: raise EasyBuildError("Unknown progress bar type: %s", bar_type) @@ -388,3 +381,13 @@ def print_checks(checks_data): console_print(table) else: print('\n'.join(lines)) + + +# this constant must be defined at the end, since functions used as values need to be defined +PROGRESS_BAR_TYPES = { + PROGRESS_BAR_DOWNLOAD_ALL: download_all_progress_bar, + PROGRESS_BAR_DOWNLOAD_ONE: download_one_progress_bar, + PROGRESS_BAR_EXTENSIONS: extensions_progress_bar, + PROGRESS_BAR_EASYCONFIG: easyconfig_progress_bar, + STATUS_BAR: status_bar, +} diff --git a/test/framework/output.py b/test/framework/output.py index a34c3d2de1..ea574d043a 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -34,7 +34,8 @@ import easybuild.tools.output from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_output_style, update_build_option -from easybuild.tools.output import PROGRESS_BAR_EXTENSIONS, DummyRich, colorize, get_progress_bar, show_progress_bars +from easybuild.tools.output import PROGRESS_BAR_EXTENSIONS, PROGRESS_BAR_TYPES +from easybuild.tools.output import DummyRich, colorize, get_progress_bar, show_progress_bars from easybuild.tools.output import start_progress_bar, status_bar, stop_progress_bar, update_progress_bar, use_rich try: @@ -140,6 +141,21 @@ def test_colorize(self): self.assertErrorRegex(EasyBuildError, "Unknown color: nosuchcolor", colorize, 'test', 'nosuchcolor') + def test_get_progress_bar(self): + """ + Test get_progress_bar. + """ + # restore default configuration to show progress bars (disabled to avoid mangled test output), + # to ensure we'll get actual Progress instances when Rich is available + update_build_option('show_progress_bar', True) + + for pbar_type in PROGRESS_BAR_TYPES: + pbar = get_progress_bar(pbar_type, ignore_cache=True) + if HAVE_RICH: + self.assertTrue(isinstance(pbar, rich.progress.Progress)) + else: + self.assertTrue(isinstance(pbar, DummyRich)) + def test_get_start_update_stop_progress_bar(self): """ Test starting/updating/stopping of progress bars. @@ -165,12 +181,6 @@ def test_get_start_update_stop_progress_bar(self): update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="test123", progress_size=5) stop_progress_bar(PROGRESS_BAR_EXTENSIONS) - pbar = get_progress_bar(PROGRESS_BAR_EXTENSIONS) - if HAVE_RICH: - self.assertTrue(isinstance(pbar, rich.progress.Progress)) - else: - self.assertTrue(isinstance(pbar, DummyRich)) - def suite(): """ returns all the testcases in this module """ From fdc8a1aa15dc92dcf6e9623eac7912515d0c653a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 22 Oct 2021 10:04:26 +0200 Subject: [PATCH 646/864] only update extensions progress bar in init_ext_instances if there actually are extensions --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index b22322544c..e4b1254d0e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2483,8 +2483,6 @@ def init_ext_instances(self): """ exts_list = self.cfg.get_ref('exts_list') - self.update_exts_progress_bar("creating internal datastructures for extensions") - # early exit if there are no extensions if not exts_list: return @@ -2510,6 +2508,8 @@ def init_ext_instances(self): exts_cnt = len(self.exts) + self.update_exts_progress_bar("creating internal datastructures for extensions") + for idx, ext in enumerate(self.exts): ext_name = ext['name'] self.log.debug("Creating class instance for extension %s...", ext_name) From bdf1e208b1e0d00068ce9d8f16a7236f05e74f77 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 22 Oct 2021 11:00:35 +0200 Subject: [PATCH 647/864] drop sleep from failing test command in test_run_cmd_async --- test/framework/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/run.py b/test/framework/run.py index cfcaac2805..a6a88b638c 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -609,7 +609,7 @@ def test_run_cmd_async(self): self.assertEqual(res, {'done': True, 'exit_code': 0, 'output': 'sleeping...\ntest123\n'}) # check asynchronous running of failing command - error_test_cmd = "sleep 2; echo 'FAIL!' >&2; exit 123" + error_test_cmd = "echo 'FAIL!' >&2; exit 123" cmd_info = run_cmd(error_test_cmd, asynchronous=True) error_pattern = 'cmd ".*" exited with exit code 123' self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info) From 78552da47d0a3044de08c361b8360f96335b0c85 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 22 Oct 2021 11:57:49 +0200 Subject: [PATCH 648/864] add test for insecure downloading with filetools.download_file --- test/framework/filetools.py | 79 ++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index dd46966071..0425f10ac1 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -47,7 +47,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import IGNORE, ERROR, update_build_option from easybuild.tools.multidiff import multidiff -from easybuild.tools.py2vs3 import std_urllib +from easybuild.tools.py2vs3 import StringIO, std_urllib class FileToolsTest(EnhancedTestCase): @@ -66,12 +66,18 @@ def setUp(self): super(FileToolsTest, self).setUp() self.orig_filetools_std_urllib_urlopen = ft.std_urllib.urlopen + if ft.HAVE_REQUESTS: + self.orig_filetools_requests_get = ft.requests.get + self.orig_filetools_HAVE_REQUESTS = ft.HAVE_REQUESTS def tearDown(self): """Cleanup.""" super(FileToolsTest, self).tearDown() ft.std_urllib.urlopen = self.orig_filetools_std_urllib_urlopen + ft.HAVE_REQUESTS = self.orig_filetools_HAVE_REQUESTS + if ft.HAVE_REQUESTS: + ft.requests.get = self.orig_filetools_requests_get def test_extract_cmd(self): """Test various extract commands.""" @@ -508,7 +514,6 @@ def fake_urllib_open(*args, **kwargs): # replaceurlopen with function that raises HTTP error 403 def fake_urllib_open(*args, **kwargs): - from easybuild.tools.py2vs3 import StringIO raise ft.std_urllib.HTTPError(url, 403, "Forbidden", "", StringIO()) ft.std_urllib.urlopen = fake_urllib_open @@ -523,6 +528,76 @@ def fake_urllib_open(*args, **kwargs): ft.HAVE_REQUESTS = False self.assertErrorRegex(EasyBuildError, "SSL issues with urllib2", ft.download_file, fn, url, target) + def test_download_file_insecure(self): + """ + Test downloading of file via insecure URL + """ + # replace urlopen with function that raises IOError + def fake_urllib_open(url, *args, **kwargs): + if kwargs.get('context') is None: + error_msg = " Date: Fri, 22 Oct 2021 12:05:16 +0200 Subject: [PATCH 649/864] pick up insecure_download build option directly in download_file --- easybuild/framework/easyblock.py | 19 ++++++------------- easybuild/tools/filetools.py | 4 +++- test/framework/filetools.py | 12 +++++++++--- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index dd24b1e431..2ec0ec7a7a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -388,10 +388,8 @@ def fetch_source(self, source, checksum=None, extension=False): # check if the sources can be located force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES] - insecure_download = build_option('insecure_download') path = self.obtain_file(filename, extension=extension, download_filename=download_filename, - force_download=force_download, insecure_download=insecure_download, - urls=source_urls, git_config=git_config) + force_download=force_download, urls=source_urls, git_config=git_config) if path is None: raise EasyBuildError('No file found for source %s', filename) @@ -454,9 +452,7 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None): patch_info = create_patch_info(patch_spec) force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES] - insecure_download = build_option('insecure_download') - path = self.obtain_file(patch_info['name'], extension=extension, force_download=force_download, - insecure_download=insecure_download) + path = self.obtain_file(patch_info['name'], extension=extension, force_download=force_download) if path: self.log.debug('File %s found for patch %s', path, patch_spec) patch_info['path'] = path @@ -501,7 +497,6 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): self.dry_run_msg("\nList of sources/patches for extensions:") force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES] - insecure_download = build_option('insecure_download') for ext in exts_list: if isinstance(ext, (list, tuple)) and ext: @@ -594,8 +589,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): if fetch_files: src_path = self.obtain_file(src_fn, extension=True, urls=source_urls, - force_download=force_download, - insecure_download=insecure_download) + force_download=force_download) if src_path: ext_src.update({'src': src_path}) else: @@ -669,7 +663,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): return exts_sources def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False, - insecure_download=False, git_config=None): + git_config=None): """ Locate the file with the given name - searches in different subdirectories of source path @@ -679,7 +673,6 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No :param urls: list of source URLs where this file may be available :param download_filename: filename with which the file should be downloaded, and then renamed to :param force_download: always try to download file, even if it's already available in source path - :param insecure_download: don't check the server certificate against the available certificate authorities :param git_config: dictionary to define how to download a git repository """ srcpaths = source_paths() @@ -711,7 +704,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No self.log.info("Found file %s at %s, no need to download it", filename, filepath) return fullpath - if download_file(filename, url, fullpath, insecure=insecure_download): + if download_file(filename, url, fullpath): return fullpath except IOError as err: @@ -838,7 +831,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No self.log.debug("Trying to download file %s from %s to %s ..." % (filename, fullurl, targetpath)) downloaded = False try: - if download_file(filename, fullurl, targetpath, insecure=insecure_download): + if download_file(filename, fullurl, targetpath): downloaded = True except IOError as err: diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 7c5439e6c1..b313928df1 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -738,9 +738,11 @@ def det_file_size(http_header): return res -def download_file(filename, url, path, forced=False, insecure=False): +def download_file(filename, url, path, forced=False): """Download a file from the given URL, to the specified path.""" + insecure = build_option('insecure_download') + _log.debug("Trying to download %s from %s to %s", filename, url, path) timeout = build_option('download_timeout') diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 0425f10ac1..1438a5482a 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -45,7 +45,7 @@ from easybuild.tools import run import easybuild.tools.filetools as ft from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import IGNORE, ERROR, update_build_option +from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import StringIO, std_urllib @@ -532,6 +532,9 @@ def test_download_file_insecure(self): """ Test downloading of file via insecure URL """ + + self.assertFalse(build_option('insecure_download')) + # replace urlopen with function that raises IOError def fake_urllib_open(url, *args, **kwargs): if kwargs.get('context') is None: @@ -554,8 +557,9 @@ def fake_urllib_open(url, *args, **kwargs): res = ft.download_file(fn, url, target_path) self.assertEqual(res, None) + update_build_option('insecure_download', True) self.mock_stderr(True) - res = ft.download_file(fn, url, target_path, insecure=True) + res = ft.download_file(fn, url, target_path) stderr = self.get_stderr() self.mock_stderr(False) @@ -586,11 +590,13 @@ def fake_requests_get(url, *args, **kwargs): ft.requests.get = fake_requests_get + update_build_option('insecure_download', False) res = ft.download_file(fn, url, target_path) self.assertEqual(res, None) + update_build_option('insecure_download', True) self.mock_stderr(True) - res = ft.download_file(fn, url, target_path, insecure=True) + res = ft.download_file(fn, url, target_path) stderr = self.get_stderr() self.mock_stderr(False) From 85273d039ccb8926d95102045321efd7005e111e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 22 Oct 2021 14:10:18 +0200 Subject: [PATCH 650/864] use check_async_cmd in Extension.async_cmd_check --- easybuild/framework/extension.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index eda2393fa0..024261c332 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -40,7 +40,7 @@ from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.tools.build_log import EasyBuildError, raise_nosupport from easybuild.tools.filetools import change_dir -from easybuild.tools.run import complete_cmd, get_output_from_process, run_cmd +from easybuild.tools.run import check_async_cmd, complete_cmd, run_cmd from easybuild.tools.py2vs3 import string_type @@ -196,23 +196,21 @@ def async_cmd_check(self): raise EasyBuildError("No installation command running asynchronously for %s", self.name) else: self.log.debug("Checking on installation of extension %s...", self.name) - proc = self.async_cmd_info[0] # use small read size, to avoid waiting for a long time until sufficient output is produced - self.async_cmd_output += get_output_from_process(proc, read_size=self.async_cmd_read_size) - ec = proc.poll() - if ec is None: - res = False + res = check_async_cmd(*self.async_cmd_info, output_read_size=self.async_cmd_read_size) + self.async_cmd_output += res['output'] + if res['done']: + self.log.info("Installation of extension %s completed!", self.name) + else: self.async_cmd_check_cnt += 1 + self.log.debug("Installation of extension %s still running (checked %d times)", + self.name, self.async_cmd_check_cnt) # increase read size after sufficient checks, # to avoid that installation hangs due to output buffer filling up... if self.async_cmd_check_cnt % 10 == 0 and self.async_cmd_read_size < (1024 ** 2): self.async_cmd_read_size *= 2 - else: - self.log.debug("Completing installation of extension %s...", self.name) - self.async_cmd_output, _ = complete_cmd(*self.async_cmd_info, output=self.async_cmd_output) - res = True - return res + return res['done'] @property def required_deps(self): From b2938dde42c2848df277fc8b2db401ff80f68dfe Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 22 Oct 2021 14:27:29 +0200 Subject: [PATCH 651/864] remove import for unused complete_cmd from framework/extension.py --- easybuild/framework/extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 024261c332..f78d1c63e6 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -40,7 +40,7 @@ from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.tools.build_log import EasyBuildError, raise_nosupport from easybuild.tools.filetools import change_dir -from easybuild.tools.run import check_async_cmd, complete_cmd, run_cmd +from easybuild.tools.run import check_async_cmd, run_cmd from easybuild.tools.py2vs3 import string_type From b15a880e8eb01df4f41fb2b27e52893c07a987d4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 22 Oct 2021 17:33:46 +0200 Subject: [PATCH 652/864] fix typo in comment in filetools test Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com> --- test/framework/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 1438a5482a..e511f31eb3 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -575,7 +575,7 @@ def fake_urllib_open(url, *args, **kwargs): fn = os.path.basename(url) target_path = os.path.join(self.test_prefix, fn) - # replaceurlopen with function that raises HTTP error 403 + # replace urlopen with function that raises HTTP error 403 def fake_urllib_open(url, *args, **kwargs): raise ft.std_urllib.HTTPError(url, 403, "Forbidden", "", StringIO()) From 343f4f009a382913b19cdf295aac58a0c8ef2001 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 22 Oct 2021 17:34:17 +0200 Subject: [PATCH 653/864] keep order of options alphabetically sorted by moving down insecure-download --- easybuild/tools/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 4c2f7091d9..5ef43c952c 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -407,8 +407,6 @@ def override_options(self): 'force-download': ("Force re-downloading of sources and/or patches, " "even if they are available already in source path", 'choice', 'store_or_None', DEFAULT_FORCE_DOWNLOAD, FORCE_DOWNLOAD_CHOICES), - 'insecure-download': ("Don't check the server certificate against the available certificate authorities.", - None, 'store_true', False), 'generate-devel-module': ("Generate a develop module file, implies --force if disabled", None, 'store_true', True), 'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None), @@ -428,6 +426,8 @@ def override_options(self): 'ignore-checksums': ("Ignore failing checksum verification", None, 'store_true', False), 'ignore-test-failure': ("Ignore a failing test step", None, 'store_true', False), 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), + 'insecure-download': ("Don't check the server certificate against the available certificate authorities.", + None, 'store_true', False), 'install-latest-eb-release': ("Install latest known version of easybuild", None, 'store_true', False), 'lib-lib64-symlink': ("Automatically create symlinks for lib/ pointing to lib64/ if the former is missing", None, 'store_true', True), From 447c1da420e7ae24e73adef41cfc6e74f05c3ce1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 25 Oct 2021 09:53:07 +0200 Subject: [PATCH 654/864] fix occasional failure in test_run_cmd_async --- test/framework/run.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/framework/run.py b/test/framework/run.py index a6a88b638c..24128aad6b 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -616,6 +616,9 @@ def test_run_cmd_async(self): cmd_info = run_cmd(error_test_cmd, asynchronous=True) res = check_async_cmd(*cmd_info, fail_on_error=False) + # keep checking until command is fully done + while not res['done']: + res = check_async_cmd(*cmd_info, fail_on_error=False) self.assertEqual(res, {'done': True, 'exit_code': 123, 'output': "FAIL!\n"}) # also test with a command that produces a lot of output, From d82ade9348cce77368ea1baca5a37de9a33a6b29 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 25 Oct 2021 09:53:26 +0200 Subject: [PATCH 655/864] check early for opt-in to using experimental feature when --parallel-extensions-install is used --- easybuild/tools/options.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 99ed51721b..2142a50f4a 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -892,6 +892,10 @@ def postprocess(self): # set tmpdir self.tmpdir = set_tmpdir(self.options.tmpdir) + # early check for opt-in to installing extensions in parallel (experimental feature) + if self.options.parallel_extensions_install: + self.log.experimental("installing extensions in parallel") + # take --include options into account (unless instructed otherwise) if self.with_include: self._postprocess_include() From 42c0bb3ad4fac2acb2d1ae6fc8601c4fe3ffa802 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 25 Oct 2021 09:57:07 +0200 Subject: [PATCH 656/864] tweak extensions progress bar label to also show 'X/Y done' when installing extensions in parallel --- easybuild/framework/easyblock.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e4b1254d0e..2f31ea464b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1743,14 +1743,16 @@ def update_exts_progress_bar_helper(running_exts, progress_size): """Helper function to update extensions progress bar.""" running_exts_cnt = len(running_exts) if running_exts_cnt > 1: - progress_info = "Installing %d extensions: " % running_exts_cnt + progress_info = "Installing %d extensions" % running_exts_cnt elif running_exts_cnt == 1: progress_info = "Installing extension " else: progress_info = "Not installing extensions (yet)" - progress_info += ', '.join(e.name for e in running_exts) - progress_info += " (%d/%d done)" % (len(installed_ext_names), exts_cnt) + if running_exts_cnt: + progress_info += " (%d/%d done): " % (len(installed_ext_names), exts_cnt) + progress_info += ', '.join(e.name for e in running_exts) + self.update_exts_progress_bar(progress_info, progress_size=progress_size) iter_id = 0 From 283404b268bb360b138a581182b3f877a4221467 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Tue, 26 Oct 2021 10:48:34 +0100 Subject: [PATCH 657/864] remove '--depth 1' from git clone --- easybuild/tools/filetools.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index b313928df1..062d47c737 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2598,11 +2598,6 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # compose 'git clone' command, and run it clone_cmd = ['git', 'clone'] - if not keep_git_dir: - # Speed up cloning by only fetching the most recent commit, not the whole history - # When we don't want to keep the .git folder there won't be a difference in the result - clone_cmd.extend(['--depth', '1']) - if tag: clone_cmd.extend(['--branch', tag]) if recursive: From 0aae4d7561b5081254654c47399e281799bb26c6 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Tue, 26 Oct 2021 11:20:48 +0100 Subject: [PATCH 658/864] only disable when using a commit and tests --- easybuild/tools/filetools.py | 5 +++++ test/framework/filetools.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 062d47c737..148db6352a 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2598,6 +2598,11 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # compose 'git clone' command, and run it clone_cmd = ['git', 'clone'] + if not keep_git_dir and not commit: + # Speed up cloning by only fetching the most recent commit, not the whole history + # When we don't want to keep the .git folder there won't be a difference in the result + clone_cmd.extend(['--depth', '1']) + if tag: clone_cmd.extend(['--branch', tag]) if recursive: diff --git a/test/framework/filetools.py b/test/framework/filetools.py index e511f31eb3..1e61f997ef 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2750,7 +2750,7 @@ def run_check(): del git_config['tag'] git_config['commit'] = '8456f86' expected = '\n'.join([ - r' running command "git clone --depth 1 --no-checkout %(git_repo)s"', + r' running command "git clone --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "git checkout 8456f86 && git submodule update --init --recursive"', r" \(in testrepository\)", @@ -2761,7 +2761,7 @@ def run_check(): del git_config['recursive'] expected = '\n'.join([ - r' running command "git clone --depth 1 --no-checkout %(git_repo)s"', + r' running command "git clone --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "git checkout 8456f86"', r" \(in testrepository\)", From cb4074dc75d10511dbabf4c864cab88b08e45b36 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Oct 2021 13:57:43 +0200 Subject: [PATCH 659/864] enhance test_get_source_tarball_from_git to trigger fixed bug w.r.t. cloning depth --- test/framework/filetools.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index e511f31eb3..cfd56c38af 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2784,6 +2784,7 @@ def run_check(): test_file = os.path.join(target_dir, 'test.tar.gz') self.assertEqual(res, test_file) self.assertTrue(os.path.isfile(test_file)) + test_tar_gzs = [os.path.basename(test_file)] self.assertEqual(os.listdir(target_dir), ['test.tar.gz']) # Check that we indeed downloaded the right tag extracted_dir = tempfile.mkdtemp(prefix='extracted_dir') @@ -2805,12 +2806,21 @@ def run_check(): self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'this-is-a-tag.txt'))) del git_config['tag'] - git_config['commit'] = '8456f86' + git_config['commit'] = '90366ea' res = ft.get_source_tarball_from_git('test2.tar.gz', target_dir, git_config) test_file = os.path.join(target_dir, 'test2.tar.gz') self.assertEqual(res, test_file) self.assertTrue(os.path.isfile(test_file)) - self.assertEqual(sorted(os.listdir(target_dir)), ['test.tar.gz', 'test2.tar.gz']) + test_tar_gzs.append(os.path.basename(test_file)) + self.assertEqual(sorted(os.listdir(target_dir)), test_tar_gzs) + + git_config['keep_git_dir'] = True + res = ft.get_source_tarball_from_git('test3.tar.gz', target_dir, git_config) + test_file = os.path.join(target_dir, 'test3.tar.gz') + self.assertEqual(res, test_file) + self.assertTrue(os.path.isfile(test_file)) + test_tar_gzs.append(os.path.basename(test_file)) + self.assertEqual(sorted(os.listdir(target_dir)), test_tar_gzs) except EasyBuildError as err: if "Network is down" in str(err): From c5d598f19cc9824ea6b945605132eb12f992fe22 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Oct 2021 14:20:28 +0200 Subject: [PATCH 660/864] inject short sleep before checking status of failing asynchronous command --- test/framework/run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/run.py b/test/framework/run.py index 24128aad6b..da4448d741 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -611,6 +611,7 @@ def test_run_cmd_async(self): # check asynchronous running of failing command error_test_cmd = "echo 'FAIL!' >&2; exit 123" cmd_info = run_cmd(error_test_cmd, asynchronous=True) + time.sleep(1) error_pattern = 'cmd ".*" exited with exit code 123' self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info) From 73af4253f6d642ba8162ef1bf8e5a32d30e9ecd4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Oct 2021 14:58:40 +0200 Subject: [PATCH 661/864] drop 'parallel' argument for install_extensions, to avoid having to opt-in to support for installing extensions in parallel in various easyblocks --- easybuild/framework/easyblock.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 2f31ea464b..468eddf764 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1642,17 +1642,16 @@ def skip_extensions(self): self.ext_instances = res - def install_extensions(self, install=True, parallel=False): + def install_extensions(self, install=True): """ Install extensions. :param install: actually install extensions, don't just prepare environment for installing - :param parallel: install extensions in parallel """ self.log.debug("List of loaded modules: %s", self.modules_tool.list()) - if build_option('parallel_extensions_install') and parallel: + if build_option('parallel_extensions_install'): self.log.experimental("installing extensions in parallel") self.install_extensions_parallel(install=install) else: From 16e02eae6fce1e3d8878ac68efc135f83046c9f3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Oct 2021 15:25:16 +0200 Subject: [PATCH 662/864] add run_async method to install extension asynchronously --- easybuild/framework/easyblock.py | 2 +- easybuild/framework/extension.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 468eddf764..ff8a561226 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1831,7 +1831,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size): rpath_filter_dirs=self.rpath_filter_dirs) if install: ext.prerun() - ext.run(asynchronous=True) + ext.run_async() running_exts.append(ext) self.log.info("Started installation of extension %s in the background...", ext.name) update_exts_progress_bar_helper(running_exts, 0) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index f78d1c63e6..251eed6afe 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -168,10 +168,16 @@ def prerun(self): def run(self, *args, **kwargs): """ - Actual installation of a extension. + Actual installation of an extension. """ pass + def run_async(self, *args, **kwargs): + """ + Asynchronous installation of an extension. + """ + raise NotImplementedError + def postrun(self): """ Stuff to do after installing a extension. From 60c5d1537b3bfbb0f1b3a85676331df952e154f0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Oct 2021 20:07:35 +0200 Subject: [PATCH 663/864] move printing of progress info on installing extensions in parallel after every iteration, and only when not showing progress bars --- easybuild/framework/easyblock.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index ff8a561226..58f4303aa3 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -91,7 +91,7 @@ from easybuild.tools.modules import Lmod, curr_module_paths, invalidate_module_caches_for, get_software_root from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ALL, PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS -from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar +from easybuild.tools.output import show_progress_bars, start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.package.utilities import package from easybuild.tools.py2vs3 import extract_method_name, string_type from easybuild.tools.repository.repository import init_repository @@ -1775,16 +1775,6 @@ def update_exts_progress_bar_helper(running_exts, progress_size): else: self.log.debug("Installation of %s is still running...", ext.name) - # print progress info every now and then - if iter_id % 1 == 0: - msg = "%d out of %d extensions installed (%d queued, %d running: %s)" - installed_cnt, queued_cnt, running_cnt = len(installed_ext_names), len(exts_queue), len(running_exts) - if running_cnt <= 3: - running_ext_names = ', '.join(x.name for x in running_exts) - else: - running_ext_names = ', '.join(x.name for x in running_exts[:3]) + ", ..." - print_msg(msg % (installed_cnt, exts_cnt, queued_cnt, running_cnt, running_ext_names), log=self.log) - # try to start as many extension installations as we can, taking into account number of available cores, # but only consider first 100 extensions still in the queue max_iter = min(100, len(exts_queue)) @@ -1836,6 +1826,16 @@ def update_exts_progress_bar_helper(running_exts, progress_size): self.log.info("Started installation of extension %s in the background...", ext.name) update_exts_progress_bar_helper(running_exts, 0) + # print progress info after every iteration (unless that info is already shown via progress bar) + if not show_progress_bars(): + msg = "%d out of %d extensions installed (%d queued, %d running: %s)" + installed_cnt, queued_cnt, running_cnt = len(installed_ext_names), len(exts_queue), len(running_exts) + if running_cnt <= 3: + running_ext_names = ', '.join(x.name for x in running_exts) + else: + running_ext_names = ', '.join(x.name for x in running_exts[:3]) + ", ..." + print_msg(msg % (installed_cnt, exts_cnt, queued_cnt, running_cnt, running_ext_names), log=self.log) + # # MISCELLANEOUS UTILITY FUNCTIONS # From 3e186fb061c7e862e9966be31e959125b6a477a1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Oct 2021 20:09:48 +0200 Subject: [PATCH 664/864] return True in Extension.async_cmd_check if async_cmd_info is set to False, which indicates that no asynchronous command was started --- easybuild/framework/extension.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 251eed6afe..54ea39f544 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -200,6 +200,9 @@ def async_cmd_check(self): """ if self.async_cmd_info is None: raise EasyBuildError("No installation command running asynchronously for %s", self.name) + elif self.async_cmd_info is False: + self.log.info("No asynchronous command was started for extension %s", self.name) + return True else: self.log.debug("Checking on installation of extension %s...", self.name) # use small read size, to avoid waiting for a long time until sufficient output is produced @@ -207,6 +210,7 @@ def async_cmd_check(self): self.async_cmd_output += res['output'] if res['done']: self.log.info("Installation of extension %s completed!", self.name) + self.async_cmd_info = None else: self.async_cmd_check_cnt += 1 self.log.debug("Installation of extension %s still running (checked %d times)", From 0dd9061e65edbe7611e5b0d9a32aaf5637c942ac Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Oct 2021 20:27:04 +0200 Subject: [PATCH 665/864] add test for installing extensions in parallel --- .../easyblocks/generic/toy_extension.py | 54 ++++++++++++++++--- .../sandbox/easybuild/easyblocks/t/toy.py | 54 +++++++++++++++---- test/framework/toy_build.py | 39 ++++++++++++++ 3 files changed, 131 insertions(+), 16 deletions(-) diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index bbb792e7ee..603346efe0 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -30,7 +30,8 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock -from easybuild.easyblocks.toy import EB_toy +from easybuild.easyblocks.toy import EB_toy, compose_toy_build_cmd +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.run import run_cmd @@ -45,20 +46,59 @@ def extra_options(): } return ExtensionEasyBlock.extra_options(extra_vars=extra_vars) - def run(self): - """Build toy extension.""" + @property + def required_deps(self): + """Return list of required dependencies for this extension.""" + deps = { + 'bar': [], + 'barbar': ['bar'], + 'ls': [], + } + if self.name in deps: + return deps[self.name] + else: + raise EasyBuildError("Dependencies for %s are unknown!", self.name) + + def run(self, *args, **kwargs): + """ + Install toy extension. + """ if self.src: - super(Toy_Extension, self).run(unpack_src=True) - EB_toy.configure_step(self.master, name=self.name) EB_toy.build_step(self.master, name=self.name, buildopts=self.cfg['buildopts']) if self.cfg['toy_ext_param']: run_cmd(self.cfg['toy_ext_param']) - EB_toy.install_step(self.master, name=self.name) - return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper(), self.name) + def prerun(self): + """ + Prepare installation of toy extension. + """ + super(Toy_Extension, self).prerun() + + if self.src: + super(Toy_Extension, self).run(unpack_src=True) + EB_toy.configure_step(self.master, name=self.name) + + def run_async(self): + """ + Install toy extension asynchronously. + """ + if self.src: + cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) + self.async_cmd_start(cmd) + else: + self.async_cmd_info = False + + def postrun(self): + """ + Wrap up installation of toy extension. + """ + super(Toy_Extension, self).postrun() + + EB_toy.install_step(self.master, name=self.name) + def sanity_check_step(self, *args, **kwargs): """Custom sanity check for toy extensions.""" self.log.info("Loaded modules: %s", self.modules_tool.list()) diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index f9a9f7a8c5..bec0e7fe42 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -41,6 +41,19 @@ from easybuild.tools.run import run_cmd +def compose_toy_build_cmd(cfg, name, prebuildopts, buildopts): + """ + Compose command to build toy. + """ + + cmd = "%(prebuildopts)s gcc %(name)s.c -o %(name)s %(buildopts)s" % { + 'name': name, + 'prebuildopts': prebuildopts, + 'buildopts': buildopts, + } + return cmd + + class EB_toy(ExtensionEasyBlock): """Support for building/installing toy.""" @@ -92,17 +105,13 @@ def configure_step(self, name=None): def build_step(self, name=None, buildopts=None): """Build toy.""" - if buildopts is None: buildopts = self.cfg['buildopts'] - if name is None: name = self.name - run_cmd('%(prebuildopts)s gcc %(name)s.c -o %(name)s %(buildopts)s' % { - 'name': name, - 'prebuildopts': self.cfg['prebuildopts'], - 'buildopts': buildopts, - }) + + cmd = compose_toy_build_cmd(self.cfg, name, self.cfg['prebuildopts'], buildopts) + run_cmd(cmd) def install_step(self, name=None): """Install toy.""" @@ -118,11 +127,38 @@ def install_step(self, name=None): mkdir(libdir, parents=True) write_file(os.path.join(libdir, 'lib%s.a' % name), name.upper()) - def run(self): - """Install toy as extension.""" + @property + def required_deps(self): + """Return list of required dependencies for this extension.""" + if self.name == 'toy': + return ['bar', 'barbar'] + else: + raise EasyBuildError("Dependencies for %s are unknown!", self.name) + + def prerun(self): + """ + Prepare installation of toy as extension. + """ super(EB_toy, self).run(unpack_src=True) self.configure_step() + + def run(self): + """ + Install toy as extension. + """ self.build_step() + + def run_async(self): + """ + Asynchronous installation of toy as extension. + """ + cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) + self.async_cmd_start(cmd) + + def postrun(self): + """ + Wrap up installation of toy as extension. + """ self.install_step() def make_module_step(self, fake=False): diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 1a68de2dcb..5b75116ee1 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1777,6 +1777,45 @@ def test_module_only_extensions(self): self.eb_main([test_ec, '--module-only', '--force'], do_build=True, raise_error=True) self.assertTrue(os.path.exists(toy_mod)) + def test_toy_exts_parallel(self): + """ + Test parallel installation of extensions (--parallel-extensions-install) + """ + topdir = os.path.abspath(os.path.dirname(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_mod += '.lua' + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\n' + '\n'.join([ + "exts_list = [", + " ('ls'),", + " ('bar', '0.0'),", + " ('barbar', '0.0', {", + " 'start_dir': 'src',", + " }),", + " ('toy', '0.0'),", + "]", + "sanity_check_commands = ['barbar', 'toy']", + "sanity_check_paths = {'files': ['bin/barbar', 'bin/toy'], 'dirs': ['bin']}", + ]) + write_file(test_ec, test_ec_txt) + + args = ['--parallel-extensions-install', '--experimental', '--force'] + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + self.assertEqual(stderr, '') + expected_stdout = '\n'.join([ + "== 0 out of 4 extensions installed (2 queued, 2 running: ls, bar)", + "== 2 out of 4 extensions installed (1 queued, 1 running: barbar)", + "== 3 out of 4 extensions installed (0 queued, 1 running: toy)", + "== 4 out of 4 extensions installed (0 queued, 0 running: )", + '', + ]) + self.assertEqual(stdout, expected_stdout) + def test_backup_modules(self): """Test use of backing up of modules with --module-only.""" From 0c7a7dbd8b1b4a5f576d03d008e8a8feffe8bf05 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Oct 2021 22:51:26 +0200 Subject: [PATCH 666/864] tweak error when Extension.required_deps is not implemented --- easybuild/framework/extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 54ea39f544..5333627c29 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -225,7 +225,7 @@ def async_cmd_check(self): @property def required_deps(self): """Return list of required dependencies for this extension.""" - raise NotImplementedError("Don't know how to determine required dependencies for %s" % self.name) + raise NotImplementedError("Don't know how to determine required dependencies for extension '%s'" % self.name) @property def toolchain(self): From 3179ddbab7cc8dfa3c0a74444dfad01b5c8cb9ac Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Oct 2021 22:51:43 +0200 Subject: [PATCH 667/864] remove unused iter_id in EasyBlock.install_extensions_parallel --- easybuild/framework/easyblock.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 58f4303aa3..d41801c3e9 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1754,11 +1754,8 @@ def update_exts_progress_bar_helper(running_exts, progress_size): self.update_exts_progress_bar(progress_info, progress_size=progress_size) - iter_id = 0 while exts_queue or running_exts: - iter_id += 1 - # always go back to original work dir to avoid running stuff from a dir that no longer exists change_dir(self.orig_workdir) From be818bf832f4d18854b44d9eeab2f9018a7d6d83 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Oct 2021 08:47:13 +0200 Subject: [PATCH 668/864] fix typo in warning message on suppressing duplicate paths in append_paths/prepend_paths in ModuleGenerator (+ add test to check for warning message) --- easybuild/tools/module_generator.py | 2 +- test/framework/module_generator.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 59444b273e..ef34ca62d3 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -226,7 +226,7 @@ def _filter_paths(self, key, paths): filtered_paths = [x for x in paths if x not in added_paths and not added_paths.add(x)] if filtered_paths != paths: removed_paths = paths if filtered_paths is None else [x for x in paths if x not in filtered_paths] - print_warning("Supressed adding the following path(s) to $%s of the module as they were already added: %s", + print_warning("Suppressed adding the following path(s) to $%s of the module as they were already added: %s", key, removed_paths, log=self.log) if not filtered_paths: diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index c046180883..998fbddd70 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -719,6 +719,17 @@ def append_paths(*args, **kwargs): "which only expects relative paths." % self.modgen.app.installdir, append_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir]) + # check for warning that is printed when same path is added multiple times + with self.modgen.start_module_creation(): + self.modgen.append_paths('TEST', 'path1') + self.mock_stderr(True) + self.modgen.append_paths('TEST', 'path1') + stderr = self.get_stderr() + self.mock_stderr(False) + expected_warning = "\nWARNING: Suppressed adding the following path(s) to $TEST of the module " + expected_warning += "as they were already added: path1\n\n" + self.assertEqual(stderr, expected_warning) + def test_module_extensions(self): """test the extensions() for extensions""" # not supported for Tcl modules @@ -798,6 +809,17 @@ def prepend_paths(*args, **kwargs): "which only expects relative paths." % self.modgen.app.installdir, prepend_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir]) + # check for warning that is printed when same path is added multiple times + with self.modgen.start_module_creation(): + self.modgen.prepend_paths('TEST', 'path1') + self.mock_stderr(True) + self.modgen.prepend_paths('TEST', 'path1') + stderr = self.get_stderr() + self.mock_stderr(False) + expected_warning = "\nWARNING: Suppressed adding the following path(s) to $TEST of the module " + expected_warning += "as they were already added: path1\n\n" + self.assertEqual(stderr, expected_warning) + def test_det_user_modpath(self): """Test for generic det_user_modpath method.""" # None by default From 36f583ab25a165467d4d0dd00133b536ad549dcb Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 27 Oct 2021 12:09:24 +0200 Subject: [PATCH 669/864] Make sure correct include directory is used for FlexiBLAS --- easybuild/toolchains/linalg/flexiblas.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py index 26250570b1..c77476218f 100644 --- a/easybuild/toolchains/linalg/flexiblas.py +++ b/easybuild/toolchains/linalg/flexiblas.py @@ -27,6 +27,7 @@ :author: Kenneth Hoste (Ghent University) """ +import os import re from easybuild.tools.toolchain.linalg import LinAlg @@ -67,10 +68,12 @@ class FlexiBLAS(LinAlg): """ BLAS_MODULE_NAME = ['FlexiBLAS'] BLAS_LIB = ['flexiblas'] + BLAS_INCLUDE_DIR = [os.path.join('include', 'flexiblas')] BLAS_FAMILY = TC_CONSTANT_FLEXIBLAS LAPACK_MODULE_NAME = ['FlexiBLAS'] LAPACK_IS_BLAS = True + LAPACK_INCLUDE_DIR = [os.path.join('include', 'flexiblas')] LAPACK_FAMILY = TC_CONSTANT_FLEXIBLAS def banned_linked_shared_libs(self): From cd8951dfb99d816afb69d361bf55e76a834c57c4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Oct 2021 13:55:45 +0200 Subject: [PATCH 670/864] deprecate old toolchain versions: GCC(core) < 8.0, iccifort < 2019.0, gompi/iimpi/iompi/foss/intel/iomkl < 2019 --- easybuild/toolchains/foss.py | 8 +++----- easybuild/toolchains/gcc.py | 13 +++++++++++++ easybuild/toolchains/gcccore.py | 13 +++++++++++++ easybuild/toolchains/gompi.py | 12 +++--------- easybuild/toolchains/iccifort.py | 7 ++++--- easybuild/toolchains/iimpi.py | 11 ++++------- easybuild/toolchains/intel.py | 8 +++----- easybuild/toolchains/iomkl.py | 18 ++++++++++++++++++ easybuild/toolchains/iompi.py | 16 ++++++++++++++++ 9 files changed, 77 insertions(+), 29 deletions(-) diff --git a/easybuild/toolchains/foss.py b/easybuild/toolchains/foss.py index e8e093ae09..d4bfbb671b 100644 --- a/easybuild/toolchains/foss.py +++ b/easybuild/toolchains/foss.py @@ -46,7 +46,7 @@ def __init__(self, *args, **kwargs): """Toolchain constructor.""" super(Foss, self).__init__(*args, **kwargs) - # need to transform a version like '2016a' with something that is safe to compare with '2000' + # need to transform a version like '2018b' with something that is safe to compare with '2019' # comparing subversions that include letters causes TypeErrors in Python 3 # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) version = self.version.replace('a', '.01').replace('b', '.07') @@ -84,10 +84,8 @@ def banned_linked_shared_libs(self): def is_deprecated(self): """Return whether or not this toolchain is deprecated.""" - # foss toolchains older than foss/2016a are deprecated - # take into account that foss/2016.x is always < foss/2016a according to LooseVersion; - # foss/2016.01 & co are not deprecated yet... - if self.looseversion < LooseVersion('2016.01'): + # foss toolchains older than foss/2019a are deprecated since EasyBuild v4.5.0; + if self.looseversion < LooseVersion('2019'): deprecated = True else: deprecated = False diff --git a/easybuild/toolchains/gcc.py b/easybuild/toolchains/gcc.py index 30f3891f58..ab9d58774b 100644 --- a/easybuild/toolchains/gcc.py +++ b/easybuild/toolchains/gcc.py @@ -27,6 +27,8 @@ :author: Kenneth Hoste (Ghent University) """ +from distutils.version import LooseVersion +import re from easybuild.toolchains.gcccore import GCCcore from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME @@ -38,3 +40,14 @@ class GccToolchain(GCCcore): COMPILER_MODULE_NAME = [NAME] SUBTOOLCHAIN = [GCCcore.NAME, SYSTEM_TOOLCHAIN_NAME] OPTIONAL = False + + def is_deprecated(self): + """Return whether or not this toolchain is deprecated.""" + # GCC toolchains older than GCC version 8.x are deprecated since EasyBuild v4.5.0 + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', self.version) and LooseVersion(self.version) < LooseVersion('8.0'): + deprecated = True + else: + deprecated = False + + return deprecated diff --git a/easybuild/toolchains/gcccore.py b/easybuild/toolchains/gcccore.py index a95f0dcbdb..49a190ca28 100644 --- a/easybuild/toolchains/gcccore.py +++ b/easybuild/toolchains/gcccore.py @@ -27,6 +27,8 @@ :author: Kenneth Hoste (Ghent University) """ +from distutils.version import LooseVersion +import re from easybuild.toolchains.compiler.gcc import Gcc from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME @@ -41,3 +43,14 @@ class GCCcore(Gcc): # GCCcore is only guaranteed to be present in recent toolchains # for old versions of some toolchains (GCC, intel) it is not part of the hierarchy and hence optional OPTIONAL = True + + def is_deprecated(self): + """Return whether or not this toolchain is deprecated.""" + # GCC toolchains older than GCC version 8.x are deprecated since EasyBuild v4.5.0 + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', self.version) and LooseVersion(self.version) < LooseVersion('8.0'): + deprecated = True + else: + deprecated = False + + return deprecated diff --git a/easybuild/toolchains/gompi.py b/easybuild/toolchains/gompi.py index a5bbc4c7b9..f9ed1ab1bc 100644 --- a/easybuild/toolchains/gompi.py +++ b/easybuild/toolchains/gompi.py @@ -41,7 +41,7 @@ class Gompi(GccToolchain, OpenMPI): def is_deprecated(self): """Return whether or not this toolchain is deprecated.""" - # need to transform a version like '2016a' with something that is safe to compare with '2000' + # need to transform a version like '2018b' with something that is safe to compare with '2019' # comparing subversions that include letters causes TypeErrors in Python 3 # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) version = self.version.replace('a', '.01').replace('b', '.07') @@ -50,14 +50,8 @@ def is_deprecated(self): # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion if re.match('^[0-9]', version): - gompi_ver = LooseVersion(version) - # deprecate oldest gompi toolchains (versions 1.x) - if gompi_ver < LooseVersion('2000'): - deprecated = True - # gompi toolchains older than gompi/2016a are deprecated - # take into account that gompi/2016.x is always < gompi/2016a according to LooseVersion; - # gompi/2016.01 & co are not deprecated yet... - elif gompi_ver < LooseVersion('2016.01'): + # gompi toolchains older than gompi/2019a are deprecated since EasyBuild v4.5.0 + if LooseVersion(version) < LooseVersion('2019'): deprecated = True return deprecated diff --git a/easybuild/toolchains/iccifort.py b/easybuild/toolchains/iccifort.py index ee6f39846c..cb44c0408b 100644 --- a/easybuild/toolchains/iccifort.py +++ b/easybuild/toolchains/iccifort.py @@ -50,14 +50,15 @@ class IccIfort(IntelIccIfort): def is_deprecated(self): """Return whether or not this toolchain is deprecated.""" - # need to transform a version like '2016a' with something that is safe to compare with '2016.01' + # need to transform a version like '2018b' with something that is safe to compare with '2019.0' # comparing subversions that include letters causes TypeErrors in Python 3 # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) version = self.version.replace('a', '.01').replace('b', '.07') - # iccifort toolchains older than iccifort/2016.1.150-* are deprecated + # iccifort toolchains older than iccifort/2019.0.117-* are deprecated; + # note: intel/2019a uses iccifort 2019.1.144; # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion - if re.match('^[0-9]', version) and LooseVersion(version) < LooseVersion('2016.1'): + if re.match('^[0-9]', version) and LooseVersion(version) < LooseVersion('2019.0'): deprecated = True else: deprecated = False diff --git a/easybuild/toolchains/iimpi.py b/easybuild/toolchains/iimpi.py index 0a2104872b..f89d17cc3a 100644 --- a/easybuild/toolchains/iimpi.py +++ b/easybuild/toolchains/iimpi.py @@ -53,11 +53,12 @@ def __init__(self, *args, **kwargs): # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion if re.match('^[0-9]', self.version): - # need to transform a version like '2016a' with something that is safe to compare with '8.0', '2016.01' + # need to transform a version like '2018b' with something that is safe to compare with '2019' # comparing subversions that include letters causes TypeErrors in Python 3 # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) # (good enough for this purpose) self.iimpi_ver = self.version.replace('a', '.01').replace('b', '.07') + if LooseVersion(self.iimpi_ver) >= LooseVersion('2020.12'): self.oneapi_gen = True self.SUBTOOLCHAIN = IntelCompilersToolchain.NAME @@ -77,12 +78,8 @@ def is_deprecated(self): # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion if re.match('^[0-9]', str(self.iimpi_ver)): - loosever = LooseVersion(self.iimpi_ver) - # iimpi toolchains older than iimpi/2016.01 are deprecated - # iimpi 8.1.5 is an exception, since it used in intel/2016a (which is not deprecated yet) - if loosever < LooseVersion('8.0'): - deprecated = True - elif loosever > LooseVersion('2000') and loosever < LooseVersion('2016.01'): + # iimpi toolchains older than iimpi/2019a are deprecated since EasyBuild v4.5.0 + if LooseVersion(self.iimpi_ver) < LooseVersion('2019'): deprecated = True return deprecated diff --git a/easybuild/toolchains/intel.py b/easybuild/toolchains/intel.py index bb0550baee..3639f7bca1 100644 --- a/easybuild/toolchains/intel.py +++ b/easybuild/toolchains/intel.py @@ -48,16 +48,14 @@ class Intel(Iimpi, IntelMKL, IntelFFTW): def is_deprecated(self): """Return whether or not this toolchain is deprecated.""" - # need to transform a version like '2016a' with something that is safe to compare with '2016.01' + # need to transform a version like '2018b' with something that is safe to compare with '2019' # comparing subversions that include letters causes TypeErrors in Python 3 # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) version = self.version.replace('a', '.01').replace('b', '.07') - # intel toolchains older than intel/2016a are deprecated - # take into account that intel/2016.x is always < intel/2016a according to LooseVersion; - # intel/2016.01 & co are not deprecated yet... + # intel toolchains older than intel/2019a are deprecated since EasyBuild v4.5.0 # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion - if re.match('^[0-9]', version) and LooseVersion(version) < LooseVersion('2016.01'): + if re.match('^[0-9]', version) and LooseVersion(version) < LooseVersion('2019'): deprecated = True else: deprecated = False diff --git a/easybuild/toolchains/iomkl.py b/easybuild/toolchains/iomkl.py index bc68ae7b24..6c0b0e01a4 100644 --- a/easybuild/toolchains/iomkl.py +++ b/easybuild/toolchains/iomkl.py @@ -29,6 +29,8 @@ :author: Stijn De Weirdt (Ghent University) :author: Kenneth Hoste (Ghent University) """ +from distutils.version import LooseVersion +import re from easybuild.toolchains.iompi import Iompi from easybuild.toolchains.iimkl import Iimkl @@ -43,3 +45,19 @@ class Iomkl(Iompi, IntelMKL, IntelFFTW): """ NAME = 'iomkl' SUBTOOLCHAIN = [Iompi.NAME, Iimkl.NAME] + + def is_deprecated(self): + """Return whether or not this toolchain is deprecated.""" + # need to transform a version like '2018b' with something that is safe to compare with '2019' + # comparing subversions that include letters causes TypeErrors in Python 3 + # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) + version = self.version.replace('a', '.01').replace('b', '.07') + + # iomkl toolchains older than iomkl/2019a are deprecated since EasyBuild v4.5.0 + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', version) and LooseVersion(version) < LooseVersion('2019'): + deprecated = True + else: + deprecated = False + + return deprecated diff --git a/easybuild/toolchains/iompi.py b/easybuild/toolchains/iompi.py index 05664cb484..e3c82cb907 100644 --- a/easybuild/toolchains/iompi.py +++ b/easybuild/toolchains/iompi.py @@ -92,3 +92,19 @@ def set_variables(self): IntelCompilersToolchain.set_variables(self) else: IccIfort.set_variables(self) + + def is_deprecated(self): + """Return whether or not this toolchain is deprecated.""" + # need to transform a version like '2018b' with something that is safe to compare with '2019' + # comparing subversions that include letters causes TypeErrors in Python 3 + # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) + version = self.version.replace('a', '.01').replace('b', '.07') + + # iompi toolchains older than iompi/2019a are deprecated since EasyBuild v4.5.0 + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', version) and LooseVersion(version) < LooseVersion('2019'): + deprecated = True + else: + deprecated = False + + return deprecated From dece32de5c13642423f13b19f9229acd6a363214 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Oct 2021 15:22:37 +0200 Subject: [PATCH 671/864] add --unit-testing-mode configuration option, to allow use of deprecated toolchain versions when running EasyBuild framework test suite --- easybuild/framework/easyconfig/easyconfig.py | 5 ++++- easybuild/tools/config.py | 1 + easybuild/tools/options.py | 1 + test/framework/utilities.py | 7 ++++++- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 0cbcb11ef3..3dc255e117 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -837,7 +837,10 @@ def check_deprecated(self, path): raise EasyBuildError("Wrong type for value of 'deprecated' easyconfig parameter: %s", type(deprecated)) if self.toolchain.is_deprecated(): - depr_msgs.append("toolchain '%(name)s/%(version)s' is marked as deprecated" % self['toolchain']) + # allow use of deprecated toolchains when running unit tests, + # because test easyconfigs/modules often use old toolchain versions (and updating them is far from trivial) + if not build_option('unit_testing_mode'): + depr_msgs.append("toolchain '%(name)s/%(version)s' is marked as deprecated" % self['toolchain']) if depr_msgs: depr_msg = ', '.join(depr_msgs) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 7b669764f4..a66f688215 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -285,6 +285,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'generate_devel_module', 'sticky_bit', 'trace', + 'unit_testing_mode', 'upload_test_report', 'update_modules_tool_cache', 'use_ccache', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 5ef43c952c..ea6fab61c7 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -489,6 +489,7 @@ def override_options(self): None, 'store', None), 'update-modules-tool-cache': ("Update modules tool cache file(s) after generating module file", None, 'store_true', False), + 'unit-testing-mode': ("Run in unit test mode", None, 'store_true', False), 'use-ccache': ("Enable use of ccache to speed up compilation, with specified cache dir", str, 'store', False, {'metavar': "PATH"}), 'use-f90cache': ("Enable use of f90cache to speed up compilation, with specified cache dir", diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 14248243d3..f105b7b81f 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -289,6 +289,10 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos """Helper method to call EasyBuild main function.""" cleanup() + # always run main in unit testing mode (which for example allows for using deprecated toolchains); + # note: don't change 'args' value, which is passed by reference! + main_args = args + ['--unit-testing-mode'] + myerr = False if logfile is None: logfile = self.logfile @@ -306,7 +310,7 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos modtool = None else: modtool = self.modtool - main(args=args, logfile=logfile, do_build=do_build, testing=testing, modtool=modtool) + main(args=main_args, logfile=logfile, do_build=do_build, testing=testing, modtool=modtool) except SystemExit as err: if raise_systemexit: raise err @@ -476,6 +480,7 @@ def init_config(args=None, build_options=None, with_include=True): 'local_var_naming_check': 'error', 'silence_deprecation_warnings': eb_go.options.silence_deprecation_warnings, 'suffix_modules_path': GENERAL_CLASS, + 'unit_testing_mode': True, 'valid_module_classes': module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } From 19bf460c0cc06a23e8ae4bb7061625aba201afc5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Oct 2021 19:42:51 +0200 Subject: [PATCH 672/864] restore resolving of non-broken symlinks in copy_file --- easybuild/tools/filetools.py | 12 ++++++++---- test/framework/filetools.py | 20 +++++++++++++++----- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 148db6352a..99546c329d 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2365,8 +2365,10 @@ def copy_file(path, target_path, force_in_dry_run=False): raise EasyBuildError("Could not copy '%s' it does not exist!", path) else: try: + # check whether path to copy exists (we could be copying a broken symlink, which is supported) + path_exists = os.path.exists(path) target_exists = os.path.exists(target_path) - if target_exists and os.path.samefile(path, target_path): + if target_exists and path_exists and os.path.samefile(path, target_path): _log.debug("Not copying %s to %s since files are identical", path, target_path) # if target file exists and is owned by someone else than the current user, # try using shutil.copyfile to just copy the file contents @@ -2376,7 +2378,10 @@ def copy_file(path, target_path, force_in_dry_run=False): _log.info("Copied contents of file %s to %s", path, target_path) else: mkdir(os.path.dirname(target_path), parents=True) - if os.path.islink(path): + if path_exists: + shutil.copy2(path, target_path) + _log.info("%s copied to %s", path, target_path) + elif os.path.islink(path): if os.path.isdir(target_path): target_path = os.path.join(target_path, os.path.basename(path)) _log.info("target_path changed to %s", target_path) @@ -2385,8 +2390,7 @@ def copy_file(path, target_path, force_in_dry_run=False): symlink(link_target, target_path, use_abspath_source=False) _log.info("created symlink %s to %s", link_target, target_path) else: - shutil.copy2(path, target_path) - _log.info("%s copied to %s", path, target_path) + raise EasyBuildError("Specified path %s is not an existing file or a symbolic link!", path) except (IOError, OSError, shutil.Error) as err: raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 4f22b69883..8c54f5881f 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1739,13 +1739,23 @@ def test_copy_file(self): # Make sure it doesn't fail if path is a symlink and target_path is a dir toy_link_fn = 'toy-link-0.0.eb' toy_link = os.path.join(self.test_prefix, toy_link_fn) - ft.symlink(toy_ec, toy_link) + ft.symlink(target_path, toy_link) dir_target_path = os.path.join(self.test_prefix, 'subdir') ft.mkdir(dir_target_path) ft.copy_file(toy_link, dir_target_path) - self.assertTrue(os.path.islink(os.path.join(dir_target_path, toy_link_fn))) + copied_file = os.path.join(dir_target_path, toy_link_fn) + # symlinks that point to an existing file are resolved on copy (symlink itself is not copied) + self.assertTrue(os.path.exists(copied_file), "%s should exist" % copied_file) + self.assertTrue(os.path.isfile(copied_file), "%s should be a file" % copied_file) + ft.remove_file(copied_file) + + # test copying of a broken symbolic link: copy_file should not fail, but copy it! + ft.remove_file(target_path) + ft.copy_file(toy_link, dir_target_path) + self.assertTrue(os.path.islink(copied_file), "%s should be a broken symbolic link" % copied_file) + self.assertFalse(os.path.exists(copied_file), "%s should be a broken symbolic link" % copied_file) self.assertEqual(os.readlink(os.path.join(dir_target_path, toy_link_fn)), os.readlink(toy_link)) - os.remove(os.path.join(dir_target_path, toy_link)) + ft.remove_file(copied_file) # clean error when trying to copy a directory with copy_file src, target = os.path.dirname(toy_ec), os.path.join(self.test_prefix, 'toy') @@ -1781,8 +1791,8 @@ def test_copy_file(self): } init_config(build_options=build_options) - # remove target file, it shouldn't get copied under dry run - os.remove(target_path) + # make sure target file is not there, it shouldn't get copied under dry run + self.assertFalse(os.path.exists(target_path)) self.mock_stdout(True) ft.copy_file(toy_ec, target_path) From bfc96cb8b50353c62881a194a966081bf68cd2a1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Oct 2021 21:43:22 +0200 Subject: [PATCH 673/864] explictly disable rebase when pulling develop branch to create a merge commit, since not specifying how to reconcile divergent branches is an error with Git 2.33.1 and newer (fixes #3873) --- easybuild/tools/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 5773d39a49..43fa6f07ae 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -2409,7 +2409,7 @@ def sync_with_develop(git_repo, branch_name, github_account, github_repo): remote = create_remote(git_repo, github_account, github_repo, https=True) # fetch latest version of develop branch - pull_out = git_repo.git.pull(remote.name, GITHUB_DEVELOP_BRANCH) + pull_out = git_repo.git.pull(remote.name, GITHUB_DEVELOP_BRANCH, no_rebase=True) _log.debug("Output of 'git pull %s %s': %s", remote.name, GITHUB_DEVELOP_BRANCH, pull_out) # fetch to make sure we can check out the 'develop' branch From bb1a51cf0c94b5cda640df29b132689e8c4a2212 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 Oct 2021 16:54:01 +0200 Subject: [PATCH 674/864] make check for running failing command asynchronously more robust w.r.t. obtaining full output --- test/framework/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/run.py b/test/framework/run.py index da4448d741..6c298890d5 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -619,7 +619,7 @@ def test_run_cmd_async(self): res = check_async_cmd(*cmd_info, fail_on_error=False) # keep checking until command is fully done while not res['done']: - res = check_async_cmd(*cmd_info, fail_on_error=False) + res = check_async_cmd(*cmd_info, fail_on_error=False, output=res['output']) self.assertEqual(res, {'done': True, 'exit_code': 123, 'output': "FAIL!\n"}) # also test with a command that produces a lot of output, From 8d1d8c6518b3303d18455f775d0a2dea1c916ffc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 Oct 2021 18:10:31 +0200 Subject: [PATCH 675/864] correct total count for extensions progress bar after skipping already installed extensions --- easybuild/framework/easyblock.py | 5 +++-- easybuild/tools/output.py | 4 +++- test/framework/output.py | 1 + 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index d41801c3e9..f298f4189c 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1641,6 +1641,7 @@ def skip_extensions(self): self.update_exts_progress_bar("skipping installed extensions (%d/%d checked)" % (idx + 1, exts_cnt)) self.ext_instances = res + self.update_exts_progress_bar("already installed extensions filtered out", total=len(self.ext_instances)) def install_extensions(self, install=True): """ @@ -2573,11 +2574,11 @@ def init_ext_instances(self): pbar_label += "(%d/%d done)" % (idx + 1, exts_cnt) self.update_exts_progress_bar(pbar_label) - def update_exts_progress_bar(self, info, progress_size=0): + def update_exts_progress_bar(self, info, progress_size=0, total=None): """ Update extensions progress bar with specified info and amount of progress made """ - update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=info, progress_size=progress_size) + update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=info, progress_size=progress_size, total=total) def extensions_step(self, fetch=False, install=True): """ diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 6882af3c6b..207a658569 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -286,7 +286,7 @@ def start_progress_bar(bar_type, size, label=None): pbar.update(task_id, description=label) -def update_progress_bar(bar_type, label=None, progress_size=1): +def update_progress_bar(bar_type, label=None, progress_size=1, total=None): """ Update progress bar of given type (if it was started), add progress of given size. @@ -300,6 +300,8 @@ def update_progress_bar(bar_type, label=None, progress_size=1): pbar.update(task_id, description=label) if progress_size: pbar.update(task_id, advance=progress_size) + if total: + pbar.update(task_id, total=total) def stop_progress_bar(bar_type, visible=False): diff --git a/test/framework/output.py b/test/framework/output.py index ea574d043a..fdcba7038f 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -178,6 +178,7 @@ def test_get_start_update_stop_progress_bar(self): # also test normal cycle: start, update, stop start_progress_bar(PROGRESS_BAR_EXTENSIONS, 100) update_progress_bar(PROGRESS_BAR_EXTENSIONS) # single step progress + update_progress_bar(PROGRESS_BAR_EXTENSIONS, total=50) update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="test123", progress_size=5) stop_progress_bar(PROGRESS_BAR_EXTENSIONS) From fe9b9b1d300d9e1ef4e5edc4fbdf2de37639e85e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 Oct 2021 19:17:49 +0200 Subject: [PATCH 676/864] don't try to exclude skipped steps from total step count (just always update easyconfig progress bar), self.skip_step only really works after check_readiness_step has been run... --- easybuild/framework/easyblock.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f298f4189c..608ee0ec5a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3745,8 +3745,7 @@ def run_all_steps(self, run_test_cases): # figure out how many steps will actually be run (not be skipped) step_cnt = 0 for (step_name, _, _, skippable) in steps: - if not self.skip_step(step_name, skippable): - step_cnt += 1 + step_cnt += 1 if self.cfg['stop'] == step_name: break @@ -3793,7 +3792,7 @@ def run_all_steps(self, run_test_cases): elif self.logdebug or build_option('trace'): print_msg("... (took < 1 sec)", log=self.log, silent=self.silent) - update_progress_bar(PROGRESS_BAR_EASYCONFIG) + update_progress_bar(PROGRESS_BAR_EASYCONFIG) except StopException: pass From 5db0645dbc96106639db06760bb8e73985fbca16 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 Oct 2021 19:59:21 +0200 Subject: [PATCH 677/864] tweak test_http_header_fields_urlpat to download from sources.easybuild.io rather than https://ftp.gnu.org --- test/framework/options.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 4e78999742..4823d16f1e 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2756,9 +2756,14 @@ def test_http_header_fields_urlpat(self): """Test use of --http-header-fields-urlpat.""" tmpdir = tempfile.mkdtemp() test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - ec_file = os.path.join(test_ecs_dir, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb') + gzip_ec = os.path.join(test_ecs_dir, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb') + gzip_ec_txt = read_file(gzip_ec) + regex = re.compile('^source_urls = .*', re.M) + test_ec_txt = regex.sub("source_urls = ['https://sources.easybuild.io/g/gzip']", gzip_ec_txt) + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) common_args = [ - ec_file, + test_ec, '--stop=fetch', '--debug', '--force', @@ -2800,7 +2805,7 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None): # A: simple direct case (all is logged because passed directly via EasyBuild configuration options) args = list(common_args) args.extend([ - '--http-header-fields-urlpat=gnu.org::%s:%s' % (testdohdr, testdoval), + '--http-header-fields-urlpat=easybuild.io::%s:%s' % (testdohdr, testdoval), '--http-header-fields-urlpat=nomatch.com::%s:%s' % (testdonthdr, testdontval), ]) # expect to find everything passed on cmdline @@ -2813,7 +2818,7 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None): # B: simple file case (secrets in file are not logged) txt = '\n'.join([ - 'gnu.org::%s: %s' % (testdohdr, testdoval), + 'easybuild.io::%s: %s' % (testdohdr, testdoval), 'nomatch.com::%s: %s' % (testdonthdr, testdontval), '', ]) @@ -2825,7 +2830,7 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None): # C: recursion one: header value is another file txt = '\n'.join([ - 'gnu.org::%s: %s' % (testdohdr, testincfile), + 'easybuild.io::%s: %s' % (testdohdr, testincfile), 'nomatch.com::%s: %s' % (testdonthdr, testexcfile), '', ]) @@ -2839,7 +2844,11 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None): run_and_assert(args, "case C", expected, not_expected) # D: recursion two: header field+value is another file, - write_file(testcmdfile, '\n'.join(['gnu.org::%s' % (testinchdrfile), 'nomatch.com::%s' % (testexchdrfile), ''])) + write_file(testcmdfile, '\n'.join([ + 'easybuild.io::%s' % (testinchdrfile), + 'nomatch.com::%s' % (testexchdrfile), + '', + ])) write_file(testinchdrfile, '%s: %s\n' % (testdohdr, testdoval)) write_file(testexchdrfile, '%s: %s\n' % (testdonthdr, testdontval)) # expect to find only the header key (and the literal filename) and only for the appropriate url @@ -2851,7 +2860,7 @@ def run_and_assert(args, msg, words_expected=None, words_unexpected=None): # E: recursion three: url pattern + header field + value in another file write_file(testcmdfile, '%s\n' % (testurlpatfile)) txt = '\n'.join([ - 'gnu.org::%s: %s' % (testdohdr, testdoval), + 'easybuild.io::%s: %s' % (testdohdr, testdoval), 'nomatch.com::%s: %s' % (testdonthdr, testdontval), '', ]) From 6e33e605a67a5a64fb50b1d430e42bd1dde5294e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 Oct 2021 20:01:47 +0200 Subject: [PATCH 678/864] drop unused 'skippable' local variable when determining step count in run_all_steps Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com> --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 608ee0ec5a..59bf0b7e6a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3744,7 +3744,7 @@ def run_all_steps(self, run_test_cases): # figure out how many steps will actually be run (not be skipped) step_cnt = 0 - for (step_name, _, _, skippable) in steps: + for (step_name, _, _, _) in steps: step_cnt += 1 if self.cfg['stop'] == step_name: break From ec3f5f6f111aae2c53bc23e8deca3eee86cf669f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 Oct 2021 21:26:29 +0200 Subject: [PATCH 679/864] also deprecate old versions of toolchains that include CUDA component + gimpi --- easybuild/toolchains/gcccuda.py | 19 +++++++++++++++++++ easybuild/toolchains/gimpi.py | 19 +++++++++++++++++++ easybuild/toolchains/iimpic.py | 19 +++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/easybuild/toolchains/gcccuda.py b/easybuild/toolchains/gcccuda.py index e7a1d5b8c8..2ac83412be 100644 --- a/easybuild/toolchains/gcccuda.py +++ b/easybuild/toolchains/gcccuda.py @@ -27,6 +27,8 @@ :author: Kenneth Hoste (Ghent University) """ +import re +from distutils.version import LooseVersion from easybuild.toolchains.compiler.cuda import Cuda from easybuild.toolchains.gcc import GccToolchain @@ -38,3 +40,20 @@ class GccCUDA(GccToolchain, Cuda): COMPILER_MODULE_NAME = ['GCC', 'CUDA'] SUBTOOLCHAIN = GccToolchain.NAME + + def is_deprecated(self): + """Return whether or not this toolchain is deprecated.""" + # need to transform a version like '2018b' with something that is safe to compare with '2019' + # comparing subversions that include letters causes TypeErrors in Python 3 + # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) + version = self.version.replace('a', '.01').replace('b', '.07') + + deprecated = False + + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', version): + # gompi toolchains older than gompi/2019a are deprecated since EasyBuild v4.5.0 + if LooseVersion(version) < LooseVersion('2019'): + deprecated = True + + return deprecated diff --git a/easybuild/toolchains/gimpi.py b/easybuild/toolchains/gimpi.py index ce93b4acd2..8c42e3f9cc 100644 --- a/easybuild/toolchains/gimpi.py +++ b/easybuild/toolchains/gimpi.py @@ -28,6 +28,8 @@ :author: Stijn De Weirdt (Ghent University) :author: Kenneth Hoste (Ghent University) """ +import re +from distutils.version import LooseVersion from easybuild.toolchains.gcc import GccToolchain from easybuild.toolchains.mpi.intelmpi import IntelMPI @@ -37,3 +39,20 @@ class Gimpi(GccToolchain, IntelMPI): """Compiler toolchain with GCC and Intel MPI.""" NAME = 'gimpi' SUBTOOLCHAIN = GccToolchain.NAME + + def is_deprecated(self): + """Return whether or not this toolchain is deprecated.""" + # need to transform a version like '2018b' with something that is safe to compare with '2019' + # comparing subversions that include letters causes TypeErrors in Python 3 + # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) + version = self.version.replace('a', '.01').replace('b', '.07') + + deprecated = False + + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', version): + # gompi toolchains older than gompi/2019a are deprecated since EasyBuild v4.5.0 + if LooseVersion(version) < LooseVersion('2019'): + deprecated = True + + return deprecated diff --git a/easybuild/toolchains/iimpic.py b/easybuild/toolchains/iimpic.py index 1940e20a30..c50bc7e0f7 100644 --- a/easybuild/toolchains/iimpic.py +++ b/easybuild/toolchains/iimpic.py @@ -27,6 +27,8 @@ :author: Ake Sandgren (HPC2N) """ +import re +from distutils.version import LooseVersion from easybuild.toolchains.iccifortcuda import IccIfortCUDA from easybuild.toolchains.mpi.intelmpi import IntelMPI @@ -36,3 +38,20 @@ class Iimpic(IccIfortCUDA, IntelMPI): """Compiler toolchain with Intel compilers (icc/ifort), Intel MPI and CUDA.""" NAME = 'iimpic' SUBTOOLCHAIN = IccIfortCUDA.NAME + + def is_deprecated(self): + """Return whether or not this toolchain is deprecated.""" + # need to transform a version like '2018b' with something that is safe to compare with '2019' + # comparing subversions that include letters causes TypeErrors in Python 3 + # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) + version = self.version.replace('a', '.01').replace('b', '.07') + + deprecated = False + + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', version): + # gompi toolchains older than gompi/2019a are deprecated since EasyBuild v4.5.0 + if LooseVersion(version) < LooseVersion('2019'): + deprecated = True + + return deprecated From 5ff67516dadd8fdb3c0c917ad4aa4cb59edea886 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 Oct 2021 21:38:14 +0200 Subject: [PATCH 680/864] fix typo in comments for gimpi and iimpic toolchains Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com> --- easybuild/toolchains/gimpi.py | 2 +- easybuild/toolchains/iimpic.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/toolchains/gimpi.py b/easybuild/toolchains/gimpi.py index 8c42e3f9cc..56555cc42f 100644 --- a/easybuild/toolchains/gimpi.py +++ b/easybuild/toolchains/gimpi.py @@ -51,7 +51,7 @@ def is_deprecated(self): # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion if re.match('^[0-9]', version): - # gompi toolchains older than gompi/2019a are deprecated since EasyBuild v4.5.0 + # gimpi toolchains older than gimpi/2019a are deprecated since EasyBuild v4.5.0 if LooseVersion(version) < LooseVersion('2019'): deprecated = True diff --git a/easybuild/toolchains/iimpic.py b/easybuild/toolchains/iimpic.py index c50bc7e0f7..0673ffb81b 100644 --- a/easybuild/toolchains/iimpic.py +++ b/easybuild/toolchains/iimpic.py @@ -50,7 +50,7 @@ def is_deprecated(self): # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion if re.match('^[0-9]', version): - # gompi toolchains older than gompi/2019a are deprecated since EasyBuild v4.5.0 + # iimpic toolchains older than iimpic/2019a are deprecated since EasyBuild v4.5.0 if LooseVersion(version) < LooseVersion('2019'): deprecated = True From 155f88d65f0c8bfedbf2d61d874a06f3cd82d4d1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 Oct 2021 21:38:58 +0200 Subject: [PATCH 681/864] fix typo in comments for gcccuda toolchain Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com> --- easybuild/toolchains/gcccuda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/gcccuda.py b/easybuild/toolchains/gcccuda.py index 2ac83412be..6d2fa190ed 100644 --- a/easybuild/toolchains/gcccuda.py +++ b/easybuild/toolchains/gcccuda.py @@ -52,7 +52,7 @@ def is_deprecated(self): # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion if re.match('^[0-9]', version): - # gompi toolchains older than gompi/2019a are deprecated since EasyBuild v4.5.0 + # gcccuda toolchains older than gcccuda/2019a are deprecated since EasyBuild v4.5.0 if LooseVersion(version) < LooseVersion('2019'): deprecated = True From 923ee946bd815f5640e60afee1355a734e02da42 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Fri, 29 Oct 2021 10:15:05 +0800 Subject: [PATCH 682/864] prepare release notes for EasyBuild v4.5.0 + bump version to 4.5.0 --- RELEASE_NOTES | 47 ++++++++++++++++++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index e622dd198b..eb1979fed9 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,53 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.5.0 (October 29th 2021) +-------------------------- + +feature release + +- various enhancements, including: + - add --review-pr-max and --review-pr-filter options to limit easyconfigs shown in multi-diff + retain order of easyconfigs being compared with (#3754) + - use Rich (if available) to show progress bar when installing easyconfigs (#3823) + - expand progress bar to full screen width (#3826) + - add --output-style configuration option, which can be used to disable use of Rich or type of any colored output (#3833) + - disable progress bars when running the tests to avoid messing up test suite output (#3835) + - use separate different progress bars for different aspects of the installations being performed (#3844) + - fixes & tweaks for overall & easyconfig progress bars (#3864) + - make update_progress_bar a bit more robust by just doing nothing if the corresponding progress bar was not started (and making stopping of a non-started progress bar fatal) (#3867) + - fix easyconfig + extensions progress bars when --skip is used (#3882) + - add support for checking required/optional EasyBuild dependencies via 'eb --check-eb-deps' (#3829) + - add support for collecting GPU info (via nvidia-smi), and include it in --show-system-info and test report (#3851) + - added support for --insecure-download configuration option (#3859) + - make intelfftw toolchain component aware of imkl-FFTW module (#3861) + - make sure the contrib/hooks tree is included in the distribution (#3862) + - add check_async_cmd function to facilitate checking on asynchronously running commands (#3865) + - add initial/experimental support for installing extensions in parallel (#3667) + - filter out duplicate paths added to module files (#3770) +- various bug fixes, including: + - ensure that path configuration options have absolute path values (#3832) + - fix broken test by taking into account changed error raised by Python 3.9.7 when copying directory via shutil.copyfile (#3840) + - ensure newer location of CUDA stubs is taken into account by RPATH filter (#3850) + - replace which by command -v to avoid dependency on which (#3852) + - fix copy_file so it doesn't fail when copying a symbolic link if the target path is an existing directory (#3855) + - refactor EasyBlock to decouple collecting of information on extension source/patch files from downloading them (#3860) + - fix library paths to add to $LDFLAGS for intel-compilers toolchain component (#3866) + - remove '--depth 1' from git clone when 'commit' specified (#3871) + - enhance test_get_source_tarball_from_git to trigger fixed bug w.r.t. cloning depth (#3872) + - fix typo in warning message on suppressing duplicate paths in append_paths/prepend_paths in ModuleGenerator (+ add test to check for warning message) (#3874) + - make sure correct include directory is used for FlexiBLAS (#3875) + - restore resolving of non-broken symlinks in copy_file (#3877) + - clarify error message when determining required dependencies for extension fails (#3878) + - explictly disable rebase when pulling develop branch to create a merge commit, since not specifying how to reconcile divergent branches is an error with Git 2.33.1 and newer (#3879) + - make check for running failing command asynchronously more robust w.r.t. obtaining full output (#3881) + - tweak test_http_header_fields_urlpat to download from sources.easybuild.io rather than https://ftp.gnu.org (#3883) +- other changes: + - change copy_file function to raise an error when trying to copy non-existing file (#3836) + - only print the hook messages if EasyBuild is running in debug mode (#3843) + - deprecate old toolchain versions (pre-2019a common toolchains) (#3876) + - also deprecate old versions of toolchains that include CUDA component + gimpi (#3884) + + v4.4.2 (September 7th 2021) --------------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 2216e1f42d..3aeeffa234 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.4.3.dev0') +VERSION = LooseVersion('4.5.0') UNKNOWN = 'UNKNOWN' From 37c4c1b5193d5356b9017c78e3fa7be2c234233b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 29 Oct 2021 08:53:52 +0200 Subject: [PATCH 683/864] tweak release notes for v4.5.0 release --- RELEASE_NOTES | 40 +++++++++++++++------------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index eb1979fed9..5fbbbd2ef9 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -10,45 +10,35 @@ v4.5.0 (October 29th 2021) feature release - various enhancements, including: - - add --review-pr-max and --review-pr-filter options to limit easyconfigs shown in multi-diff + retain order of easyconfigs being compared with (#3754) - - use Rich (if available) to show progress bar when installing easyconfigs (#3823) - - expand progress bar to full screen width (#3826) - - add --output-style configuration option, which can be used to disable use of Rich or type of any colored output (#3833) - - disable progress bars when running the tests to avoid messing up test suite output (#3835) - - use separate different progress bars for different aspects of the installations being performed (#3844) - - fixes & tweaks for overall & easyconfig progress bars (#3864) - - make update_progress_bar a bit more robust by just doing nothing if the corresponding progress bar was not started (and making stopping of a non-started progress bar fatal) (#3867) - - fix easyconfig + extensions progress bars when --skip is used (#3882) + - add --review-pr-max and --review-pr-filter options to limit easyconfigs used by --review-pr + retain order of easyconfigs being compared with (#3754) + - use Rich (if available) to show progress bars when installing easyconfigs (#3823, #3826, #3833, #3835, #3844, #3864, #3867, #3882) + - see also https://docs.easybuild.io/en/latest/Progress_bars.html - add support for checking required/optional EasyBuild dependencies via 'eb --check-eb-deps' (#3829) - add support for collecting GPU info (via nvidia-smi), and include it in --show-system-info and test report (#3851) - added support for --insecure-download configuration option (#3859) - make intelfftw toolchain component aware of imkl-FFTW module (#3861) - - make sure the contrib/hooks tree is included in the distribution (#3862) - - add check_async_cmd function to facilitate checking on asynchronously running commands (#3865) - - add initial/experimental support for installing extensions in parallel (#3667) - - filter out duplicate paths added to module files (#3770) + - make sure the contrib/hooks tree is included in the source tarball for easybuild-framework (#3862) + - add check_async_cmd function to facilitate checking on asynchronously running commands (#3865, #3881) + - add initial/experimental support for installing extensions in parallel (#3667, #3878) + - see also https://docs.easybuild.io/en/latest/Installing_extensions_in_parallel.html + - filter out duplicate paths added to module files, and print warning when they occur (#3770, #3874) - various bug fixes, including: - ensure that path configuration options have absolute path values (#3832) - fix broken test by taking into account changed error raised by Python 3.9.7 when copying directory via shutil.copyfile (#3840) - ensure newer location of CUDA stubs is taken into account by RPATH filter (#3850) - - replace which by command -v to avoid dependency on which (#3852) - - fix copy_file so it doesn't fail when copying a symbolic link if the target path is an existing directory (#3855) + - replace 'which' by 'command -v' in 'eb' wrapper script to avoid dependency on 'which' command (#3852) - refactor EasyBlock to decouple collecting of information on extension source/patch files from downloading them (#3860) + - in this context, the EasyBlock.fetch_extension_sources method was deprecated, and replaced by EasyBlock.collect_exts_file_info - fix library paths to add to $LDFLAGS for intel-compilers toolchain component (#3866) - - remove '--depth 1' from git clone when 'commit' specified (#3871) - - enhance test_get_source_tarball_from_git to trigger fixed bug w.r.t. cloning depth (#3872) - - fix typo in warning message on suppressing duplicate paths in append_paths/prepend_paths in ModuleGenerator (+ add test to check for warning message) (#3874) - - make sure correct include directory is used for FlexiBLAS (#3875) - - restore resolving of non-broken symlinks in copy_file (#3877) - - clarify error message when determining required dependencies for extension fails (#3878) + - remove '--depth 1' from git clone when 'commit' specified (#3871, #3872) + - make sure correct include directory is used for FlexiBLAS toolchain component (#3875) - explictly disable rebase when pulling develop branch to create a merge commit, since not specifying how to reconcile divergent branches is an error with Git 2.33.1 and newer (#3879) - - make check for running failing command asynchronously more robust w.r.t. obtaining full output (#3881) - tweak test_http_header_fields_urlpat to download from sources.easybuild.io rather than https://ftp.gnu.org (#3883) - other changes: - - change copy_file function to raise an error when trying to copy non-existing file (#3836) + - change copy_file function to raise an error when trying to copy non-existing file (#3836, #3855, #3877) - only print the hook messages if EasyBuild is running in debug mode (#3843) - - deprecate old toolchain versions (pre-2019a common toolchains) (#3876) - - also deprecate old versions of toolchains that include CUDA component + gimpi (#3884) + - deprecate old toolchain versions (pre-2019a common toolchains) (#3876, #3884) + - see also https://docs.easybuild.io/en/latest/Deprecated-easyconfigs.html#deprecated-toolchains v4.4.2 (September 7th 2021) From d41ce1c435cdecd90c302f4a00ae0368d07db648 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 29 Oct 2021 16:17:32 +0200 Subject: [PATCH 684/864] bump version to 4.5.1dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 3aeeffa234..6939144849 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.5.0') +VERSION = LooseVersion('4.5.1.dev0') UNKNOWN = 'UNKNOWN' From 4baee22ff689fbf0dc176a04f2e1b44db5f499cd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 30 Oct 2021 21:10:42 +0200 Subject: [PATCH 685/864] only remove lock if it was created in the same EasyBuild session (not if it existed already) --- easybuild/framework/easyblock.py | 5 ++++- test/framework/toy_build.py | 15 +++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 59bf0b7e6a..d12f9bde6d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3756,6 +3756,7 @@ def run_all_steps(self, run_test_cases): ignore_locks = build_option('ignore_locks') + lock_created = False try: if ignore_locks: self.log.info("Ignoring locks...") @@ -3768,6 +3769,7 @@ def run_all_steps(self, run_test_cases): # create lock to avoid that another installation running in parallel messes things up create_lock(lock_name) + lock_created = True for step_name, descr, step_methods, skippable in steps: if self.skip_step(step_name, skippable): @@ -3797,7 +3799,8 @@ def run_all_steps(self, run_test_cases): except StopException: pass finally: - if not ignore_locks: + # remove lock, but only if it was created in this session (not if it was there already) + if lock_created: remove_lock(lock_name) stop_progress_bar(PROGRESS_BAR_EASYCONFIG) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 5b75116ee1..22cd86d06e 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3198,6 +3198,13 @@ def test_toy_build_lock(self): error_pattern = "Lock .*_software_toy_0.0.lock already exists, aborting!" self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, raise_error=True, verbose=False) + # lock should still be there after it was hit + self.assertTrue(os.path.exists(toy_lock_path)) + + # trying again should give same result + self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, raise_error=True, verbose=False) + self.assertTrue(os.path.exists(toy_lock_path)) + locks_dir = os.path.join(self.test_prefix, 'locks') # no lock in place, so installation proceeds as normal @@ -3216,7 +3223,7 @@ def test_toy_build_lock(self): orig_sigalrm_handler = signal.getsignal(signal.SIGALRM) # define a context manager that remove a lock after a while, so we can check the use of --wait-for-lock - class remove_lock_after(object): + class RemoveLockAfter(object): def __init__(self, seconds, lock_fp): self.seconds = seconds self.lock_fp = lock_fp @@ -3264,7 +3271,7 @@ def __exit__(self, type, value, traceback): all_args = extra_args + opts # use context manager to remove lock after 3 seconds - with remove_lock_after(3, toy_lock_path): + with RemoveLockAfter(3, toy_lock_path): self.mock_stderr(True) self.mock_stdout(True) self.test_toy_build(extra_args=all_args, verify=False, raise_error=True, testing=False) @@ -3332,7 +3339,7 @@ def test_toy_lock_cleanup_signals(self): orig_sigalrm_handler = signal.getsignal(signal.SIGALRM) # context manager which stops the function being called with the specified signal - class wait_and_signal(object): + class WaitAndSignal(object): def __init__(self, seconds, signum): self.seconds = seconds self.signum = signum @@ -3367,7 +3374,7 @@ def __exit__(self, type, value, traceback): # avoid recycling stderr of previous test stderr = '' - with wait_and_signal(1, signum): + with WaitAndSignal(1, signum): # change back to original working directory before each test change_dir(orig_wd) From ec6b1ab2149548c49e301a678a778971b5a373db Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 31 Oct 2021 14:15:34 +0100 Subject: [PATCH 686/864] determine which extensions can be skipped in parallel (if EasyBuild is configured with --parallel-extensions-install) --- easybuild/framework/easyblock.py | 87 ++++++++++++++++++++++++++++---- test/framework/toy_build.py | 16 +++++- 2 files changed, 92 insertions(+), 11 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 59bf0b7e6a..95d5997ec5 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -83,7 +83,7 @@ from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP from easybuild.tools.hooks import MODULE_WRITE, load_hooks, run_hook -from easybuild.tools.run import run_cmd +from easybuild.tools.run import check_async_cmd, run_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version @@ -1612,9 +1612,10 @@ def prepare_for_extensions(self): def skip_extensions(self): """ - Called when self.skip is True - - use this to detect existing extensions and to remove them from self.ext_instances - - based on initial R version + Skip already installed extensions, + by removing them from list of Extension instances to install (self.ext_instances). + + This is done in parallel when EasyBuild is configured to install extensions in parallel. """ self.update_exts_progress_bar("skipping installed extensions") @@ -1624,25 +1625,91 @@ def skip_extensions(self): if not exts_filter or len(exts_filter) == 0: raise EasyBuildError("Skipping of extensions, but no exts_filter set in easyconfig") + if build_option('parallel_extensions_install'): + self.skip_extensions_parallel(exts_filter) + else: + self.skip_extensions_sequential(exts_filter) + + def skip_extensions_sequential(self, exts_filter): + """ + Skip already installed extensions (checking sequentially), + by removing them from list of Extension instances to install (self.ext_instances). + """ + print_msg("skipping installed extensions (sequentially)", log=self.log) + exts_cnt = len(self.ext_instances) res = [] for idx, ext_inst in enumerate(self.ext_instances): cmd, stdin = resolve_exts_filter_template(exts_filter, ext_inst) - (cmdstdouterr, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False) - self.log.info("exts_filter result %s %s", cmdstdouterr, ec) - if ec: + (out, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, + regexp=False, trace=False) + self.log.info("exts_filter result for %s: exit code %s; output: %s", ext_inst.name, ec, out) + if ec == 0: + print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log) + else: self.log.info("Not skipping %s", ext_inst.name) - self.log.debug("exit code: %s, stdout/err: %s", ec, cmdstdouterr) res.append(ext_inst) - else: - print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log) self.update_exts_progress_bar("skipping installed extensions (%d/%d checked)" % (idx + 1, exts_cnt)) self.ext_instances = res self.update_exts_progress_bar("already installed extensions filtered out", total=len(self.ext_instances)) + def skip_extensions_parallel(self, exts_filter): + """ + Skip already installed extensions (checking in parallel), + by removing them from list of Extension instances to install (self.ext_instances). + """ + self.log.experimental("Skipping installed extensions in parallel") + print_msg("skipping installed extensions (in parallel)", log=self.log) + + async_cmd_info_cache = {} + running_checks_ids = [] + installed_exts_ids = [] + exts_queue = list(enumerate(self.ext_instances[:])) + checked_exts_cnt = 0 + exts_cnt = len(self.ext_instances) + + # asynchronously run checks to see whether extensions are already installed + while exts_queue or running_checks_ids: + + # first handle completed checks + for idx in running_checks_ids[:]: + ext_name = self.ext_instances[idx].name + # don't read any output, just check whether command completed + async_cmd_info = check_async_cmd(*async_cmd_info_cache[idx], output_read_size=0, fail_on_error=False) + if async_cmd_info['done']: + out, ec = async_cmd_info['output'], async_cmd_info['exit_code'] + self.log.info("exts_filter result for %s: exit code %s; output: %s", ext_name, ec, out) + running_checks_ids.remove(idx) + if ec == 0: + print_msg("skipping extension %s" % ext_name, log=self.log) + installed_exts_ids.append(idx) + + checked_exts_cnt += 1 + exts_pbar_label = "skipping installed extensions " + exts_pbar_label += "(%d/%d checked)" % (checked_exts_cnt, exts_cnt) + self.update_exts_progress_bar(exts_pbar_label) + + # start additional checks asynchronously + while exts_queue and len(running_checks_ids) < self.cfg['parallel']: + idx, ext = exts_queue.pop(0) + cmd, stdin = resolve_exts_filter_template(exts_filter, ext) + async_cmd_info_cache[idx] = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, + regexp=False, trace=False, asynchronous=True) + running_checks_ids.append(idx) + + # compose new list of extensions, skip over the ones that are already installed; + # note: original order in extensions list should be preserved! + retained_ext_instances = [] + for idx, ext in enumerate(self.ext_instances): + if idx not in installed_exts_ids: + retained_ext_instances.append(ext) + self.log.info("Not skipping %s", ext.name) + + self.ext_instances = retained_ext_instances + def install_extensions(self, install=True): """ Install extensions. diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 5b75116ee1..96f9f5ec7d 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1804,7 +1804,7 @@ def test_toy_exts_parallel(self): ]) write_file(test_ec, test_ec_txt) - args = ['--parallel-extensions-install', '--experimental', '--force'] + args = ['--parallel-extensions-install', '--experimental', '--force', '--parallel=3'] stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) self.assertEqual(stderr, '') expected_stdout = '\n'.join([ @@ -1816,6 +1816,20 @@ def test_toy_exts_parallel(self): ]) self.assertEqual(stdout, expected_stdout) + # also test skipping of extensions in parallel + args.append('--skip') + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + self.assertEqual(stderr, '') + expected_stdout = '\n'.join([ + "skipping installed extensions (in parallel)", + "== skipping extension ls", + "== skipping extension bar", + "== skipping extension barbar", + "== skipping extension toy", + ]) + error_msg = "Expected output '%s' should be found in %s'" % (expected_stdout, stdout) + self.assertTrue(expected_stdout in stdout, error_msg) + def test_backup_modules(self): """Test use of backing up of modules with --module-only.""" From 40fa1c5eea8a1cdd20f547d6d5bf72c783cae65a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 31 Oct 2021 19:00:07 +0100 Subject: [PATCH 687/864] check expected output lines one by one when testing skipping extensions in parallel --- test/framework/toy_build.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 96f9f5ec7d..234b72241c 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1820,15 +1820,19 @@ def test_toy_exts_parallel(self): args.append('--skip') stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) self.assertEqual(stderr, '') - expected_stdout = '\n'.join([ - "skipping installed extensions (in parallel)", - "== skipping extension ls", - "== skipping extension bar", - "== skipping extension barbar", - "== skipping extension toy", - ]) - error_msg = "Expected output '%s' should be found in %s'" % (expected_stdout, stdout) - self.assertTrue(expected_stdout in stdout, error_msg) + + # order in which these patterns occur is not fixed, so check them one by one + patterns = [ + r"^== skipping installed extensions \(in parallel\)$", + r"^== skipping extension ls$", + r"^== skipping extension bar$", + r"^== skipping extension barbar$", + r"^== skipping extension toy$", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout) + self.assertTrue(regex.search(stdout), error_msg) def test_backup_modules(self): """Test use of backing up of modules with --module-only.""" From 2937c91d0c3145c1b12b76c90035ca5782b84863 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 7 Nov 2021 15:24:27 +0100 Subject: [PATCH 688/864] take into account that repositorypath could specify remote repository via git@:/.git (fixes #3892) --- easybuild/tools/options.py | 2 +- test/framework/options.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index e0d7b47324..ac4554f0d7 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1067,7 +1067,7 @@ def _postprocess_checks(self): def get_cfg_opt_abs_path(self, opt_name, path): """Get path value of configuration option as absolute path.""" - if os.path.isabs(path): + if os.path.isabs(path) or path.startswith('git@'): abs_path = path else: abs_path = os.path.abspath(path) diff --git a/test/framework/options.py b/test/framework/options.py index ad60d8f7eb..a828eb0e1d 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6361,6 +6361,25 @@ def test_config_abs_path(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (pattern, txt)) + def test_config_repositorypath(self): + """Test how special repositorypath values are handled.""" + + repositorypath = 'git@github.com:boegel/my_easyconfigs.git' + args = [ + '--repositorypath=%s' % repositorypath, + '--show-config', + ] + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) + + regex = re.compile(r'repositorypath\s+\(C\) = %s' % repositorypath, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) + + args[0] = '--repositorypath=%s,some/subdir' % repositorypath + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) + + regex = re.compile(r"repositorypath\s+\(C\) = \('%s', 'some/subdir'\)" % repositorypath, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) + # end-to-end testing of unknown filename def test_easystack_wrong_read(self): """Test for --easystack when wrong name is provided""" From e8c5f964d6ad393be7403194a22ed53b5b3accb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 8 Nov 2021 15:40:29 +0100 Subject: [PATCH 689/864] fix typo in Agreement --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index a828eb0e1d..0ff930f4dc 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6242,7 +6242,7 @@ def test_accept_eula_for(self): # by default, no EULAs are accepted at all args = [test_ec, '--force'] - error_pattern = r"The End User License Argreement \(EULA\) for toy is currently not accepted!" + error_pattern = r"The End User License Agreement \(EULA\) for toy is currently not accepted!" self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True) toy_modfile = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') if get_module_syntax() == 'Lua': From 6cde2f95902c2358ebb3dc48be9b2a611cfc5930 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 8 Nov 2021 15:41:44 +0100 Subject: [PATCH 690/864] fix typo in Agreement --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f5c60aa81c..417f9e6c55 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1895,7 +1895,7 @@ def check_accepted_eula(self, name=None, more_info=None): self.log.info("EULA for %s is accepted", name) else: error_lines = [ - "The End User License Argreement (EULA) for %(name)s is currently not accepted!", + "The End User License Agreement (EULA) for %(name)s is currently not accepted!", ] if more_info: error_lines.append("(see %s for more information)" % more_info) From 8de5b7d4deba55d63e8b65b90d593623efa91862 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 18 Nov 2021 15:29:46 +0100 Subject: [PATCH 691/864] don't try to ensure absolute path for path part of repositorypath --- easybuild/tools/options.py | 20 +++++++++----------- test/framework/options.py | 2 +- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index ac4554f0d7..6d123508a2 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1092,22 +1092,20 @@ def _ensure_abs_path(self, opt_name): def _postprocess_config(self): """Postprocessing of configuration options""" - # resolve relative paths for configuration options that specify a location; + # resolve relative paths for configuration options that specify a location, + # to avoid incorrect paths being used when EasyBuild changes the current working directory + # (see https://github.com/easybuilders/easybuild-framework/issues/3619); # ensuring absolute paths for 'robot' is handled separately below, - # because we need to be careful with the argument pass to --robot + # because we need to be careful with the argument pass to --robot; + # note: repositorypath is purposely not listed here, because it's a special case: + # - the value could consist of a 2-tuple (, ); + # - the could also specify the location of a *remote* (Git( repository, + # which can be done in variety of formats (git@:/), https://, etc.) + # (see also https://github.com/easybuilders/easybuild-framework/issues/3892); path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath', 'installpath_modules', 'installpath_software', 'prefix', 'packagepath', 'robot_paths', 'sourcepath'] - # repositorypath is a special case: only first part is a path; - # 2nd (optional) part is a relative subdir and should not be resolved to an absolute path! - repositorypath = self.options.repositorypath - if isinstance(repositorypath, (list, tuple)) and len(repositorypath) == 2: - abs_path = self.get_cfg_opt_abs_path('repositorypath', repositorypath[0]) - self.options.repositorypath = (abs_path, repositorypath[1]) - else: - path_opt_names.append('repositorypath') - for opt_name in path_opt_names: self._ensure_abs_path(opt_name) diff --git a/test/framework/options.py b/test/framework/options.py index 0ff930f4dc..cc6138f9c4 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6377,7 +6377,7 @@ def test_config_repositorypath(self): args[0] = '--repositorypath=%s,some/subdir' % repositorypath txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) - regex = re.compile(r"repositorypath\s+\(C\) = \('%s', 'some/subdir'\)" % repositorypath, re.M) + regex = re.compile(r"repositorypath\s+\(C\) = %s, some/subdir" % repositorypath, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) # end-to-end testing of unknown filename From d68e31f76f2656d3394b0ca5f032e6d326f9c8e7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 18 Nov 2021 16:52:10 +0100 Subject: [PATCH 692/864] fix broken tests now that repositorypath is a list (again), rather than a tuple --- test/framework/config.py | 2 +- test/framework/options.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/config.py b/test/framework/config.py index 0c4489a412..cb13d348a5 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -302,7 +302,7 @@ def test_generaloption_config_file(self): self.assertEqual(install_path('mod'), installpath_modules), # via config file self.assertEqual(source_paths(), [testpath2]) # via command line self.assertEqual(build_path(), testpath1) # via config file - self.assertEqual(get_repositorypath(), (os.path.join(topdir, 'ebfiles_repo'), 'somesubdir')) # via config file + self.assertEqual(get_repositorypath(), [os.path.join(topdir, 'ebfiles_repo'), 'somesubdir']) # via config file # hardcoded first entry self.assertEqual(options.robot_paths[0], '/tmp/foo') diff --git a/test/framework/options.py b/test/framework/options.py index cc6138f9c4..ffcd583c86 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6334,7 +6334,7 @@ def test_config_abs_path(self): r"^containerpath\s+\(F\) = /.*/test_topdir/test_middle_dir$", r"^installpath\s+\(E\) = /.*/test_topdir$", r"^prefix\s+\(C\) = /.*/test_topdir/test_middle_dir$", - r"^repositorypath\s+\(F\) = \('/apps/easyconfigs_archive', ' somesubdir'\)$", + r"^repositorypath\s+\(F\) = /apps/easyconfigs_archive,\s+somesubdir$", r"^sourcepath\s+\(C\) = /.*/test_topdir/test_middle_dir/test_subdir$", r"^robot-paths\s+\(E\) = /.*/test_topdir$", ] From 007edebeaa9557b57d7271ebf6709d96c1bddee7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Nov 2021 09:03:04 +0100 Subject: [PATCH 693/864] fix builddir value when --sanity-check-only is used and buildininstalldir is enabled --- easybuild/framework/easyblock.py | 8 +++++--- test/framework/options.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index aba8791970..4e7ec9d7e9 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -268,6 +268,11 @@ def __init__(self, ec): # generate build/install directories self.gen_builddir() self.gen_installdir() + if self.build_in_installdir: + # self.builddir is set by self.gen_builddir(), + # but needs to be correct if the build is performed in the installation directory + self.log.info("Changing build dir to %s", self.installdir) + self.builddir = self.installdir self.ignored_errors = False @@ -941,9 +946,6 @@ def make_builddir(self): raise EasyBuildError("self.builddir not set, make sure gen_builddir() is called first!") self.log.debug("Creating the build directory %s (cleanup: %s)", self.builddir, self.cfg['cleanupoldbuild']) else: - self.log.info("Changing build dir to %s" % self.installdir) - self.builddir = self.installdir - self.log.info("Overriding 'cleanupoldinstall' (to False), 'cleanupoldbuild' (to True) " "and 'keeppreviousinstall' because we're building in the installation directory.") # force cleanup before installation diff --git a/test/framework/options.py b/test/framework/options.py index ffcd583c86..8b5c326202 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6037,6 +6037,21 @@ def test_sanity_check_only(self): stdout = self.mocked_main(args + ['--trace'], do_build=True, raise_error=True, testing=False) + # check whether %(builddir)s value is correct + # when build_in_installdir is enabled and --sanity-check-only is used + # (see https://github.com/easybuilders/easybuild-framework/issues/3895) + test_ec_txt += '\n' + '\n'.join([ + "buildininstalldir = True", + "sanity_check_commands = [", + # build and install directory should be the same path + " 'test %(builddir)s = %(installdir)s',", + # build/install directory must exist (even though step that creates build dir was never run) + " 'test -d %(builddir)s',", + "]", + ]) + write_file(test_ec, test_ec_txt) + self.eb_main(args, do_build=True, raise_error=True) + def test_skip_extensions(self): """Test use of --skip-extensions.""" topdir = os.path.abspath(os.path.dirname(__file__)) From bc5cdec39c0694219afc05ca2fef1e5d1951f3ad Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 20 Nov 2021 09:13:30 +0100 Subject: [PATCH 694/864] fix broken test for buildininstalldir --- test/framework/easyconfig.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index c62b767069..3626908654 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1335,8 +1335,6 @@ def test_buildininstalldir(self): self.prep() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) - eb.gen_builddir() - eb.gen_installdir() eb.make_builddir() eb.make_installdir() self.assertEqual(eb.builddir, eb.installdir) From 5ac33392ceb16d06ffa1b32e54c4d675d374cd1c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 27 Nov 2021 10:16:31 +0100 Subject: [PATCH 695/864] also show download progress bar when using --inject-checksums --- easybuild/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index a6680d83ea..2203636a3b 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -531,7 +531,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): dump_env_script(easyconfigs) elif options.inject_checksums: - inject_checksums(ordered_ecs, options.inject_checksums) + with rich_live_cm(): + inject_checksums(ordered_ecs, options.inject_checksums) # cleanup and exit after dry run, searching easyconfigs or submitting regression test stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums] From 7c04252f00a1a24d5b8aef2aef5293041c41bc7d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 27 Nov 2021 12:27:57 +0100 Subject: [PATCH 696/864] return None if required dependencies for an extension could not be determined, and only install it when preceding extensions have been installed --- easybuild/framework/easyblock.py | 23 +++++++++++++++++++++-- easybuild/framework/extension.py | 3 ++- test/framework/toy_build.py | 28 ++++++++++++++++++++++++++-- 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index aba8791970..29c1495f40 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1852,7 +1852,26 @@ def update_exts_progress_bar_helper(running_exts, progress_size): # check whether extension at top of the queue is ready to install ext = exts_queue.pop(0) - pending_deps = [x for x in ext.required_deps if x not in installed_ext_names] + required_deps = ext.required_deps + if required_deps is None: + pending_deps = None + self.log.info("Required dependencies for %s are unknown!", ext.name) + else: + self.log.info("Required dependencies for %s: %s", ext.name, ', '.join(required_deps)) + pending_deps = [x for x in required_deps if x not in installed_ext_names] + self.log.info("Missing required dependencies for %s: %s", ext.name, ', '.join(pending_deps)) + + # if required dependencies could not be determined, wait until all preceding extensions are installed + if pending_deps is None: + if running_exts: + # add extension back at top of the queue, + # since we need to preverse installation order of extensions; + # break out of for loop since there is no point to keep checking + # until running installations have been completed + exts_queue.insert(0, ext) + break + else: + pending_deps = [] if self.dry_run: tup = (ext.name, ext.version, ext.__class__.__name__) @@ -1865,7 +1884,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size): # make sure all required dependencies are actually going to be installed, # to avoid getting stuck in an infinite loop! - missing_deps = [x for x in ext.required_deps if x not in all_ext_names] + missing_deps = [x for x in required_deps if x not in all_ext_names] if missing_deps: raise EasyBuildError("Missing required dependencies for %s are not going to be installed: %s", ext.name, ', '.join(missing_deps)) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 5333627c29..27dd812e20 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -225,7 +225,8 @@ def async_cmd_check(self): @property def required_deps(self): """Return list of required dependencies for this extension.""" - raise NotImplementedError("Don't know how to determine required dependencies for extension '%s'" % self.name) + self.log.info("Don't know how to determine required dependencies for extension '%s'", self.name) + return None @property def toolchain(self): diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 60d66c2304..49a406fc59 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1816,8 +1816,32 @@ def test_toy_exts_parallel(self): ]) self.assertEqual(stdout, expected_stdout) - # also test skipping of extensions in parallel - args.append('--skip') + # check behaviour when using Toy_Extension easyblock that doesn't implement required_deps method; + # framework should fall back to installing extensions sequentially + toy_ext_eb = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'toy_extension.py') + copy_file(toy_ext_eb, self.test_prefix) + toy_ext_eb = os.path.join(self.test_prefix, 'toy_extension.py') + toy_ext_eb_txt = read_file(toy_ext_eb) + toy_ext_eb_txt = toy_ext_eb_txt.replace('def required_deps', 'def xxx_required_deps') + write_file(toy_ext_eb, toy_ext_eb_txt) + + args.append('--include-easyblocks=%s' % toy_ext_eb) + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + self.assertEqual(stderr, '') + expected_stdout = '\n'.join([ + "== 0 out of 4 extensions installed (3 queued, 1 running: ls)", + "== 1 out of 4 extensions installed (2 queued, 1 running: bar)", + "== 2 out of 4 extensions installed (1 queued, 1 running: barbar)", + "== 3 out of 4 extensions installed (0 queued, 1 running: toy)", + "== 4 out of 4 extensions installed (0 queued, 0 running: )", + '', + ]) + self.assertEqual(stdout, expected_stdout) + + remove_file(toy_ext_eb) + + # also test skipping of extensions in parallel (so do not use patched you easyblock) + args[-1] = '--skip' stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) self.assertEqual(stderr, '') From d008310a1fb40c39444171ae43eb775546c79457 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 30 Nov 2021 16:37:26 +0100 Subject: [PATCH 697/864] pick up custom extract_cmd specified for extension --- easybuild/framework/easyblock.py | 8 +++++-- easybuild/framework/extension.py | 1 + easybuild/framework/extensioneasyblock.py | 2 +- easybuild/tools/filetools.py | 9 +++++--- test/framework/toy_build.py | 28 +++++++++++++++++++++++ 5 files changed, 42 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index aba8791970..892b2f9b14 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -572,8 +572,12 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): if fetch_files: src = self.fetch_source(source, checksums, extension=True) - # copy 'path' entry to 'src' for use with extensions - ext_src.update({'src': src['path']}) + ext_src.update({ + # keep track of custom extract command (if any) + 'extract_cmd': src['cmd'], + # copy 'path' entry to 'src' for use with extensions + 'src': src['path'], + }) else: # use default template for name of source file if none is specified diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 5333627c29..3f3b6f53f6 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -119,6 +119,7 @@ def __init__(self, mself, ext, extra_params=None): # list of source/patch files: we use an empty list as default value like in EasyBlock self.src = resolve_template(self.ext.get('src', []), self.cfg.template_values) + self.src_extract_cmd = self.ext.get('extract_cmd', None) self.patches = resolve_template(self.ext.get('patches', []), self.cfg.template_values) self.options = resolve_template(copy.deepcopy(self.ext.get('options', {})), self.cfg.template_values) diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index c3b5c7a9fb..3c427c5a9f 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -118,7 +118,7 @@ def run(self, unpack_src=False): if unpack_src: targetdir = os.path.join(self.master.builddir, remove_unwanted_chars(self.name)) self.ext_dir = extract_file(self.src, targetdir, extra_options=self.unpack_options, - change_into_dir=False) + change_into_dir=False, cmd=self.src_extract_cmd) # setting start dir must be done from unpacked source directory for extension, # because start_dir value is usually a relative path (if it is set) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 99546c329d..3db2b1fc43 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -456,11 +456,14 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced _log.debug("Unpacking %s in directory %s", fn, abs_dest) cwd = change_dir(abs_dest) - if not cmd: - cmd = extract_cmd(fn, overwrite=overwrite) - else: + if cmd: # complete command template with filename cmd = cmd % fn + _log.debug("Using specified command to unpack %s: %s", fn, cmd) + else: + cmd = extract_cmd(fn, overwrite=overwrite) + _log.debug("Using command derived from file extension to unpack %s: %s", fn, cmd) + if not cmd: raise EasyBuildError("Can't extract file %s with unknown filetype", fn) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 60d66c2304..c6b2a90f67 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1362,6 +1362,34 @@ def test_toy_extension_sources(self): write_file(test_ec, test_ec_txt) self.test_toy_build(ec_file=test_ec, raise_error=True) + def test_toy_extension_extract_cmd(self): + """Test for custom extract_cmd specified for an extension.""" + test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + toy_ec_txt = read_file(toy_ec) + + # create file that we'll copy via 'patches' + write_file(os.path.join(self.test_prefix, 'test.txt'), 'test123') + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = '\n'.join([ + toy_ec_txt, + 'exts_list = [', + ' ("bar", "0.0", {', + # deliberately incorrect custom extract command, just to verify that it's picked up + ' "sources": [{', + ' "filename": "bar-%(version)s.tar.gz",', + ' "extract_cmd": "unzip %s",', + ' }],', + ' }),', + ']', + ]) + write_file(test_ec, test_ec_txt) + + error_pattern = "unzip .*/bar-0.0.tar.gz.* exited with exit code [1-9]" + self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec, + raise_error=True, verbose=False) + def test_toy_extension_sources_git_config(self): """Test install toy that includes extensions with 'sources' spec including 'git_config'.""" test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') From ff98040fb7de571ecc211622d687af52e8d1ba29 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 3 Dec 2021 20:50:00 +0100 Subject: [PATCH 698/864] fix broken test_toy_exts_parallel with Python 2 by testing last with custom toy easyblock that doesn't implement required_deps --- test/framework/toy_build.py | 40 ++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 49a406fc59..dc3e168e7d 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1816,6 +1816,24 @@ def test_toy_exts_parallel(self): ]) self.assertEqual(stdout, expected_stdout) + # also test skipping of extensions in parallel + args.append('--skip') + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + self.assertEqual(stderr, '') + + # order in which these patterns occur is not fixed, so check them one by one + patterns = [ + r"^== skipping installed extensions \(in parallel\)$", + r"^== skipping extension ls$", + r"^== skipping extension bar$", + r"^== skipping extension barbar$", + r"^== skipping extension toy$", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout) + self.assertTrue(regex.search(stdout), error_msg) + # check behaviour when using Toy_Extension easyblock that doesn't implement required_deps method; # framework should fall back to installing extensions sequentially toy_ext_eb = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'toy_extension.py') @@ -1825,7 +1843,7 @@ def test_toy_exts_parallel(self): toy_ext_eb_txt = toy_ext_eb_txt.replace('def required_deps', 'def xxx_required_deps') write_file(toy_ext_eb, toy_ext_eb_txt) - args.append('--include-easyblocks=%s' % toy_ext_eb) + args[-1] = '--include-easyblocks=%s' % toy_ext_eb stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) self.assertEqual(stderr, '') expected_stdout = '\n'.join([ @@ -1838,26 +1856,6 @@ def test_toy_exts_parallel(self): ]) self.assertEqual(stdout, expected_stdout) - remove_file(toy_ext_eb) - - # also test skipping of extensions in parallel (so do not use patched you easyblock) - args[-1] = '--skip' - stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) - self.assertEqual(stderr, '') - - # order in which these patterns occur is not fixed, so check them one by one - patterns = [ - r"^== skipping installed extensions \(in parallel\)$", - r"^== skipping extension ls$", - r"^== skipping extension bar$", - r"^== skipping extension barbar$", - r"^== skipping extension toy$", - ] - for pattern in patterns: - regex = re.compile(pattern, re.M) - error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout) - self.assertTrue(regex.search(stdout), error_msg) - def test_backup_modules(self): """Test use of backing up of modules with --module-only.""" From 7575509086db4feee3d92d12371dc76ec63e4e5b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 4 Dec 2021 11:05:24 +0100 Subject: [PATCH 699/864] make test_run_cmd_async more robust against fluke failures --- test/framework/run.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/framework/run.py b/test/framework/run.py index 6c298890d5..a821a67d3c 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -605,7 +605,9 @@ def test_run_cmd_async(self): self.assertEqual(res, {'done': False, 'exit_code': None, 'output': 'sleeping...\n'}) # 2nd check with default output size (1024) gets full output - res = check_async_cmd(*cmd_info, output=res['output']) + # (keep checking until command is fully done) + while not res['done']: + res = check_async_cmd(*cmd_info, output=res['output']) self.assertEqual(res, {'done': True, 'exit_code': 0, 'output': 'sleeping...\ntest123\n'}) # check asynchronous running of failing command From eace8e3caca00069f8e39087cdfeecaf48f0b848 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 4 Dec 2021 14:24:49 +0100 Subject: [PATCH 700/864] extend test_sanity_check_only to also check whether build_in_installdir that is set by easyblock is correctly taken into account --- test/framework/options.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 8b5c326202..f1ff815c54 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6038,7 +6038,7 @@ def test_sanity_check_only(self): stdout = self.mocked_main(args + ['--trace'], do_build=True, raise_error=True, testing=False) # check whether %(builddir)s value is correct - # when build_in_installdir is enabled and --sanity-check-only is used + # when buildininstalldir is enabled in easyconfig and --sanity-check-only is used # (see https://github.com/easybuilders/easybuild-framework/issues/3895) test_ec_txt += '\n' + '\n'.join([ "buildininstalldir = True", @@ -6052,6 +6052,26 @@ def test_sanity_check_only(self): write_file(test_ec, test_ec_txt) self.eb_main(args, do_build=True, raise_error=True) + # also check when using easyblock that enables build_in_installdir in its constructor + test_ebs = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') + toy_eb = os.path.join(test_ebs, 't', 'toy.py') + toy_eb_txt = read_file(toy_eb) + + self.assertFalse('self.build_in_installdir = True' in toy_eb_txt) + + regex = re.compile(r'^(\s+)(super\(EB_toy, self\).__init__.*)\n', re.M) + toy_eb_txt = regex.sub(r'\1\2\n\1self.build_in_installdir = True', toy_eb_txt) + self.assertTrue('self.build_in_installdir = True' in toy_eb_txt) + + toy_eb = os.path.join(self.test_prefix, 'toy.py') + write_file(toy_eb, toy_eb_txt) + + test_ec_txt = test_ec_txt.replace('buildininstalldir = True', '') + write_file(test_ec, test_ec_txt) + + args.append('--include-easyblocks=%s' % toy_eb) + self.eb_main(args, do_build=True, raise_error=True) + def test_skip_extensions(self): """Test use of --skip-extensions.""" topdir = os.path.abspath(os.path.dirname(__file__)) From 96236290d34803d50aea4f74bd96ca7c5597033f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 4 Dec 2021 14:36:25 +0100 Subject: [PATCH 701/864] introduce post_init method to ensure that builddir EasyBlock variable is correctly set based on build_in_installdir (which may be defined dynamically by easyblock in constructor) --- easybuild/framework/easyblock.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 4e7ec9d7e9..4a74dc9839 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -268,11 +268,6 @@ def __init__(self, ec): # generate build/install directories self.gen_builddir() self.gen_installdir() - if self.build_in_installdir: - # self.builddir is set by self.gen_builddir(), - # but needs to be correct if the build is performed in the installation directory - self.log.info("Changing build dir to %s", self.installdir) - self.builddir = self.installdir self.ignored_errors = False @@ -281,6 +276,16 @@ def __init__(self, ec): self.log.info("Init completed for application name %s version %s" % (self.name, self.version)) + def post_init(self): + """ + Run post-initialization tasks. + """ + if self.build_in_installdir: + # self.builddir is set by self.gen_builddir(), + # but needs to be correct if the build is performed in the installation directory + self.log.info("Changing build dir to %s", self.installdir) + self.builddir = self.installdir + # INIT/CLOSE LOG def _init_log(self): """ @@ -3840,6 +3845,9 @@ def run_all_steps(self, run_test_cases): create_lock(lock_name) lock_created = True + # run post-initialization tasks first, before running any steps + self.post_init() + for step_name, descr, step_methods, skippable in steps: if self.skip_step(step_name, skippable): print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent) From 1d4e45492981b80698aaa0702895191732b5ce3c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 4 Dec 2021 14:58:34 +0100 Subject: [PATCH 702/864] improve error message for unknown file extensions, add comment to clarify behaviour of find_extension in extract_cmd, extend test_extract_cmd to check case of unknown extension --- easybuild/tools/filetools.py | 4 +++- test/framework/filetools.py | 3 +++ test/framework/toy_build.py | 3 --- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 3db2b1fc43..bde3d1156f 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1369,7 +1369,7 @@ def find_extension(filename): if res: ext = res.group('ext') else: - raise EasyBuildError('Unknown file type for file %s', filename) + raise EasyBuildError("%s has unknown file extension", filename) return ext @@ -1382,7 +1382,9 @@ def extract_cmd(filepath, overwrite=False): ext = find_extension(filename) target = filename[:-len(ext)] + # find_extension will either return an extension listed in EXTRACT_CMDS, or raise an error cmd_tmpl = EXTRACT_CMDS[ext.lower()] + if overwrite: if 'unzip -qq' in cmd_tmpl: cmd_tmpl = cmd_tmpl.replace('unzip -qq', 'unzip -qq -o') diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 8c54f5881f..57f5d18975 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -114,6 +114,9 @@ def test_extract_cmd(self): self.assertEqual("unzip -qq -o test.zip", ft.extract_cmd('test.zip', True)) + error_pattern = "test.foo has unknown file extension" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.extract_cmd, 'test.foo') + def test_find_extension(self): """Test find_extension function.""" tests = [ diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index c6b2a90f67..6e33a0d47e 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1368,9 +1368,6 @@ def test_toy_extension_extract_cmd(self): toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') toy_ec_txt = read_file(toy_ec) - # create file that we'll copy via 'patches' - write_file(os.path.join(self.test_prefix, 'test.txt'), 'test123') - test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = '\n'.join([ toy_ec_txt, From e4344d0c7e5a2ae0cba49207edde670acd222357 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 4 Dec 2021 20:21:29 +0100 Subject: [PATCH 703/864] clean up included toy easyblock in test_sanity_check_only --- test/framework/options.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index f1ff815c54..7b82cb8111 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6072,6 +6072,9 @@ def test_sanity_check_only(self): args.append('--include-easyblocks=%s' % toy_eb) self.eb_main(args, do_build=True, raise_error=True) + # undo import of the toy easyblock, to avoid problems with other tests + del sys.modules['easybuild.easyblocks.toy'] + def test_skip_extensions(self): """Test use of --skip-extensions.""" topdir = os.path.abspath(os.path.dirname(__file__)) From d0a759ef5d05771d9fa328c1d0d8366a09616c5e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 Dec 2021 19:41:15 +0100 Subject: [PATCH 704/864] fix broken test_buildininstalldir by calling also post_init method after creating EasyBlock instance --- test/framework/easyconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 3626908654..d0718ac9ff 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1335,6 +1335,7 @@ def test_buildininstalldir(self): self.prep() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) + eb.post_init() eb.make_builddir() eb.make_installdir() self.assertEqual(eb.builddir, eb.installdir) From 7066e03f3e863f53a27fae7080e4a357ae132848 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 Dec 2021 19:46:59 +0100 Subject: [PATCH 705/864] correctly undo included toy toolchain in test_sanity_check_only --- test/framework/options.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 7b82cb8111..41b29f0dec 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6069,11 +6069,20 @@ def test_sanity_check_only(self): test_ec_txt = test_ec_txt.replace('buildininstalldir = True', '') write_file(test_ec, test_ec_txt) + orig_local_sys_path = sys.path[:] args.append('--include-easyblocks=%s' % toy_eb) self.eb_main(args, do_build=True, raise_error=True) # undo import of the toy easyblock, to avoid problems with other tests del sys.modules['easybuild.easyblocks.toy'] + sys.path = orig_local_sys_path + import easybuild.easyblocks + reload(easybuild.easyblocks) + import easybuild.easyblocks.toy + reload(easybuild.easyblocks.toy) + # need to reload toy_extension, which imports EB_toy, to ensure right EB_toy is picked up in later tests + import easybuild.easyblocks.generic.toy_extension + reload(easybuild.easyblocks.generic.toy_extension) def test_skip_extensions(self): """Test use of --skip-extensions.""" From 18b0499710a264051cb8e293d1a65b3dd56fcac8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 Dec 2021 21:00:00 +0100 Subject: [PATCH 706/864] allow oversubscription in sanity check for OpenMPI-based toolchains --- easybuild/framework/easyblock.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 29c1495f40..c9bd499d89 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -52,6 +52,7 @@ from distutils.version import LooseVersion import easybuild.tools.environment as env +import easybuild.tools.toolchain as toolchain from easybuild.base import fancylogger from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easyconfig.easyconfig import ITERATE_OPTIONS, EasyConfig, ActiveMNS, get_easyblock_class @@ -3352,7 +3353,12 @@ def xs2str(xs): if extra_modules: self.log.info("Loading extra modules for sanity check: %s", ', '.join(extra_modules)) - # chdir to installdir (better environment for running tests) + # allow oversubscription of P processes on C cores (P>C) for software installed on top of Open MPI; + # this is useful to avoid failing of sanity check commands that involve MPI + if self.toolchain.mpi_family() in toolchain.OPENMPI: + env.setvar('OMPI_MCA_rmaps_base_oversubscribe', '1') + + # change to install directory (better environment for running tests) if os.path.isdir(self.installdir): change_dir(self.installdir) From fd2b6a0e6c128d1b90850e0210fc6d9a5503fd28 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Dec 2021 15:44:54 +0100 Subject: [PATCH 707/864] fix check to see whether toolchain includes OpenMPI Co-authored-by: Miguel Dias Costa --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index c9bd499d89..949ab3d3cb 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3355,7 +3355,7 @@ def xs2str(xs): # allow oversubscription of P processes on C cores (P>C) for software installed on top of Open MPI; # this is useful to avoid failing of sanity check commands that involve MPI - if self.toolchain.mpi_family() in toolchain.OPENMPI: + if self.toolchain.mpi_family() and self.toolchain.mpi_family() in toolchain.OPENMPI: env.setvar('OMPI_MCA_rmaps_base_oversubscribe', '1') # change to install directory (better environment for running tests) From 28c2905c884974451bc02bc9b69e1421ebc215c5 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Fri, 10 Dec 2021 14:19:27 +0800 Subject: [PATCH 708/864] prepare release notes for EasyBuild v4.5.1 + bump version to 4.5.1 --- RELEASE_NOTES | 21 +++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 5fbbbd2ef9..7724a0faf5 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,27 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.5.1 (December 13th 2021) +--------------------------- + +update/bugfix release + +- various enhancements, including: + - determine which extensions can be skipped in parallel (if --parallel-extensions-install is enabled) (#3890) + - fall back to sequential installation for extensions with unknown dependencies when using --parallel-extensions-install (#3906) + - allow oversubscription in sanity check for OpenMPI-based toolchains (#3909) +- various bug fixes, including: + - also dump environment to reprod directory (#3374) + - take into account that repositorypath could specify remote repository via git@:/.git (#3893) + - fix typo in EULA agreement error message (#3894) + - only remove lock if it was created in the same EasyBuild session (not if it existed already) (#3889) + - don't try to ensure absolute path for path part of repositorypath (#3899) + - introduce EasyBlock.post_init method to correctly define builddir variable when build-in-installdir mode is enabled in easyconfig or easyblock (#3900) + - also show download progress bar when using --inject-checksums (#3905) + - pick up custom extract_cmd specified for extension (#3907) + - make test_run_cmd_async more robust against fluke failures (#3908) + + v4.5.0 (October 29th 2021) -------------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 6939144849..d05df8a975 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.5.1.dev0') +VERSION = LooseVersion('4.5.1') UNKNOWN = 'UNKNOWN' From 313e5e89f1e7eebfd9fa2d2932977723f04923ed Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Dec 2021 11:55:25 +0100 Subject: [PATCH 709/864] tweak release notes for v4.5.1 release --- RELEASE_NOTES | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 7724a0faf5..a09c9f6859 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -10,15 +10,14 @@ v4.5.1 (December 13th 2021) update/bugfix release - various enhancements, including: + - also dump environment to reprod directory (#3374) - determine which extensions can be skipped in parallel (if --parallel-extensions-install is enabled) (#3890) - fall back to sequential installation for extensions with unknown dependencies when using --parallel-extensions-install (#3906) - allow oversubscription in sanity check for OpenMPI-based toolchains (#3909) - various bug fixes, including: - - also dump environment to reprod directory (#3374) - - take into account that repositorypath could specify remote repository via git@:/.git (#3893) + - don't try to ensure absolute path for path part of repositorypath (#3893, #3899) - fix typo in EULA agreement error message (#3894) - only remove lock if it was created in the same EasyBuild session (not if it existed already) (#3889) - - don't try to ensure absolute path for path part of repositorypath (#3899) - introduce EasyBlock.post_init method to correctly define builddir variable when build-in-installdir mode is enabled in easyconfig or easyblock (#3900) - also show download progress bar when using --inject-checksums (#3905) - pick up custom extract_cmd specified for extension (#3907) From 015493f7b92e2d70dfd8545961943d9e6c3199b7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Dec 2021 15:15:19 +0100 Subject: [PATCH 710/864] bump version to 4.5.2dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index d05df8a975..21bcecc715 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.5.1') +VERSION = LooseVersion('4.5.2.dev0') UNKNOWN = 'UNKNOWN' From 8c9757a234fc24f52e47d797df57e1765646cd07 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 18 Dec 2021 18:25:50 +0100 Subject: [PATCH 711/864] automatically prepend env-for-shebang value with sysroot --- easybuild/framework/easyblock.py | 12 ++++++- test/framework/toy_build.py | 61 ++++++++++++++++++++++++-------- 2 files changed, 57 insertions(+), 16 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 391f58e3a0..4ea2b44f0a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2758,6 +2758,16 @@ def package_step(self): def fix_shebang(self): """Fix shebang lines for specified files.""" + + env_for_shebang = build_option('env_for_shebang') + sysroot = build_option('sysroot') + if sysroot and not env_for_shebang.startswith(sysroot): + env_for_shebang = os.path.join(sysroot, env_for_shebang.lstrip('/')) + if os.path.exists(env_for_shebang.split(' ')[0]): + self.log.info("Path to 'env' command to use in patched shebang lines: %s", env_for_shebang) + else: + raise EasyBuildError("Path to 'env' command to use in shebang lines does not exist: %s", env_for_shebang) + for lang in ['bash', 'perl', 'python']: shebang_regex = re.compile(r'^#![ ]*.*[/ ]%s.*' % lang) fix_shebang_for = self.cfg['fix_%s_shebang_for' % lang] @@ -2765,7 +2775,7 @@ def fix_shebang(self): if isinstance(fix_shebang_for, string_type): fix_shebang_for = [fix_shebang_for] - shebang = '#!%s %s' % (build_option('env_for_shebang'), lang) + shebang = '#!%s %s' % (env_for_shebang, lang) for glob_pattern in fix_shebang_for: paths = glob.glob(os.path.join(self.installdir, glob_pattern)) self.log.info("Fixing '%s' shebang to '%s' for files that match '%s': %s", diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 3f46f2f4c3..5bcaef1761 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3209,23 +3209,54 @@ def test_fix_shebang(self): # exact same file as original binary (untouched) self.assertEqual(toy_txt, fn_txt) - regexes_S = {} + regexes_shebang = {} + + def check_shebangs(): + for ext in ['sh', 'pl', 'py']: + for script in scripts[ext]: + bin_path = os.path.join(toy_bindir, script) + bin_txt = read_file(bin_path) + # the scripts b1.py, b1.pl, b1.sh, b2.sh should keep their original shebang + if script.startswith('b'): + self.assertTrue(regexes[ext].match(bin_txt), + "Pattern '%s' found in %s: %s" % (regexes[ext].pattern, bin_path, bin_txt)) + else: + regex_shebang = regexes_shebang[ext] + self.assertTrue(regex_shebang.match(bin_txt), + "Pattern '%s' found in %s: %s" % (regex_shebang.pattern, bin_path, bin_txt)) + # no re.M, this should match at start of file! - regexes_S['py'] = re.compile(r'^#!/usr/bin/env -S python\n# test$') - regexes_S['pl'] = re.compile(r'^#!/usr/bin/env -S perl\n# test$') - regexes_S['sh'] = re.compile(r'^#!/usr/bin/env -S bash\n# test$') + regexes_shebang['py'] = re.compile(r'^#!/usr/bin/env -S python\n# test$') + regexes_shebang['pl'] = re.compile(r'^#!/usr/bin/env -S perl\n# test$') + regexes_shebang['sh'] = re.compile(r'^#!/usr/bin/env -S bash\n# test$') - for ext in ['sh', 'pl', 'py']: - for script in scripts[ext]: - bin_path = os.path.join(toy_bindir, script) - bin_txt = read_file(bin_path) - # the scripts b1.py, b1.pl, b1.sh, b2.sh should keep their original shebang - if script.startswith('b'): - self.assertTrue(regexes[ext].match(bin_txt), - "Pattern '%s' found in %s: %s" % (regexes[ext].pattern, bin_path, bin_txt)) - else: - self.assertTrue(regexes_S[ext].match(bin_txt), - "Pattern '%s' found in %s: %s" % (regexes_S[ext].pattern, bin_path, bin_txt)) + check_shebangs() + + # test again with EasyBuild configured with sysroot, which should be prepended + # automatically to env-for-shebang value (unless it's prefixed with sysroot already) + extra_args = [ + '--env-for-shebang=/usr/bin/env -S', + '--sysroot=/usr/../', # sysroot must exist, and so must /usr/bin/env when appended to it + ] + self.test_toy_build(ec_file=test_ec, extra_args=extra_args, raise_error=True) + + regexes_shebang['py'] = re.compile(r'^#!/usr/../usr/bin/env -S python\n# test$') + regexes_shebang['pl'] = re.compile(r'^#!/usr/../usr/bin/env -S perl\n# test$') + regexes_shebang['sh'] = re.compile(r'^#!/usr/../usr/bin/env -S bash\n# test$') + + check_shebangs() + + extra_args = [ + '--env-for-shebang=/usr/../usr/../usr/bin/env -S', + '--sysroot=/usr/../', # sysroot must exist, and so must /usr/bin/env when appended to it + ] + self.test_toy_build(ec_file=test_ec, extra_args=extra_args, raise_error=True) + + regexes_shebang['py'] = re.compile(r'^#!/usr/../usr/../usr/bin/env -S python\n# test$') + regexes_shebang['pl'] = re.compile(r'^#!/usr/../usr/../usr/bin/env -S perl\n# test$') + regexes_shebang['sh'] = re.compile(r'^#!/usr/../usr/../usr/bin/env -S bash\n# test$') + + check_shebangs() def test_toy_system_toolchain_alias(self): """Test use of 'system' toolchain alias.""" From 364187f25b9eaa0215079eb518c6101a61176469 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Tue, 28 Dec 2021 15:28:32 +0100 Subject: [PATCH 712/864] add deprecation warning for patches without .patch suffix --- easybuild/tools/filetools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index bde3d1156f..6096f74847 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1493,6 +1493,9 @@ def create_patch_info(patch_spec): str(patch_spec)) elif isinstance(patch_spec, string_type): + if not patch_spec.endswith('.patch'): + _log.deprecated("Add '.patch' suffix to patch file, or use 2-element list/tuple to specify " + "path to where non-patch file should be copied: %s" % patch_spec, '5.0') patch_info = {'name': patch_spec} else: error_msg = "Wrong patch spec, should be string of 2-tuple with patch name + argument: %s" From 1b9b24ff64747149ec1f65e181892d004bd13829 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Tue, 28 Dec 2021 16:41:06 +0100 Subject: [PATCH 713/864] hopefully fix tests --- test/framework/easyconfig.py | 2 +- test/framework/filetools.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index d0718ac9ff..37b57f408b 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -461,7 +461,7 @@ def test_exts_list(self): ' ("ext1", "1.0"),', ' ("ext2", "2.0", {', ' "source_urls": [("http://example.com", "suffix")],' - ' "patches": ["toy-0.0.eb"],', # dummy patch to avoid downloading fail + ' "patches": [("toy-0.0.eb", '.')],', # dummy patch to avoid downloading fail ' "checksums": [', # SHA256 checksum for source (gzip-1.4.eb) ' "6a5abcab719cefa95dca4af0db0d2a9d205d68f775a33b452ec0f2b75b6a3a45",', diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 57f5d18975..a409b25f39 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1628,11 +1628,14 @@ def test_create_patch_info(self): """Test create_patch_info function.""" self.assertEqual(ft.create_patch_info('foo.patch'), {'name': 'foo.patch'}) - self.assertEqual(ft.create_patch_info('foo.txt'), {'name': 'foo.txt'}) self.assertEqual(ft.create_patch_info(('foo.patch', 1)), {'name': 'foo.patch', 'level': 1}) self.assertEqual(ft.create_patch_info(('foo.patch', 'subdir')), {'name': 'foo.patch', 'sourcepath': 'subdir'}) self.assertEqual(ft.create_patch_info(('foo.txt', 'subdir')), {'name': 'foo.txt', 'copy': 'subdir'}) + error_pattern = "Add '.patch' suffix to patch file, or use 2-element list/tuple to specify " + error_pattern += "path to where non-patch file should be copied: foo.txt" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.create_patch_info, 'foo.txt') + # faulty input error_msg = "Wrong patch spec" self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, None) From 501ccf4a6368712618abe919372d1c739f884bd2 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Tue, 28 Dec 2021 16:43:34 +0100 Subject: [PATCH 714/864] add comment --- test/framework/filetools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index a409b25f39..9ed19e60f0 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1632,6 +1632,7 @@ def test_create_patch_info(self): self.assertEqual(ft.create_patch_info(('foo.patch', 'subdir')), {'name': 'foo.patch', 'sourcepath': 'subdir'}) self.assertEqual(ft.create_patch_info(('foo.txt', 'subdir')), {'name': 'foo.txt', 'copy': 'subdir'}) + # deprecation warning (which is an error in this context) error_pattern = "Add '.patch' suffix to patch file, or use 2-element list/tuple to specify " error_pattern += "path to where non-patch file should be copied: foo.txt" self.assertErrorRegex(EasyBuildError, error_pattern, ft.create_patch_info, 'foo.txt') From 3dff4f84d5598329cf466f7debcc4f9cf6a81370 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Tue, 28 Dec 2021 16:45:11 +0100 Subject: [PATCH 715/864] fix syntax error --- test/framework/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 37b57f408b..cb51bf5768 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -461,7 +461,7 @@ def test_exts_list(self): ' ("ext1", "1.0"),', ' ("ext2", "2.0", {', ' "source_urls": [("http://example.com", "suffix")],' - ' "patches": [("toy-0.0.eb", '.')],', # dummy patch to avoid downloading fail + ' "patches": [("toy-0.0.eb", ".")],', # dummy patch to avoid downloading fail ' "checksums": [', # SHA256 checksum for source (gzip-1.4.eb) ' "6a5abcab719cefa95dca4af0db0d2a9d205d68f775a33b452ec0f2b75b6a3a45",', From a941d71f577419dde420b406fc1e096cb4b5787a Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Tue, 28 Dec 2021 17:34:15 +0100 Subject: [PATCH 716/864] more test fixes --- test/framework/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index cb51bf5768..6a11966c68 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -488,7 +488,7 @@ def test_exts_list(self): self.assertEqual(exts_sources[1]['options'], { 'checksums': ['6a5abcab719cefa95dca4af0db0d2a9d205d68f775a33b452ec0f2b75b6a3a45', '2d964e0e8f05a7cce0dd83a3e68c9737da14b87b61b8b8b0291d58d4c8d1031c'], - 'patches': ['toy-0.0.eb'], + 'patches': [('toy-0.0.eb', '.')], 'source_tmpl': 'gzip-1.4.eb', 'source_urls': [('http://example.com', 'suffix')], }) From 0764aeb7ef4e0c874ae30ed2f2e5e13a0742dc5f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 4 Jan 2022 20:35:52 +0100 Subject: [PATCH 717/864] tweak deprecation warning for patch files that have filename not ending with .patch --- easybuild/tools/filetools.py | 3 +-- test/framework/filetools.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 6096f74847..b47d5504f7 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1494,8 +1494,7 @@ def create_patch_info(patch_spec): elif isinstance(patch_spec, string_type): if not patch_spec.endswith('.patch'): - _log.deprecated("Add '.patch' suffix to patch file, or use 2-element list/tuple to specify " - "path to where non-patch file should be copied: %s" % patch_spec, '5.0') + _log.deprecated("Use of patch file with filename that doesn't end with .patch: %s" % patch_spec, '5.0') patch_info = {'name': patch_spec} else: error_msg = "Wrong patch spec, should be string of 2-tuple with patch name + argument: %s" diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 9ed19e60f0..995b731e25 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1632,10 +1632,17 @@ def test_create_patch_info(self): self.assertEqual(ft.create_patch_info(('foo.patch', 'subdir')), {'name': 'foo.patch', 'sourcepath': 'subdir'}) self.assertEqual(ft.create_patch_info(('foo.txt', 'subdir')), {'name': 'foo.txt', 'copy': 'subdir'}) - # deprecation warning (which is an error in this context) - error_pattern = "Add '.patch' suffix to patch file, or use 2-element list/tuple to specify " - error_pattern += "path to where non-patch file should be copied: foo.txt" - self.assertErrorRegex(EasyBuildError, error_pattern, ft.create_patch_info, 'foo.txt') + self.allow_deprecated_behaviour() + self.mock_stderr(True) + self.assertEqual(ft.create_patch_info('foo.txt'), {'name': 'foo.txt'}) + stderr = self.get_stderr() + self.mock_stderr(False) + self.disallow_deprecated_behaviour() + expected_warning = "Use of patch file with filename that doesn't end with .patch: foo.txt" + self.assertTrue(expected_warning in stderr) + + # deprecation warning is treated as an error in context of unit test suite + self.assertErrorRegex(EasyBuildError, expected_warning, ft.create_patch_info, 'foo.txt') # faulty input error_msg = "Wrong patch spec" From 0185938a224723d8da07874b4ba2d8ac8e41ae7b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 10 Jan 2022 09:49:39 +0100 Subject: [PATCH 718/864] relax pattern checks in test_toy_exts_parallel (fixes #3914) --- test/framework/toy_build.py | 41 +++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 5bcaef1761..c27d9fabb2 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1832,14 +1832,20 @@ def test_toy_exts_parallel(self): args = ['--parallel-extensions-install', '--experimental', '--force', '--parallel=3'] stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) self.assertEqual(stderr, '') - expected_stdout = '\n'.join([ - "== 0 out of 4 extensions installed (2 queued, 2 running: ls, bar)", - "== 2 out of 4 extensions installed (1 queued, 1 running: barbar)", - "== 3 out of 4 extensions installed (0 queued, 1 running: toy)", - "== 4 out of 4 extensions installed (0 queued, 0 running: )", + + # take into account that each of these lines may appear multiple times, + # in case no progress was made between checks + patterns = [ + r"== 0 out of 4 extensions installed \(2 queued, 2 running: ls, bar\)$", + r"== 2 out of 4 extensions installed \(1 queued, 1 running: barbar\)$", + r"== 3 out of 4 extensions installed \(0 queued, 1 running: toy\)$", + r"== 4 out of 4 extensions installed \(0 queued, 0 running: \)$", '', - ]) - self.assertEqual(stdout, expected_stdout) + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout) + self.assertTrue(regex.search(stdout), error_msg) # also test skipping of extensions in parallel args.append('--skip') @@ -1871,15 +1877,20 @@ def test_toy_exts_parallel(self): args[-1] = '--include-easyblocks=%s' % toy_ext_eb stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) self.assertEqual(stderr, '') - expected_stdout = '\n'.join([ - "== 0 out of 4 extensions installed (3 queued, 1 running: ls)", - "== 1 out of 4 extensions installed (2 queued, 1 running: bar)", - "== 2 out of 4 extensions installed (1 queued, 1 running: barbar)", - "== 3 out of 4 extensions installed (0 queued, 1 running: toy)", - "== 4 out of 4 extensions installed (0 queued, 0 running: )", + # take into account that each of these lines may appear multiple times, + # in case no progress was made between checks + patterns = [ + r"^== 0 out of 4 extensions installed \(3 queued, 1 running: ls\)$", + r"^== 1 out of 4 extensions installed \(2 queued, 1 running: bar\)$", + r"^== 2 out of 4 extensions installed \(1 queued, 1 running: barbar\)$", + r"^== 3 out of 4 extensions installed \(0 queued, 1 running: toy\)$", + r"^== 4 out of 4 extensions installed \(0 queued, 0 running: \)$", '', - ]) - self.assertEqual(stdout, expected_stdout) + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout) + self.assertTrue(regex.search(stdout), error_msg) def test_backup_modules(self): """Test use of backing up of modules with --module-only.""" From a8bbbd7df3c08b234ba91dd6ff1e9c31962dd051 Mon Sep 17 00:00:00 2001 From: casparl Date: Tue, 11 Jan 2022 18:18:05 +0100 Subject: [PATCH 719/864] Altered the original update_build_option(key,value) so that it now returns the original value. This allows us to easily restore it. Also, added an update_build_options function that takes a dictionary and calls update_build_option for each key-value pair in the dictionary --- easybuild/tools/config.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index f12b7510e4..edffb4d7a9 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -593,6 +593,23 @@ def update_build_option(key, value): build_options._FrozenDict__dict[key] = value _log.warning("Build option '%s' was updated to: %s", key, build_option(key)) + # Return original value, so they can be restored later if needed + return orig_value + +def update_build_options(key_value_dict): + """ + Update all build options in the key_value_dict with the value given in that dictionary, + by calling update_build_option(key, value) repeatedly. + This function can be used e.g. when EasyConfig-specific build options are passed in an EasyStack file. + See https://github.com/easybuilders/easybuild-framework/issues/3513#issuecomment-986990195 + """ + orig_key_value_dict = {} + for key, value in key_value_dict.items(): + orig_key_value_dict[key] = update_build_option(key, value) + + # Return original key-value pairs in a dictionary. + # This way, they can later be restored by a single call to update_build_options(orig_key_value_dict) + return orig_key_value_dict def build_path(): """ From fc609bfd18c3d431710b51c72ccb00c0d90ad242 Mon Sep 17 00:00:00 2001 From: casparl Date: Tue, 11 Jan 2022 18:31:13 +0100 Subject: [PATCH 720/864] Forgot to actually set orig_value... Also, made the hound happy with an extra white line after the function ends. --- easybuild/tools/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index edffb4d7a9..0504af5481 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -590,12 +590,14 @@ def update_build_option(key, value): """ # BuildOptions() is a (singleton) frozen dict, so this is less straightforward that it seems... build_options = BuildOptions() + orig_value = build_options.__FrozenDict__dict[key] build_options._FrozenDict__dict[key] = value _log.warning("Build option '%s' was updated to: %s", key, build_option(key)) # Return original value, so they can be restored later if needed return orig_value + def update_build_options(key_value_dict): """ Update all build options in the key_value_dict with the value given in that dictionary, @@ -611,6 +613,7 @@ def update_build_options(key_value_dict): # This way, they can later be restored by a single call to update_build_options(orig_key_value_dict) return orig_key_value_dict + def build_path(): """ Return the build path From 4fe7af2df11e34b316598b99877b7fb4c50ddcfc Mon Sep 17 00:00:00 2001 From: casparl Date: Tue, 11 Jan 2022 18:37:09 +0100 Subject: [PATCH 721/864] Removed one underscore that was too many... --- easybuild/tools/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 0504af5481..b0c8aa5236 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -590,7 +590,7 @@ def update_build_option(key, value): """ # BuildOptions() is a (singleton) frozen dict, so this is less straightforward that it seems... build_options = BuildOptions() - orig_value = build_options.__FrozenDict__dict[key] + orig_value = build_options._FrozenDict__dict[key] build_options._FrozenDict__dict[key] = value _log.warning("Build option '%s' was updated to: %s", key, build_option(key)) From 718f472a8badda03eaa083cf559a3bd5f9c34368 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Thu, 13 Jan 2022 11:21:22 +0100 Subject: [PATCH 722/864] Update easybuild/tools/config.py Correct grammar Co-authored-by: Kenneth Hoste --- easybuild/tools/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index b0c8aa5236..d62fab6008 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -594,7 +594,7 @@ def update_build_option(key, value): build_options._FrozenDict__dict[key] = value _log.warning("Build option '%s' was updated to: %s", key, build_option(key)) - # Return original value, so they can be restored later if needed + # Return original value, so it can be restored later if needed return orig_value From c3897e8410845bef42fd70e854cceb98322c7572 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Thu, 13 Jan 2022 11:22:58 +0100 Subject: [PATCH 723/864] Update easybuild/tools/config.py Make function description more generic for maintainability Co-authored-by: Kenneth Hoste --- easybuild/tools/config.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index d62fab6008..7213e6673f 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -600,10 +600,8 @@ def update_build_option(key, value): def update_build_options(key_value_dict): """ - Update all build options in the key_value_dict with the value given in that dictionary, - by calling update_build_option(key, value) repeatedly. - This function can be used e.g. when EasyConfig-specific build options are passed in an EasyStack file. - See https://github.com/easybuilders/easybuild-framework/issues/3513#issuecomment-986990195 +Update build options as specified by the given dictionary (where keys are assumed to be build option names). +Returns dictionary with original values for the updated build options. """ orig_key_value_dict = {} for key, value in key_value_dict.items(): From 77c000bf6cc98a6d8f997529026ef21406d76cdf Mon Sep 17 00:00:00 2001 From: casparl Date: Thu, 13 Jan 2022 11:37:12 +0100 Subject: [PATCH 724/864] Updated the update_build_option test to check the return value, and added a update_build_options unit test --- test/framework/config.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/test/framework/config.py b/test/framework/config.py index cb13d348a5..213bf21c6e 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -681,8 +681,29 @@ def test_update_build_option(self): """Test updating of a build option.""" self.assertEqual(build_option('banned_linked_shared_libs'), None) - update_build_option('banned_linked_shared_libs', '/usr/lib64/libssl.so.1.1') + old_val = update_build_option('banned_linked_shared_libs', '/usr/lib64/libssl.so.1.1') self.assertEqual(build_option('banned_linked_shared_libs'), '/usr/lib64/libssl.so.1.1') + self.assertEqual(old_val, None) + + def test_update_build_options(self): + """Test updating of a dictionary of build options.""" + self.assertEqual(build_option('banned_linked_shared_libs'), None) + self.assertEqual(build_option('filter_env_vars'), None) + + new_opt_dict = { + 'banned_linked_shared_libs': '/usr/lib64/libssl.so.1.1', + 'filter_env_vars': 'LD_LIBRARY_PATH', + } + + original_opt_dict = update_build_options(new_opt_dict) + self.assertEqual(build_option('banned_linked_shared_libs'), '/usr/lib64/libssl.so.1.1') + self.assertEqual(build_option('filter_env_vars'), 'LD_LIBRARY_PATH') + + # Check the returned dictionary by simply restoring the variables and checking if the build + # options have their original values again + update_build_options(original_opt_dict) + self.assertEqual(build_option('banned_linked_shared_libs'), None) + self.assertEqual(build_option('filter_env_vars'), None) def suite(): From b5a56868229ec5ed7f06438714fe63a675f85fa9 Mon Sep 17 00:00:00 2001 From: casparl Date: Thu, 13 Jan 2022 11:41:20 +0100 Subject: [PATCH 725/864] Need to fix my vimrc... silly mixing of tabs and spaces --- test/framework/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/framework/config.py b/test/framework/config.py index 213bf21c6e..009c1531c1 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -688,20 +688,20 @@ def test_update_build_option(self): def test_update_build_options(self): """Test updating of a dictionary of build options.""" self.assertEqual(build_option('banned_linked_shared_libs'), None) - self.assertEqual(build_option('filter_env_vars'), None) + self.assertEqual(build_option('filter_env_vars'), None) new_opt_dict = { 'banned_linked_shared_libs': '/usr/lib64/libssl.so.1.1', 'filter_env_vars': 'LD_LIBRARY_PATH', } - original_opt_dict = update_build_options(new_opt_dict) + original_opt_dict = update_build_options(new_opt_dict) self.assertEqual(build_option('banned_linked_shared_libs'), '/usr/lib64/libssl.so.1.1') - self.assertEqual(build_option('filter_env_vars'), 'LD_LIBRARY_PATH') + self.assertEqual(build_option('filter_env_vars'), 'LD_LIBRARY_PATH') # Check the returned dictionary by simply restoring the variables and checking if the build # options have their original values again - update_build_options(original_opt_dict) + update_build_options(original_opt_dict) self.assertEqual(build_option('banned_linked_shared_libs'), None) self.assertEqual(build_option('filter_env_vars'), None) From 0fbe134d8f2794d89a5d75ba8d6ba2e23a715c43 Mon Sep 17 00:00:00 2001 From: casparl Date: Thu, 13 Jan 2022 11:42:33 +0100 Subject: [PATCH 726/864] Need to import update_build_options function in config.py --- test/framework/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/config.py b/test/framework/config.py index 009c1531c1..8287005490 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -40,7 +40,7 @@ from easybuild.tools import run from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, build_path, get_build_log_path, get_log_filename, get_repositorypath -from easybuild.tools.config import install_path, log_file_format, log_path, source_paths, update_build_option +from easybuild.tools.config import install_path, log_file_format, log_path, source_paths, update_build_option, update_build_options from easybuild.tools.config import BuildOptions, ConfigurationVariables from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, init_build_options from easybuild.tools.filetools import copy_dir, mkdir, write_file From 35f2d0f2ecb07778c3dddd3b6c0a6bbf1449fc76 Mon Sep 17 00:00:00 2001 From: casparl Date: Thu, 13 Jan 2022 11:43:16 +0100 Subject: [PATCH 727/864] Fixed too long line --- test/framework/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/config.py b/test/framework/config.py index 8287005490..75c9fd03f9 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -40,7 +40,8 @@ from easybuild.tools import run from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, build_path, get_build_log_path, get_log_filename, get_repositorypath -from easybuild.tools.config import install_path, log_file_format, log_path, source_paths, update_build_option, update_build_options +from easybuild.tools.config import install_path, log_file_format, log_path, source_paths +from easybuild.tools.config import update_build_option, update_build_options from easybuild.tools.config import BuildOptions, ConfigurationVariables from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, init_build_options from easybuild.tools.filetools import copy_dir, mkdir, write_file From 474a0f3321da1f6875ca770e29210a834a1d6ae6 Mon Sep 17 00:00:00 2001 From: casparl Date: Thu, 13 Jan 2022 16:09:08 +0100 Subject: [PATCH 728/864] Fixed indentations --- easybuild/tools/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 7213e6673f..36a6c981ce 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -600,8 +600,8 @@ def update_build_option(key, value): def update_build_options(key_value_dict): """ -Update build options as specified by the given dictionary (where keys are assumed to be build option names). -Returns dictionary with original values for the updated build options. + Update build options as specified by the given dictionary (where keys are assumed to be build option names). + Returns dictionary with original values for the updated build options. """ orig_key_value_dict = {} for key, value in key_value_dict.items(): From 446c8cef85efa58421a032646db5805f7a57a732 Mon Sep 17 00:00:00 2001 From: casparl Date: Thu, 13 Jan 2022 16:23:43 +0100 Subject: [PATCH 729/864] Also update config opts that are NOT None by default --- test/framework/config.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/test/framework/config.py b/test/framework/config.py index 75c9fd03f9..10dc43da19 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -680,31 +680,45 @@ def test_get_build_log_path(self): def test_update_build_option(self): """Test updating of a build option.""" + # Test if new build option is set: self.assertEqual(build_option('banned_linked_shared_libs'), None) - - old_val = update_build_option('banned_linked_shared_libs', '/usr/lib64/libssl.so.1.1') + update_build_option('banned_linked_shared_libs', '/usr/lib64/libssl.so.1.1') self.assertEqual(build_option('banned_linked_shared_libs'), '/usr/lib64/libssl.so.1.1') - self.assertEqual(old_val, None) + + # Test also if the correct old value is returned + self.assertTrue(build_option('cleanup_builddir')) + orig_cleanup_builddir = update_build_option('cleanup_builddir', False) + self.assertFalse(build_option('cleanup_builddir') + self.assertTrue(orig_cleanup_builddir) def test_update_build_options(self): """Test updating of a dictionary of build options.""" + # Check if original defaults are as expected: self.assertEqual(build_option('banned_linked_shared_libs'), None) self.assertEqual(build_option('filter_env_vars'), None) + self.assertTrue(build_option('cleanup_builddir') + self.assertEqual(build_option('pr_target_account'), 'easybuilders') + # Update build options based on dictionary new_opt_dict = { 'banned_linked_shared_libs': '/usr/lib64/libssl.so.1.1', 'filter_env_vars': 'LD_LIBRARY_PATH', + 'cleanup_builddir': False, + 'pr_target_account': 'john_doe', } - original_opt_dict = update_build_options(new_opt_dict) self.assertEqual(build_option('banned_linked_shared_libs'), '/usr/lib64/libssl.so.1.1') self.assertEqual(build_option('filter_env_vars'), 'LD_LIBRARY_PATH') + self.assertFalse(build_option('cleanup_builddir')) + self.assertEqual(build_option('pr_target_account'), 'john_doe') # Check the returned dictionary by simply restoring the variables and checking if the build # options have their original values again update_build_options(original_opt_dict) self.assertEqual(build_option('banned_linked_shared_libs'), None) self.assertEqual(build_option('filter_env_vars'), None) + self.assertTrue(build_option('cleanup_builddir')) + self.assertEqual(build_option('pr_target_account'), 'easybuilders') def suite(): From 898ef7d264868d5afc5f03ed9fbf61c8faa4b627 Mon Sep 17 00:00:00 2001 From: casparl Date: Thu, 13 Jan 2022 16:27:03 +0100 Subject: [PATCH 730/864] Added one more case (pr_target_account) to the update_build_option test --- test/framework/config.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/framework/config.py b/test/framework/config.py index 10dc43da19..d992efe0fe 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -691,6 +691,11 @@ def test_update_build_option(self): self.assertFalse(build_option('cleanup_builddir') self.assertTrue(orig_cleanup_builddir) + self.assertEqual(build_option('pr_target_account'), 'easybuilders') + orig_pr_target_account = update_build_option('pr_target_account', 'test_pr_target_account') + self.assertEqual(build_option('pr_target_account'), 'test_pr_target_account') + self.assertEqual(orig_pr_target_account, 'easybuilders') + def test_update_build_options(self): """Test updating of a dictionary of build options.""" # Check if original defaults are as expected: @@ -704,13 +709,13 @@ def test_update_build_options(self): 'banned_linked_shared_libs': '/usr/lib64/libssl.so.1.1', 'filter_env_vars': 'LD_LIBRARY_PATH', 'cleanup_builddir': False, - 'pr_target_account': 'john_doe', + 'pr_target_account': 'test_pr_target_account', } original_opt_dict = update_build_options(new_opt_dict) self.assertEqual(build_option('banned_linked_shared_libs'), '/usr/lib64/libssl.so.1.1') self.assertEqual(build_option('filter_env_vars'), 'LD_LIBRARY_PATH') self.assertFalse(build_option('cleanup_builddir')) - self.assertEqual(build_option('pr_target_account'), 'john_doe') + self.assertEqual(build_option('pr_target_account'), 'test_pr_target_account') # Check the returned dictionary by simply restoring the variables and checking if the build # options have their original values again From a11907f8b93719cd3d705cdea4ec0aaf2a25eda4 Mon Sep 17 00:00:00 2001 From: casparl Date: Thu, 13 Jan 2022 16:28:42 +0100 Subject: [PATCH 731/864] No reason to not also keep checking the return option of the first test case... --- test/framework/config.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/framework/config.py b/test/framework/config.py index d992efe0fe..5c55e34e5e 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -680,12 +680,11 @@ def test_get_build_log_path(self): def test_update_build_option(self): """Test updating of a build option.""" - # Test if new build option is set: self.assertEqual(build_option('banned_linked_shared_libs'), None) - update_build_option('banned_linked_shared_libs', '/usr/lib64/libssl.so.1.1') + orig_banned_linked_shared_libs = update_build_option('banned_linked_shared_libs', '/usr/lib64/libssl.so.1.1') self.assertEqual(build_option('banned_linked_shared_libs'), '/usr/lib64/libssl.so.1.1') + self.assertEqual(orig_banned_linked_shared_libs, None) - # Test also if the correct old value is returned self.assertTrue(build_option('cleanup_builddir')) orig_cleanup_builddir = update_build_option('cleanup_builddir', False) self.assertFalse(build_option('cleanup_builddir') From 3962879a5797389f5afabac426886ecd9c6a9230 Mon Sep 17 00:00:00 2001 From: casparl Date: Thu, 13 Jan 2022 17:00:18 +0100 Subject: [PATCH 732/864] Forgot bracket... --- test/framework/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/config.py b/test/framework/config.py index 5c55e34e5e..72024905f6 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -687,7 +687,7 @@ def test_update_build_option(self): self.assertTrue(build_option('cleanup_builddir')) orig_cleanup_builddir = update_build_option('cleanup_builddir', False) - self.assertFalse(build_option('cleanup_builddir') + self.assertFalse(build_option('cleanup_builddir')) self.assertTrue(orig_cleanup_builddir) self.assertEqual(build_option('pr_target_account'), 'easybuilders') @@ -700,7 +700,7 @@ def test_update_build_options(self): # Check if original defaults are as expected: self.assertEqual(build_option('banned_linked_shared_libs'), None) self.assertEqual(build_option('filter_env_vars'), None) - self.assertTrue(build_option('cleanup_builddir') + self.assertTrue(build_option('cleanup_builddir')) self.assertEqual(build_option('pr_target_account'), 'easybuilders') # Update build options based on dictionary From 9174cf03f54d956aa6e6de32148b45c56d4015d0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 14 Jan 2022 19:58:48 +0100 Subject: [PATCH 733/864] remove version restriction for keyring in requirements.txt to test with latest version --- .github/workflows/unit_tests.yml | 2 +- requirements.txt | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index ce18b787e6..b3f815d681 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -116,7 +116,7 @@ jobs: if [ ! -z $GITHUB_TOKEN ]; then if [ "x${{matrix.python}}" == 'x2.6' ]; then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())"; - else SET_KEYRING="import keyrings; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; + else SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; fi; python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; fi diff --git a/requirements.txt b/requirements.txt index 591fc502f3..5fac14563b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,6 @@ # keyring is required to provide GitHub token to EasyBuild; -# keyring v5.7.1 is last version to be compatible with py2.6; # for recent versions of keyring, keyrings.alt must be installed too -keyring==5.7.1; python_version < '2.7' -keyring<=9.1; python_version >= '2.7' +keyring; python_version >= '2.7' keyrings.alt; python_version >= '2.7' # GitDB 4.0.1 no longer supports Python 2.6 From 91daa9a391a5ed8fa83b1bae805fa938f7d6d410 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 14 Jan 2022 20:55:11 +0100 Subject: [PATCH 734/864] add extra ignore pattern for produced output by unit test suite --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index b3f815d681..dcec310755 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -193,7 +193,7 @@ jobs: # run test suite python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2" + IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2|from cryptography.utils import int_from_bytes" # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1) From 49bc35af98b0ef6a44ef111a418e9c2a8d3f0013 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 14 Jan 2022 22:35:49 +0100 Subject: [PATCH 735/864] also run tests with Python 3.10 --- .github/workflows/unit_tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index dcec310755..d3a577ce91 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -62,6 +62,12 @@ jobs: - python: 3.9 modules_tool: ${{needs.setup.outputs.lmod8}} module_syntax: Tcl + - python: '3.10' + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Lua + - python: '3.10' + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Tcl # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7) - python: 3.6 From 130d84964bd7e70e39cfd7074840ee52603957f3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 14 Jan 2022 22:36:42 +0100 Subject: [PATCH 736/864] import Mapping from collections.abc for Python 3 --- easybuild/base/frozendict.py | 2 +- easybuild/tools/py2vs3/py3.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/base/frozendict.py b/easybuild/base/frozendict.py index b7546b0b9d..6bfe91a82b 100644 --- a/easybuild/base/frozendict.py +++ b/easybuild/base/frozendict.py @@ -21,10 +21,10 @@ It can be used as a drop-in replacement for dictionaries where immutability is desired. """ import operator -from collections import Mapping from functools import reduce from easybuild.base import fancylogger +from easybuild.tools.py2vs3 import Mapping # minor adjustments: diff --git a/easybuild/tools/py2vs3/py3.py b/easybuild/tools/py2vs3/py3.py index ba5f46043f..56f9fbef8d 100644 --- a/easybuild/tools/py2vs3/py3.py +++ b/easybuild/tools/py2vs3/py3.py @@ -36,6 +36,7 @@ import sys import urllib.request as std_urllib # noqa from collections import OrderedDict # noqa +from collections.abc import Mapping # noqa from distutils.version import LooseVersion from functools import cmp_to_key from html.parser import HTMLParser # noqa From d14188aedeb7967a056f40eb738af2e3aef28861 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 14 Jan 2022 22:38:37 +0100 Subject: [PATCH 737/864] remove easybuild/tools/ordereddict.py, which was only needed to support Python 2.6 (which is no longer supported since EasyBuild v4.4.0) --- easybuild/tools/ordereddict.py | 276 --------------------------------- easybuild/tools/py2vs3/py2.py | 7 +- 2 files changed, 1 insertion(+), 282 deletions(-) delete mode 100644 easybuild/tools/ordereddict.py diff --git a/easybuild/tools/ordereddict.py b/easybuild/tools/ordereddict.py deleted file mode 100644 index 34b2a771fa..0000000000 --- a/easybuild/tools/ordereddict.py +++ /dev/null @@ -1,276 +0,0 @@ -# http://code.activestate.com/recipes/576693/ (r9) -# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. -# Passes Python2.7's test suite and incorporates all the latest updates. - -# Copyright (C) 2009 Raymond Hettinger - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -""" -Module provided an ordered dictionary class. - -:author: Raymond Hettinger -""" -try: - from thread import get_ident as _get_ident -except ImportError: - from dummy_thread import get_ident as _get_ident - -try: - from _abcoll import KeysView, ValuesView, ItemsView -except ImportError: - pass - - -class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as for regular dictionaries. - - # The internal self.__map dictionary maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. - - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. Signature is the same as for - regular dictionaries, but keyword arguments are not recommended - because their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link which goes at the end of the linked - # list, and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which is - # then removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, key = self.__map.pop(key) - link_prev[1] = link_next - link_next[0] = link_prev - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - root = self.__root - curr = root[1] - while curr is not root: - yield curr[2] - curr = curr[1] - - def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - root = self.__root - curr = root[0] - while curr is not root: - yield curr[2] - curr = curr[0] - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - try: - for node in self.__map.itervalues(): - del node[:] - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - except AttributeError: - pass - dict.clear(self) - - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - root = self.__root - if last: - link = root[0] - link_prev = link[0] - link_prev[1] = root - root[0] = link_prev - else: - link = root[1] - link_next = link[1] - root[1] = link_next - link_next[0] = root - key = link[2] - del self.__map[key] - value = dict.pop(self, key) - return key, value - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) items in od' - for k in self: - yield (k, self[k]) - - def update(self, *args, **kwds): - '''od.update(E, **F) -> None. Update od from dict/iterable E and F. - - If E is a dict instance, does: for k in E: od[k] = E[k] - If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] - Or if E is an iterable of items, does: for k, v in E: od[k] = v - In either case, this is followed by: for k, v in F.items(): od[k] = v - - ''' - if len(args) > 1: - raise TypeError('update() takes at most 2 positional ' - 'arguments (%d given)' % (1 + len(args),)) - # Make progressively weaker assumptions about "other" - other = () - if len(args) == 1: - other = args[0] - if isinstance(other, dict): - for key in other: - self[key] = other[key] - elif hasattr(other, 'keys'): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value - for key, value in kwds.items(): - self[key] = value - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default - - def __repr__(self, _repr_running={}): - 'od.__repr__() <==> repr(od)' - call_key = id(self), _get_ident() - if call_key in _repr_running: - return '...' - _repr_running[call_key] = 1 - try: - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - finally: - del _repr_running[call_key] - - def __reduce__(self): - 'Return state information for pickling' - items = [[k, self[k]] for k in self] - inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S - and values equal to v (which defaults to None). - - ''' - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return len(self) == len(other) and self.items() == other.items() - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other - - # -- the following methods are only used in Python 2.7 -- - - def viewkeys(self): - "od.viewkeys() -> a set-like object providing a view on od's keys" - return KeysView(self) - - def viewvalues(self): - "od.viewvalues() -> an object providing a view on od's values" - return ValuesView(self) - - def viewitems(self): - "od.viewitems() -> a set-like object providing a view on od's items" - return ItemsView(self) diff --git a/easybuild/tools/py2vs3/py2.py b/easybuild/tools/py2vs3/py2.py index 00c7382316..a1b5252ef7 100644 --- a/easybuild/tools/py2vs3/py2.py +++ b/easybuild/tools/py2vs3/py2.py @@ -34,6 +34,7 @@ import json import subprocess import urllib2 as std_urllib # noqa +from collections import Mapping, OrderedDict # noqa from HTMLParser import HTMLParser # noqa from string import letters as ascii_letters # noqa from string import lowercase as ascii_lowercase # noqa @@ -41,12 +42,6 @@ from urllib import urlencode # noqa from urllib2 import HTTPError, HTTPSHandler, Request, URLError, build_opener, urlopen # noqa -try: - # Python 2.7 - from collections import OrderedDict # noqa -except ImportError: - # only needed to keep supporting Python 2.6 - from easybuild.tools.ordereddict import OrderedDict # noqa # reload function (built-in in Python 2) reload = reload From 978a238dfc25f12eccc1614ac475b344ddca2432 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 14 Jan 2022 23:15:56 +0100 Subject: [PATCH 738/864] fix error message patterns to match those produced by Python 3.10 --- test/framework/easyconfig.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 6a11966c68..b9fd16795f 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -197,7 +197,13 @@ def test_validation(self): self.contents += "\nsyntax_error'" self.prep() - error_pattern = "Parsing easyconfig file failed: EOL while scanning string literal" + + # exact error message depends on Python version (different starting with Python 3.10) + if sys.version_info >= (3, 10): + error_pattern = "Parsing easyconfig file failed: unterminated string literal" + else: + error_pattern = "Parsing easyconfig file failed: EOL while scanning string literal" + self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, self.eb_file) # introduce "TypeError: format requires mapping" issue" @@ -3588,7 +3594,9 @@ def test_not_an_easyconfig(self): # cfr. https://github.com/easybuilders/easybuild-framework/issues/2383 not_an_ec = os.path.join(os.path.dirname(test_ecs_dir), 'sandbox', 'not_an_easyconfig.eb') - error_pattern = "Parsing easyconfig file failed: invalid syntax" + # from Python 3.10 onwards: invalid decimal literal + # older Python versions: invalid syntax + error_pattern = "Parsing easyconfig file failed: invalid" self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, not_an_ec) def test_check_sha256_checksums(self): From 9d676691e6ed1f1887b3b35ff75d18eb99e340f8 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Sat, 15 Jan 2022 08:28:34 +0000 Subject: [PATCH 739/864] simplify requirements.txt by removing python 2.6 support --- requirements.txt | 50 +++++++++++++----------------------------------- 1 file changed, 13 insertions(+), 37 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5fac14563b..69f2933a49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,51 +1,28 @@ # keyring is required to provide GitHub token to EasyBuild; # for recent versions of keyring, keyrings.alt must be installed too -keyring; python_version >= '2.7' -keyrings.alt; python_version >= '2.7' +keyring +keyrings.alt -# GitDB 4.0.1 no longer supports Python 2.6 -gitdb==0.6.4; python_version < '2.7' -gitdb; python_version >= '2.7' +# GitDB +gitdb # GitPython 3.1.15 deprecates Python 3.5 GitPython==3.1.14; python_version >= '3.0' and python_version < '3.6' -# GitPython 2.1.9 no longer supports Python 2.6 -GitPython==2.1.8; python_version < '2.7' -GitPython; python_version >= '3.6' or (python_version <= '3.0' and python_version >= '2.7') +GitPython; python_version >= '3.6' or python_version <= '3.0' +# pydot +pydot -# pydot (dep for python-graph-dot) 1.2.0 and more recent doesn't work with Python 2.6 -pydot==1.1.0; python_version < '2.7' -pydot; python_version >= '2.7' +# autopep8 +autopep8 -# pycparser 2.19 (dep for paramiko) doesn't work with Python 2.6 -pycparser<2.19; python_version < '2.7' - -# idna 2.8 (dep for paramiko) & more recent doesn't work with Python 2.6 -idna<2.8; python_version < '2.7' - -# paramiko 2.4.0 (dep for GC3Pie) & more recent doesn't work with Python 2.6 -paramiko<2.4.0; python_version < '2.7' - -# SQLAlchemy 1.2.0 (dep for GC3Pie) & more recent doesn't work with Python 2.6 -SQLAlchemy<1.2.0; python_version < '2.7' - -# python 2.0 (dep for GC3Pie) & more recent doesn't work with Python 2.6 -python-daemon<2.0; python_version < '2.7' - -# autopep8 1.3.4 is last one to support Python 2.6 -autopep8<1.3.5; python_version < '2.7' -autopep8; python_version >= '2.7' - -# PyYAML 5.x no longer supports Python 2.6 -PyYAML<5.0; python_version < '2.7' -PyYAML; python_version >= '2.7' +# PyYAML +PyYAML # optional Python packages for EasyBuild # flake8 is a superset of pycodestyle -pycodestyle; python_version < '2.7' -flake8; python_version >= '2.7' +flake8 # 2.6.7 uses invalid Python 2 syntax GC3Pie!=2.6.7; python_version < '3.0' @@ -54,9 +31,8 @@ python-graph-dot python-hglib requests -archspec; python_version >= '2.7' +archspec -# cryptography is not needed at all for Python 2.6 # cryptography 3.4.0 no longer supports Python 2.7 cryptography==3.3.2; python_version == '2.7' cryptography; python_version >= '3.5' From a920e858e1ca34f41399acb7e7ca6b0cf55b31fa Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Sat, 15 Jan 2022 09:19:03 +0000 Subject: [PATCH 740/864] separate listing of pydot no longer required --- requirements.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 69f2933a49..9c405e73cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,9 +10,6 @@ gitdb GitPython==3.1.14; python_version >= '3.0' and python_version < '3.6' GitPython; python_version >= '3.6' or python_version <= '3.0' -# pydot -pydot - # autopep8 autopep8 From fbcc462e73216ddcc95c66eeab910d672f9761c6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 Jan 2022 11:13:26 +0100 Subject: [PATCH 741/864] use sources.easybuild.io as backup URL for downloading Environment Modules sources to run test suite --- easybuild/scripts/install_eb_dep.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/scripts/install_eb_dep.sh b/easybuild/scripts/install_eb_dep.sh index 82c34b774d..484cbe9371 100755 --- a/easybuild/scripts/install_eb_dep.sh +++ b/easybuild/scripts/install_eb_dep.sh @@ -18,12 +18,13 @@ PRECONFIG_CMD= if [ "$PKG_NAME" == 'modules' ] && [ "$PKG_VERSION" == '3.2.10' ]; then PKG_URL="http://prdownloads.sourceforge.net/modules/${PKG}.tar.gz" - BACKUP_PKG_URL="https://easybuilders.github.io/easybuild/files/${PKG}.tar.gz" + BACKUP_PKG_URL="https://sources.easybuild.io/e/EnvironmentModules/${PKG}.tar.gz" export PATH="$PREFIX/Modules/$PKG_VERSION/bin:$PATH" export MOD_INIT="$PREFIX/Modules/$PKG_VERSION/init/bash" elif [ "$PKG_NAME" == 'modules' ]; then PKG_URL="http://prdownloads.sourceforge.net/modules/${PKG}.tar.gz" + BACKUP_PKG_URL="https://sources.easybuild.io/e/EnvironmentModules/${PKG}.tar.gz" export PATH="$PREFIX/bin:$PATH" export MOD_INIT="$PREFIX/init/bash" From 9658e4c4ab323a10f36f83bfeef27ecda2999743 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 Jan 2022 11:29:47 +0100 Subject: [PATCH 742/864] clean up code block for Environment Modules versions in script to install EB dependencies --- easybuild/scripts/install_eb_dep.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/easybuild/scripts/install_eb_dep.sh b/easybuild/scripts/install_eb_dep.sh index 484cbe9371..1de3a1712e 100755 --- a/easybuild/scripts/install_eb_dep.sh +++ b/easybuild/scripts/install_eb_dep.sh @@ -16,17 +16,16 @@ PKG_VERSION="${PKG##*-}" CONFIG_OPTIONS= PRECONFIG_CMD= -if [ "$PKG_NAME" == 'modules' ] && [ "$PKG_VERSION" == '3.2.10' ]; then +if [ "$PKG_NAME" == 'modules' ]; then PKG_URL="http://prdownloads.sourceforge.net/modules/${PKG}.tar.gz" BACKUP_PKG_URL="https://sources.easybuild.io/e/EnvironmentModules/${PKG}.tar.gz" - export PATH="$PREFIX/Modules/$PKG_VERSION/bin:$PATH" - export MOD_INIT="$PREFIX/Modules/$PKG_VERSION/init/bash" - -elif [ "$PKG_NAME" == 'modules' ]; then - PKG_URL="http://prdownloads.sourceforge.net/modules/${PKG}.tar.gz" - BACKUP_PKG_URL="https://sources.easybuild.io/e/EnvironmentModules/${PKG}.tar.gz" - export PATH="$PREFIX/bin:$PATH" - export MOD_INIT="$PREFIX/init/bash" + if [ "$PKG_VERSION" == '3.2.10' ]; then + export PATH="$PREFIX/Modules/$PKG_VERSION/bin:$PATH" + export MOD_INIT="$PREFIX/Modules/$PKG_VERSION/init/bash" + else + export PATH="$PREFIX/bin:$PATH" + export MOD_INIT="$PREFIX/init/bash" + fi elif [ "$PKG_NAME" == 'lua' ]; then PKG_URL="http://downloads.sourceforge.net/project/lmod/${PKG}.tar.gz" From c9988dda5b7ad17d982f68633b938c24acc735f0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 Jan 2022 13:18:45 +0100 Subject: [PATCH 743/864] no need to explicitely list GitDB dependency (required by GitPython) --- requirements.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9c405e73cb..cc7c9b4fb7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,9 +3,6 @@ keyring keyrings.alt -# GitDB -gitdb - # GitPython 3.1.15 deprecates Python 3.5 GitPython==3.1.14; python_version >= '3.0' and python_version < '3.6' GitPython; python_version >= '3.6' or python_version <= '3.0' From 08bbdef0170e55fec49af17b772f5ce1b76fc851 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 18 Jan 2022 11:52:09 +0100 Subject: [PATCH 744/864] enhance get_cpu_architecture and get_cpu_family to be aware of RISC-V --- easybuild/tools/systemtools.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 6783b49eaa..17a2afc4b3 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -86,6 +86,8 @@ AARCH64 = 'AArch64' POWER = 'POWER' X86_64 = 'x86_64' +RISCV32 = 'RISC-V-32' +RISCV64 = 'RISC-V-64' ARCH_KEY_PREFIX = 'arch=' @@ -106,6 +108,7 @@ # Family constants POWER_LE = 'POWER little-endian' +RISCV = 'RISC-V' # OS constants LINUX = 'Linux' @@ -117,8 +120,8 @@ PROC_CPUINFO_FP = '/proc/cpuinfo' PROC_MEMINFO_FP = '/proc/meminfo' -CPU_ARCHITECTURES = [AARCH32, AARCH64, POWER, X86_64] -CPU_FAMILIES = [AMD, ARM, INTEL, POWER, POWER_LE] +CPU_ARCHITECTURES = [AARCH32, AARCH64, POWER, RISCV32, RISCV64, X86_64] +CPU_FAMILIES = [AMD, ARM, INTEL, POWER, POWER_LE, RISCV] CPU_VENDORS = [AMD, APM, ARM, BROADCOM, CAVIUM, DEC, IBM, INTEL, MARVELL, MOTOROLA, NVIDIA, QUALCOMM] # ARM implementer IDs (i.e., the hexadeximal keys) taken from ARMv8-A Architecture Reference Manual # (ARM DDI 0487A.j, Section G6.2.102, Page G6-4493) @@ -312,9 +315,11 @@ def get_cpu_architecture(): :return: a value from the CPU_ARCHITECTURES list """ - power_regex = re.compile("ppc64.*") - aarch64_regex = re.compile("aarch64.*") aarch32_regex = re.compile("arm.*") + aarch64_regex = re.compile("aarch64.*") + power_regex = re.compile("ppc64.*") + riscv32_regex = re.compile("riscv32.*") + riscv64_regex = re.compile("riscv64.*") system, node, release, version, machine, processor = platform.uname() @@ -327,6 +332,10 @@ def get_cpu_architecture(): arch = AARCH64 elif aarch32_regex.match(machine): arch = AARCH32 + elif riscv64_regex.match(machine): + arch = RISCV64 + elif riscv32_regex.match(machine): + arch = RISCV32 if arch == UNKNOWN: _log.warning("Failed to determine CPU architecture, returning %s", arch) @@ -410,6 +419,9 @@ def get_cpu_family(): if powerle_regex.search(machine): family = POWER_LE + elif arch in [RISCV32, RISCV64]: + family = RISCV + if family is None: family = UNKNOWN _log.warning("Failed to determine CPU family, returning %s" % family) From 90c010121281c6c38296f3b5ea0e8ab73cc06b8c Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Tue, 18 Jan 2022 12:00:20 +0100 Subject: [PATCH 745/864] Also allow intelfftw on GCCcore --- easybuild/toolchains/fft/intelfftw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py index dca03a0476..af44be187f 100644 --- a/easybuild/toolchains/fft/intelfftw.py +++ b/easybuild/toolchains/fft/intelfftw.py @@ -62,7 +62,7 @@ def _set_fftw_variables(self): compsuff = '_intel' elif get_software_root('PGI'): compsuff = '_pgi' - elif get_software_root('GCC'): + elif get_software_root('GCC') or get_software_root('GCCcore'): compsuff = '_gnu' else: error_msg = "Not using Intel compilers, PGI nor GCC, don't know compiler suffix for FFTW libraries." From 18901728ba78091772eec8096dcef5a6e9aa840b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 18 Jan 2022 12:09:04 +0100 Subject: [PATCH 746/864] also take /etc/os-release into account in get_os_name and get_os_version --- easybuild/tools/systemtools.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 6783b49eaa..ce7a41005c 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -113,6 +113,7 @@ UNKNOWN = 'UNKNOWN' +ETC_OS_RELEASE = '/etc/os-release' MAX_FREQ_FP = '/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq' PROC_CPUINFO_FP = '/proc/cpuinfo' PROC_MEMINFO_FP = '/proc/meminfo' @@ -669,11 +670,17 @@ def get_os_name(): if hasattr(platform, 'linux_distribution'): # platform.linux_distribution is more useful, but only available since Python 2.6 # this allows to differentiate between Fedora, CentOS, RHEL and Scientific Linux (Rocks is just CentOS) - os_name = platform.linux_distribution()[0].strip().lower() + os_name = platform.linux_distribution()[0].strip() elif HAVE_DISTRO: # distro package is the recommended alternative to platform.linux_distribution, # see https://pypi.org/project/distro os_name = distro.name() + elif os.path.exists(ETC_OS_RELEASE): + os_release_txt = read_file(ETC_OS_RELEASE) + name_regex = re.compile('^NAME="?(?P[^"\n]+)"?$', re.M) + res = name_regex.search(os_release_txt) + if res: + os_name = res.group('name') else: # no easy way to determine name of Linux distribution os_name = None @@ -687,7 +694,7 @@ def get_os_name(): } if os_name: - return os_name_map.get(os_name, os_name) + return os_name_map.get(os_name.lower(), os_name) else: return UNKNOWN @@ -700,6 +707,19 @@ def get_os_version(): os_version = platform.dist()[1] elif HAVE_DISTRO: os_version = distro.version() + elif os.path.exists(ETC_OS_RELEASE): + os_release_txt = read_file(ETC_OS_RELEASE) + version_regex = re.compile('^VERSION="?(?P[^"\n]+)"?$', re.M) + res = version_regex.search(os_release_txt) + if res: + os_version = res.group('version') + else: + # VERSION may not always be defined (for example on Gentoo), + # fall back to VERSION_ID in that case + version_regex = re.compile('^VERSION_ID="?(?P[^"\n]+)"?$', re.M) + res = version_regex.search(os_release_txt) + if res: + os_version = res.group('version') else: os_version = None From 82cfcc84170ad11f0b7b32727f144bb1ef919ad7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 18 Jan 2022 14:09:02 +0100 Subject: [PATCH 747/864] take into account that platform.dist* may return empty strings as OS name/version --- easybuild/tools/systemtools.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index ce7a41005c..fcddb6d9f0 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -671,11 +671,15 @@ def get_os_name(): # platform.linux_distribution is more useful, but only available since Python 2.6 # this allows to differentiate between Fedora, CentOS, RHEL and Scientific Linux (Rocks is just CentOS) os_name = platform.linux_distribution()[0].strip() - elif HAVE_DISTRO: + + # take into account that on some OSs, platform.distribution returns an empty string as OS name, + # for example on OpenSUSE Leap 15.2 + if not os_name and HAVE_DISTRO: # distro package is the recommended alternative to platform.linux_distribution, # see https://pypi.org/project/distro os_name = distro.name() - elif os.path.exists(ETC_OS_RELEASE): + + if not os_name and os.path.exists(ETC_OS_RELEASE): os_release_txt = read_file(ETC_OS_RELEASE) name_regex = re.compile('^NAME="?(?P[^"\n]+)"?$', re.M) res = name_regex.search(os_release_txt) @@ -705,9 +709,13 @@ def get_os_version(): # platform.dist was removed in Python 3.8 if hasattr(platform, 'dist'): os_version = platform.dist()[1] - elif HAVE_DISTRO: + + # take into account that on some OSs, platform.dist returns an empty string as OS version, + # for example on OpenSUSE Leap 15.2 + if not os_version and HAVE_DISTRO: os_version = distro.version() - elif os.path.exists(ETC_OS_RELEASE): + + if not os_version and os.path.exists(ETC_OS_RELEASE): os_release_txt = read_file(ETC_OS_RELEASE) version_regex = re.compile('^VERSION="?(?P[^"\n]+)"?$', re.M) res = version_regex.search(os_release_txt) From 17a17a94ac0c3188e3aaedd970c7ce4111c35d41 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 18 Jan 2022 16:45:50 +0100 Subject: [PATCH 748/864] always initialize os_version in get_os_version Co-authored-by: SebastianAchilles --- easybuild/tools/systemtools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index fcddb6d9f0..a31dc471f7 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -706,6 +706,8 @@ def get_os_name(): def get_os_version(): """Determine system version.""" + os_version = None + # platform.dist was removed in Python 3.8 if hasattr(platform, 'dist'): os_version = platform.dist()[1] From ba7d509fa6868e57c33b8282c2421e89632c63ea Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 18 Jan 2022 16:47:42 +0100 Subject: [PATCH 749/864] also run tests for 'eb' command with Python 3.10 --- .github/workflows/eb_command.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml index d0b72e3079..baed42182e 100644 --- a/.github/workflows/eb_command.yml +++ b/.github/workflows/eb_command.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-18.04 strategy: matrix: - python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] + python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10'] fail-fast: false steps: - uses: actions/checkout@v2 From a658ede07bb9f1316bd402045c69dd0ce5bb66f5 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Sat, 22 Jan 2022 12:20:14 +0800 Subject: [PATCH 750/864] prepare release notes for EasyBuild v4.5.2 + bump version to 4.5.2 --- RELEASE_NOTES | 22 ++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index a09c9f6859..2fce78b6e4 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,28 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.5.2 (January 24th 2022) +-------------------------- + +update/bugfix release + +- various enhancements, including: + - automatically prepend env-for-shebang value with sysroot (#3919) + - add support for update_build_options + enhance update_build_options to return original value (#3923) + - fix compatibility with Python 3.10 (#3926) + - make intelfftw toolchain component aware of GCCcore (#3929) + - also take /etc/os-release into account in get_os_name and get_os_version (#3930) + - enhance get_cpu_architecture and get_cpu_family to be aware of RISC-V (#3931) + - also run tests for 'eb' command with Python 3.10 (#3933) +- various bug fixes, including: + - relax pattern checks in test_toy_exts_parallel (#3921) + - use sources.easybuild.io as backup URL for downloading Environment Modules sources to run test suite (#3928) +- other changes: + - deprecate use of patch files that don't have a filename ending with .patch (#3920) + - remove version restriction for keyring in requirements.txt to test with latest version (#3925) + - simplify requirements.txt by removing python 2.6 support (#3927) + + v4.5.1 (December 13th 2021) --------------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 21bcecc715..d13e587d12 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.5.2.dev0') +VERSION = LooseVersion('4.5.2') UNKNOWN = 'UNKNOWN' From 22a0394f6f7a5af69d064ceb6a88369e02032203 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 23 Jan 2022 15:37:04 +0100 Subject: [PATCH 751/864] minor tweak release notes for v4.5.2 --- RELEASE_NOTES | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 2fce78b6e4..14980d7d43 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -10,20 +10,19 @@ v4.5.2 (January 24th 2022) update/bugfix release - various enhancements, including: - - automatically prepend env-for-shebang value with sysroot (#3919) - - add support for update_build_options + enhance update_build_options to return original value (#3923) - - fix compatibility with Python 3.10 (#3926) - - make intelfftw toolchain component aware of GCCcore (#3929) + - automatically prepend value for env-for-shebang configuration setting with sysroot (#3919) + - enhance update_build_option to return original value + add support for update_build_options (#3923) + - fix compatibility with Python 3.10 (#3926, #3933) - also take /etc/os-release into account in get_os_name and get_os_version (#3930) - enhance get_cpu_architecture and get_cpu_family to be aware of RISC-V (#3931) - - also run tests for 'eb' command with Python 3.10 (#3933) - various bug fixes, including: - relax pattern checks in test_toy_exts_parallel (#3921) - use sources.easybuild.io as backup URL for downloading Environment Modules sources to run test suite (#3928) + - make intelfftw toolchain component aware of GCCcore (#3929) - other changes: - - deprecate use of patch files that don't have a filename ending with .patch (#3920) + - deprecate use of (actual) patch files that don't have a filename ending with .patch (#3920) - remove version restriction for keyring in requirements.txt to test with latest version (#3925) - - simplify requirements.txt by removing python 2.6 support (#3927) + - simplify requirements.txt by removing Python 2.6 support (#3927) v4.5.1 (December 13th 2021) From 1b303db5f6dddfc5aa2122a32189d6202633d207 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 23 Jan 2022 15:58:37 +0100 Subject: [PATCH 752/864] take into account that patch files can also be zipped when checking filename extension for patches --- easybuild/tools/filetools.py | 11 ++++++++--- test/framework/filetools.py | 9 ++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index b47d5504f7..9f8ead13a3 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -163,6 +163,8 @@ '.sh': "cp -a %(filepath)s .", } +ZIPPED_PATCH_EXTS = ('.bz2', '.gz', '.xz') + # global set of names of locks that were created in this session global_lock_names = set() @@ -1493,8 +1495,11 @@ def create_patch_info(patch_spec): str(patch_spec)) elif isinstance(patch_spec, string_type): - if not patch_spec.endswith('.patch'): - _log.deprecated("Use of patch file with filename that doesn't end with .patch: %s" % patch_spec, '5.0') + allowed_patch_exts = ['.patch' + x for x in ('',) + ZIPPED_PATCH_EXTS] + if not any(patch_spec.endswith(x) for x in allowed_patch_exts): + msg = "Use of patch file with filename that doesn't end with correct extension: %s " % patch_spec + msg += "(should be any of: %s)" % (', '.join(allowed_patch_exts)) + _log.deprecated(msg, '5.0') patch_info = {'name': patch_spec} else: error_msg = "Wrong patch spec, should be string of 2-tuple with patch name + argument: %s" @@ -1548,7 +1553,7 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=Fa # split in stem (filename w/o extension) + extension patch_stem, patch_extension = os.path.splitext(os.path.split(abs_patch_file)[1]) # Supports only bz2, gz and xz. zip can be archives which are not supported. - if patch_extension in ['.gz', '.bz2', '.xz']: + if patch_extension in ZIPPED_PATCH_EXTS: # split again to get the second extension patch_subextension = os.path.splitext(patch_stem)[1] if patch_subextension == ".patch": diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 995b731e25..da9a19fa89 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1638,11 +1638,14 @@ def test_create_patch_info(self): stderr = self.get_stderr() self.mock_stderr(False) self.disallow_deprecated_behaviour() - expected_warning = "Use of patch file with filename that doesn't end with .patch: foo.txt" - self.assertTrue(expected_warning in stderr) + expected_warning = "Use of patch file with filename that doesn't end with correct extension: foo.txt " + expected_warning += "(should be any of: .patch, .patch.bz2, .patch.gz, .patch.xz)" + fail_msg = "Warning '%s' should appear in stderr output: %s" % (expected_warning, stderr) + self.assertTrue(expected_warning in stderr, fail_msg) # deprecation warning is treated as an error in context of unit test suite - self.assertErrorRegex(EasyBuildError, expected_warning, ft.create_patch_info, 'foo.txt') + expected_error = expected_warning.replace('(', '\\(').replace(')', '\\)') + self.assertErrorRegex(EasyBuildError, expected_error, ft.create_patch_info, 'foo.txt') # faulty input error_msg = "Wrong patch spec" From 9888eb9e6e9e4d48f33b6170a0e69de478a6ac71 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 23 Jan 2022 21:26:00 +0100 Subject: [PATCH 753/864] only run GitHub tests when testing with Lua module syntax, to avoid hitting GitHub rate limit when running tests --- .github/workflows/unit_tests.yml | 3 ++- RELEASE_NOTES | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index d3a577ce91..dc748edbab 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -116,9 +116,10 @@ jobs: GITHUB_TOKEN: ${{secrets.TEST_GITHUB_TOKEN}} run: | # don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools, + # and only when testing with Lua as module syntax, # to avoid hitting GitHub rate limit; # tests that require a GitHub token are skipped automatically when no GitHub token is available - if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]]; then + if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]] && [[ "${{matrix.modules_syntax}}" == 'Lua' ]]; then if [ ! -z $GITHUB_TOKEN ]; then if [ "x${{matrix.python}}" == 'x2.6' ]; then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())"; diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 14980d7d43..65a91ff1bb 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -23,6 +23,7 @@ update/bugfix release - deprecate use of (actual) patch files that don't have a filename ending with .patch (#3920) - remove version restriction for keyring in requirements.txt to test with latest version (#3925) - simplify requirements.txt by removing Python 2.6 support (#3927) + - only run GitHub tests when testing with Lua module syntax, to avoid hitting GitHub rate limit when running tests (#3938) v4.5.1 (December 13th 2021) From e7afc7a4d5f2f2e78fd9cfaba95fa611c4d0980c Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 24 Jan 2022 11:25:31 +0800 Subject: [PATCH 754/864] initialize BACKUP_PKG_URL with empty string --- easybuild/scripts/install_eb_dep.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/scripts/install_eb_dep.sh b/easybuild/scripts/install_eb_dep.sh index 1de3a1712e..0b6fb49d11 100755 --- a/easybuild/scripts/install_eb_dep.sh +++ b/easybuild/scripts/install_eb_dep.sh @@ -16,6 +16,8 @@ PKG_VERSION="${PKG##*-}" CONFIG_OPTIONS= PRECONFIG_CMD= +BACKUP_PKG_URL= + if [ "$PKG_NAME" == 'modules' ]; then PKG_URL="http://prdownloads.sourceforge.net/modules/${PKG}.tar.gz" BACKUP_PKG_URL="https://sources.easybuild.io/e/EnvironmentModules/${PKG}.tar.gz" From 79de23a80af0324e576f94d3680938b49e13118d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 24 Jan 2022 07:30:48 +0100 Subject: [PATCH 755/864] indicate compatibility with Python 3.10 in setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 4b2d936ec6..e9c42436b5 100644 --- a/setup.py +++ b/setup.py @@ -116,6 +116,7 @@ def find_rel_test(): "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Topic :: Software Development :: Build Tools", ], platforms="Linux", From 4824ef98dc4e0d653e663da42f6dd03241be973d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 24 Jan 2022 07:31:19 +0100 Subject: [PATCH 756/864] bump version to 4.5.3dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index d13e587d12..878e7b5b2b 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.5.2') +VERSION = LooseVersion('4.5.3.dev0') UNKNOWN = 'UNKNOWN' From 0a53b869f9fd632f007e518248491c261a690248 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Tue, 25 Jan 2022 22:03:44 +0100 Subject: [PATCH 757/864] Fix get_os_name and get_os_version --- easybuild/tools/systemtools.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 1e534cb101..b78f8319c5 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -697,9 +697,6 @@ def get_os_name(): res = name_regex.search(os_release_txt) if res: os_name = res.group('name') - else: - # no easy way to determine name of Linux distribution - os_name = None os_name_map = { 'red hat enterprise linux server': 'RHEL', @@ -742,8 +739,6 @@ def get_os_version(): res = version_regex.search(os_release_txt) if res: os_version = res.group('version') - else: - os_version = None if os_version: if get_os_name() in ["suse", "SLES"]: From ed3a42204770ff8d2011c181b5fdadb290edf706 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 27 Jan 2022 10:28:53 +0100 Subject: [PATCH 758/864] consistently use actions/setup-python@v2 in CI workflows --- .github/workflows/linting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 55ada2170c..cf7c666470 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -7,7 +7,7 @@ jobs: - uses: actions/checkout@v2 - name: set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: 3.8 From 1d189b95d1850c59cde02c53be0e368adc898222 Mon Sep 17 00:00:00 2001 From: scimerman Date: Thu, 27 Jan 2022 15:52:30 +0100 Subject: [PATCH 759/864] Python 2 does not work, version 3 does --- easybuild/tools/containers/singularity.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/containers/singularity.py b/easybuild/tools/containers/singularity.py index 98c416414c..dab55b525f 100644 --- a/easybuild/tools/containers/singularity.py +++ b/easybuild/tools/containers/singularity.py @@ -272,9 +272,9 @@ def resolve_template_data(self): # EPEL is required for installing Lmod & python-pip 'epel-release', # EasyBuild requirements - 'python setuptools Lmod', - # pip is used to install EasyBuild packages - 'python-pip', + 'python3 setuptools Lmod', + # pip3 is used to install EasyBuild packages + 'python3-pip', # useful utilities 'bzip2 gzip tar zip unzip xz', # extracting sources 'curl wget', # downloading @@ -308,13 +308,13 @@ def resolve_template_data(self): template_data['install_os_deps'] = '\n'.join(install_os_deps) # install (latest) EasyBuild in container image - # use 'pip install', unless custom commands are specified via 'install_eb' keyword + # use 'pip3 install', unless custom commands are specified via 'install_eb' keyword if 'install_eb' not in template_data: template_data['install_eb'] = '\n'.join([ - "# install EasyBuild using pip", + "# install EasyBuild using pip3", # upgrade pip - "pip install -U pip", - "pip install easybuild", + "pip3 install -U pip", + "pip3 install easybuild", ]) # if no custom value is specified for 'post_commands' keyword, From 021d8f38bcbfdac3691cacff3c2c1960340a3dbc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 29 Jan 2022 19:32:56 +0100 Subject: [PATCH 760/864] specify easybuild.io as EasyBuild homepage in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4b2d936ec6..bca6cc85f9 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ def find_rel_test(): implement support for installing particular (groups of) software packages.""", license="GPLv2", keywords="software build building installation installing compilation HPC scientific", - url="https://easybuilders.github.io/easybuild", + url="https://easybuild.io", packages=easybuild_packages, package_dir={'test.framework': 'test/framework'}, package_data={'test.framework': find_rel_test()}, From 6296b35e70c8c13067aec54f8494aa0ed0aeda7f Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Mon, 31 Jan 2022 12:44:36 +0100 Subject: [PATCH 761/864] Add a "clone_into" field to git_config source specification. There is at least one software (avogadrolibs) that assumes that the directory the repo is cloned into has a different name than the upstream repo. --- easybuild/tools/filetools.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index b47d5504f7..c53e1acf48 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2581,6 +2581,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): repo_name = git_config.pop('repo_name', None) commit = git_config.pop('commit', None) recursive = git_config.pop('recursive', False) + clone_into = git_config.pop('clone_into', False) keep_git_dir = git_config.pop('keep_git_dir', False) # input validation of git_config dict @@ -2624,10 +2625,17 @@ def get_source_tarball_from_git(filename, targetdir, git_config): clone_cmd.append('%s/%s.git' % (url, repo_name)) + if clone_into: + clone_cmd.append('%s' % clone_into) + tmpdir = tempfile.mkdtemp() cwd = change_dir(tmpdir) run.run_cmd(' '.join(clone_cmd), log_all=True, simple=True, regexp=False) + # If the clone is done into a specified name, change repo_name + if clone_into: + repo_name = clone_into + # if a specific commit is asked for, check it out if commit: checkout_cmd = ['git', 'checkout', commit] From 946454f1e7280aaf0bb55b86f0a5c618580569c6 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 4 Feb 2022 11:42:56 +0100 Subject: [PATCH 762/864] Update containers.py --- test/framework/containers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/containers.py b/test/framework/containers.py index da7b5c32d3..e73ae7d558 100644 --- a/test/framework/containers.py +++ b/test/framework/containers.py @@ -141,8 +141,8 @@ def test_end2end_singularity_recipe_config(self): self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) pip_patterns = [ - # EasyBuild is installed with pip by default - "pip install easybuild", + # EasyBuild is installed with pip3 by default + "pip3 install easybuild", ] post_commands_patterns = [ # easybuild user is added if it doesn't exist yet From 30ac78d32099e38b2fd7d7ec71637c9a89b22c54 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 4 Feb 2022 11:49:54 +0100 Subject: [PATCH 763/864] Update default Ubuntu image --- easybuild/tools/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 36a6c981ce..abcfb155be 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -134,7 +134,7 @@ JOB_DEPS_TYPE_ABORT_ON_ERROR = 'abort_on_error' JOB_DEPS_TYPE_ALWAYS_RUN = 'always_run' -DOCKER_BASE_IMAGE_UBUNTU = 'ubuntu:16.04' +DOCKER_BASE_IMAGE_UBUNTU = 'ubuntu:20.04' DOCKER_BASE_IMAGE_CENTOS = 'centos:7' LOCAL_VAR_NAMING_CHECK_ERROR = 'error' From 0f3f29d38ffb43197285b193fd9d4f0fbf2dc8d0 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 4 Feb 2022 13:48:17 +0100 Subject: [PATCH 764/864] Also update docker container recipes --- easybuild/tools/containers/docker.py | 20 +++++++++++++------- test/framework/containers.py | 6 +++--- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/containers/docker.py b/easybuild/tools/containers/docker.py index 104a96ec86..21e6d46c84 100644 --- a/easybuild/tools/containers/docker.py +++ b/easybuild/tools/containers/docker.py @@ -44,9 +44,9 @@ """ DOCKER_INSTALL_EASYBUILD = """\ -RUN pip install -U pip setuptools && \\ - hash -r pip && \\ - pip install -U easybuild +RUN pip3 install -U pip setuptools && \\ + hash -r pip3&& \\ + pip3 install -U easybuild RUN mkdir /app && \\ mkdir /scratch && \\ @@ -72,9 +72,12 @@ CMD ["/bin/bash", "-l"] """ -DOCKER_UBUNTU1604_INSTALL_DEPS = """\ +DOCKER_UBUNTU2004_INSTALL_DEPS = """\ RUN apt-get update && \\ - apt-get install -y python python-pip lmod curl wget + apt-get install -y python3 python3-pip lmod curl wget git \\ + bzip2 gzip tar zip unzip xz-utils \\ + patch automake git debianutils \\ + g++ libdata-dump-perl libthread-queue-any-perl RUN OS_DEPS='%(os_deps)s' && \\ test -n "${OS_DEPS}" && \\ @@ -83,7 +86,10 @@ DOCKER_CENTOS7_INSTALL_DEPS = """\ RUN yum install -y epel-release && \\ - yum install -y python python-pip Lmod curl wget git + yum install -y python3 python3-pip Lmod curl wget git \\ + bzip2 gzip tar zip unzip xz \\ + patch makefile git which \\ + gcc-c++ perl-Data-Dumper perl-Thread-Queue RUN OS_DEPS='%(os_deps)s' && \\ test -n "${OS_DEPS}" && \\ @@ -91,7 +97,7 @@ """ DOCKER_OS_INSTALL_DEPS_TMPLS = { - DOCKER_BASE_IMAGE_UBUNTU: DOCKER_UBUNTU1604_INSTALL_DEPS, + DOCKER_BASE_IMAGE_UBUNTU: DOCKER_UBUNTU2004_INSTALL_DEPS, DOCKER_BASE_IMAGE_CENTOS: DOCKER_CENTOS7_INSTALL_DEPS, } diff --git a/test/framework/containers.py b/test/framework/containers.py index e73ae7d558..b513517db5 100644 --- a/test/framework/containers.py +++ b/test/framework/containers.py @@ -386,7 +386,7 @@ def test_end2end_dockerfile(self): base_args + ['--container-config=not-supported'], raise_error=True) - for cont_base in ['ubuntu:16.04', 'centos:7']: + for cont_base in ['ubuntu:20.04', 'centos:7']: stdout, stderr = self.run_main(base_args + ['--container-config=%s' % cont_base]) self.assertFalse(stderr) regexs = ["^== Dockerfile definition file created at %s/containers/Dockerfile.toy-0.0" % self.test_prefix] @@ -406,10 +406,10 @@ def test_end2end_dockerfile(self): remove_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0')) base_args.insert(1, os.path.join(test_ecs, 'g', 'GCC', 'GCC-4.9.2.eb')) - self.run_main(base_args + ['--container-config=ubuntu:16.04']) + self.run_main(base_args + ['--container-config=ubuntu:20.04']) def_file = read_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0')) regexs = [ - "FROM ubuntu:16.04", + "FROM ubuntu:20.04", "eb toy-0.0.eb GCC-4.9.2.eb", "module load toy/0.0 GCC/4.9.2", ] From 2090d5ba8aa0e66557b710f56e6ea812365940fb Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 4 Feb 2022 13:52:48 +0100 Subject: [PATCH 765/864] Add openssl devel headers --- easybuild/tools/containers/docker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/containers/docker.py b/easybuild/tools/containers/docker.py index 21e6d46c84..e8af752358 100644 --- a/easybuild/tools/containers/docker.py +++ b/easybuild/tools/containers/docker.py @@ -77,7 +77,7 @@ apt-get install -y python3 python3-pip lmod curl wget git \\ bzip2 gzip tar zip unzip xz-utils \\ patch automake git debianutils \\ - g++ libdata-dump-perl libthread-queue-any-perl + g++ libdata-dump-perl libthread-queue-any-perl libssl-dev RUN OS_DEPS='%(os_deps)s' && \\ test -n "${OS_DEPS}" && \\ @@ -89,7 +89,7 @@ yum install -y python3 python3-pip Lmod curl wget git \\ bzip2 gzip tar zip unzip xz \\ patch makefile git which \\ - gcc-c++ perl-Data-Dumper perl-Thread-Queue + gcc-c++ perl-Data-Dumper perl-Thread-Queue openssl-dev RUN OS_DEPS='%(os_deps)s' && \\ test -n "${OS_DEPS}" && \\ From affbf2cb438ef739970184ba1b1a93243150419d Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 4 Feb 2022 16:08:12 +0100 Subject: [PATCH 766/864] Update Docker support --- easybuild/tools/containers/docker.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/easybuild/tools/containers/docker.py b/easybuild/tools/containers/docker.py index e8af752358..9712ee777a 100644 --- a/easybuild/tools/containers/docker.py +++ b/easybuild/tools/containers/docker.py @@ -35,6 +35,7 @@ from easybuild.tools.containers.base import ContainerGenerator from easybuild.tools.containers.utils import det_os_deps from easybuild.tools.filetools import remove_dir +from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS from easybuild.tools.run import run_cmd @@ -61,26 +62,26 @@ RUN set -x && \\ . /usr/share/lmod/lmod/init/sh && \\ - eb %(eb_opts)s --installpath=/app/ --prefix=/scratch --tmpdir=/scratch/tmp + eb --robot %(eb_opts)s --installpath=/app/ --prefix=/scratch --tmpdir=/scratch/tmp -RUN touch ${HOME}/.profile && \\ - echo '\\n# Added by easybuild docker packaging' >> ${HOME}/.profile && \\ - echo 'source /usr/share/lmod/lmod/init/bash' >> ${HOME}/.profile && \\ - echo 'module use %(init_modulepath)s' >> ${HOME}/.profile && \\ - echo 'module load %(mod_names)s' >> ${HOME}/.profile +RUN touch ${HOME}/.bashrc && \\ + echo '' >> ${HOME}/.bashrc && \\ + echo '# Added by easybuild docker packaging' >> ${HOME}/.bashrc && \\ + echo 'source /usr/share/lmod/lmod/init/bash' >> ${HOME}/.bashrc && \\ + echo 'module use %(init_modulepath)s' >> ${HOME}/.bashrc && \\ + echo 'module load %(mod_names)s' >> ${HOME}/.bashrc CMD ["/bin/bash", "-l"] """ DOCKER_UBUNTU2004_INSTALL_DEPS = """\ RUN apt-get update && \\ - apt-get install -y python3 python3-pip lmod curl wget git \\ - bzip2 gzip tar zip unzip xz-utils \\ + DEBIAN_FRONTEND=noninteractive apt-get install -y python3 python3-pip lmod \\ + curl wget git bzip2 gzip tar zip unzip xz-utils \\ patch automake git debianutils \\ g++ libdata-dump-perl libthread-queue-any-perl libssl-dev RUN OS_DEPS='%(os_deps)s' && \\ - test -n "${OS_DEPS}" && \\ for dep in ${OS_DEPS}; do apt-get -qq install ${dep} || true; done """ @@ -88,7 +89,7 @@ RUN yum install -y epel-release && \\ yum install -y python3 python3-pip Lmod curl wget git \\ bzip2 gzip tar zip unzip xz \\ - patch makefile git which \\ + patch make git which \\ gcc-c++ perl-Data-Dumper perl-Thread-Queue openssl-dev RUN OS_DEPS='%(os_deps)s' && \\ @@ -131,9 +132,12 @@ def resolve_template_data(self): ec = self.easyconfigs[-1]['ec'] - init_modulepath = os.path.join("/app/modules/all", *self.mns.det_init_modulepaths(ec)) + # We are using the default MNS inside the container + docker_mns = EasyBuildMNS() + + init_modulepath = os.path.join("/app/modules/all", *docker_mns.det_init_modulepaths(ec)) - mod_names = [e['ec'].full_mod_name for e in self.easyconfigs] + mod_names = [docker_mns.det_full_module_name(e['ec']) for e in self.easyconfigs] eb_opts = [os.path.basename(e['spec']) for e in self.easyconfigs] From ae920389d1e8b1d62b08856e33424cbe98ec7784 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 4 Feb 2022 17:06:21 +0100 Subject: [PATCH 767/864] Missed an Ubuntu 16.04 ref --- test/framework/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/containers.py b/test/framework/containers.py index b513517db5..42efadb9fd 100644 --- a/test/framework/containers.py +++ b/test/framework/containers.py @@ -435,7 +435,7 @@ def test_end2end_docker_image(self): '-C', # equivalent with --containerize '--experimental', '--container-type=docker', - '--container-config=ubuntu:16.04', + '--container-config=ubuntu:20.04', '--container-build-image', ] From c20e0b50298e68983e843a5738cfab75ecb5b2e7 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Sat, 5 Feb 2022 14:53:19 +0100 Subject: [PATCH 768/864] Update containers.py --- test/framework/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/containers.py b/test/framework/containers.py index 42efadb9fd..133754bcf7 100644 --- a/test/framework/containers.py +++ b/test/framework/containers.py @@ -410,7 +410,7 @@ def test_end2end_dockerfile(self): def_file = read_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0')) regexs = [ "FROM ubuntu:20.04", - "eb toy-0.0.eb GCC-4.9.2.eb", + "eb --robot toy-0.0.eb GCC-4.9.2.eb", "module load toy/0.0 GCC/4.9.2", ] self.check_regexs(regexs, def_file) From 2e5513a9ba7fd7c0ef8760ddaf8d1771e910fc37 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Sun, 6 Feb 2022 13:28:36 +0100 Subject: [PATCH 769/864] Add bash completition for easyconfigs from local dir but not robot search path --- eb_bash_completion_local.bash | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 eb_bash_completion_local.bash diff --git a/eb_bash_completion_local.bash b/eb_bash_completion_local.bash new file mode 100644 index 0000000000..c85ca740b8 --- /dev/null +++ b/eb_bash_completion_local.bash @@ -0,0 +1,12 @@ +_eb() +{ + local cur prev quoted + _get_comp_words_by_ref cur prev + _quote_readline_by_ref "$cur" quoted + + case $cur in + --*) _optcomplete "$@"; return 0 ;; + *) COMPREPLY=( $(compgen -f -X '!*.eb' -- $cur ) ) ;; + esac +} +complete -F _eb eb From 14e70c1ddb10cbdf068023263561cce7f613451f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 6 Feb 2022 21:18:57 +0100 Subject: [PATCH 770/864] also check for git in --check-eb-deps --- easybuild/tools/systemtools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index b78f8319c5..8214b0d8b9 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -168,6 +168,7 @@ '7z': "extracting sources (.iso)", 'bunzip2': "decompressing sources (.bz2, .tbz, .tbz2, ...)", DPKG: "checking OS dependencies (Debian, Ubuntu, ...)", + 'git': "downloading sources using 'git clone'", 'gunzip': "decompressing source files (.gz, .tgz, ...)", 'make': "build tool", 'patch': "applying patch files", From b2284d70ddd54c2f1ab638acd73ec717dd0fe2f4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 8 Feb 2022 23:09:41 +0100 Subject: [PATCH 771/864] avoid crash in get_os_version on modern SLES-based OSs (fixes #3952) --- easybuild/tools/systemtools.py | 57 ++++++++++++++-------------------- test/framework/systemtools.py | 22 +++++++++++++ 2 files changed, 46 insertions(+), 33 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 8214b0d8b9..37411e4b86 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -742,40 +742,31 @@ def get_os_version(): os_version = res.group('version') if os_version: - if get_os_name() in ["suse", "SLES"]: - - # SLES subversions can only be told apart based on kernel version, - # see http://wiki.novell.com/index.php/Kernel_versions - version_suffixes = { - '11': [ - ('2.6.27', ''), - ('2.6.32', '_SP1'), - ('3.0.101-63', '_SP4'), - # not 100% correct, since early SP3 had 3.0.76 - 3.0.93, but close enough? - ('3.0.101', '_SP3'), - # SP2 kernel versions range from 3.0.13 - 3.0.101 - ('3.0', '_SP2'), - ], - - '12': [ - ('3.12.28', ''), - ('3.12.49', '_SP1'), - ], - } - + # older SLES subversions can only be told apart based on kernel version, + # see http://wiki.novell.com/index.php/Kernel_versions + sles_version_suffixes = { + '11': [ + ('2.6.27', ''), + ('2.6.32', '_SP1'), + ('3.0.101-63', '_SP4'), + # not 100% correct, since early SP3 had 3.0.76 - 3.0.93, but close enough? + ('3.0.101', '_SP3'), + # SP2 kernel versions range from 3.0.13 - 3.0.101 + ('3.0', '_SP2'), + ], + + '12': [ + ('3.12.28', ''), + ('3.12.49', '_SP1'), + ], + } + if get_os_name() in ['suse', 'SLES'] and os_version in sles_version_suffixes: # append suitable suffix to system version - if os_version in version_suffixes.keys(): - kernel_version = platform.uname()[2] - known_sp = False - for (kver, suff) in version_suffixes[os_version]: - if kernel_version.startswith(kver): - os_version += suff - known_sp = True - break - if not known_sp: - suff = '_UNKNOWN_SP' - else: - raise EasyBuildError("Don't know how to determine subversions for SLES %s", os_version) + kernel_version = platform.uname()[2] + for (kver, suff) in sles_version_suffixes[os_version]: + if kernel_version.startswith(kver): + os_version += suff + break return os_version else: diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index fa49515eda..cbbf8f9509 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -366,10 +366,13 @@ def setUp(self): self.orig_is_readable = st.is_readable self.orig_read_file = st.read_file self.orig_run_cmd = st.run_cmd + self.orig_platform_dist = st.platform.dist if hasattr(st.platform, 'dist') else None self.orig_platform_uname = st.platform.uname self.orig_get_tool_version = st.get_tool_version self.orig_sys_version_info = st.sys.version_info self.orig_HAVE_ARCHSPEC = st.HAVE_ARCHSPEC + self.orig_HAVE_DISTRO = st.HAVE_DISTRO + self.orig_ETC_OS_RELEASE = st.ETC_OS_RELEASE if hasattr(st, 'archspec_cpu_host'): self.orig_archspec_cpu_host = st.archspec_cpu_host else: @@ -383,10 +386,14 @@ def tearDown(self): st.get_os_name = self.orig_get_os_name st.get_os_type = self.orig_get_os_type st.run_cmd = self.orig_run_cmd + if self.orig_platform_dist is not None: + st.platform.dist = self.orig_platform_dist st.platform.uname = self.orig_platform_uname st.get_tool_version = self.orig_get_tool_version st.sys.version_info = self.orig_sys_version_info st.HAVE_ARCHSPEC = self.orig_HAVE_ARCHSPEC + st.HAVE_DISTRO = self.orig_HAVE_DISTRO + st.ETC_OS_RELEASE = self.orig_ETC_OS_RELEASE if self.orig_archspec_cpu_host is not None: st.archspec_cpu_host = self.orig_archspec_cpu_host super(SystemToolsTest, self).tearDown() @@ -738,6 +745,21 @@ def test_os_version(self): os_version = get_os_version() self.assertTrue(isinstance(os_version, string_type) or os_version == UNKNOWN) + # make sure that bug fixed in https://github.com/easybuilders/easybuild-framework/issues/3952 + # does not surface again, by mocking what's needed to make get_os_version fall into SLES-specific path + + if hasattr(st.platform, 'dist'): + st.platform.dist = lambda: (None, None) + st.HAVE_DISTRO = False + + st.get_os_name = lambda: 'SLES' + fake_etc_os_release = os.path.join(self.test_prefix, 'os-release') + write_file(fake_etc_os_release, 'VERSION="15-SP1"') + st.ETC_OS_RELEASE = fake_etc_os_release + + os_version = get_os_version() + self.assertEqual(os_version, '15-SP1') + def test_gcc_version_native(self): """Test getting gcc version.""" gcc_version = get_gcc_version() From 9290f4a0bf34eec5504f2d1f685eaed3f18f5343 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 29 Jan 2022 21:17:15 +0100 Subject: [PATCH 772/864] add end2end test for 'eb --containerize' --- .github/workflows/container_tests.yml | 92 +++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 .github/workflows/container_tests.yml diff --git a/.github/workflows/container_tests.yml b/.github/workflows/container_tests.yml new file mode 100644 index 0000000000..da30fcb7c0 --- /dev/null +++ b/.github/workflows/container_tests.yml @@ -0,0 +1,92 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Tests for container support +on: [push, pull_request] +jobs: + build: + # stick to Ubuntu 18.04, where we can still easily install yum via 'apt-get install' + runs-on: ubuntu-18.04 + strategy: + matrix: + python: [2.7, 3.6] + fail-fast: false + steps: + - uses: actions/checkout@v2 + + - name: set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{matrix.python}} + architecture: x64 + + - name: install OS & Python packages + run: | + # for building Singularity images + sudo apt-get install rpm + sudo apt-get install yum + # for modules tool + sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev + # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 + # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists + if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then + sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so + fi + + - name: install Lmod + run: | + # avoid downloading modules tool sources into easybuild-framework dir + cd $HOME + export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh + # install Lmod + source $INSTALL_DEP Lmod-8.4.27 $HOME + # changes in environment are not passed to other steps, so need to create files... + echo $MOD_INIT > mod_init + echo $PATH > path + if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi + + # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181 + - name: install Singularity + run: | + # install alien, which can be used to convert RPMs to Debian packages + sudo apt-get install alien + alien --version + # determine latest version of Singularity available in EPEL, and download RPM + singularity_rpm=$(curl -sL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/ | grep singularity | sed 's/.*singularity/singularity/g' | sed 's/rpm.*/rpm/g') + curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/${singularity_rpm} + # convert Singularity RPM to Debian package, and install it + sudo alien -d ${singularity_rpm} + sudo apt install ./singularity*.deb + singularity --version + + - name: install sources + run: | + # install from source distribution tarball, to test release as published on PyPI + python setup.py sdist + ls dist + export PREFIX=/tmp/$USER/$GITHUB_SHA + pip install --prefix $PREFIX dist/easybuild-framework*tar.gz + pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz + + - name: run test + run: | + # run tests *outside* of checked out easybuild-framework directory, + # to ensure we're testing installed version (see previous step) + cd $HOME + # initialize environment for modules tool + if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi + source $(cat $HOME/mod_init); type module + # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that); + # also pick up changes to $PATH set by sourcing $MOD_INIT + export PREFIX=/tmp/$USER/$GITHUB_SHA + export PATH=$PREFIX/bin:$(cat $HOME/path) + export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH + eb --version + # create $HOME/.rpmmacros, see also https://github.com/apptainer/singularity/issues/241 + echo '%_var /var' > $HOME/.rpmmacros + echo '%_dbpath %{_var}/lib/rpm' >> $HOME/.rpmmacros + # build CentOS 7 container image for bzip2 1.0.8 using EasyBuild; + # see https://docs.easybuild.io/en/latest/Containers.html + curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb + export EASYBUILD_CONTAINERPATH=$PWD + export EASYBUILD_CONTAINER_CONFIG='bootstrap=yum,osversion=7' + eb bzip2-1.0.8.eb --containerize --experimental --container-build-image + test -f bzip2-1.0.8.sif From bdd34dd248f608ac04946faf375b339820770e94 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 10 Feb 2022 08:34:53 +0100 Subject: [PATCH 773/864] also try running bzip2 in workflow to test container building --- .github/workflows/container_tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_tests.yml b/.github/workflows/container_tests.yml index da30fcb7c0..cc8cafe298 100644 --- a/.github/workflows/container_tests.yml +++ b/.github/workflows/container_tests.yml @@ -89,4 +89,5 @@ jobs: export EASYBUILD_CONTAINERPATH=$PWD export EASYBUILD_CONTAINER_CONFIG='bootstrap=yum,osversion=7' eb bzip2-1.0.8.eb --containerize --experimental --container-build-image - test -f bzip2-1.0.8.sif + singularity exec bzip2-1.0.8.sif command -v bzip2 | grep '/app/software/bzip2/1.0.8/bin/bzip2' || (echo "Path to bzip2 '$which_bzip2' is not correct" && exit 1) + singularity exec bzip2-1.0.8.sif bzip2 --help From e232a9fc7a9fbd4a557d66a1343005787375a734 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Fri, 11 Feb 2022 09:55:01 +0800 Subject: [PATCH 774/864] prepare release notes for EasyBuild v4.5.3 + bump version to 4.5.3 --- RELEASE_NOTES | 20 ++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 65a91ff1bb..3f608e4b32 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,26 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.5.3 (February 11th 2022) +--------------------------- + +update/bugfix release + +- various enhancements, including: + - also check for git in --check-eb-deps (#3954) + - add end2end test for 'eb --containerize' (#3958) +- various bug fixes, including: + - take into account that patch files can also be zipped when checking filename extension for patches (#3936) + - initialize BACKUP_PKG_URL in install_eb_dep.sh with empty string (#3939) + - fix get_os_name and get_os_version to avoid reporting UNKNOWN in output of eb --show-system-info (#3942) + - consistently use actions/setup-python (#3944) + - switch to using pip3 for installing EasyBuild in Singularity definition file generated by EasyBuild (#3945) + - specify easybuild.io as EasyBuild homepage in setup.py (#3947) + - avoid crash in get_os_version on modern SLES-based OSs (#3955) +- other changes: + - indicate compatibility with Python 3.10 in setup.py (#3940) + + v4.5.2 (January 24th 2022) -------------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 878e7b5b2b..090fb28792 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.5.3.dev0') +VERSION = LooseVersion('4.5.3') UNKNOWN = 'UNKNOWN' From ddb3250802419b51a0ee3f63cc481de66627f1ea Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Feb 2022 07:29:16 +0100 Subject: [PATCH 775/864] minor tweak release notes for v4.5.3 --- RELEASE_NOTES | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 3f608e4b32..c90361c413 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -14,10 +14,10 @@ update/bugfix release - add end2end test for 'eb --containerize' (#3958) - various bug fixes, including: - take into account that patch files can also be zipped when checking filename extension for patches (#3936) - - initialize BACKUP_PKG_URL in install_eb_dep.sh with empty string (#3939) + - initialize BACKUP_PKG_URL with empty string in install_eb_dep.sh script (#3939) - fix get_os_name and get_os_version to avoid reporting UNKNOWN in output of eb --show-system-info (#3942) - - consistently use actions/setup-python (#3944) - - switch to using pip3 for installing EasyBuild in Singularity definition file generated by EasyBuild (#3945) + - consistently use actions/setup-python@v2 in CI workflows (#3944) + - switch to using pip3 for installing EasyBuild in container recipes generated by EasyBuild (#3945) - specify easybuild.io as EasyBuild homepage in setup.py (#3947) - avoid crash in get_os_version on modern SLES-based OSs (#3955) - other changes: From 67dab820090a78c27fca26b81e5861b4b68b9043 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Feb 2022 11:54:42 +0100 Subject: [PATCH 776/864] bump version to 4.5.4dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 090fb28792..1396a1c4a1 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.5.3') +VERSION = LooseVersion('4.5.4.dev0') UNKNOWN = 'UNKNOWN' From f066b53154815891cfc3cbfc7f9fd4b513bde02a Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Mon, 14 Feb 2022 14:36:51 +0100 Subject: [PATCH 777/864] also test on new python 3.10 --- .github/workflows/linting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 3902d127fc..045a79bc18 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -5,7 +5,7 @@ jobs: runs-on: ubuntu-18.04 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10] steps: - uses: actions/checkout@v2 From 0f0dd1b1c8bbf9aa845bc52876946796f1d3af9e Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Mon, 14 Feb 2022 14:38:37 +0100 Subject: [PATCH 778/864] 3.10 is 3.1 so specify '3.10' --- .github/workflows/linting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 045a79bc18..2f5c9d68ab 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -5,7 +5,7 @@ jobs: runs-on: ubuntu-18.04 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10'] steps: - uses: actions/checkout@v2 From 78204417e98e4485c7608d3077af43e79301a595 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Mon, 14 Feb 2022 17:03:38 +0100 Subject: [PATCH 779/864] Update linting.yml py3 functions will always fail in py2, so ignore for linting for now. c --- .github/workflows/linting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 2f5c9d68ab..cc26e5be28 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -22,4 +22,4 @@ jobs: - name: Run flake8 to verify PEP8-compliance of Python code run: | - flake8 + flake8 --exclude ./easybuild/tools/py2vs3/py3.py From ef5b9e3db6a0500f537633cc42bea43941a263da Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Mon, 14 Feb 2022 17:14:44 +0100 Subject: [PATCH 780/864] fix linting warnings --- easybuild/framework/easyblock.py | 11 +++++------ easybuild/tools/configobj.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 4ea2b44f0a..f507b8b357 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1485,12 +1485,11 @@ def make_module_req(self): if key in keys_requiring_files: # only retain paths that contain at least one file recursive = keys_requiring_files[key] - retained_paths = [ - path - for path, fullpath in ((path, os.path.join(self.installdir, path)) for path in paths) - if os.path.isdir(fullpath) - and dir_contains_files(fullpath, recursive=recursive) - ] + retained_paths = [] + for pth in paths: + fullpath = os.path.join(self.installdir, path) + if os.path.isdir(fullpath) and dir_contains_files(fullpath, recursive=recursive): + retained_paths.append(pth) if retained_paths != paths: self.log.info("Only retaining paths for %s that contain at least one file: %s -> %s", key, paths, retained_paths) diff --git a/easybuild/tools/configobj.py b/easybuild/tools/configobj.py index 7931f2ca05..433238b95f 100644 --- a/easybuild/tools/configobj.py +++ b/easybuild/tools/configobj.py @@ -2023,7 +2023,7 @@ def write(self, outfile=None, section=None): # might need to encode # NOTE: This will *screw* UTF16, each line will start with the BOM if self.encoding: - out = [line.encode(self.encoding) for line in out] + out = [lne.encode(self.encoding) for lne in out] if (self.BOM and ((self.encoding is None) or (BOM_LIST.get(self.encoding.lower()) == 'utf_8'))): # Add the UTF8 BOM From 00db387f245c3fd41ce8d4a7d29a108220449b27 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Mon, 14 Feb 2022 17:19:56 +0100 Subject: [PATCH 781/864] Update easyblock.py --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f507b8b357..e71ee973fc 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1487,7 +1487,7 @@ def make_module_req(self): recursive = keys_requiring_files[key] retained_paths = [] for pth in paths: - fullpath = os.path.join(self.installdir, path) + fullpath = os.path.join(self.installdir, pth) if os.path.isdir(fullpath) and dir_contains_files(fullpath, recursive=recursive): retained_paths.append(pth) if retained_paths != paths: From 799c3b4e96d1f86353c1faab760a440912a47e87 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 15 Feb 2022 12:43:33 +0800 Subject: [PATCH 782/864] check if patch exists in repo --- easybuild/tools/github.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index f58d14f28c..68ade9fa5f 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1783,10 +1783,10 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): res = new_branch_github(paths, ecs, commit_msg=commit_msg) file_info, deleted_paths, _, branch_name, diff_stat, pr_target_repo = res - for ec in file_info['ecs']: + for ec, ec_path in zip(file_info['ecs'], file_info['paths_in_repo']): for patch in ec.asdict()['patches']: - if patch not in paths['patch_files']: - print_warning("%s, referenced by %s, is not included in this PR" % (patch, ec.filename())) + if patch not in paths['patch_files'] and not os.path.isfile(os.path.join(os.path.dirname(ec_path), patch)): + print_warning("new patch %s, referenced by %s, is not included in this PR" % (patch, ec.filename())) new_pr_from_branch(branch_name, title=title, descr=descr, pr_target_repo=pr_target_repo, pr_metadata=(file_info, deleted_paths, diff_stat), commit_msg=commit_msg) From 4666c1c854b98870ffe29be56bd40e304aa3162c Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 15 Feb 2022 16:37:17 +0800 Subject: [PATCH 783/864] clarify warning about missing patch file --- easybuild/tools/github.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 68ade9fa5f..0eac379302 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1786,7 +1786,8 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): for ec, ec_path in zip(file_info['ecs'], file_info['paths_in_repo']): for patch in ec.asdict()['patches']: if patch not in paths['patch_files'] and not os.path.isfile(os.path.join(os.path.dirname(ec_path), patch)): - print_warning("new patch %s, referenced by %s, is not included in this PR" % (patch, ec.filename())) + print_warning("new patch file %s, referenced by %s, is not included in this PR" % + (patch, ec.filename())) new_pr_from_branch(branch_name, title=title, descr=descr, pr_target_repo=pr_target_repo, pr_metadata=(file_info, deleted_paths, diff_stat), commit_msg=commit_msg) From 72dbc8929d3c149a47696e0b299b550a520835c6 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 15 Feb 2022 18:20:28 +0800 Subject: [PATCH 784/864] take into account that 'patches' also takes tuples --- easybuild/tools/github.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 0eac379302..066d795922 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1785,6 +1785,8 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): for ec, ec_path in zip(file_info['ecs'], file_info['paths_in_repo']): for patch in ec.asdict()['patches']: + if isinstance(patch, tuple): + patch = patch[0] if patch not in paths['patch_files'] and not os.path.isfile(os.path.join(os.path.dirname(ec_path), patch)): print_warning("new patch file %s, referenced by %s, is not included in this PR" % (patch, ec.filename())) From f76074c945abe414a701b8243cc098fccedbb6fd Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 19 Feb 2022 11:46:26 +0000 Subject: [PATCH 785/864] Add 'sync pr' message when the PR has a mergeable state but is showing a failed status for the test suite on the last commit --- easybuild/tools/github.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 066d795922..ca906deb2f 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1155,12 +1155,14 @@ def not_eligible(msg): # check test suite result, Travis must give green light msg_tmpl = "* test suite passes: %s" + failed_status_last_commit = False if pr_data['status_last_commit'] == STATUS_SUCCESS: print_msg(msg_tmpl % 'OK', prefix=False) elif pr_data['status_last_commit'] == STATUS_PENDING: res = not_eligible(msg_tmpl % "pending...") else: res = not_eligible(msg_tmpl % "(status: %s)" % pr_data['status_last_commit']) + failed_status_last_commit = True if pr_data['base']['repo']['name'] == GITHUB_EASYCONFIGS_REPO: # check for successful test report (checked in reverse order) @@ -1221,14 +1223,19 @@ def not_eligible(msg): # check github mergeable state msg_tmpl = "* mergeable state is clean: %s" + mergeable = False if pr_data['merged']: print_msg(msg_tmpl % "PR is already merged", prefix=False) elif pr_data['mergeable_state'] == GITHUB_MERGEABLE_STATE_CLEAN: print_msg(msg_tmpl % "OK", prefix=False) + mergeable = True else: reason = "FAILED (mergeable state is '%s')" % pr_data['mergeable_state'] res = not_eligible(msg_tmpl % reason) + if failed_status_last_commit and mergeable: + print_msg("\nThis PR is mergeable but the test suite has a failed status. Try syncing the PR.", prefix=False) + return res From 4c4a6a679fdaaa3f9833c742fcccb308b13f3eaa Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 19 Feb 2022 11:53:43 +0000 Subject: [PATCH 786/864] Only check for --new-pr for patches in teh easyconfigs repository --- easybuild/tools/github.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 066d795922..61d86a367d 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1783,13 +1783,15 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): res = new_branch_github(paths, ecs, commit_msg=commit_msg) file_info, deleted_paths, _, branch_name, diff_stat, pr_target_repo = res - for ec, ec_path in zip(file_info['ecs'], file_info['paths_in_repo']): - for patch in ec.asdict()['patches']: - if isinstance(patch, tuple): - patch = patch[0] - if patch not in paths['patch_files'] and not os.path.isfile(os.path.join(os.path.dirname(ec_path), patch)): - print_warning("new patch file %s, referenced by %s, is not included in this PR" % - (patch, ec.filename())) + if pr_target_repo == GITHUB_EASYCONFIGS_REPO: + for ec, ec_path in zip(file_info['ecs'], file_info['paths_in_repo']): + for patch in ec.asdict()['patches']: + if isinstance(patch, tuple): + patch = patch[0] + if patch not in paths['patch_files'] and not os.path.isfile(os.path.join(os.path.dirname(ec_path), + patch)): + print_warning("new patch file %s, referenced by %s, is not included in this PR" % + (patch, ec.filename())) new_pr_from_branch(branch_name, title=title, descr=descr, pr_target_repo=pr_target_repo, pr_metadata=(file_info, deleted_paths, diff_stat), commit_msg=commit_msg) From 7802570d308ea6f09b5b6bf9827f75d8fc1f173d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 19 Feb 2022 13:47:56 +0100 Subject: [PATCH 787/864] add test for opening an easyblock PR via --new-pr --- test/framework/options.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 41b29f0dec..e6d4da01c5 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4568,6 +4568,37 @@ def test_github_new_pr_dependencies(self): self._assert_regexs(regexs, txt) + def test_new_pr_easyblock(self): + """ + Test using --new-pr to open an easyblocks PR + """ + + if self.github_token is None: + print("Skipping test_new_pr_easyblock, no GitHub token available?") + return + + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_eb = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') + self.assertTrue(os.path.exists(toy_eb)) + + args = [ + '--github-user=%s' % GITHUB_TEST_ACCOUNT, + '--new-pr', + toy_eb, + '-D', + ] + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) + + patterns = [ + r'target: easybuilders/easybuild-easyblocks:develop', + r'from: easybuild_test/easybuild-easyblocks:[0-9]+_new_pr_toy', + r'title: "new easyblock for toy"', + r'easybuild/easyblocks/t/toy.py', + ] + for pattern in patterns: + regex = re.compile(pattern) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) + def test_github_merge_pr(self): """ Test use of --merge-pr (dry run)""" From a2ec58b8a3f4f5bd1c8e1ac0e5a4e82f3c4f579c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 19 Feb 2022 18:07:50 +0100 Subject: [PATCH 788/864] don't check py2vs3/py3.py when running lint test with Python 2, and vice versa --- .github/workflows/linting.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index cc26e5be28..becc9cc1cd 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -22,4 +22,10 @@ jobs: - name: Run flake8 to verify PEP8-compliance of Python code run: | - flake8 --exclude ./easybuild/tools/py2vs3/py3.py + # don't check py2vs3/py3.py when testing with Python 2, and vice versa + if [[ "${{ matrix.python-version }}" =~ "2." ]]; then + py_excl=py3 + else + py_excl=py2 + fi + flake8 --exclude ./easybuild/tools/py2vs3/${py_excl}.py From 0b1c933bbf84a8e08db1108814165ffec820469f Mon Sep 17 00:00:00 2001 From: Robert Mijakovic Date: Tue, 22 Feb 2022 12:15:52 +0100 Subject: [PATCH 789/864] Adds nvompi toolchain. The idea is to replicate what gompi/foss/GCC(core) do --- easybuild/toolchains/nvompi.py | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 easybuild/toolchains/nvompi.py diff --git a/easybuild/toolchains/nvompi.py b/easybuild/toolchains/nvompi.py new file mode 100644 index 0000000000..c24245b0cc --- /dev/null +++ b/easybuild/toolchains/nvompi.py @@ -0,0 +1,39 @@ +## +# Copyright 2013-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for nvompi compiler toolchain (NVHPC + Open MPI). + +:author: Maxime Boissonneault (Universite Laval, Calcul Quebec, Compute Canada) +:author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) +""" + +from easybuild.toolchains.nvhpc import NVHPCToolchain +from easybuild.toolchains.mpi.openmpi import OpenMPI + + +class Nvompi(NVHPCToolchain, OpenMPI): + """Compiler toolchain with NVHPC and Open MPI.""" + NAME = 'nvompi' + SUBTOOLCHAIN = NVHPCToolchain.NAME From abff48ec61ff4152aac7bdbc315a80255ef297fd Mon Sep 17 00:00:00 2001 From: Robert Mijakovic Date: Tue, 22 Feb 2022 12:17:20 +0100 Subject: [PATCH 790/864] Adds nvpsmpi toolchain. The idea is to replicate what gompi/foss/GCC(core) do. --- easybuild/toolchains/nvpsmpi.py | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 easybuild/toolchains/nvpsmpi.py diff --git a/easybuild/toolchains/nvpsmpi.py b/easybuild/toolchains/nvpsmpi.py new file mode 100644 index 0000000000..bf0f536099 --- /dev/null +++ b/easybuild/toolchains/nvpsmpi.py @@ -0,0 +1,41 @@ +## +# Copyright 2016-2021 Ghent University +# Copyright 2016-2021 Forschungszentrum Juelich +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for npsmpi compiler toolchain (includes NVHPC and ParaStationMPI). + +:author: Damian Alvarez (Forschungszentrum Juelich) +:author: Sebastian Achilles (Forschungszentrum Juelich) +""" + +from easybuild.toolchains.nvhpc import NVHPCToolchain +from easybuild.toolchains.mpi.psmpi import Psmpi + + +# Order matters! +class NVpsmpi(NVHPCToolchain, Psmpi): + """Compiler toolchain with NVHPC and ParaStationMPI.""" + NAME = 'nvpsmpi' + SUBTOOLCHAIN = NVHPCToolchain.NAME From 1c0c77a22e7b8e56b86c1f29eae0a25bb6738f97 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 23 Feb 2022 09:13:56 +0100 Subject: [PATCH 791/864] Add GCC+MPItrampoline toolchain --- easybuild/toolchains/gmpit.py | 56 ++++++++++++++++ easybuild/toolchains/mpi/mpitrampoline.py | 80 +++++++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 easybuild/toolchains/gmpit.py create mode 100644 easybuild/toolchains/mpi/mpitrampoline.py diff --git a/easybuild/toolchains/gmpit.py b/easybuild/toolchains/gmpit.py new file mode 100644 index 0000000000..6ab7b8f041 --- /dev/null +++ b/easybuild/toolchains/gmpit.py @@ -0,0 +1,56 @@ +## +# Copyright 2022-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gmpit compiler toolchain (includes GCC and MPItrampoline). + +:author: Alan O'Cais (CECAM) +""" +from distutils.version import LooseVersion +import re + +from easybuild.toolchains.gcc import GccToolchain +from easybuild.toolchains.mpi.mpitrampoline import MPItrampoline + + +class Gmpit(GccToolchain, MPItrampoline): + """Compiler toolchain with GCC and MPItrampoline.""" + NAME = 'gmpit' + SUBTOOLCHAIN = GccToolchain.NAME + + def is_deprecated(self): + """Return whether or not this toolchain is deprecated.""" + # need to transform a version like '2018b' with something that is safe to compare with '2019' + # comparing subversions that include letters causes TypeErrors in Python 3 + # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) + version = self.version.replace('a', '.01').replace('b', '.07') + + deprecated = False + + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', version): + if LooseVersion(version) < LooseVersion('2019'): + pass + + return deprecated diff --git a/easybuild/toolchains/mpi/mpitrampoline.py b/easybuild/toolchains/mpi/mpitrampoline.py new file mode 100644 index 0000000000..1df3e4d3a0 --- /dev/null +++ b/easybuild/toolchains/mpi/mpitrampoline.py @@ -0,0 +1,80 @@ +## +# Copyright 2022-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for MPItrampoline as toolchain MPI library. + +:author: Alan O'Cais (CECAM) +""" +import os +import shutil +import tempfile +from distutils.version import LooseVersion + +import easybuild.tools.environment as env +from easybuild.tools.build_log import print_warning +from easybuild.tools.config import build_option +from easybuild.tools.toolchain.constants import COMPILER_VARIABLES, MPI_COMPILER_VARIABLES +from easybuild.tools.toolchain.mpi import Mpi +from easybuild.tools.toolchain.variables import CommandFlagList + + +TC_CONSTANT_MPITRAMPOLINE = "MPItrampoline" +TC_CONSTANT_MPI_TYPE_MPITRAMPOLINE= "MPI_TYPE_MPITRAMPOLINE" + + +class MPItrampoline(Mpi): + """MPItrampoline MPI class""" + MPI_MODULE_NAME = ['MPItrampoline'] + MPI_FAMILY = TC_CONSTANT_MPITRAMPOLINE + MPI_TYPE = TC_CONSTANT_MPI_TYPE_MPITRAMPOLINE + + MPI_LIBRARY_NAME = 'mpi' + + # version-dependent, so defined at runtime + MPI_COMPILER_MPIF77 = None + MPI_COMPILER_MPIF90 = None + MPI_COMPILER_MPIFC = None + + # MPItrampoline reads from CC etc env variables + MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES]) + + MPI_LINK_INFO_OPTION = '-showme:link' + + def __init__(self, *args, **kwargs): + """Toolchain constructor""" + super(MPItrampoline, self).__init__(*args, **kwargs) + + def _set_mpi_compiler_variables(self): + """Define MPI wrapper commands (depends on OpenMPI version) and add OMPI_* variables to set.""" + + self.MPI_COMPILER_MPIF77 = 'mpifort' + self.MPI_COMPILER_MPIF90 = 'mpifort' + self.MPI_COMPILER_MPIFC = 'mpifort' + + # this needs to be done first, otherwise e.g., CC is set to MPICC if the usempi toolchain option is enabled + for var, _ in COMPILER_VARIABLES: + self.variables.nappend('MPITRAMPOLINE_%s' % var, str(self.variables[var].get_first()), var_class=CommandFlagList) + + super(MPItrampoline, self)._set_mpi_compiler_variables() From 6281370abc56847cc624f1f53cb54ac211e72e30 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 23 Feb 2022 09:16:48 +0100 Subject: [PATCH 792/864] Hound --- easybuild/toolchains/mpi/mpitrampoline.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easybuild/toolchains/mpi/mpitrampoline.py b/easybuild/toolchains/mpi/mpitrampoline.py index 1df3e4d3a0..c70808dd03 100644 --- a/easybuild/toolchains/mpi/mpitrampoline.py +++ b/easybuild/toolchains/mpi/mpitrampoline.py @@ -75,6 +75,9 @@ def _set_mpi_compiler_variables(self): # this needs to be done first, otherwise e.g., CC is set to MPICC if the usempi toolchain option is enabled for var, _ in COMPILER_VARIABLES: - self.variables.nappend('MPITRAMPOLINE_%s' % var, str(self.variables[var].get_first()), var_class=CommandFlagList) + self.variables.nappend( + 'MPITRAMPOLINE_%s' % var, str(self.variables[var].get_first()), + var_class=CommandFlagList + ) super(MPItrampoline, self)._set_mpi_compiler_variables() From 06b3a0735f17ef1724f1918cddb61325ed020d7d Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 23 Feb 2022 09:17:46 +0100 Subject: [PATCH 793/864] Hound --- easybuild/toolchains/mpi/mpitrampoline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/toolchains/mpi/mpitrampoline.py b/easybuild/toolchains/mpi/mpitrampoline.py index c70808dd03..756ceb18bd 100644 --- a/easybuild/toolchains/mpi/mpitrampoline.py +++ b/easybuild/toolchains/mpi/mpitrampoline.py @@ -27,7 +27,6 @@ :author: Alan O'Cais (CECAM) """ -import os import shutil import tempfile from distutils.version import LooseVersion From edc77575c9948becc3dc38be73eaa817c196a167 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 23 Feb 2022 09:19:24 +0100 Subject: [PATCH 794/864] Hound --- easybuild/toolchains/mpi/mpitrampoline.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/easybuild/toolchains/mpi/mpitrampoline.py b/easybuild/toolchains/mpi/mpitrampoline.py index 756ceb18bd..613108742e 100644 --- a/easybuild/toolchains/mpi/mpitrampoline.py +++ b/easybuild/toolchains/mpi/mpitrampoline.py @@ -27,20 +27,15 @@ :author: Alan O'Cais (CECAM) """ -import shutil -import tempfile -from distutils.version import LooseVersion import easybuild.tools.environment as env -from easybuild.tools.build_log import print_warning -from easybuild.tools.config import build_option from easybuild.tools.toolchain.constants import COMPILER_VARIABLES, MPI_COMPILER_VARIABLES from easybuild.tools.toolchain.mpi import Mpi from easybuild.tools.toolchain.variables import CommandFlagList TC_CONSTANT_MPITRAMPOLINE = "MPItrampoline" -TC_CONSTANT_MPI_TYPE_MPITRAMPOLINE= "MPI_TYPE_MPITRAMPOLINE" +TC_CONSTANT_MPI_TYPE_MPITRAMPOLINE = "MPI_TYPE_MPITRAMPOLINE" class MPItrampoline(Mpi): @@ -75,7 +70,7 @@ def _set_mpi_compiler_variables(self): # this needs to be done first, otherwise e.g., CC is set to MPICC if the usempi toolchain option is enabled for var, _ in COMPILER_VARIABLES: self.variables.nappend( - 'MPITRAMPOLINE_%s' % var, str(self.variables[var].get_first()), + 'MPITRAMPOLINE_%s' % var, str(self.variables[var].get_first()), var_class=CommandFlagList ) From f3ea414825884b41a5e995c648650347aba9066b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 23 Feb 2022 09:20:06 +0100 Subject: [PATCH 795/864] Hound --- easybuild/toolchains/mpi/mpitrampoline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/toolchains/mpi/mpitrampoline.py b/easybuild/toolchains/mpi/mpitrampoline.py index 613108742e..4314592aed 100644 --- a/easybuild/toolchains/mpi/mpitrampoline.py +++ b/easybuild/toolchains/mpi/mpitrampoline.py @@ -28,7 +28,6 @@ :author: Alan O'Cais (CECAM) """ -import easybuild.tools.environment as env from easybuild.tools.toolchain.constants import COMPILER_VARIABLES, MPI_COMPILER_VARIABLES from easybuild.tools.toolchain.mpi import Mpi from easybuild.tools.toolchain.variables import CommandFlagList From 165dce87196d5980f988a03fe2f0e2ceaf7b5645 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 23 Feb 2022 13:57:03 +0100 Subject: [PATCH 796/864] Add MPItrampoline command template --- easybuild/tools/toolchain/mpi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/tools/toolchain/mpi.py b/easybuild/tools/toolchain/mpi.py index ac9da42114..9f1b896a18 100644 --- a/easybuild/tools/toolchain/mpi.py +++ b/easybuild/tools/toolchain/mpi.py @@ -68,6 +68,7 @@ def get_mpi_cmd_template(mpi_family, params, mpi_version=None): toolchain.MVAPICH2: mpirun_n_cmd, toolchain.MPICH: mpirun_n_cmd, toolchain.MPICH2: mpirun_n_cmd, + toolchain.MPITRAMPOLINE: "mpiexec -n %(nr_ranks)s %(cmd)s", } # Intel MPI mpirun needs more work From e83cfa6a9974ac41160fb36886aac456c6a72cfb Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 23 Feb 2022 14:53:33 +0100 Subject: [PATCH 797/864] Update gmpit.py --- easybuild/toolchains/gmpit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/gmpit.py b/easybuild/toolchains/gmpit.py index 6ab7b8f041..5541f688fb 100644 --- a/easybuild/toolchains/gmpit.py +++ b/easybuild/toolchains/gmpit.py @@ -51,6 +51,6 @@ def is_deprecated(self): # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion if re.match('^[0-9]', version): if LooseVersion(version) < LooseVersion('2019'): - pass + deprecated = True return deprecated From 515573f3b9584a02956f406f1cab2d6935deeef0 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 24 Feb 2022 08:36:40 +0100 Subject: [PATCH 798/864] Update mpitrampoline.py --- easybuild/toolchains/mpi/mpitrampoline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/toolchains/mpi/mpitrampoline.py b/easybuild/toolchains/mpi/mpitrampoline.py index 4314592aed..7dcd0c44a1 100644 --- a/easybuild/toolchains/mpi/mpitrampoline.py +++ b/easybuild/toolchains/mpi/mpitrampoline.py @@ -45,7 +45,7 @@ class MPItrampoline(Mpi): MPI_LIBRARY_NAME = 'mpi' - # version-dependent, so defined at runtime + # May be version-dependent, so defined at runtime MPI_COMPILER_MPIF77 = None MPI_COMPILER_MPIF90 = None MPI_COMPILER_MPIFC = None @@ -60,7 +60,7 @@ def __init__(self, *args, **kwargs): super(MPItrampoline, self).__init__(*args, **kwargs) def _set_mpi_compiler_variables(self): - """Define MPI wrapper commands (depends on OpenMPI version) and add OMPI_* variables to set.""" + """Define MPI wrapper commands and add MPITRAMPOLINE_* variables to set.""" self.MPI_COMPILER_MPIF77 = 'mpifort' self.MPI_COMPILER_MPIF90 = 'mpifort' From 941829083dd3322d81f6dce6c9b7e831c0580ca4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 27 Feb 2022 16:04:21 +0100 Subject: [PATCH 799/864] use 'zypper search -i' to check whether specified OS dependency is installed on openSUSE --- easybuild/tools/systemtools.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 37411e4b86..e681929798 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -163,6 +163,7 @@ # OS package handler name constants RPM = 'rpm' DPKG = 'dpkg' +ZYPPER = 'zypper' SYSTEM_TOOLS = { '7z': "extracting sources (.iso)", @@ -178,6 +179,7 @@ 'tar': "unpacking source files (.tar)", 'unxz': "decompressing source files (.xz, .txz)", 'unzip': "decompressing files (.zip)", + ZYPPER: "checking OS dependencies (openSUSE)", } SYSTEM_TOOL_CMDS = { @@ -785,14 +787,17 @@ def check_os_dependency(dep): os_to_pkg_cmd_map = { 'centos': RPM, 'debian': DPKG, + 'opensuse': ZYPPER, 'redhat': RPM, + 'rhel': RPM, 'ubuntu': DPKG, } pkg_cmd_flag = { DPKG: '-s', RPM: '-q', + ZYPPER: 'search -i', } - os_name = get_os_name() + os_name = get_os_name().lower().split(' ')[0] if os_name in os_to_pkg_cmd_map: pkg_cmds = [os_to_pkg_cmd_map[os_name]] else: From d176bd191afa9a62449be7abdc789f9a23c72b48 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 1 Mar 2022 14:58:18 +0100 Subject: [PATCH 800/864] add support for post-install patches --- easybuild/framework/easyblock.py | 28 ++++++++++++++++--- easybuild/framework/easyconfig/default.py | 1 + .../sources/toy/toy-0.0_fix-README.patch | 5 ++++ test/framework/toy_build.py | 19 +++++++++++++ 4 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 test/framework/sandbox/sources/toy/toy-0.0_fix-README.patch diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e71ee973fc..7232e74d09 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -455,12 +455,16 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None): All patches will be checked if a file exists (or can be located) """ if patch_specs is None: - patch_specs = self.cfg['patches'] + post_install_patches = self.cfg['postinstallpatches'] + patch_specs = self.cfg['patches'] + post_install_patches + else: + post_install_patches = [] patches = [] for index, patch_spec in enumerate(patch_specs): patch_info = create_patch_info(patch_spec) + patch_info['postinstall'] = patch_spec in post_install_patches force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES] path = self.obtain_file(patch_info['name'], extension=extension, force_download=force_download) @@ -2230,7 +2234,7 @@ def fetch_step(self, skip_checksums=False): self.dry_run_msg("\nList of patches:") # fetch patches - if self.cfg['patches']: + if self.cfg['patches'] + self.cfg['postinstallpatches']: if isinstance(self.cfg['checksums'], (list, tuple)): # if checksums are provided as a list, first entries are assumed to be for sources patches_checksums = self.cfg['checksums'][len(self.cfg['sources']):] @@ -2402,11 +2406,15 @@ def extract_step(self): else: raise EasyBuildError("Unpacking source %s failed", src['name']) - def patch_step(self, beginpath=None): + def patch_step(self, beginpath=None, patches=None): """ Apply the patches """ - for patch in self.patches: + if patches is None: + # if no patches are specified, use all non-post-install patches + patches = [p for p in self.patches if not p['postinstall']] + + for patch in patches: self.log.info("Applying patch %s" % patch['name']) trace_msg("applying patch %s" % patch['name']) @@ -2821,6 +2829,17 @@ def run_post_install_commands(self, commands=None): raise EasyBuildError("Invalid element in 'postinstallcmds', not a string: %s", cmd) run_cmd(cmd, simple=True, log_ok=True, log_all=True) + def apply_post_install_patches(self, patches=None): + """ + Apply post-install patch files that are specified via the 'postinstallpatches' easyconfig parameter. + """ + if patches is None: + patches = [p for p in self.patches if p['postinstall']] + + self.log.debug("Post-install patches to apply: %s", patches) + if patches: + self.patch_step(beginpath=self.installdir, patches=patches) + def post_install_step(self): """ Do some postprocessing @@ -2828,6 +2847,7 @@ def post_install_step(self): """ self.run_post_install_commands() + self.apply_post_install_patches() self.fix_shebang() diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 70571c12d5..eeed425420 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -114,6 +114,7 @@ 'preinstallopts': ['', 'Extra prefix options for installation.', BUILD], 'pretestopts': ['', 'Extra prefix options for test.', BUILD], 'postinstallcmds': [[], 'Commands to run after the install step.', BUILD], + 'postinstallpatches': [[], 'Patch files to apply after running the install step.', BUILD], 'required_linked_shared_libs': [[], "List of shared libraries (names, file names, or paths) which must be " "linked in all installed binaries/libraries", BUILD], 'runtest': [None, ('Indicates if a test should be run after make; should specify argument ' diff --git a/test/framework/sandbox/sources/toy/toy-0.0_fix-README.patch b/test/framework/sandbox/sources/toy/toy-0.0_fix-README.patch new file mode 100644 index 0000000000..71adf829c4 --- /dev/null +++ b/test/framework/sandbox/sources/toy/toy-0.0_fix-README.patch @@ -0,0 +1,5 @@ +--- README.old 2022-03-01 14:28:43.000000000 +0100 ++++ README 2022-03-01 14:29:01.000000000 +0100 +@@ -1 +1 @@ +-TOY ++toy 0.0, a toy test program diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index c27d9fabb2..9a8ca0dabe 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3721,6 +3721,25 @@ def test_toy_ignore_test_failure(self): self.assertTrue("Build succeeded (with --ignore-test-failure) for 1 out of 1" in stdout) self.assertFalse(stderr) + def test_toy_post_install_patches(self): + """ + Test use of post-install patches + """ + test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + + test_ec_txt = read_file(toy_ec) + test_ec_txt += "\npostinstallpatches = ['toy-0.0_fix-README.patch']" + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + + self.test_toy_build(ec_file=test_ec) + + toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') + toy_readme_txt = read_file(os.path.join(toy_installdir, 'README')) + # verify whether patch indeed got applied + self.assertEqual(toy_readme_txt, "toy 0.0, a toy test program\n") + def suite(): """ return all the tests in this file """ From f899f521ce45e2e9feed02b6b5efe800d5f9fc96 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 1 Mar 2022 18:48:36 +0100 Subject: [PATCH 801/864] enhance test_get_source_tarball_from_git to also check use of 'clone_into' --- test/framework/filetools.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index da9a19fa89..ff369bdb49 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2755,6 +2755,16 @@ def run_check(): ]) % git_repo run_check() + git_config['clone_into'] = 'test123' + expected = '\n'.join([ + r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"', + r" \(in .*/tmp.*\)", + r' running command "tar cfvz .*/target/test.tar.gz --exclude .git test123"', + r" \(in .*/tmp.*\)", + ]) % git_repo + run_check() + del git_config['clone_into'] + git_config['recursive'] = True expected = '\n'.join([ r' running command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', From 20fdb74ddbb76b262f4c1850f5ad251de80f15af Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 2 Mar 2022 10:39:27 +0000 Subject: [PATCH 802/864] improve message --- easybuild/tools/github.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index f6599b1881..a8ae5a0a97 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1234,7 +1234,8 @@ def not_eligible(msg): res = not_eligible(msg_tmpl % reason) if failed_status_last_commit and mergeable: - print_msg("\nThis PR is mergeable but the test suite has a failed status. Try syncing the PR.", prefix=False) + print_msg("\nThis PR is mergeable but the test suite has a failed status. Try syncing the PR with the " + "develop branch using 'eb --sync-pr-with-develop %s'" % pr_data['number'], prefix=False) return res From 9896bbbd4937b22e239fc03d05f4108ca38188fe Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 2 Mar 2022 19:13:26 +0100 Subject: [PATCH 803/864] small change to defining post_install_patches in EasyBlock.fetch_patches --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 7232e74d09..f471b38251 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -454,11 +454,11 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None): Add a list of patches. All patches will be checked if a file exists (or can be located) """ + post_install_patches = [] if patch_specs is None: + # if no patch_specs are specified, use all pre-install and post-install patches post_install_patches = self.cfg['postinstallpatches'] patch_specs = self.cfg['patches'] + post_install_patches - else: - post_install_patches = [] patches = [] for index, patch_spec in enumerate(patch_specs): From 623e188a3c9c3cb83b1f021c0caed3a8bb8319be Mon Sep 17 00:00:00 2001 From: Orient Date: Tue, 8 Mar 2022 16:19:19 +0100 Subject: [PATCH 804/864] Added new parameter key to specify some download or installation steps for user in case of complicated way of obtaining needed files --- easybuild/framework/easyblock.py | 14 ++++++++++++-- easybuild/framework/easyconfig/default.py | 1 + 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f471b38251..c2ed7bce6f 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -402,7 +402,15 @@ def fetch_source(self, source, checksum=None, extension=False): path = self.obtain_file(filename, extension=extension, download_filename=download_filename, force_download=force_download, urls=source_urls, git_config=git_config) if path is None: - raise EasyBuildError('No file found for source %s', filename) + if download_instructions is None: + download_instructions = self.cfg['download_instructions'] + + if self.cfg['download_instructions']: + print_msg(self.cfg['download_instructions']) + print_msg("Please make file available via active 'sourcepath' configuration setting: %s" % build_option('sourcepath')) + raise EasyBuildError("File %s is missing, see download instructions above" % source) + else: + raise EasyBuildError('No file found for source %s', filename) self.log.debug('File %s found for source %s' % (path, filename)) @@ -586,7 +594,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): source['source_urls'] = source_urls if fetch_files: - src = self.fetch_source(source, checksums, extension=True) + src = self.fetch_source(source, checksums, extension=True, download_instructions=ext_options.get('download_instructions')) ext_src.update({ # keep track of custom extract command (if any) 'extract_cmd': src['cmd'], @@ -869,6 +877,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No self.dry_run_msg(" * %s (MISSING)", filename) return filename else: + if self.cfg['download_instructions'] != None: + print_msg(self.cfg['download_instructions']) raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... " "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index eeed425420..a1f1fef043 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -90,6 +90,7 @@ 'checksums': [[], "Checksums for sources and patches", BUILD], 'configopts': ['', 'Extra options passed to configure (default already has --prefix)', BUILD], 'cuda_compute_capabilities': [[], "List of CUDA compute capabilities to build with (if supported)", BUILD], + 'download_instructions': ['', "Specify steps to aquire necessary file, if obtaining it is difficult", BUILD], 'easyblock': [None, "EasyBlock to use for building; if set to None, an easyblock is selected " "based on the software name", BUILD], 'easybuild_version': [None, "EasyBuild-version this spec-file was written for", BUILD], From db42fd9b377cf2ea113718277cdbfb6febbc2e3a Mon Sep 17 00:00:00 2001 From: ItIsI-Orient <62220698+ItIsI-Orient@users.noreply.github.com> Date: Tue, 8 Mar 2022 16:35:49 +0100 Subject: [PATCH 805/864] fix of errors --- easybuild/framework/easyblock.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index c2ed7bce6f..f3c2e8370c 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -402,14 +402,6 @@ def fetch_source(self, source, checksum=None, extension=False): path = self.obtain_file(filename, extension=extension, download_filename=download_filename, force_download=force_download, urls=source_urls, git_config=git_config) if path is None: - if download_instructions is None: - download_instructions = self.cfg['download_instructions'] - - if self.cfg['download_instructions']: - print_msg(self.cfg['download_instructions']) - print_msg("Please make file available via active 'sourcepath' configuration setting: %s" % build_option('sourcepath')) - raise EasyBuildError("File %s is missing, see download instructions above" % source) - else: raise EasyBuildError('No file found for source %s', filename) self.log.debug('File %s found for source %s' % (path, filename)) @@ -594,7 +586,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): source['source_urls'] = source_urls if fetch_files: - src = self.fetch_source(source, checksums, extension=True, download_instructions=ext_options.get('download_instructions')) + src = self.fetch_source(source, checksums, extension=True, + download_instructions=ext_options.get('download_instructions')) ext_src.update({ # keep track of custom extract command (if any) 'extract_cmd': src['cmd'], @@ -877,7 +870,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No self.dry_run_msg(" * %s (MISSING)", filename) return filename else: - if self.cfg['download_instructions'] != None: + if self.cfg['download_instructions'] is not None: print_msg(self.cfg['download_instructions']) raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... " "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) From 1f99dccf0ca251c144774b212a9a0bde53a29d88 Mon Sep 17 00:00:00 2001 From: ItIsI-Orient <62220698+ItIsI-Orient@users.noreply.github.com> Date: Tue, 8 Mar 2022 16:39:43 +0100 Subject: [PATCH 806/864] fix of errors --- easybuild/framework/easyblock.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f3c2e8370c..209915acf0 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -402,7 +402,7 @@ def fetch_source(self, source, checksum=None, extension=False): path = self.obtain_file(filename, extension=extension, download_filename=download_filename, force_download=force_download, urls=source_urls, git_config=git_config) if path is None: - raise EasyBuildError('No file found for source %s', filename) + raise EasyBuildError('No file found for source %s', filename) self.log.debug('File %s found for source %s' % (path, filename)) @@ -586,8 +586,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): source['source_urls'] = source_urls if fetch_files: - src = self.fetch_source(source, checksums, extension=True, - download_instructions=ext_options.get('download_instructions')) + src = self.fetch_source(source, checksums, extension=True, + download_instructions=ext_options.get('download_instructions')) ext_src.update({ # keep track of custom extract command (if any) 'extract_cmd': src['cmd'], From 4c3e30d32710d9b6e3a79b876815c1ddb52cb8b2 Mon Sep 17 00:00:00 2001 From: ItIsI-Orient <62220698+ItIsI-Orient@users.noreply.github.com> Date: Tue, 8 Mar 2022 16:43:04 +0100 Subject: [PATCH 807/864] fix of errors --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 209915acf0..36bc1faf3a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -587,7 +587,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): if fetch_files: src = self.fetch_source(source, checksums, extension=True, - download_instructions=ext_options.get('download_instructions')) + download_instructions=ext_options.get('download_instructions')) ext_src.update({ # keep track of custom extract command (if any) 'extract_cmd': src['cmd'], From c890250810c645584f921a9c746aa0a3eab7fb55 Mon Sep 17 00:00:00 2001 From: ItIsI-Orient <62220698+ItIsI-Orient@users.noreply.github.com> Date: Tue, 8 Mar 2022 17:32:39 +0100 Subject: [PATCH 808/864] Update easyblock.py --- easybuild/framework/easyblock.py | 33 +++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 36bc1faf3a..d9a61a0bc4 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -364,7 +364,7 @@ def get_checksum_for(self, checksums, filename=None, index=None): else: raise EasyBuildError("Invalid type for checksums (%s), should be list, tuple or None.", type(checksums)) - def fetch_source(self, source, checksum=None, extension=False): + def fetch_source(self, source, checksum=None, extension=False, download_instructions=None): """ Get a specific source (tarball, iso, url) Will be tested for existence or can be located @@ -400,9 +400,19 @@ def fetch_source(self, source, checksum=None, extension=False): # check if the sources can be located force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES] path = self.obtain_file(filename, extension=extension, download_filename=download_filename, - force_download=force_download, urls=source_urls, git_config=git_config) + force_download=force_download, urls=source_urls, git_config=git_config, + download_instructions=download_instructions) if path is None: - raise EasyBuildError('No file found for source %s', filename) + if download_instructions is None: + download_instructions = self.cfg['download_instructions'] + + if self.cfg['download_instructions']: + print_msg(self.cfg['download_instructions']) + print_msg("Please make file available via active" + "'sourcepath' configuration setting: %s" % build_option('sourcepath')) + raise EasyBuildError("file %s missing, see download instructions above" % source) + else: + raise EasyBuildError('No file found for source %s', filename) self.log.debug('File %s found for source %s' % (path, filename)) @@ -587,7 +597,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): if fetch_files: src = self.fetch_source(source, checksums, extension=True, - download_instructions=ext_options.get('download_instructions')) + download_instructions=ext_options.get('download_instructions')) ext_src.update({ # keep track of custom extract command (if any) 'extract_cmd': src['cmd'], @@ -683,7 +693,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): return exts_sources def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False, - git_config=None): + git_config=None, download_instructions=None): """ Locate the file with the given name - searches in different subdirectories of source path @@ -870,10 +880,15 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No self.dry_run_msg(" * %s (MISSING)", filename) return filename else: - if self.cfg['download_instructions'] is not None: - print_msg(self.cfg['download_instructions']) - raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... " - "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + if download_instructions is None: + download_instructions = self.cfg['download_instructions'] + if download_instructions: + print_msg(download_instructions) + raise EasyBuildError("Couldn't find file %s, please follow the instructions above. " + "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + else: + raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... " + "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) # # GETTER/SETTER UTILITY FUNCTIONS From 10a3614f21de9cda7768778c862e286baf3156bd Mon Sep 17 00:00:00 2001 From: ItIsI-Orient <62220698+ItIsI-Orient@users.noreply.github.com> Date: Tue, 8 Mar 2022 17:35:08 +0100 Subject: [PATCH 809/864] Fix indentations --- easybuild/framework/easyblock.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index d9a61a0bc4..3e6ccf3e9d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -404,15 +404,15 @@ def fetch_source(self, source, checksum=None, extension=False, download_instruct download_instructions=download_instructions) if path is None: if download_instructions is None: - download_instructions = self.cfg['download_instructions'] + download_instructions = self.cfg['download_instructions'] if self.cfg['download_instructions']: - print_msg(self.cfg['download_instructions']) - print_msg("Please make file available via active" - "'sourcepath' configuration setting: %s" % build_option('sourcepath')) - raise EasyBuildError("file %s missing, see download instructions above" % source) + print_msg(self.cfg['download_instructions']) + print_msg("Please make file available via active" + "'sourcepath' configuration setting: %s" % build_option('sourcepath')) + raise EasyBuildError("file %s missing, see download instructions above" % source) else: - raise EasyBuildError('No file found for source %s', filename) + raise EasyBuildError('No file found for source %s', filename) self.log.debug('File %s found for source %s' % (path, filename)) @@ -597,7 +597,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): if fetch_files: src = self.fetch_source(source, checksums, extension=True, - download_instructions=ext_options.get('download_instructions')) + download_instructions=ext_options.get('download_instructions')) ext_src.update({ # keep track of custom extract command (if any) 'extract_cmd': src['cmd'], @@ -885,10 +885,10 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No if download_instructions: print_msg(download_instructions) raise EasyBuildError("Couldn't find file %s, please follow the instructions above. " - "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) else: raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... " - "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) # # GETTER/SETTER UTILITY FUNCTIONS From 422649035025bf87307e3da678ed1bca39e5d7a1 Mon Sep 17 00:00:00 2001 From: ItIsI-Orient <62220698+ItIsI-Orient@users.noreply.github.com> Date: Tue, 8 Mar 2022 17:37:01 +0100 Subject: [PATCH 810/864] fix indentations? --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 3e6ccf3e9d..4212c94f60 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -885,10 +885,10 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No if download_instructions: print_msg(download_instructions) raise EasyBuildError("Couldn't find file %s, please follow the instructions above. " - "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) else: raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... " - "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) # # GETTER/SETTER UTILITY FUNCTIONS From 96089fcb115b681f6a8ff6dc9aaa710b01ba38ed Mon Sep 17 00:00:00 2001 From: ItIsI-Orient <62220698+ItIsI-Orient@users.noreply.github.com> Date: Tue, 8 Mar 2022 17:39:02 +0100 Subject: [PATCH 811/864] indentation fix --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 4212c94f60..545ad28883 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -885,10 +885,10 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No if download_instructions: print_msg(download_instructions) raise EasyBuildError("Couldn't find file %s, please follow the instructions above. " - "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) else: raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... " - "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) # # GETTER/SETTER UTILITY FUNCTIONS From e96ce73fb80bb3dd58ec9677832cc529ff2fad9f Mon Sep 17 00:00:00 2001 From: ItIsI-Orient <62220698+ItIsI-Orient@users.noreply.github.com> Date: Tue, 8 Mar 2022 17:41:04 +0100 Subject: [PATCH 812/864] last indent fix (hopefully) --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 545ad28883..b73b075cde 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -409,7 +409,7 @@ def fetch_source(self, source, checksum=None, extension=False, download_instruct if self.cfg['download_instructions']: print_msg(self.cfg['download_instructions']) print_msg("Please make file available via active" - "'sourcepath' configuration setting: %s" % build_option('sourcepath')) + "'sourcepath' configuration setting: %s" % build_option('sourcepath')) raise EasyBuildError("file %s missing, see download instructions above" % source) else: raise EasyBuildError('No file found for source %s', filename) From acf1044631de31f35e9a0029b10eda0b1fd401c8 Mon Sep 17 00:00:00 2001 From: ItIsI-Orient <62220698+ItIsI-Orient@users.noreply.github.com> Date: Wed, 9 Mar 2022 14:15:57 +0100 Subject: [PATCH 813/864] Message improvement + quick condition fix --- easybuild/framework/easyblock.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index b73b075cde..839c58fcd0 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -882,10 +882,9 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No else: if download_instructions is None: download_instructions = self.cfg['download_instructions'] - if download_instructions: + if download_instructions is not None: print_msg(download_instructions) - raise EasyBuildError("Couldn't find file %s, please follow the instructions above. " - "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + raise EasyBuildError("Couldn't find file %s, please follow the instructions above. ", filename) else: raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... " "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) From d7451608f7154d0625b5ca5ee564600412487828 Mon Sep 17 00:00:00 2001 From: ItIsI-Orient <62220698+ItIsI-Orient@users.noreply.github.com> Date: Wed, 9 Mar 2022 17:24:40 +0100 Subject: [PATCH 814/864] bugfix --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 839c58fcd0..771b734794 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -882,7 +882,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No else: if download_instructions is None: download_instructions = self.cfg['download_instructions'] - if download_instructions is not None: + if download_instructions is not None and download_instructions != "": print_msg(download_instructions) raise EasyBuildError("Couldn't find file %s, please follow the instructions above. ", filename) else: From 013303c24bac99aaa38a48121b627da31c8cfafc Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Thu, 10 Mar 2022 18:12:52 +0100 Subject: [PATCH 815/864] add support for collecting GPU info (via rocm-smi) --- easybuild/tools/systemtools.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 37411e4b86..1a17dbfdfd 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -610,6 +610,29 @@ def get_gpu_info(): except Exception as err: _log.debug("Exception was raised when running nvidia-smi: %s", err) _log.info("No NVIDIA GPUs detected") + + try: + cmd = "rocm-smi --showdriverversion --csv" + _log.debug("Trying to determine AMD GPU driver on Linux via cmd '%s'", cmd) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) + if ec == 0: + amd_driver = out.strip().split('\n')[1].split(',')[1] + + cmd = "rocm-smi --showproductname --csv" + _log.debug("Trying to determine AMD GPU info on Linux via cmd '%s'", cmd) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) + if ec == 0: + for line in out.strip().split('\n')[1:]: + amd_card_model = line.split(',')[2] + amd_gpu = ', '.join([amd_card_model,amd_driver]) + amd_gpu_info = gpu_info.setdefault('AMD', {}) + amd_gpu_info.setdefault(amd_gpu, 0) + amd_gpu_info[amd_gpu] += 1 + else: + _log.debug("None zero exit (%s) from rocm-smi: %s", ec, out) + except Exception as err: + _log.debug("Exception was raised when running rocm-smi: %s", err) + _log.info("No AMD GPUs detected") else: _log.info("Only know how to get GPU info on Linux, assuming no GPUs are present") From 5e79222d1ae778b74b973f8becd641c906a97e07 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Thu, 10 Mar 2022 18:17:58 +0100 Subject: [PATCH 816/864] add support for collecting GPU info (via rocm-smi) --- easybuild/tools/systemtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 1a17dbfdfd..83da914a44 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -612,7 +612,7 @@ def get_gpu_info(): _log.info("No NVIDIA GPUs detected") try: - cmd = "rocm-smi --showdriverversion --csv" + cmd = "rocm-smi --showdriverversion --csv" _log.debug("Trying to determine AMD GPU driver on Linux via cmd '%s'", cmd) out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) if ec == 0: From 3222a830e2a0f2368339e07883e16d7583a6128a Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Thu, 10 Mar 2022 18:19:34 +0100 Subject: [PATCH 817/864] add support for collecting GPU info (via rocm-smi) --- easybuild/tools/systemtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 83da914a44..55de973f19 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -624,7 +624,7 @@ def get_gpu_info(): if ec == 0: for line in out.strip().split('\n')[1:]: amd_card_model = line.split(',')[2] - amd_gpu = ', '.join([amd_card_model,amd_driver]) + amd_gpu = ', '.join([amd_card_model, amd_driver]) amd_gpu_info = gpu_info.setdefault('AMD', {}) amd_gpu_info.setdefault(amd_gpu, 0) amd_gpu_info[amd_gpu] += 1 From 1f5ef16927107d550cd679cd3fcc738a59d41644 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 15 Mar 2022 20:19:11 +0100 Subject: [PATCH 818/864] tweak printing of download instructions in obtain_file + remove dead code dealing with download instructions in fetch_source --- easybuild/framework/easyblock.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 771b734794..1e7d360be3 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -403,16 +403,7 @@ def fetch_source(self, source, checksum=None, extension=False, download_instruct force_download=force_download, urls=source_urls, git_config=git_config, download_instructions=download_instructions) if path is None: - if download_instructions is None: - download_instructions = self.cfg['download_instructions'] - - if self.cfg['download_instructions']: - print_msg(self.cfg['download_instructions']) - print_msg("Please make file available via active" - "'sourcepath' configuration setting: %s" % build_option('sourcepath')) - raise EasyBuildError("file %s missing, see download instructions above" % source) - else: - raise EasyBuildError('No file found for source %s', filename) + raise EasyBuildError('No file found for source %s', filename) self.log.debug('File %s found for source %s' % (path, filename)) @@ -880,14 +871,19 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No self.dry_run_msg(" * %s (MISSING)", filename) return filename else: + error_msg = "Couldn't find file %s anywhere, " if download_instructions is None: download_instructions = self.cfg['download_instructions'] if download_instructions is not None and download_instructions != "": - print_msg(download_instructions) - raise EasyBuildError("Couldn't find file %s, please follow the instructions above. ", filename) + msg = "\nDownload instructions:\n\n" + download_instructions + '\n' + print_msg(msg, prefix=False, stderr=True) + error_msg += "please follow the download instructions above, and make the file available " + error_msg += "in the active source path (%s)" % ':'.join(source_paths()) else: - raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... " - "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + error_msg += "and downloading it didn't work either... " + error_msg += "Paths attempted (in order): %s " % ', '.join(failedpaths) + + raise EasyBuildError(error_msg, filename) # # GETTER/SETTER UTILITY FUNCTIONS From 16f55f825f5f036f4c0aeadc1b9f52a364b3210c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 15 Mar 2022 20:26:17 +0100 Subject: [PATCH 819/864] add test to verify use of download_instructions easyconfig parameter --- test/framework/easyblock.py | 104 ++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 6da8ec6815..960af782bf 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1480,6 +1480,110 @@ def test_fetch_sources(self): error_pattern = "Found one or more unexpected keys in 'sources' specification: {'nosuchkey': 'foobar'}" self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_sources, sources, checksums=[]) + def test_download_instructions(self): + """Test use of download_instructions easyconfig parameter.""" + orig_test_ec = '\n'.join([ + "easyblock = 'ConfigureMake'", + "name = 'software_with_missing_sources'", + "version = '0.0'", + "homepage = 'https://example.com'", + "description = 'test'", + "toolchain = SYSTEM", + "sources = [SOURCE_TAR_GZ]", + "exts_list = [", + " ('ext_with_missing_sources', '0.0', {", + " 'sources': [SOURCE_TAR_GZ],", + " }),", + "]", + ]) + self.contents = orig_test_ec + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + + common_error_pattern = "^Couldn't find file software_with_missing_sources-0.0.tar.gz anywhere" + error_pattern = common_error_pattern + ", and downloading it didn't work either" + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + + download_instructions = "download_instructions = 'Manual download from example.com required'" + sources = "sources = [SOURCE_TAR_GZ]" + self.contents = self.contents.replace(sources, download_instructions + '\n' + sources) + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + + error_pattern = common_error_pattern + ", please follow the download instructions above" + self.mock_stderr(True) + self.mock_stdout(True) + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + stderr = self.get_stderr().strip() + stdout = self.get_stdout().strip() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, "Download instructions:\n\nManual download from example.com required") + self.assertEqual(stdout, '') + + # create dummy source file + write_file(os.path.join(os.path.dirname(self.eb_file), 'software_with_missing_sources-0.0.tar.gz'), '') + + # now downloading of sources for extension should fail + # top-level download instructions are printed (because there's nothing else) + error_pattern = "^Couldn't find file ext_with_missing_sources-0.0.tar.gz anywhere" + self.mock_stderr(True) + self.mock_stdout(True) + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + stderr = self.get_stderr().strip() + stdout = self.get_stdout().strip() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, "Download instructions:\n\nManual download from example.com required") + self.assertEqual(stdout, '') + + # wipe top-level download instructions, try again + self.contents = self.contents.replace(download_instructions, '') + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + + # no download instructions printed anymore now + self.mock_stderr(True) + self.mock_stdout(True) + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + stderr = self.get_stderr().strip() + stdout = self.get_stdout().strip() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stdout, '') + + # inject download instructions for extension + download_instructions = ' ' * 8 + "'download_instructions': " + download_instructions += "'Extension sources must be downloaded via example.com'," + sources = "'sources': [SOURCE_TAR_GZ]," + self.contents = self.contents.replace(sources, sources + '\n' + download_instructions) + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + + self.mock_stderr(True) + self.mock_stdout(True) + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + stderr = self.get_stderr().strip() + stdout = self.get_stdout().strip() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, "Download instructions:\n\nExtension sources must be downloaded via example.com") + self.assertEqual(stdout, '') + + # create dummy source file for extension + write_file(os.path.join(os.path.dirname(self.eb_file), 'ext_with_missing_sources-0.0.tar.gz'), '') + + # no more errors, all source files found (so no download instructions printed either) + self.mock_stderr(True) + self.mock_stdout(True) + eb.fetch_step() + stderr = self.get_stderr().strip() + stdout = self.get_stdout().strip() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, '') + self.assertEqual(stdout, '') + def test_fetch_patches(self): """Test fetch_patches method.""" testdir = os.path.abspath(os.path.dirname(__file__)) From 37b0d8657bf78637fc5a8e3b5ebe0133f92c5f23 Mon Sep 17 00:00:00 2001 From: Jasper Grimm Date: Thu, 17 Mar 2022 15:32:12 +0000 Subject: [PATCH 820/864] ensure --review-pr can find dependencies included in PR --- easybuild/framework/easyconfig/tools.py | 7 ++++++- easybuild/tools/options.py | 8 +++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index be6ff5fe01..3aaed01c6f 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -308,7 +308,7 @@ def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None): return paths -def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None): +def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None, review_pr=None): """Obtain alternative paths for easyconfig files.""" # paths where tweaked easyconfigs will be placed, easyconfigs listed on the command line take priority and will be @@ -322,6 +322,11 @@ def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None): # paths where files touched in PRs will be downloaded to, # which are picked up via 'pr_paths' build option in fetch_files_from_pr pr_paths = None + if from_prs and review_pr: + from_prs.append(review_pr) if review_pr not in from_prs + elif review_pr: + from_prs = [review_pr] + if from_prs: pr_paths = [os.path.join(tmpdir, 'files_pr%s' % pr) for pr in from_prs] diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 6d123508a2..1fd18ceedc 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1547,10 +1547,16 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): except ValueError: raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") + try: + review_pr = int(eb_go.options.review_pr) + except ValueError: + raise EasyBuildError("Argument to --review-pr must be an integer PR #.") + # determine robot path # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs tweaked_ecs = try_to_generate and build_specs - tweaked_ecs_paths, pr_paths = alt_easyconfig_paths(tmpdir, tweaked_ecs=tweaked_ecs, from_prs=from_prs) + tweaked_ecs_paths, pr_paths = alt_easyconfig_paths(tmpdir, tweaked_ecs=tweaked_ecs, from_prs=from_prs, + review_pr=review_pr) auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_paths, auto_robot=auto_robot) log.debug("Full robot path: %s", robot_path) From b87c8a3761bb89eeeead706a6013ff806902cc5f Mon Sep 17 00:00:00 2001 From: Jasper <65227842+jfgrimm@users.noreply.github.com> Date: Thu, 17 Mar 2022 15:46:02 +0000 Subject: [PATCH 821/864] fix syntax --- easybuild/framework/easyconfig/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 3aaed01c6f..66d181c1d9 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -323,7 +323,7 @@ def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None, review_pr=Non # which are picked up via 'pr_paths' build option in fetch_files_from_pr pr_paths = None if from_prs and review_pr: - from_prs.append(review_pr) if review_pr not in from_prs + from_prs.append(review_pr) if review_pr not in from_prs else from_prs elif review_pr: from_prs = [review_pr] From fa629682fe9dcef98d920c50c2b0873b1df0526a Mon Sep 17 00:00:00 2001 From: Jasper <65227842+jfgrimm@users.noreply.github.com> Date: Mon, 21 Mar 2022 11:06:32 +0000 Subject: [PATCH 822/864] guard review_pr assignment to avoid attempting to convert None to int --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 1fd18ceedc..126dd701cc 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1548,7 +1548,7 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") try: - review_pr = int(eb_go.options.review_pr) + review_pr = (lambda x: int(x) if x else None)(eb_go.options.review_pr) except ValueError: raise EasyBuildError("Argument to --review-pr must be an integer PR #.") From 602439a0f74fd849ba3119c85a1904df15b4d690 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 25 Mar 2022 11:18:11 +0100 Subject: [PATCH 823/864] also print download instructions for extensions that use source_tmpl to specify name of source file --- easybuild/framework/easyblock.py | 7 +++++-- test/framework/easyblock.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 1e7d360be3..2822e0a2c9 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -556,6 +556,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): source_urls = ext_options.get('source_urls', []) checksums = ext_options.get('checksums', []) + download_instructions = ext_options.get('download_instructions') + if ext_options.get('nosource', None): self.log.debug("No sources for extension %s, as indicated by 'nosource'", ext_name) @@ -588,7 +590,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): if fetch_files: src = self.fetch_source(source, checksums, extension=True, - download_instructions=ext_options.get('download_instructions')) + download_instructions=download_instructions) ext_src.update({ # keep track of custom extract command (if any) 'extract_cmd': src['cmd'], @@ -610,7 +612,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): if fetch_files: src_path = self.obtain_file(src_fn, extension=True, urls=source_urls, - force_download=force_download) + force_download=force_download, + download_instructions=download_instructions) if src_path: ext_src.update({'src': src_path}) else: diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 960af782bf..efbc743f0e 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1570,6 +1570,21 @@ def test_download_instructions(self): self.assertEqual(stderr, "Download instructions:\n\nExtension sources must be downloaded via example.com") self.assertEqual(stdout, '') + # download instructions should also be printed if 'source_tmpl' is used to specify extension sources + self.contents = self.contents.replace(sources, "'source_tmpl': SOURCE_TAR_GZ,") + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + + self.mock_stderr(True) + self.mock_stdout(True) + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + stderr = self.get_stderr().strip() + stdout = self.get_stdout().strip() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, "Download instructions:\n\nExtension sources must be downloaded via example.com") + self.assertEqual(stdout, '') + # create dummy source file for extension write_file(os.path.join(os.path.dirname(self.eb_file), 'ext_with_missing_sources-0.0.tar.gz'), '') From 366a65d7f61ad3adb4784ec45702bf03251353c5 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Sat, 26 Mar 2022 16:09:05 +0100 Subject: [PATCH 824/864] enhance GPU info (via rocm-smi): report card series in addition --- easybuild/tools/systemtools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 882673769c..4d26066a18 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -625,8 +625,9 @@ def get_gpu_info(): out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) if ec == 0: for line in out.strip().split('\n')[1:]: + amd_card_series = line.split(',')[1] amd_card_model = line.split(',')[2] - amd_gpu = ', '.join([amd_card_model, amd_driver]) + amd_gpu = "%s (model: %s, driver: %s)" % (amd_card_series, amd_card_model, amd_driver) amd_gpu_info = gpu_info.setdefault('AMD', {}) amd_gpu_info.setdefault(amd_gpu, 0) amd_gpu_info[amd_gpu] += 1 From fa605e69d944bcecee0093acaeb3b3bf0a837a34 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Mon, 28 Mar 2022 14:03:26 +0200 Subject: [PATCH 825/864] prepare release notes for EasyBuild v4.5.4 + bump version to 4.5.4 --- RELEASE_NOTES | 23 +++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index c90361c413..9cd3aca54e 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,29 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.5.4 (March 28th 2022) +--------------------------- + +update/bugfix release + +- various enhancements, including: + - warn about potentially missing patches in --new-pr (#3759) + - add support for 'clone_into' field in git_config source spec to specify different name for top-level directory (#3949) + - add bash completion for easyconfigs from local dir but not robot search path (#3953) + - add a 'sync pr' message when the PR has a mergeable state but is showing a failed status for the test suite on the last commit (#3967) + - add gmpit toolchain definition (GCC + MPItrampoline) (#3971) + - use 'zypper search -i' to check whether specified OS dependency is installed on openSUSE + make sure that rpm is considered for checking OS dependencies on RHEL8 (#3973) + - add support for post-install patches (#3974) + - support 'download_instructions' easyconfig parameter key to specify some download or installation steps for user in case of complicated way of obtaining needed files (#3976) + - add support for collecting GPU info (via rocm-smi) (#3978) + - enhance GPU info (via rocm-smi): report card series in addition (#3982) +- various bug fixes, including: + - only check for patches when using --new-pr if easyconfigs repository is target (#3966) + - also print download instructions for extensions that use source_tmpl to specify name of source file (#3981) +- other changes: + - enable code linting check for all supported Python versions (#3725) + + v4.5.3 (February 11th 2022) --------------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 1396a1c4a1..2375cb9000 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.5.4.dev0') +VERSION = LooseVersion('4.5.4') UNKNOWN = 'UNKNOWN' From 40646e7f58dbabf1fc0f4febfa648e07bbb3a182 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Mon, 28 Mar 2022 14:22:53 +0200 Subject: [PATCH 826/864] minor tweak release notes for EasyBuild v4.5.4 --- RELEASE_NOTES | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 9cd3aca54e..6ff4e486f6 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -18,8 +18,7 @@ update/bugfix release - use 'zypper search -i' to check whether specified OS dependency is installed on openSUSE + make sure that rpm is considered for checking OS dependencies on RHEL8 (#3973) - add support for post-install patches (#3974) - support 'download_instructions' easyconfig parameter key to specify some download or installation steps for user in case of complicated way of obtaining needed files (#3976) - - add support for collecting GPU info (via rocm-smi) (#3978) - - enhance GPU info (via rocm-smi): report card series in addition (#3982) + - add support for collecting AMD GPU info (via rocm-smi) (#3978, #3982) - various bug fixes, including: - only check for patches when using --new-pr if easyconfigs repository is target (#3966) - also print download instructions for extensions that use source_tmpl to specify name of source file (#3981) From 08abb62435d23ff28083e3883f1dcd090f9f669f Mon Sep 17 00:00:00 2001 From: Jasper <65227842+jfgrimm@users.noreply.github.com> Date: Mon, 28 Mar 2022 15:39:37 +0100 Subject: [PATCH 827/864] update package list before installing --- .github/workflows/container_tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/container_tests.yml b/.github/workflows/container_tests.yml index cc8cafe298..d3bc6e05fb 100644 --- a/.github/workflows/container_tests.yml +++ b/.github/workflows/container_tests.yml @@ -20,6 +20,8 @@ jobs: - name: install OS & Python packages run: | + # ensure package list is up to date to avoid 404's for old packages + sudo apt-get update -yqq # for building Singularity images sudo apt-get install rpm sudo apt-get install yum From deb08681c5802321aea1c1524f1d980ac472f563 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 28 Mar 2022 17:47:24 +0200 Subject: [PATCH 828/864] update copyright lines for 2022 --- easybuild/__init__.py | 2 +- easybuild/base/exceptions.py | 2 +- easybuild/base/fancylogger.py | 2 +- easybuild/base/generaloption.py | 2 +- easybuild/base/testing.py | 2 +- easybuild/framework/__init__.py | 2 +- easybuild/framework/easyblock.py | 2 +- easybuild/framework/easyconfig/__init__.py | 2 +- easybuild/framework/easyconfig/constants.py | 2 +- easybuild/framework/easyconfig/default.py | 2 +- easybuild/framework/easyconfig/easyconfig.py | 2 +- easybuild/framework/easyconfig/format/__init__.py | 2 +- easybuild/framework/easyconfig/format/convert.py | 2 +- easybuild/framework/easyconfig/format/format.py | 2 +- easybuild/framework/easyconfig/format/one.py | 2 +- easybuild/framework/easyconfig/format/pyheaderconfigobj.py | 2 +- easybuild/framework/easyconfig/format/two.py | 2 +- easybuild/framework/easyconfig/format/version.py | 2 +- easybuild/framework/easyconfig/format/yeb.py | 2 +- easybuild/framework/easyconfig/licenses.py | 2 +- easybuild/framework/easyconfig/parser.py | 2 +- easybuild/framework/easyconfig/style.py | 2 +- easybuild/framework/easyconfig/templates.py | 2 +- easybuild/framework/easyconfig/tools.py | 2 +- easybuild/framework/easyconfig/tweak.py | 2 +- easybuild/framework/easyconfig/types.py | 2 +- easybuild/framework/easystack.py | 2 +- easybuild/framework/extension.py | 2 +- easybuild/framework/extensioneasyblock.py | 2 +- easybuild/main.py | 2 +- easybuild/scripts/bootstrap_eb.py | 2 +- easybuild/scripts/fix_docs.py | 2 +- easybuild/scripts/mk_tmpl_easyblock_for.py | 2 +- easybuild/scripts/rpath_args.py | 2 +- easybuild/scripts/rpath_wrapper_template.sh.in | 2 +- easybuild/toolchains/__init__.py | 2 +- easybuild/toolchains/cgmpich.py | 2 +- easybuild/toolchains/cgmpolf.py | 2 +- easybuild/toolchains/cgmvapich2.py | 2 +- easybuild/toolchains/cgmvolf.py | 2 +- easybuild/toolchains/cgompi.py | 2 +- easybuild/toolchains/cgoolf.py | 2 +- easybuild/toolchains/clanggcc.py | 2 +- easybuild/toolchains/compiler/__init__.py | 2 +- easybuild/toolchains/compiler/clang.py | 2 +- easybuild/toolchains/compiler/craype.py | 2 +- easybuild/toolchains/compiler/cuda.py | 2 +- easybuild/toolchains/compiler/dummycompiler.py | 2 +- easybuild/toolchains/compiler/fujitsu.py | 2 +- easybuild/toolchains/compiler/gcc.py | 2 +- easybuild/toolchains/compiler/intel_compilers.py | 2 +- easybuild/toolchains/compiler/inteliccifort.py | 2 +- easybuild/toolchains/compiler/systemcompiler.py | 2 +- easybuild/toolchains/craycce.py | 2 +- easybuild/toolchains/craygnu.py | 2 +- easybuild/toolchains/crayintel.py | 2 +- easybuild/toolchains/craypgi.py | 2 +- easybuild/toolchains/dummy.py | 2 +- easybuild/toolchains/fcc.py | 2 +- easybuild/toolchains/ffmpi.py | 2 +- easybuild/toolchains/fft/__init__.py | 2 +- easybuild/toolchains/fft/fftw.py | 2 +- easybuild/toolchains/fft/fujitsufftw.py | 2 +- easybuild/toolchains/fft/intelfftw.py | 2 +- easybuild/toolchains/foss.py | 2 +- easybuild/toolchains/fosscuda.py | 2 +- easybuild/toolchains/fujitsu.py | 2 +- easybuild/toolchains/gcc.py | 2 +- easybuild/toolchains/gcccore.py | 2 +- easybuild/toolchains/gcccuda.py | 2 +- easybuild/toolchains/gimkl.py | 2 +- easybuild/toolchains/gimpi.py | 2 +- easybuild/toolchains/gimpic.py | 2 +- easybuild/toolchains/giolf.py | 2 +- easybuild/toolchains/giolfc.py | 2 +- easybuild/toolchains/gmacml.py | 2 +- easybuild/toolchains/gmkl.py | 2 +- easybuild/toolchains/gmklc.py | 2 +- easybuild/toolchains/gmpich.py | 2 +- easybuild/toolchains/gmpich2.py | 2 +- easybuild/toolchains/gmpolf.py | 2 +- easybuild/toolchains/gmvapich2.py | 2 +- easybuild/toolchains/gmvolf.py | 2 +- easybuild/toolchains/gnu.py | 2 +- easybuild/toolchains/goalf.py | 2 +- easybuild/toolchains/gobff.py | 2 +- easybuild/toolchains/goblf.py | 2 +- easybuild/toolchains/gofbf.py | 2 +- easybuild/toolchains/golf.py | 2 +- easybuild/toolchains/golfc.py | 2 +- easybuild/toolchains/gomkl.py | 2 +- easybuild/toolchains/gomklc.py | 2 +- easybuild/toolchains/gompi.py | 2 +- easybuild/toolchains/gompic.py | 2 +- easybuild/toolchains/goolf.py | 2 +- easybuild/toolchains/goolfc.py | 2 +- easybuild/toolchains/gpsmpi.py | 2 +- easybuild/toolchains/gpsolf.py | 2 +- easybuild/toolchains/gqacml.py | 2 +- easybuild/toolchains/gsmpi.py | 2 +- easybuild/toolchains/gsolf.py | 2 +- easybuild/toolchains/iccifort.py | 2 +- easybuild/toolchains/iccifortcuda.py | 2 +- easybuild/toolchains/ictce.py | 2 +- easybuild/toolchains/iibff.py | 2 +- easybuild/toolchains/iimkl.py | 2 +- easybuild/toolchains/iimklc.py | 2 +- easybuild/toolchains/iimpi.py | 2 +- easybuild/toolchains/iimpic.py | 2 +- easybuild/toolchains/iiqmpi.py | 2 +- easybuild/toolchains/impich.py | 2 +- easybuild/toolchains/impmkl.py | 2 +- easybuild/toolchains/intel-para.py | 2 +- easybuild/toolchains/intel.py | 2 +- easybuild/toolchains/intel_compilers.py | 2 +- easybuild/toolchains/intelcuda.py | 2 +- easybuild/toolchains/iomkl.py | 2 +- easybuild/toolchains/iomklc.py | 2 +- easybuild/toolchains/iompi.py | 2 +- easybuild/toolchains/iompic.py | 2 +- easybuild/toolchains/ipsmpi.py | 2 +- easybuild/toolchains/iqacml.py | 2 +- easybuild/toolchains/ismkl.py | 2 +- easybuild/toolchains/linalg/__init__.py | 2 +- easybuild/toolchains/linalg/acml.py | 2 +- easybuild/toolchains/linalg/atlas.py | 2 +- easybuild/toolchains/linalg/blacs.py | 2 +- easybuild/toolchains/linalg/blis.py | 2 +- easybuild/toolchains/linalg/flame.py | 2 +- easybuild/toolchains/linalg/flexiblas.py | 2 +- easybuild/toolchains/linalg/fujitsussl.py | 2 +- easybuild/toolchains/linalg/gotoblas.py | 2 +- easybuild/toolchains/linalg/intelmkl.py | 2 +- easybuild/toolchains/linalg/lapack.py | 2 +- easybuild/toolchains/linalg/libsci.py | 2 +- easybuild/toolchains/linalg/openblas.py | 2 +- easybuild/toolchains/linalg/scalapack.py | 2 +- easybuild/toolchains/mpi/__init__.py | 2 +- easybuild/toolchains/mpi/craympich.py | 2 +- easybuild/toolchains/mpi/fujitsumpi.py | 2 +- easybuild/toolchains/mpi/intelmpi.py | 2 +- easybuild/toolchains/mpi/mpich.py | 2 +- easybuild/toolchains/mpi/mpich2.py | 2 +- easybuild/toolchains/mpi/mvapich2.py | 2 +- easybuild/toolchains/mpi/openmpi.py | 2 +- easybuild/toolchains/mpi/psmpi.py | 2 +- easybuild/toolchains/mpi/qlogicmpi.py | 2 +- easybuild/toolchains/mpi/spectrummpi.py | 2 +- easybuild/toolchains/nvompic.py | 4 ++-- easybuild/toolchains/nvpsmpic.py | 4 ++-- easybuild/toolchains/pmkl.py | 2 +- easybuild/toolchains/pomkl.py | 2 +- easybuild/toolchains/pompi.py | 2 +- easybuild/toolchains/system.py | 2 +- easybuild/tools/__init__.py | 2 +- easybuild/tools/asyncprocess.py | 2 +- easybuild/tools/build_details.py | 2 +- easybuild/tools/build_log.py | 2 +- easybuild/tools/config.py | 2 +- easybuild/tools/containers/__init__.py | 2 +- easybuild/tools/containers/base.py | 2 +- easybuild/tools/containers/common.py | 2 +- easybuild/tools/containers/docker.py | 2 +- easybuild/tools/containers/singularity.py | 2 +- easybuild/tools/containers/utils.py | 2 +- easybuild/tools/convert.py | 2 +- easybuild/tools/docs.py | 2 +- easybuild/tools/environment.py | 2 +- easybuild/tools/filetools.py | 2 +- easybuild/tools/github.py | 2 +- easybuild/tools/hooks.py | 2 +- easybuild/tools/include.py | 2 +- easybuild/tools/jenkins.py | 2 +- easybuild/tools/job/backend.py | 2 +- easybuild/tools/job/gc3pie.py | 2 +- easybuild/tools/job/pbs_python.py | 2 +- easybuild/tools/job/slurm.py | 2 +- easybuild/tools/module_generator.py | 2 +- easybuild/tools/module_naming_scheme/__init__.py | 2 +- easybuild/tools/module_naming_scheme/categorized_mns.py | 2 +- easybuild/tools/module_naming_scheme/easybuild_mns.py | 2 +- easybuild/tools/module_naming_scheme/hierarchical_mns.py | 2 +- .../tools/module_naming_scheme/migrate_from_eb_to_hmns.py | 2 +- easybuild/tools/module_naming_scheme/mns.py | 2 +- easybuild/tools/module_naming_scheme/toolchain.py | 2 +- easybuild/tools/module_naming_scheme/utilities.py | 2 +- easybuild/tools/modules.py | 2 +- easybuild/tools/multidiff.py | 2 +- easybuild/tools/options.py | 2 +- easybuild/tools/output.py | 2 +- .../tools/package/package_naming_scheme/easybuild_pns.py | 2 +- easybuild/tools/package/package_naming_scheme/pns.py | 2 +- easybuild/tools/package/utilities.py | 2 +- easybuild/tools/parallelbuild.py | 2 +- easybuild/tools/py2vs3/__init__.py | 2 +- easybuild/tools/py2vs3/py2.py | 2 +- easybuild/tools/py2vs3/py3.py | 2 +- easybuild/tools/repository/filerepo.py | 2 +- easybuild/tools/repository/gitrepo.py | 2 +- easybuild/tools/repository/hgrepo.py | 2 +- easybuild/tools/repository/repository.py | 2 +- easybuild/tools/repository/svnrepo.py | 2 +- easybuild/tools/robot.py | 2 +- easybuild/tools/run.py | 2 +- easybuild/tools/systemtools.py | 2 +- easybuild/tools/testing.py | 2 +- easybuild/tools/toolchain/__init__.py | 2 +- easybuild/tools/toolchain/compiler.py | 2 +- easybuild/tools/toolchain/constants.py | 2 +- easybuild/tools/toolchain/fft.py | 2 +- easybuild/tools/toolchain/linalg.py | 2 +- easybuild/tools/toolchain/mpi.py | 2 +- easybuild/tools/toolchain/options.py | 2 +- easybuild/tools/toolchain/toolchain.py | 2 +- easybuild/tools/toolchain/toolchainvariables.py | 2 +- easybuild/tools/toolchain/utilities.py | 2 +- easybuild/tools/toolchain/variables.py | 2 +- easybuild/tools/utilities.py | 2 +- easybuild/tools/variables.py | 2 +- easybuild/tools/version.py | 2 +- eb | 2 +- setup.py | 2 +- test/__init__.py | 2 +- test/framework/__init__.py | 2 +- test/framework/asyncprocess.py | 2 +- test/framework/build_log.py | 2 +- test/framework/config.py | 2 +- test/framework/containers.py | 2 +- test/framework/docs.py | 2 +- test/framework/easyblock.py | 2 +- test/framework/easyconfig.py | 2 +- test/framework/easyconfigformat.py | 2 +- test/framework/easyconfigparser.py | 2 +- test/framework/easyconfigversion.py | 2 +- test/framework/easystack.py | 2 +- test/framework/ebconfigobj.py | 2 +- test/framework/environment.py | 2 +- test/framework/filetools.py | 2 +- test/framework/format_convert.py | 2 +- test/framework/general.py | 2 +- test/framework/github.py | 2 +- test/framework/hooks.py | 2 +- test/framework/include.py | 2 +- test/framework/lib.py | 2 +- test/framework/license.py | 2 +- test/framework/module_generator.py | 2 +- test/framework/modules.py | 2 +- test/framework/modulestool.py | 2 +- test/framework/options.py | 2 +- test/framework/output.py | 2 +- test/framework/package.py | 2 +- test/framework/parallelbuild.py | 2 +- test/framework/repository.py | 2 +- test/framework/robot.py | 2 +- test/framework/run.py | 2 +- test/framework/sandbox/easybuild/easyblocks/f/fftw.py | 2 +- test/framework/sandbox/easybuild/easyblocks/f/foo.py | 2 +- test/framework/sandbox/easybuild/easyblocks/f/foofoo.py | 2 +- test/framework/sandbox/easybuild/easyblocks/g/gcc.py | 2 +- test/framework/sandbox/easybuild/easyblocks/generic/bar.py | 2 +- .../sandbox/easybuild/easyblocks/generic/configuremake.py | 2 +- .../sandbox/easybuild/easyblocks/generic/dummyextension.py | 2 +- .../sandbox/easybuild/easyblocks/generic/modulerc.py | 2 +- .../sandbox/easybuild/easyblocks/generic/toolchain.py | 2 +- .../sandbox/easybuild/easyblocks/generic/toy_extension.py | 2 +- test/framework/sandbox/easybuild/easyblocks/h/hpl.py | 2 +- test/framework/sandbox/easybuild/easyblocks/l/libtoy.py | 2 +- test/framework/sandbox/easybuild/easyblocks/o/openblas.py | 2 +- test/framework/sandbox/easybuild/easyblocks/o/openmpi.py | 2 +- test/framework/sandbox/easybuild/easyblocks/s/scalapack.py | 2 +- test/framework/sandbox/easybuild/easyblocks/t/toy.py | 2 +- test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py | 2 +- test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py | 2 +- test/framework/sandbox/easybuild/easyblocks/t/toytoy.py | 2 +- test/framework/sandbox/easybuild/tools/__init__.py | 2 +- .../sandbox/easybuild/tools/module_naming_scheme/__init__.py | 2 +- .../tools/module_naming_scheme/broken_module_naming_scheme.py | 2 +- .../tools/module_naming_scheme/test_module_naming_scheme.py | 2 +- .../module_naming_scheme/test_module_naming_scheme_more.py | 2 +- test/framework/style.py | 2 +- test/framework/suite.py | 2 +- test/framework/systemtools.py | 2 +- test/framework/toolchain.py | 2 +- test/framework/toolchainvariables.py | 2 +- test/framework/toy_build.py | 2 +- test/framework/tweak.py | 2 +- test/framework/type_checking.py | 2 +- test/framework/utilities.py | 2 +- test/framework/utilities_test.py | 2 +- test/framework/variables.py | 2 +- test/framework/yeb.py | 2 +- 291 files changed, 293 insertions(+), 293 deletions(-) diff --git a/easybuild/__init__.py b/easybuild/__init__.py index c1b9a77dde..109445bb2f 100644 --- a/easybuild/__init__.py +++ b/easybuild/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2021 Ghent University +# Copyright 2011-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/base/exceptions.py b/easybuild/base/exceptions.py index c7a8690d1b..528b4cb0ce 100644 --- a/easybuild/base/exceptions.py +++ b/easybuild/base/exceptions.py @@ -1,5 +1,5 @@ # -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/base/fancylogger.py b/easybuild/base/fancylogger.py index 94c55e250e..140346e2a7 100644 --- a/easybuild/base/fancylogger.py +++ b/easybuild/base/fancylogger.py @@ -1,5 +1,5 @@ # -# Copyright 2011-2021 Ghent University +# Copyright 2011-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 789c0ce152..c587f4fdd0 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -1,5 +1,5 @@ # -# Copyright 2011-2021 Ghent University +# Copyright 2011-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/base/testing.py b/easybuild/base/testing.py index ff855ae346..7f44c318cc 100644 --- a/easybuild/base/testing.py +++ b/easybuild/base/testing.py @@ -1,5 +1,5 @@ # -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/__init__.py b/easybuild/framework/__init__.py index c9304064b4..b2f628b039 100644 --- a/easybuild/framework/__init__.py +++ b/easybuild/framework/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 2822e0a2c9..b627b46151 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/__init__.py b/easybuild/framework/easyconfig/__init__.py index 3ae73bb0df..040a28347f 100644 --- a/easybuild/framework/easyconfig/__init__.py +++ b/easybuild/framework/easyconfig/__init__.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/constants.py b/easybuild/framework/easyconfig/constants.py index 4b5b4e4ffe..63ce5d51d7 100644 --- a/easybuild/framework/easyconfig/constants.py +++ b/easybuild/framework/easyconfig/constants.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index a1f1fef043..91e0fe7fe4 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 3dc255e117..e7abf1687d 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/__init__.py b/easybuild/framework/easyconfig/format/__init__.py index 0b0de5ca72..125c7b73fd 100644 --- a/easybuild/framework/easyconfig/format/__init__.py +++ b/easybuild/framework/easyconfig/format/__init__.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/convert.py b/easybuild/framework/easyconfig/format/convert.py index 3765df9bf5..5ba1fa7f45 100644 --- a/easybuild/framework/easyconfig/format/convert.py +++ b/easybuild/framework/easyconfig/format/convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 4fa31c84e6..c49e82fbe9 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index 2956d6a009..114cb800fd 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index 5438c031ab..48361feec5 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/two.py b/easybuild/framework/easyconfig/format/two.py index 42d8f93bf7..0c1c3c6c20 100644 --- a/easybuild/framework/easyconfig/format/two.py +++ b/easybuild/framework/easyconfig/format/two.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/version.py b/easybuild/framework/easyconfig/format/version.py index 5a643af823..8fc78e66a0 100644 --- a/easybuild/framework/easyconfig/format/version.py +++ b/easybuild/framework/easyconfig/format/version.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/yeb.py b/easybuild/framework/easyconfig/format/yeb.py index 4f63034ad2..d87b16e64f 100644 --- a/easybuild/framework/easyconfig/format/yeb.py +++ b/easybuild/framework/easyconfig/format/yeb.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/licenses.py b/easybuild/framework/easyconfig/licenses.py index 6c2ab7e70f..b7e28358af 100644 --- a/easybuild/framework/easyconfig/licenses.py +++ b/easybuild/framework/easyconfig/licenses.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index c5c9ff64af..f5ff4d9607 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/style.py b/easybuild/framework/easyconfig/style.py index 97578197b7..8bcf371d55 100644 --- a/easybuild/framework/easyconfig/style.py +++ b/easybuild/framework/easyconfig/style.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2021 Ghent University +# Copyright 2016-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 12bed87de3..d268536c63 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index be6ff5fe01..78134b00cd 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 137790a039..c3fdfa0904 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/types.py b/easybuild/framework/easyconfig/types.py index 98737328d8..77495a1f31 100644 --- a/easybuild/framework/easyconfig/types.py +++ b/easybuild/framework/easyconfig/types.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index d8078159ba..fe56253b3f 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Ghent University +# Copyright 2020-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index e85b94daec..badc42856c 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index 3c427c5a9f..8037e78507 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc). diff --git a/easybuild/main.py b/easybuild/main.py index 2203636a3b..61d27b3089 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 3274730412..c1914cdf65 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py index 6bfa02c24a..5a02fde0b0 100644 --- a/easybuild/scripts/fix_docs.py +++ b/easybuild/scripts/fix_docs.py @@ -1,5 +1,5 @@ # # -# Copyright 2016-2021 Ghent University +# Copyright 2016-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py index 5cf2b36374..928a67d8be 100644 --- a/easybuild/scripts/mk_tmpl_easyblock_for.py +++ b/easybuild/scripts/mk_tmpl_easyblock_for.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py index 23471f9344..9c4ab8b355 100755 --- a/easybuild/scripts/rpath_args.py +++ b/easybuild/scripts/rpath_args.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2016-2021 Ghent University +# Copyright 2016-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/rpath_wrapper_template.sh.in b/easybuild/scripts/rpath_wrapper_template.sh.in index 501e0aa047..c14e249076 100644 --- a/easybuild/scripts/rpath_wrapper_template.sh.in +++ b/easybuild/scripts/rpath_wrapper_template.sh.in @@ -1,6 +1,6 @@ #!/bin/bash ## -# Copyright 2016-2021 Ghent University +# Copyright 2016-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/__init__.py b/easybuild/toolchains/__init__.py index ae86d1c599..842767b3d3 100644 --- a/easybuild/toolchains/__init__.py +++ b/easybuild/toolchains/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/cgmpich.py b/easybuild/toolchains/cgmpich.py index 6348249534..675c9eb5e8 100644 --- a/easybuild/toolchains/cgmpich.py +++ b/easybuild/toolchains/cgmpich.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmpolf.py b/easybuild/toolchains/cgmpolf.py index fbd7d4fec9..ab7ac81b1d 100644 --- a/easybuild/toolchains/cgmpolf.py +++ b/easybuild/toolchains/cgmpolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmvapich2.py b/easybuild/toolchains/cgmvapich2.py index 07e5c99925..9b73f92488 100644 --- a/easybuild/toolchains/cgmvapich2.py +++ b/easybuild/toolchains/cgmvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmvolf.py b/easybuild/toolchains/cgmvolf.py index 8d88de2133..321fbee7a6 100644 --- a/easybuild/toolchains/cgmvolf.py +++ b/easybuild/toolchains/cgmvolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgompi.py b/easybuild/toolchains/cgompi.py index 5f94167ba6..5c405c710c 100644 --- a/easybuild/toolchains/cgompi.py +++ b/easybuild/toolchains/cgompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgoolf.py b/easybuild/toolchains/cgoolf.py index 7b7b91990d..6e1fff1d8a 100644 --- a/easybuild/toolchains/cgoolf.py +++ b/easybuild/toolchains/cgoolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/clanggcc.py b/easybuild/toolchains/clanggcc.py index d7cbeb9a67..15a67af146 100644 --- a/easybuild/toolchains/clanggcc.py +++ b/easybuild/toolchains/clanggcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/compiler/__init__.py b/easybuild/toolchains/compiler/__init__.py index 96308aa9a1..cff9f74a89 100644 --- a/easybuild/toolchains/compiler/__init__.py +++ b/easybuild/toolchains/compiler/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/clang.py b/easybuild/toolchains/compiler/clang.py index 8402cae0e1..c13a0978ae 100644 --- a/easybuild/toolchains/compiler/clang.py +++ b/easybuild/toolchains/compiler/clang.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/compiler/craype.py b/easybuild/toolchains/compiler/craype.py index 698a60e4e8..ad81ff0363 100644 --- a/easybuild/toolchains/compiler/craype.py +++ b/easybuild/toolchains/compiler/craype.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/cuda.py b/easybuild/toolchains/compiler/cuda.py index 5a75bc302e..2c4de92ed5 100644 --- a/easybuild/toolchains/compiler/cuda.py +++ b/easybuild/toolchains/compiler/cuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/dummycompiler.py b/easybuild/toolchains/compiler/dummycompiler.py index 4558e05b7e..5316104b7a 100644 --- a/easybuild/toolchains/compiler/dummycompiler.py +++ b/easybuild/toolchains/compiler/dummycompiler.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/fujitsu.py b/easybuild/toolchains/compiler/fujitsu.py index e12d9e060e..92efd02577 100644 --- a/easybuild/toolchains/compiler/fujitsu.py +++ b/easybuild/toolchains/compiler/fujitsu.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index affcf588b8..c7966acb7e 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/intel_compilers.py b/easybuild/toolchains/compiler/intel_compilers.py index 8bc4bdc763..b2a571a17d 100644 --- a/easybuild/toolchains/compiler/intel_compilers.py +++ b/easybuild/toolchains/compiler/intel_compilers.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2021 Ghent University +# Copyright 2021-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/inteliccifort.py b/easybuild/toolchains/compiler/inteliccifort.py index 8ac23e837b..2dc2faacbd 100644 --- a/easybuild/toolchains/compiler/inteliccifort.py +++ b/easybuild/toolchains/compiler/inteliccifort.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/systemcompiler.py b/easybuild/toolchains/compiler/systemcompiler.py index f6b4ccd979..74edcb51e9 100644 --- a/easybuild/toolchains/compiler/systemcompiler.py +++ b/easybuild/toolchains/compiler/systemcompiler.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2021 Ghent University +# Copyright 2019-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/craycce.py b/easybuild/toolchains/craycce.py index 799ed94853..93dd97c440 100644 --- a/easybuild/toolchains/craycce.py +++ b/easybuild/toolchains/craycce.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/craygnu.py b/easybuild/toolchains/craygnu.py index 9e69e9a1f2..a53e90f931 100644 --- a/easybuild/toolchains/craygnu.py +++ b/easybuild/toolchains/craygnu.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/crayintel.py b/easybuild/toolchains/crayintel.py index c56bd4e0da..89e40cdc32 100644 --- a/easybuild/toolchains/crayintel.py +++ b/easybuild/toolchains/crayintel.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/craypgi.py b/easybuild/toolchains/craypgi.py index ed5a8e66ba..9cd0735dc2 100644 --- a/easybuild/toolchains/craypgi.py +++ b/easybuild/toolchains/craypgi.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/dummy.py b/easybuild/toolchains/dummy.py index 44eddfa402..fd48d650a3 100644 --- a/easybuild/toolchains/dummy.py +++ b/easybuild/toolchains/dummy.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fcc.py b/easybuild/toolchains/fcc.py index 0ca29695fa..eb20230f81 100644 --- a/easybuild/toolchains/fcc.py +++ b/easybuild/toolchains/fcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ffmpi.py b/easybuild/toolchains/ffmpi.py index a951f8c0f1..68e1799389 100644 --- a/easybuild/toolchains/ffmpi.py +++ b/easybuild/toolchains/ffmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/__init__.py b/easybuild/toolchains/fft/__init__.py index b8d6cfce97..4353145f26 100644 --- a/easybuild/toolchains/fft/__init__.py +++ b/easybuild/toolchains/fft/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/fftw.py b/easybuild/toolchains/fft/fftw.py index ece375253a..3003634ac8 100644 --- a/easybuild/toolchains/fft/fftw.py +++ b/easybuild/toolchains/fft/fftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/fujitsufftw.py b/easybuild/toolchains/fft/fujitsufftw.py index 4b49b09932..33eb185708 100644 --- a/easybuild/toolchains/fft/fujitsufftw.py +++ b/easybuild/toolchains/fft/fujitsufftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py index 0e878f9e76..4c1a29194c 100644 --- a/easybuild/toolchains/fft/intelfftw.py +++ b/easybuild/toolchains/fft/intelfftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/foss.py b/easybuild/toolchains/foss.py index d4bfbb671b..970a2ea9b3 100644 --- a/easybuild/toolchains/foss.py +++ b/easybuild/toolchains/foss.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fosscuda.py b/easybuild/toolchains/fosscuda.py index e2e98d54f1..9edea341d2 100644 --- a/easybuild/toolchains/fosscuda.py +++ b/easybuild/toolchains/fosscuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fujitsu.py b/easybuild/toolchains/fujitsu.py index a4fedfa1a3..ef0663e6af 100644 --- a/easybuild/toolchains/fujitsu.py +++ b/easybuild/toolchains/fujitsu.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gcc.py b/easybuild/toolchains/gcc.py index ab9d58774b..df610d745b 100644 --- a/easybuild/toolchains/gcc.py +++ b/easybuild/toolchains/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gcccore.py b/easybuild/toolchains/gcccore.py index 49a190ca28..ace116975a 100644 --- a/easybuild/toolchains/gcccore.py +++ b/easybuild/toolchains/gcccore.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gcccuda.py b/easybuild/toolchains/gcccuda.py index 6d2fa190ed..babf11b776 100644 --- a/easybuild/toolchains/gcccuda.py +++ b/easybuild/toolchains/gcccuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gimkl.py b/easybuild/toolchains/gimkl.py index 986bdb978a..3afdb9659e 100644 --- a/easybuild/toolchains/gimkl.py +++ b/easybuild/toolchains/gimkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gimpi.py b/easybuild/toolchains/gimpi.py index 56555cc42f..346e5d3055 100644 --- a/easybuild/toolchains/gimpi.py +++ b/easybuild/toolchains/gimpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gimpic.py b/easybuild/toolchains/gimpic.py index a4fc060503..eddf0d6cb1 100644 --- a/easybuild/toolchains/gimpic.py +++ b/easybuild/toolchains/gimpic.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/giolf.py b/easybuild/toolchains/giolf.py index 75b6bf01f0..f42b8e6a41 100644 --- a/easybuild/toolchains/giolf.py +++ b/easybuild/toolchains/giolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/giolfc.py b/easybuild/toolchains/giolfc.py index 6c6d08dc3b..af78d412c8 100644 --- a/easybuild/toolchains/giolfc.py +++ b/easybuild/toolchains/giolfc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmacml.py b/easybuild/toolchains/gmacml.py index 4e876b1b7a..6bcc9593ab 100644 --- a/easybuild/toolchains/gmacml.py +++ b/easybuild/toolchains/gmacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmkl.py b/easybuild/toolchains/gmkl.py index 3505cd1e66..f23a7befcb 100644 --- a/easybuild/toolchains/gmkl.py +++ b/easybuild/toolchains/gmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmklc.py b/easybuild/toolchains/gmklc.py index 8105e056cd..6481685708 100644 --- a/easybuild/toolchains/gmklc.py +++ b/easybuild/toolchains/gmklc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpich.py b/easybuild/toolchains/gmpich.py index 338c548154..2484163c5d 100644 --- a/easybuild/toolchains/gmpich.py +++ b/easybuild/toolchains/gmpich.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpich2.py b/easybuild/toolchains/gmpich2.py index 8a58f4fa03..562426b138 100644 --- a/easybuild/toolchains/gmpich2.py +++ b/easybuild/toolchains/gmpich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpolf.py b/easybuild/toolchains/gmpolf.py index f36f46a9cb..4686a77ca8 100644 --- a/easybuild/toolchains/gmpolf.py +++ b/easybuild/toolchains/gmpolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gmvapich2.py b/easybuild/toolchains/gmvapich2.py index 7c2d615105..f2e36d1df8 100644 --- a/easybuild/toolchains/gmvapich2.py +++ b/easybuild/toolchains/gmvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmvolf.py b/easybuild/toolchains/gmvolf.py index 7e555dddad..075b714c87 100644 --- a/easybuild/toolchains/gmvolf.py +++ b/easybuild/toolchains/gmvolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gnu.py b/easybuild/toolchains/gnu.py index e4b7db94fa..1366d333f3 100644 --- a/easybuild/toolchains/gnu.py +++ b/easybuild/toolchains/gnu.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goalf.py b/easybuild/toolchains/goalf.py index c7d1771d5b..c6003aa334 100644 --- a/easybuild/toolchains/goalf.py +++ b/easybuild/toolchains/goalf.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gobff.py b/easybuild/toolchains/gobff.py index 26474f281d..4597d28e48 100644 --- a/easybuild/toolchains/gobff.py +++ b/easybuild/toolchains/gobff.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goblf.py b/easybuild/toolchains/goblf.py index 3bf2d50e81..b4cb71e4d4 100644 --- a/easybuild/toolchains/goblf.py +++ b/easybuild/toolchains/goblf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gofbf.py b/easybuild/toolchains/gofbf.py index 294f62cd03..8e448c89a1 100644 --- a/easybuild/toolchains/gofbf.py +++ b/easybuild/toolchains/gofbf.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2021 Ghent University +# Copyright 2021-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/golf.py b/easybuild/toolchains/golf.py index dbabc3c63a..f01d6ea7d7 100644 --- a/easybuild/toolchains/golf.py +++ b/easybuild/toolchains/golf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/golfc.py b/easybuild/toolchains/golfc.py index 9542026168..7d4073f1a8 100644 --- a/easybuild/toolchains/golfc.py +++ b/easybuild/toolchains/golfc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gomkl.py b/easybuild/toolchains/gomkl.py index 48794536e2..c240628884 100644 --- a/easybuild/toolchains/gomkl.py +++ b/easybuild/toolchains/gomkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gomklc.py b/easybuild/toolchains/gomklc.py index 2ae5a40ee4..42e490e631 100644 --- a/easybuild/toolchains/gomklc.py +++ b/easybuild/toolchains/gomklc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gompi.py b/easybuild/toolchains/gompi.py index f9ed1ab1bc..7493539c5a 100644 --- a/easybuild/toolchains/gompi.py +++ b/easybuild/toolchains/gompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gompic.py b/easybuild/toolchains/gompic.py index 65ca8e1f65..4f23b81b40 100644 --- a/easybuild/toolchains/gompic.py +++ b/easybuild/toolchains/gompic.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goolf.py b/easybuild/toolchains/goolf.py index 0cf23145ce..fdf564e376 100644 --- a/easybuild/toolchains/goolf.py +++ b/easybuild/toolchains/goolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goolfc.py b/easybuild/toolchains/goolfc.py index e32d342780..7eae7f4dbd 100644 --- a/easybuild/toolchains/goolfc.py +++ b/easybuild/toolchains/goolfc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gpsmpi.py b/easybuild/toolchains/gpsmpi.py index cd9e019246..ecea4e4620 100644 --- a/easybuild/toolchains/gpsmpi.py +++ b/easybuild/toolchains/gpsmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gpsolf.py b/easybuild/toolchains/gpsolf.py index 24a313eb2d..282e7c8c00 100644 --- a/easybuild/toolchains/gpsolf.py +++ b/easybuild/toolchains/gpsolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gqacml.py b/easybuild/toolchains/gqacml.py index 8a438f5462..ee03ad92bb 100644 --- a/easybuild/toolchains/gqacml.py +++ b/easybuild/toolchains/gqacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gsmpi.py b/easybuild/toolchains/gsmpi.py index 071b4ee0b9..7925f6fad0 100644 --- a/easybuild/toolchains/gsmpi.py +++ b/easybuild/toolchains/gsmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gsolf.py b/easybuild/toolchains/gsolf.py index 22e84fef4e..98c412beb6 100644 --- a/easybuild/toolchains/gsolf.py +++ b/easybuild/toolchains/gsolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iccifort.py b/easybuild/toolchains/iccifort.py index cb44c0408b..0b281c9c27 100644 --- a/easybuild/toolchains/iccifort.py +++ b/easybuild/toolchains/iccifort.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iccifortcuda.py b/easybuild/toolchains/iccifortcuda.py index 2fd32e4224..add77b3a0c 100644 --- a/easybuild/toolchains/iccifortcuda.py +++ b/easybuild/toolchains/iccifortcuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ictce.py b/easybuild/toolchains/ictce.py index 38fa8ffb76..4b26095d9b 100644 --- a/easybuild/toolchains/ictce.py +++ b/easybuild/toolchains/ictce.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iibff.py b/easybuild/toolchains/iibff.py index 71dde7534f..25a761ff2c 100644 --- a/easybuild/toolchains/iibff.py +++ b/easybuild/toolchains/iibff.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimkl.py b/easybuild/toolchains/iimkl.py index 6a23f1912f..ba6292ffb2 100644 --- a/easybuild/toolchains/iimkl.py +++ b/easybuild/toolchains/iimkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimklc.py b/easybuild/toolchains/iimklc.py index 9a1c0e5d36..08ebaf99cc 100644 --- a/easybuild/toolchains/iimklc.py +++ b/easybuild/toolchains/iimklc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimpi.py b/easybuild/toolchains/iimpi.py index f89d17cc3a..ef19c0a5e0 100644 --- a/easybuild/toolchains/iimpi.py +++ b/easybuild/toolchains/iimpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimpic.py b/easybuild/toolchains/iimpic.py index 0673ffb81b..a27c9eb97a 100644 --- a/easybuild/toolchains/iimpic.py +++ b/easybuild/toolchains/iimpic.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iiqmpi.py b/easybuild/toolchains/iiqmpi.py index b76fd58696..5de35f89bf 100644 --- a/easybuild/toolchains/iiqmpi.py +++ b/easybuild/toolchains/iiqmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/impich.py b/easybuild/toolchains/impich.py index 86d0f1ebb3..063a1f2c73 100644 --- a/easybuild/toolchains/impich.py +++ b/easybuild/toolchains/impich.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/impmkl.py b/easybuild/toolchains/impmkl.py index 166de95d5b..bb550d1051 100644 --- a/easybuild/toolchains/impmkl.py +++ b/easybuild/toolchains/impmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intel-para.py b/easybuild/toolchains/intel-para.py index 96b1f0372f..cbb02d98a4 100644 --- a/easybuild/toolchains/intel-para.py +++ b/easybuild/toolchains/intel-para.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intel.py b/easybuild/toolchains/intel.py index 3639f7bca1..773cdf12a7 100644 --- a/easybuild/toolchains/intel.py +++ b/easybuild/toolchains/intel.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intel_compilers.py b/easybuild/toolchains/intel_compilers.py index 21b118f291..00f3c874ac 100644 --- a/easybuild/toolchains/intel_compilers.py +++ b/easybuild/toolchains/intel_compilers.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2021 Ghent University +# Copyright 2021-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intelcuda.py b/easybuild/toolchains/intelcuda.py index 3b5a1538aa..ffc9899b03 100644 --- a/easybuild/toolchains/intelcuda.py +++ b/easybuild/toolchains/intelcuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iomkl.py b/easybuild/toolchains/iomkl.py index 6c0b0e01a4..3f6c2910f5 100644 --- a/easybuild/toolchains/iomkl.py +++ b/easybuild/toolchains/iomkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iomklc.py b/easybuild/toolchains/iomklc.py index 785a884616..d1399c503f 100644 --- a/easybuild/toolchains/iomklc.py +++ b/easybuild/toolchains/iomklc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iompi.py b/easybuild/toolchains/iompi.py index e3c82cb907..08a69f6164 100644 --- a/easybuild/toolchains/iompi.py +++ b/easybuild/toolchains/iompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iompic.py b/easybuild/toolchains/iompic.py index b2b4cab6b4..f5a47930f5 100644 --- a/easybuild/toolchains/iompic.py +++ b/easybuild/toolchains/iompic.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ipsmpi.py b/easybuild/toolchains/ipsmpi.py index 8798e746ff..ebbf638c35 100644 --- a/easybuild/toolchains/ipsmpi.py +++ b/easybuild/toolchains/ipsmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iqacml.py b/easybuild/toolchains/iqacml.py index 2ff17dee96..a7905e132e 100644 --- a/easybuild/toolchains/iqacml.py +++ b/easybuild/toolchains/iqacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ismkl.py b/easybuild/toolchains/ismkl.py index 80e0951969..db7fc3f11e 100644 --- a/easybuild/toolchains/ismkl.py +++ b/easybuild/toolchains/ismkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/__init__.py b/easybuild/toolchains/linalg/__init__.py index d77ec3e1e3..ef9096a3fd 100644 --- a/easybuild/toolchains/linalg/__init__.py +++ b/easybuild/toolchains/linalg/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/acml.py b/easybuild/toolchains/linalg/acml.py index 32de9fd6eb..16eed0fe36 100644 --- a/easybuild/toolchains/linalg/acml.py +++ b/easybuild/toolchains/linalg/acml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/atlas.py b/easybuild/toolchains/linalg/atlas.py index b77e84a5d5..f15829bfff 100644 --- a/easybuild/toolchains/linalg/atlas.py +++ b/easybuild/toolchains/linalg/atlas.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/blacs.py b/easybuild/toolchains/linalg/blacs.py index 47949dd821..dcf1950954 100644 --- a/easybuild/toolchains/linalg/blacs.py +++ b/easybuild/toolchains/linalg/blacs.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/blis.py b/easybuild/toolchains/linalg/blis.py index 924e6b1f78..d8f1c9a327 100644 --- a/easybuild/toolchains/linalg/blis.py +++ b/easybuild/toolchains/linalg/blis.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/flame.py b/easybuild/toolchains/linalg/flame.py index 998b5c39d4..8ae465b395 100644 --- a/easybuild/toolchains/linalg/flame.py +++ b/easybuild/toolchains/linalg/flame.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py index c77476218f..835a4b3191 100644 --- a/easybuild/toolchains/linalg/flexiblas.py +++ b/easybuild/toolchains/linalg/flexiblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2021 Ghent University +# Copyright 2021-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/fujitsussl.py b/easybuild/toolchains/linalg/fujitsussl.py index 3e9ec7902f..99766e56d0 100644 --- a/easybuild/toolchains/linalg/fujitsussl.py +++ b/easybuild/toolchains/linalg/fujitsussl.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/gotoblas.py b/easybuild/toolchains/linalg/gotoblas.py index 3df653fee9..93faed6328 100644 --- a/easybuild/toolchains/linalg/gotoblas.py +++ b/easybuild/toolchains/linalg/gotoblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index cefd110eac..469356bbd0 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/lapack.py b/easybuild/toolchains/linalg/lapack.py index 19052f668e..338e6dd4b1 100644 --- a/easybuild/toolchains/linalg/lapack.py +++ b/easybuild/toolchains/linalg/lapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/libsci.py b/easybuild/toolchains/linalg/libsci.py index ec3c0396e3..d3bbb48b2d 100644 --- a/easybuild/toolchains/linalg/libsci.py +++ b/easybuild/toolchains/linalg/libsci.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/openblas.py b/easybuild/toolchains/linalg/openblas.py index 4d57f51ba9..49d49de85f 100644 --- a/easybuild/toolchains/linalg/openblas.py +++ b/easybuild/toolchains/linalg/openblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/scalapack.py b/easybuild/toolchains/linalg/scalapack.py index d538dbb1b1..0a3c501d3b 100644 --- a/easybuild/toolchains/linalg/scalapack.py +++ b/easybuild/toolchains/linalg/scalapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/__init__.py b/easybuild/toolchains/mpi/__init__.py index c0fccd6239..5bb9f0bb9f 100644 --- a/easybuild/toolchains/mpi/__init__.py +++ b/easybuild/toolchains/mpi/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/craympich.py b/easybuild/toolchains/mpi/craympich.py index 43e6bb080a..ea0312d930 100644 --- a/easybuild/toolchains/mpi/craympich.py +++ b/easybuild/toolchains/mpi/craympich.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/fujitsumpi.py b/easybuild/toolchains/mpi/fujitsumpi.py index 0410339ba1..03ee6f3a41 100644 --- a/easybuild/toolchains/mpi/fujitsumpi.py +++ b/easybuild/toolchains/mpi/fujitsumpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/intelmpi.py b/easybuild/toolchains/mpi/intelmpi.py index 104320635d..7f9d7ecfe2 100644 --- a/easybuild/toolchains/mpi/intelmpi.py +++ b/easybuild/toolchains/mpi/intelmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mpich.py b/easybuild/toolchains/mpi/mpich.py index d7ee92006d..dea8450d46 100644 --- a/easybuild/toolchains/mpi/mpich.py +++ b/easybuild/toolchains/mpi/mpich.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mpich2.py b/easybuild/toolchains/mpi/mpich2.py index be2ec1a143..2038b0a98a 100644 --- a/easybuild/toolchains/mpi/mpich2.py +++ b/easybuild/toolchains/mpi/mpich2.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mvapich2.py b/easybuild/toolchains/mpi/mvapich2.py index 0a384019dc..1181b89da6 100644 --- a/easybuild/toolchains/mpi/mvapich2.py +++ b/easybuild/toolchains/mpi/mvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/openmpi.py b/easybuild/toolchains/mpi/openmpi.py index a7d5c59155..856ee13e66 100644 --- a/easybuild/toolchains/mpi/openmpi.py +++ b/easybuild/toolchains/mpi/openmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/psmpi.py b/easybuild/toolchains/mpi/psmpi.py index 38b8d22ff9..1f8df716c2 100644 --- a/easybuild/toolchains/mpi/psmpi.py +++ b/easybuild/toolchains/mpi/psmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/qlogicmpi.py b/easybuild/toolchains/mpi/qlogicmpi.py index eda309df48..8a6d9757af 100644 --- a/easybuild/toolchains/mpi/qlogicmpi.py +++ b/easybuild/toolchains/mpi/qlogicmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/spectrummpi.py b/easybuild/toolchains/mpi/spectrummpi.py index 76e87cafb8..a4adf96e7c 100644 --- a/easybuild/toolchains/mpi/spectrummpi.py +++ b/easybuild/toolchains/mpi/spectrummpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/nvompic.py b/easybuild/toolchains/nvompic.py index b411c4120f..0977bca226 100644 --- a/easybuild/toolchains/nvompic.py +++ b/easybuild/toolchains/nvompic.py @@ -1,6 +1,6 @@ ## -# Copyright 2016-2021 Ghent University -# Copyright 2016-2021 Forschungszentrum Juelich +# Copyright 2016-2022 Ghent University +# Copyright 2016-2022 Forschungszentrum Juelich # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/nvpsmpic.py b/easybuild/toolchains/nvpsmpic.py index e4120344c6..085f7da0c4 100644 --- a/easybuild/toolchains/nvpsmpic.py +++ b/easybuild/toolchains/nvpsmpic.py @@ -1,6 +1,6 @@ ## -# Copyright 2016-2021 Ghent University -# Copyright 2016-2021 Forschungszentrum Juelich +# Copyright 2016-2022 Ghent University +# Copyright 2016-2022 Forschungszentrum Juelich # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/pmkl.py b/easybuild/toolchains/pmkl.py index b0cd7e58c2..3d8a2ae1fe 100644 --- a/easybuild/toolchains/pmkl.py +++ b/easybuild/toolchains/pmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/pomkl.py b/easybuild/toolchains/pomkl.py index a00a4ea51a..eeb46e2749 100644 --- a/easybuild/toolchains/pomkl.py +++ b/easybuild/toolchains/pomkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/pompi.py b/easybuild/toolchains/pompi.py index 4557ea5db2..fd5657e0e6 100644 --- a/easybuild/toolchains/pompi.py +++ b/easybuild/toolchains/pompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/system.py b/easybuild/toolchains/system.py index 61affaa28c..61e58b9c99 100644 --- a/easybuild/toolchains/system.py +++ b/easybuild/toolchains/system.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2021 Ghent University +# Copyright 2019-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/__init__.py b/easybuild/tools/__init__.py index cfd32b007d..6c0bf4f023 100644 --- a/easybuild/tools/__init__.py +++ b/easybuild/tools/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/asyncprocess.py b/easybuild/tools/asyncprocess.py index fa1d8a1a6c..458836c1ba 100644 --- a/easybuild/tools/asyncprocess.py +++ b/easybuild/tools/asyncprocess.py @@ -1,6 +1,6 @@ ## # Copyright 2005 Josiah Carlson -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # The Asynchronous Python Subprocess recipe was originally created by Josiah Carlson. # and released under the GPL v2 on March 14, 2012 diff --git a/easybuild/tools/build_details.py b/easybuild/tools/build_details.py index 6d6da8f266..cc4ca7fb0a 100644 --- a/easybuild/tools/build_details.py +++ b/easybuild/tools/build_details.py @@ -1,4 +1,4 @@ -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 2cf97c5f2d..48fd643bc0 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index abcfb155be..4ef3a4c946 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/__init__.py b/easybuild/tools/containers/__init__.py index f5d8fca779..0ce2d21dd3 100644 --- a/easybuild/tools/containers/__init__.py +++ b/easybuild/tools/containers/__init__.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/base.py b/easybuild/tools/containers/base.py index 7fbb9edd1d..80d3641f58 100644 --- a/easybuild/tools/containers/base.py +++ b/easybuild/tools/containers/base.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/common.py b/easybuild/tools/containers/common.py index b473d7fd73..852d079d88 100644 --- a/easybuild/tools/containers/common.py +++ b/easybuild/tools/containers/common.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/docker.py b/easybuild/tools/containers/docker.py index 9712ee777a..df94375381 100644 --- a/easybuild/tools/containers/docker.py +++ b/easybuild/tools/containers/docker.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/singularity.py b/easybuild/tools/containers/singularity.py index dab55b525f..82c74a5eeb 100644 --- a/easybuild/tools/containers/singularity.py +++ b/easybuild/tools/containers/singularity.py @@ -1,4 +1,4 @@ -# Copyright 2017-2021 Ghent University +# Copyright 2017-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/utils.py b/easybuild/tools/containers/utils.py index 27b9ee1cdd..29f81fe349 100644 --- a/easybuild/tools/containers/utils.py +++ b/easybuild/tools/containers/utils.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/convert.py b/easybuild/tools/convert.py index de4d0984c1..74b3aca266 100644 --- a/easybuild/tools/convert.py +++ b/easybuild/tools/convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 3036a66a92..2f6358c02f 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py index f3329eb5bb..e99d64da2f 100644 --- a/easybuild/tools/environment.py +++ b/easybuild/tools/environment.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index f4985346db..4a4c8b060a 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index a8ae5a0a97..46fe56c703 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 6046e933f1..e12882374a 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -1,5 +1,5 @@ # # -# Copyright 2017-2021 Ghent University +# Copyright 2017-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index 31aecb9997..5aeb74d51c 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # # -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/jenkins.py b/easybuild/tools/jenkins.py index 40a6fe16e3..b299667e01 100644 --- a/easybuild/tools/jenkins.py +++ b/easybuild/tools/jenkins.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/job/backend.py b/easybuild/tools/job/backend.py index 5e5e5ceacb..868edb7fd4 100644 --- a/easybuild/tools/job/backend.py +++ b/easybuild/tools/job/backend.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/job/gc3pie.py b/easybuild/tools/job/gc3pie.py index 806d21544c..a8c921e723 100644 --- a/easybuild/tools/job/gc3pie.py +++ b/easybuild/tools/job/gc3pie.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # Copyright 2015 S3IT, University of Zurich # # This file is part of EasyBuild, diff --git a/easybuild/tools/job/pbs_python.py b/easybuild/tools/job/pbs_python.py index 28659ab91c..77fabb9ffe 100644 --- a/easybuild/tools/job/pbs_python.py +++ b/easybuild/tools/job/pbs_python.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/job/slurm.py b/easybuild/tools/job/slurm.py index 43531980d3..924c46aaf1 100644 --- a/easybuild/tools/job/slurm.py +++ b/easybuild/tools/job/slurm.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2021 Ghent University +# Copyright 2018-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index ef34ca62d3..9991f9ba01 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/__init__.py b/easybuild/tools/module_naming_scheme/__init__.py index 8a4b4975ab..c20c98fc8a 100644 --- a/easybuild/tools/module_naming_scheme/__init__.py +++ b/easybuild/tools/module_naming_scheme/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2021 Ghent University +# Copyright 2011-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/categorized_mns.py b/easybuild/tools/module_naming_scheme/categorized_mns.py index f81f9436a9..d11a7cd2bd 100644 --- a/easybuild/tools/module_naming_scheme/categorized_mns.py +++ b/easybuild/tools/module_naming_scheme/categorized_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2021 Ghent University +# Copyright 2016-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/easybuild_mns.py b/easybuild/tools/module_naming_scheme/easybuild_mns.py index b2c6c577eb..7f6c243632 100644 --- a/easybuild/tools/module_naming_scheme/easybuild_mns.py +++ b/easybuild/tools/module_naming_scheme/easybuild_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py index f9839532bc..5b5228c839 100644 --- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py +++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py b/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py index e45c257537..99bb4c6d0a 100644 --- a/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py +++ b/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/mns.py b/easybuild/tools/module_naming_scheme/mns.py index 7465691e7f..e2d95677b3 100644 --- a/easybuild/tools/module_naming_scheme/mns.py +++ b/easybuild/tools/module_naming_scheme/mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2021 Ghent University +# Copyright 2011-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/toolchain.py b/easybuild/tools/module_naming_scheme/toolchain.py index 3948b41603..4f270f1252 100644 --- a/easybuild/tools/module_naming_scheme/toolchain.py +++ b/easybuild/tools/module_naming_scheme/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/utilities.py b/easybuild/tools/module_naming_scheme/utilities.py index 0fc1e16b32..0b01037f34 100644 --- a/easybuild/tools/module_naming_scheme/utilities.py +++ b/easybuild/tools/module_naming_scheme/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 0860b810d8..1478f9299f 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/multidiff.py b/easybuild/tools/multidiff.py index a8a9d30b6c..b1f7f526b9 100644 --- a/easybuild/tools/multidiff.py +++ b/easybuild/tools/multidiff.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 6d123508a2..0c438e0e1e 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 207a658569..9968edf19f 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # # -# Copyright 2021-2021 Ghent University +# Copyright 2021-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/package/package_naming_scheme/easybuild_pns.py b/easybuild/tools/package/package_naming_scheme/easybuild_pns.py index 7116c002a4..670941b804 100644 --- a/easybuild/tools/package/package_naming_scheme/easybuild_pns.py +++ b/easybuild/tools/package/package_naming_scheme/easybuild_pns.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/package/package_naming_scheme/pns.py b/easybuild/tools/package/package_naming_scheme/pns.py index 8556abb019..bb00f550fc 100644 --- a/easybuild/tools/package/package_naming_scheme/pns.py +++ b/easybuild/tools/package/package_naming_scheme/pns.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/package/utilities.py b/easybuild/tools/package/utilities.py index 3fcd5d0ed1..97e83592b4 100644 --- a/easybuild/tools/package/utilities.py +++ b/easybuild/tools/package/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py index 0621a29516..6e96dc4cab 100644 --- a/easybuild/tools/parallelbuild.py +++ b/easybuild/tools/parallelbuild.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/py2vs3/__init__.py b/easybuild/tools/py2vs3/__init__.py index 2c6e58e8bf..dd3db6dfff 100644 --- a/easybuild/tools/py2vs3/__init__.py +++ b/easybuild/tools/py2vs3/__init__.py @@ -1,5 +1,5 @@ # -# Copyright 2019-2021 Ghent University +# Copyright 2019-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/py2vs3/py2.py b/easybuild/tools/py2vs3/py2.py index a1b5252ef7..4177c636b4 100644 --- a/easybuild/tools/py2vs3/py2.py +++ b/easybuild/tools/py2vs3/py2.py @@ -1,5 +1,5 @@ # -# Copyright 2019-2021 Ghent University +# Copyright 2019-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/py2vs3/py3.py b/easybuild/tools/py2vs3/py3.py index 56f9fbef8d..307642fdec 100644 --- a/easybuild/tools/py2vs3/py3.py +++ b/easybuild/tools/py2vs3/py3.py @@ -1,5 +1,5 @@ # -# Copyright 2019-2021 Ghent University +# Copyright 2019-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/filerepo.py b/easybuild/tools/repository/filerepo.py index 2870802d84..27d0ae9ebe 100644 --- a/easybuild/tools/repository/filerepo.py +++ b/easybuild/tools/repository/filerepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/gitrepo.py b/easybuild/tools/repository/gitrepo.py index 576afebd48..863ebf4534 100644 --- a/easybuild/tools/repository/gitrepo.py +++ b/easybuild/tools/repository/gitrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/hgrepo.py b/easybuild/tools/repository/hgrepo.py index 61c09f163a..30bebd04c1 100644 --- a/easybuild/tools/repository/hgrepo.py +++ b/easybuild/tools/repository/hgrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/repository.py b/easybuild/tools/repository/repository.py index 84d8c89abc..b67b4b2cc2 100644 --- a/easybuild/tools/repository/repository.py +++ b/easybuild/tools/repository/repository.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/svnrepo.py b/easybuild/tools/repository/svnrepo.py index 8902843184..ca2b9a26d8 100644 --- a/easybuild/tools/repository/svnrepo.py +++ b/easybuild/tools/repository/svnrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index a59f58bfe2..510cf8c23f 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 322a9fddf5..519d161747 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 4d26066a18..482e4d1364 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2021 Ghent University +# Copyright 2011-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 8f2e4d3529..8765e3a0c0 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/__init__.py b/easybuild/tools/toolchain/__init__.py index e51e6d33e0..56bb55edf0 100644 --- a/easybuild/tools/toolchain/__init__.py +++ b/easybuild/tools/toolchain/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index c0486ef03c..745e32423b 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/constants.py b/easybuild/tools/toolchain/constants.py index b334d17ae9..f6f3beb2d6 100644 --- a/easybuild/tools/toolchain/constants.py +++ b/easybuild/tools/toolchain/constants.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/fft.py b/easybuild/tools/toolchain/fft.py index 1e39953ba7..e3fcda405e 100644 --- a/easybuild/tools/toolchain/fft.py +++ b/easybuild/tools/toolchain/fft.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/linalg.py b/easybuild/tools/toolchain/linalg.py index 2e1a46db41..573d7ae750 100644 --- a/easybuild/tools/toolchain/linalg.py +++ b/easybuild/tools/toolchain/linalg.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/mpi.py b/easybuild/tools/toolchain/mpi.py index 9f1b896a18..56c76ebead 100644 --- a/easybuild/tools/toolchain/mpi.py +++ b/easybuild/tools/toolchain/mpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/options.py b/easybuild/tools/toolchain/options.py index 423d7aa339..ae6eb436f1 100644 --- a/easybuild/tools/toolchain/options.py +++ b/easybuild/tools/toolchain/options.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 45229ba722..e21bd66869 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/toolchainvariables.py b/easybuild/tools/toolchain/toolchainvariables.py index d1a25f65df..82e5550658 100644 --- a/easybuild/tools/toolchain/toolchainvariables.py +++ b/easybuild/tools/toolchain/toolchainvariables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py index ff6f01cbff..d9a6c02412 100644 --- a/easybuild/tools/toolchain/utilities.py +++ b/easybuild/tools/toolchain/utilities.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/variables.py b/easybuild/tools/toolchain/variables.py index 6758af24de..a0987694e9 100644 --- a/easybuild/tools/toolchain/variables.py +++ b/easybuild/tools/toolchain/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index c6a4512c82..b7fc9fd657 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/variables.py b/easybuild/tools/variables.py index 8cd8dcd3cb..9db0aeb6fa 100644 --- a/easybuild/tools/variables.py +++ b/easybuild/tools/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 1396a1c4a1..477b6245b8 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/eb b/eb index 880759f62c..273c21247c 100755 --- a/eb +++ b/eb @@ -1,6 +1,6 @@ #!/bin/bash ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/setup.py b/setup.py index 49de6228c0..f3ae4e58cc 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/__init__.py b/test/__init__.py index 5a26aaf317..5770afd893 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/__init__.py b/test/framework/__init__.py index ff095ace59..77c470bf68 100644 --- a/test/framework/__init__.py +++ b/test/framework/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/asyncprocess.py b/test/framework/asyncprocess.py index 1d4adb9fe3..c3cf41316c 100644 --- a/test/framework/asyncprocess.py +++ b/test/framework/asyncprocess.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/build_log.py b/test/framework/build_log.py index 26bd666632..357773e4b0 100644 --- a/test/framework/build_log.py +++ b/test/framework/build_log.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/config.py b/test/framework/config.py index 72024905f6..e0cc07991e 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/containers.py b/test/framework/containers.py index 133754bcf7..16cb5cfd34 100644 --- a/test/framework/containers.py +++ b/test/framework/containers.py @@ -1,5 +1,5 @@ # # -# Copyright 2018-2021 Ghent University +# Copyright 2018-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/docs.py b/test/framework/docs.py index b4b2496d88..fb3e2a2b44 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index efbc743f0e..dd8d6ace76 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index b9fd16795f..479557e9c3 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfigformat.py b/test/framework/easyconfigformat.py index 777693a6ea..8c2ab8aa16 100644 --- a/test/framework/easyconfigformat.py +++ b/test/framework/easyconfigformat.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfigparser.py b/test/framework/easyconfigparser.py index a7eb0098d6..1a6134c0d3 100644 --- a/test/framework/easyconfigparser.py +++ b/test/framework/easyconfigparser.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfigversion.py b/test/framework/easyconfigversion.py index c3037b8826..f119c054fa 100644 --- a/test/framework/easyconfigversion.py +++ b/test/framework/easyconfigversion.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easystack.py b/test/framework/easystack.py index 32313c8b06..78be785fb5 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/ebconfigobj.py b/test/framework/ebconfigobj.py index e7f05ab83a..3f3e0074cc 100644 --- a/test/framework/ebconfigobj.py +++ b/test/framework/ebconfigobj.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/environment.py b/test/framework/environment.py index 10f875e733..64ab3c8bad 100644 --- a/test/framework/environment.py +++ b/test/framework/environment.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/filetools.py b/test/framework/filetools.py index ff369bdb49..7d890ed7e8 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/format_convert.py b/test/framework/format_convert.py index 46d57bf1ac..1637c56768 100644 --- a/test/framework/format_convert.py +++ b/test/framework/format_convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/general.py b/test/framework/general.py index 92f082cebc..f1e231044c 100644 --- a/test/framework/general.py +++ b/test/framework/general.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/github.py b/test/framework/github.py index e6c4abf4a5..7a4fa44c85 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 1e3c44ea5f..8826c4bf37 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -1,5 +1,5 @@ # # -# Copyright 2017-2021 Ghent University +# Copyright 2017-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/include.py b/test/framework/include.py index 4e25adac4c..fbc47ab10e 100644 --- a/test/framework/include.py +++ b/test/framework/include.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/lib.py b/test/framework/lib.py index e4bbca3b15..3196ad63cf 100644 --- a/test/framework/lib.py +++ b/test/framework/lib.py @@ -1,5 +1,5 @@ # # -# Copyright 2018-2021 Ghent University +# Copyright 2018-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/license.py b/test/framework/license.py index 609bffaa3e..4244835bc9 100644 --- a/test/framework/license.py +++ b/test/framework/license.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 998fbddd70..c88eace3de 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/modules.py b/test/framework/modules.py index 9c834477ee..00d6deabc2 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index ec7a05830f..7dbe10c9ac 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/options.py b/test/framework/options.py index e6d4da01c5..126c52931b 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/output.py b/test/framework/output.py index fdcba7038f..8e69ba68eb 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -1,5 +1,5 @@ # # -# Copyright 2021-2021 Ghent University +# Copyright 2021-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/package.py b/test/framework/package.py index 5451fe64a8..3e60fba515 100644 --- a/test/framework/package.py +++ b/test/framework/package.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/parallelbuild.py b/test/framework/parallelbuild.py index 9635e8eb00..60867d01e4 100644 --- a/test/framework/parallelbuild.py +++ b/test/framework/parallelbuild.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/repository.py b/test/framework/repository.py index 05d3ede70f..528026142a 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/robot.py b/test/framework/robot.py index 4add2ada9f..0647352935 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/run.py b/test/framework/run.py index a821a67d3c..dd7332e030 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1,6 +1,6 @@ # # # -*- coding: utf-8 -*- -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/f/fftw.py b/test/framework/sandbox/easybuild/easyblocks/f/fftw.py index 7862d814a4..1df3706307 100644 --- a/test/framework/sandbox/easybuild/easyblocks/f/fftw.py +++ b/test/framework/sandbox/easybuild/easyblocks/f/fftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/f/foo.py b/test/framework/sandbox/easybuild/easyblocks/f/foo.py index 438d6dbd97..c3a52b1e4d 100644 --- a/test/framework/sandbox/easybuild/easyblocks/f/foo.py +++ b/test/framework/sandbox/easybuild/easyblocks/f/foo.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py b/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py index 48991e160c..7b1324ee66 100644 --- a/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py +++ b/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/g/gcc.py b/test/framework/sandbox/easybuild/easyblocks/g/gcc.py index cfad1ca661..ebb404cc79 100644 --- a/test/framework/sandbox/easybuild/easyblocks/g/gcc.py +++ b/test/framework/sandbox/easybuild/easyblocks/g/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py index 2ed571fbdf..d0a6da6260 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py index 74d22be579..7e2da9b53c 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py index 18ccdb1902..da7017345c 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py b/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py index c80489f418..25708d9039 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py index d07cd95300..09f330c1bf 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 603346efe0..19eaf74338 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/h/hpl.py b/test/framework/sandbox/easybuild/easyblocks/h/hpl.py index a7e6f4ba2e..0ee172e853 100644 --- a/test/framework/sandbox/easybuild/easyblocks/h/hpl.py +++ b/test/framework/sandbox/easybuild/easyblocks/h/hpl.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py index 777e8a2b40..ec76e9ae08 100644 --- a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py +++ b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2021 Ghent University +# Copyright 2021-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/o/openblas.py b/test/framework/sandbox/easybuild/easyblocks/o/openblas.py index f5ac2fd878..52ce315477 100644 --- a/test/framework/sandbox/easybuild/easyblocks/o/openblas.py +++ b/test/framework/sandbox/easybuild/easyblocks/o/openblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py b/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py index fbc13cb4fc..5fabe4eb63 100644 --- a/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py +++ b/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py b/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py index 30e4cf8f7b..0e826c3f7c 100644 --- a/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py +++ b/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index bec0e7fe42..f1d39d0af2 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py index 9d99eac45e..f730dbd8a3 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py b/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py index 6f53997f8b..0bc400e29e 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py @@ -1,5 +1,5 @@ ## -# Copyright 2020-2021 Ghent University +# Copyright 2020-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py b/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py index b303ecb849..269373e245 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/__init__.py b/test/framework/sandbox/easybuild/tools/__init__.py index 433662112e..3f77104b3a 100644 --- a/test/framework/sandbox/easybuild/tools/__init__.py +++ b/test/framework/sandbox/easybuild/tools/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py index f447c49e35..03b33c11a4 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2021 Ghent University +# Copyright 2011-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py index 3d90a37485..62e9c033af 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py index e6dc638af8..93e610b325 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py index 8c7412c5f3..5ab85d6968 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/style.py b/test/framework/style.py index 24646c5e1f..6a84b23b6a 100644 --- a/test/framework/style.py +++ b/test/framework/style.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2021 Ghent University +# Copyright 2016-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/suite.py b/test/framework/suite.py index 80bce4983f..d49d40bb6b 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -1,6 +1,6 @@ #!/usr/bin/python # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index cbbf8f9509..940f4ae3fa 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 8139b274d5..13e6743ba2 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/toolchainvariables.py b/test/framework/toolchainvariables.py index 740185d899..4547652466 100644 --- a/test/framework/toolchainvariables.py +++ b/test/framework/toolchainvariables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9a8ca0dabe..8ea6b77d2a 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 2570142c6d..314c964af3 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/type_checking.py b/test/framework/type_checking.py index d416cb6b5e..bb26d386da 100644 --- a/test/framework/type_checking.py +++ b/test/framework/type_checking.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/utilities.py b/test/framework/utilities.py index f105b7b81f..c50f544e9e 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index c05ef90539..2ef6aa6893 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/variables.py b/test/framework/variables.py index 246ad212c2..e332e6cd20 100644 --- a/test/framework/variables.py +++ b/test/framework/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/yeb.py b/test/framework/yeb.py index 39ce4117bb..80c447ddbc 100644 --- a/test/framework/yeb.py +++ b/test/framework/yeb.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), From 8935710af29223f10e704c39c96903adb07768a1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 29 Mar 2022 08:00:18 +0200 Subject: [PATCH 829/864] fix checksum for bootstrap script after updating copyright line --- .github/workflows/bootstrap_script.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bootstrap_script.yml b/.github/workflows/bootstrap_script.yml index 58c3b0ddfa..f119c4a0d4 100644 --- a/.github/workflows/bootstrap_script.yml +++ b/.github/workflows/bootstrap_script.yml @@ -107,7 +107,7 @@ jobs: EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - EB_BOOTSTRAP_EXPECTED="20210715.01 784dd29063d941be2d8b70e4c2eec12a9afe360f3ef8f753dcb518abf43ca7de" + EB_BOOTSTRAP_EXPECTED="20210715.01 0ffdc17ed7eacf78369c9cd6743728f36e61bb8bf5c1bdc1e23cf2040b1ce301" test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) # test bootstrap script From 447bdbd66afb96e6128ebf7195e38682915a88da Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 29 Mar 2022 08:44:02 +0200 Subject: [PATCH 830/864] minor tweak release notes for v4.5.4 --- RELEASE_NOTES | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 6ff4e486f6..7a4bd4d131 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,26 +4,27 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. -v4.5.4 (March 28th 2022) ---------------------------- +v4.5.4 (March 31st 2022) +------------------------ update/bugfix release - various enhancements, including: - - warn about potentially missing patches in --new-pr (#3759) + - warn about potentially missing patches in --new-pr (#3759, #3966) - add support for 'clone_into' field in git_config source spec to specify different name for top-level directory (#3949) - add bash completion for easyconfigs from local dir but not robot search path (#3953) - add a 'sync pr' message when the PR has a mergeable state but is showing a failed status for the test suite on the last commit (#3967) - add gmpit toolchain definition (GCC + MPItrampoline) (#3971) - use 'zypper search -i' to check whether specified OS dependency is installed on openSUSE + make sure that rpm is considered for checking OS dependencies on RHEL8 (#3973) - add support for post-install patches (#3974) - - support 'download_instructions' easyconfig parameter key to specify some download or installation steps for user in case of complicated way of obtaining needed files (#3976) - - add support for collecting AMD GPU info (via rocm-smi) (#3978, #3982) + - add support for 'download_instructions' easyconfig parameter key to specify some download or installation steps for user in case of complicated way of obtaining needed files (#3976, #3981) + - also try collecting AMD GPU info (via rocm-smi) for --show-system-info (#3978, #3982) - various bug fixes, including: - - only check for patches when using --new-pr if easyconfigs repository is target (#3966) - - also print download instructions for extensions that use source_tmpl to specify name of source file (#3981) + - ensure --review-pr can find dependencies included in PR (#3979) + - run 'apt-get update' in GitHub Actions workflow for container tests to update container package list before installing anything (#3985) - other changes: - enable code linting check for all supported Python versions (#3725) + - update copyright lines for 2022 (#3986) v4.5.3 (February 11th 2022) From 3bbf975051361eb4fd9a339c357f9b7ed6ac8eec Mon Sep 17 00:00:00 2001 From: Jasper Grimm Date: Tue, 29 Mar 2022 09:35:48 +0100 Subject: [PATCH 831/864] avoid appending directly to from_prs, tidy if/else logic --- easybuild/framework/easyconfig/tools.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 66d181c1d9..6796c483ec 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -321,14 +321,14 @@ def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None, review_pr=Non # paths where files touched in PRs will be downloaded to, # which are picked up via 'pr_paths' build option in fetch_files_from_pr - pr_paths = None - if from_prs and review_pr: - from_prs.append(review_pr) if review_pr not in from_prs else from_prs - elif review_pr: - from_prs = [review_pr] - + pr_paths = [] if from_prs: - pr_paths = [os.path.join(tmpdir, 'files_pr%s' % pr) for pr in from_prs] + pr_paths = from_prs + if review_pr and review_pr not in pr_paths: + pr_paths.append(review_pr) + + if pr_paths: + pr_paths = [os.path.join(tmpdir, 'files_pr%s' % pr) for pr in pr_paths] return tweaked_ecs_paths, pr_paths From 77af56f4d5ef143ae08b0275e8b68b54349ad7b3 Mon Sep 17 00:00:00 2001 From: Jasper <65227842+jfgrimm@users.noreply.github.com> Date: Tue, 29 Mar 2022 10:32:20 +0100 Subject: [PATCH 832/864] ensure pr_paths is a value copy --- easybuild/framework/easyconfig/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 6796c483ec..4fb3de77ac 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -323,7 +323,7 @@ def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None, review_pr=Non # which are picked up via 'pr_paths' build option in fetch_files_from_pr pr_paths = [] if from_prs: - pr_paths = from_prs + pr_paths = from_prs[:] if review_pr and review_pr not in pr_paths: pr_paths.append(review_pr) From f5733aee3bb7ecc25640de778730f7a7a993858b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 31 Mar 2022 10:25:33 +0200 Subject: [PATCH 833/864] bump version to 4.5.5dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index c0fa27c385..657ad3cb75 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.5.4') +VERSION = LooseVersion('4.5.5.dev0') UNKNOWN = 'UNKNOWN' From 73a1f4875136233eb66baefb55ebba535cb03338 Mon Sep 17 00:00:00 2001 From: Jasper Grimm Date: Mon, 4 Apr 2022 15:48:43 +0100 Subject: [PATCH 834/864] Use keep_available_modules for options.copy_ec --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 61d27b3089..c43c0583df 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -447,7 +447,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules - keep_available_modules = forced or dry_run_mode or options.extended_dry_run or pr_options + keep_available_modules = forced or dry_run_mode or options.extended_dry_run or pr_options or options.copy_ec keep_available_modules = keep_available_modules or options.inject_checksums or options.sanity_check_only # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping From 9b4344f9f06325b8ae541ec9996454a7c2786b1a Mon Sep 17 00:00:00 2001 From: Robert Mijakovic Date: Tue, 12 Apr 2022 17:59:22 +0200 Subject: [PATCH 835/864] Updates the author --- easybuild/toolchains/nvompi.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/toolchains/nvompi.py b/easybuild/toolchains/nvompi.py index c24245b0cc..f8bd107adb 100644 --- a/easybuild/toolchains/nvompi.py +++ b/easybuild/toolchains/nvompi.py @@ -25,8 +25,7 @@ """ EasyBuild support for nvompi compiler toolchain (NVHPC + Open MPI). -:author: Maxime Boissonneault (Universite Laval, Calcul Quebec, Compute Canada) -:author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) +:author: Robert Mijakovic (LuxProvide) """ from easybuild.toolchains.nvhpc import NVHPCToolchain From 3d29fb820aa893ba882451dd2161fe2f1c51f46f Mon Sep 17 00:00:00 2001 From: Robert Mijakovic Date: Tue, 12 Apr 2022 18:05:45 +0200 Subject: [PATCH 836/864] Updates the author, header and fixes a typo --- easybuild/toolchains/nvpsmpi.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/easybuild/toolchains/nvpsmpi.py b/easybuild/toolchains/nvpsmpi.py index bf0f536099..c68b83d7d3 100644 --- a/easybuild/toolchains/nvpsmpi.py +++ b/easybuild/toolchains/nvpsmpi.py @@ -5,11 +5,11 @@ # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), # with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be), # Flemish Research Foundation (FWO) (http://www.fwo.be/en) # and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). # -# http://github.com/hpcugent/easybuild +# https://github.com/easybuilders/easybuild # # EasyBuild is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,10 +24,9 @@ # along with EasyBuild. If not, see . ## """ -EasyBuild support for npsmpi compiler toolchain (includes NVHPC and ParaStationMPI). +EasyBuild support for nvsmpi compiler toolchain (includes NVHPC and ParaStationMPI). -:author: Damian Alvarez (Forschungszentrum Juelich) -:author: Sebastian Achilles (Forschungszentrum Juelich) +:author: Robert Mijakovic (LuxProvide) """ from easybuild.toolchains.nvhpc import NVHPCToolchain From 2a0c5c3e394fc158d5bc5648253b36c5ec21c5ff Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 8 Apr 2022 13:52:29 +0000 Subject: [PATCH 837/864] Change 'eb' command to import easybuild to check if it's working. This is a much cheaper test than "import easybuild.main" and still solves the same issue as #3610. This reduces the wall time for "eb --version" from 1.35s to 0.73s in our configuration. --- eb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb b/eb index 273c21247c..9563082b9e 100755 --- a/eb +++ b/eb @@ -80,7 +80,7 @@ for python_cmd in ${EB_PYTHON} ${EB_INSTALLPYTHON} 'python' 'python3' 'python2'; if [ ! -z $PYTHON ]; then # check whether easybuild.main is available for selected python command - $PYTHON -c "import $EASYBUILD_MAIN" 2> /dev/null + $PYTHON -c "import easybuild" 2> /dev/null if [ $? -eq 0 ]; then verbose "'$python_cmd' is able to import '$EASYBUILD_MAIN', so retaining it" else From 3cd0c8ca290d6df47fb5dffa5d7a6d55ac45a668 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 19 Apr 2022 12:54:45 +0000 Subject: [PATCH 838/864] Adjust verbose output to be consistent. --- eb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eb b/eb index 9563082b9e..2bd6e6560c 100755 --- a/eb +++ b/eb @@ -82,10 +82,10 @@ for python_cmd in ${EB_PYTHON} ${EB_INSTALLPYTHON} 'python' 'python3' 'python2'; # check whether easybuild.main is available for selected python command $PYTHON -c "import easybuild" 2> /dev/null if [ $? -eq 0 ]; then - verbose "'$python_cmd' is able to import '$EASYBUILD_MAIN', so retaining it" + verbose "'$python_cmd' is able to import 'easybuild', so retaining it" else # if easybuild.main is not available, don't use this python command, keep searching... - verbose "'$python_cmd' is NOT able to import '$EASYBUILD_MAIN', so NOT retaining it" + verbose "'$python_cmd' is NOT able to import 'easybuild' so NOT retaining it" unset PYTHON fi fi From 02bee981ffae8a390efc3667dd652a5764bd67d9 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 19 Apr 2022 13:03:37 +0000 Subject: [PATCH 839/864] Adjust test to check for "import 'easybuild'" --- .github/workflows/eb_command.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml index baed42182e..12f866287d 100644 --- a/.github/workflows/eb_command.yml +++ b/.github/workflows/eb_command.yml @@ -73,7 +73,7 @@ jobs: pymajver=$(python -c 'import sys; print(sys.version_info[0])') pymajminver=$(python -c 'import sys; print(".".join(str(x) for x in sys.version_info[:2]))') # check patterns in verbose output - for pattern in "^>> Considering .python.\.\.\." "^>> .python. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python' is able to import 'easybuild.main', so retaining it" "^>> Selected Python command: python \(.*/bin/python\)" "^This is EasyBuild 4\.[0-9.]\+"; do + for pattern in "^>> Considering .python.\.\.\." "^>> .python. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python' is able to import 'easybuild', so retaining it" "^>> Selected Python command: python \(.*/bin/python\)" "^This is EasyBuild 4\.[0-9.]\+"; do echo "Looking for pattern \"${pattern}\" in eb_version.out..." grep "$pattern" eb_version.out done @@ -81,7 +81,7 @@ jobs: for eb_python in "python${pymajver}" "python${pymajminver}"; do export EB_PYTHON="${eb_python}" eb --version | tee eb_version.out 2>&1 - for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> '${eb_python}' is able to import 'easybuild.main', so retaining it" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 4\.[0-9.]\+"; do + for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> '${eb_python}' is able to import 'easybuild', so retaining it" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 4\.[0-9.]\+"; do echo "Looking for pattern \"${pattern}\" in eb_version.out..." grep "$pattern" eb_version.out done From d954ee303b71212d3547330eb0631b11b968878a Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 19 Apr 2022 18:17:22 +0000 Subject: [PATCH 840/864] Fix comments --- eb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eb b/eb index 2bd6e6560c..ea703130a1 100755 --- a/eb +++ b/eb @@ -79,12 +79,12 @@ for python_cmd in ${EB_PYTHON} ${EB_INSTALLPYTHON} 'python' 'python3' 'python2'; fi if [ ! -z $PYTHON ]; then - # check whether easybuild.main is available for selected python command + # check whether easybuild is available for selected python command $PYTHON -c "import easybuild" 2> /dev/null if [ $? -eq 0 ]; then verbose "'$python_cmd' is able to import 'easybuild', so retaining it" else - # if easybuild.main is not available, don't use this python command, keep searching... + # if easybuild is not available, don't use this python command, keep searching... verbose "'$python_cmd' is NOT able to import 'easybuild' so NOT retaining it" unset PYTHON fi From e3ea361c1ffcd6ad1c035af64f90b47c50c7cdb3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Apr 2022 09:06:51 +0200 Subject: [PATCH 841/864] change 'eb' command to import easybuild.framework to check if EasyBuild framework is available --- .github/workflows/eb_command.yml | 4 ++-- eb | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml index 12f866287d..d86a49aa0b 100644 --- a/.github/workflows/eb_command.yml +++ b/.github/workflows/eb_command.yml @@ -73,7 +73,7 @@ jobs: pymajver=$(python -c 'import sys; print(sys.version_info[0])') pymajminver=$(python -c 'import sys; print(".".join(str(x) for x in sys.version_info[:2]))') # check patterns in verbose output - for pattern in "^>> Considering .python.\.\.\." "^>> .python. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python' is able to import 'easybuild', so retaining it" "^>> Selected Python command: python \(.*/bin/python\)" "^This is EasyBuild 4\.[0-9.]\+"; do + for pattern in "^>> Considering .python.\.\.\." "^>> .python. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python' is able to import 'easybuild.framework', so retaining it" "^>> Selected Python command: python \(.*/bin/python\)" "^This is EasyBuild 4\.[0-9.]\+"; do echo "Looking for pattern \"${pattern}\" in eb_version.out..." grep "$pattern" eb_version.out done @@ -81,7 +81,7 @@ jobs: for eb_python in "python${pymajver}" "python${pymajminver}"; do export EB_PYTHON="${eb_python}" eb --version | tee eb_version.out 2>&1 - for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> '${eb_python}' is able to import 'easybuild', so retaining it" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 4\.[0-9.]\+"; do + for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> '${eb_python}' is able to import 'easybuild.framework', so retaining it" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 4\.[0-9.]\+"; do echo "Looking for pattern \"${pattern}\" in eb_version.out..." grep "$pattern" eb_version.out done diff --git a/eb b/eb index ea703130a1..b0b625f19d 100755 --- a/eb +++ b/eb @@ -46,6 +46,10 @@ REQ_MIN_PY3VER=5 EASYBUILD_MAIN='easybuild.main' +# easybuild module to import to check whether EasyBuild framework is available; +# don't use easybuild.main here, since that's a very expensive module to import (it makes the 'eb' command slow) +EASYBUILD_IMPORT_TEST='easybuild.framework' + function verbose() { if [ ! -z ${EB_VERBOSE} ]; then echo ">> $1"; fi } @@ -79,13 +83,13 @@ for python_cmd in ${EB_PYTHON} ${EB_INSTALLPYTHON} 'python' 'python3' 'python2'; fi if [ ! -z $PYTHON ]; then - # check whether easybuild is available for selected python command - $PYTHON -c "import easybuild" 2> /dev/null + # check whether EasyBuild framework is available for selected python command + $PYTHON -c "import $EASYBUILD_IMPORT_TEST" 2> /dev/null if [ $? -eq 0 ]; then - verbose "'$python_cmd' is able to import 'easybuild', so retaining it" + verbose "'$python_cmd' is able to import '$EASYBUILD_IMPORT_TEST', so retaining it" else - # if easybuild is not available, don't use this python command, keep searching... - verbose "'$python_cmd' is NOT able to import 'easybuild' so NOT retaining it" + # if EasyBuild framework is not available, don't use this python command, keep searching... + verbose "'$python_cmd' is NOT able to import '$EASYBUILD_IMPORT_TEST' so NOT retaining it" unset PYTHON fi fi From aeeb4cd8b1e8aba11453236f8fa05c3aca73bc64 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Apr 2022 09:58:22 +0200 Subject: [PATCH 842/864] ignore deprecation warnings regarding Blowfish in test suite output when using Python 3.10 --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index dc748edbab..3f298951d9 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -200,7 +200,7 @@ jobs: # run test suite python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2|from cryptography.utils import int_from_bytes" + IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2|from cryptography.utils import int_from_bytes|Blowfish" # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1) From 332df63aaabfcb8c9e616bad8cfd81b6bbf5b3ca Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Apr 2022 11:05:10 +0200 Subject: [PATCH 843/864] also ignore cryptography deprecation warning for Python 3.6 in test suite output --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 3f298951d9..41ea5139c2 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -200,7 +200,7 @@ jobs: # run test suite python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2|from cryptography.utils import int_from_bytes|Blowfish" + IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2|from cryptography.utils import int_from_bytes|Blowfish" # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1) From 983831b387b6e97eeab3c308cde6982b8f5615e4 Mon Sep 17 00:00:00 2001 From: Adam Huffman Date: Thu, 28 Apr 2022 13:17:43 +0100 Subject: [PATCH 844/864] Fix typo in easy block.py `Skiping` -> `Skipping` --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index b627b46151..25c72a3397 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3286,7 +3286,7 @@ def _sanity_check_step_dry_run(self, custom_paths=None, custom_commands=None, ** if self.toolchain.use_rpath: self.sanity_check_rpath() else: - self.log.debug("Skiping RPATH sanity check") + self.log.debug("Skipping RPATH sanity check") def _sanity_check_step_extensions(self): """Sanity check on extensions (if any).""" @@ -3452,7 +3452,7 @@ def xs2str(xs): self.log.warning("RPATH sanity check failed!") self.sanity_check_fail_msgs.extend(rpath_fails) else: - self.log.debug("Skiping RPATH sanity check") + self.log.debug("Skipping RPATH sanity check") # pass or fail if self.sanity_check_fail_msgs: From 45615455c5efb24fdf524e72346b0513850aba42 Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Sat, 7 May 2022 02:01:43 +0200 Subject: [PATCH 845/864] fixup! tweak printing of download instructions in obtain_file + remove dead code dealing with download instructions in fetch_source Urlencoded characters (e.g. '%2F' instead of '/') caused an error while reporting download error. --- easybuild/framework/easyblock.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 25c72a3397..46764b1626 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -883,8 +883,10 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No error_msg += "please follow the download instructions above, and make the file available " error_msg += "in the active source path (%s)" % ':'.join(source_paths()) else: + # flatten list to string with '%' characters escaped (literal '%' desired in 'sprintf') + failedpaths_msg = ', '.join(failedpaths).replace('%', '%%') error_msg += "and downloading it didn't work either... " - error_msg += "Paths attempted (in order): %s " % ', '.join(failedpaths) + error_msg += "Paths attempted (in order): %s " % failedpaths_msg raise EasyBuildError(error_msg, filename) From e9083310b50685de7c56df30d7e6b59f7099b77a Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 11 May 2022 11:12:15 +0000 Subject: [PATCH 846/864] Add gfbf toolchain (GCC, FlexiBLAS, FFTW) --- easybuild/toolchains/gfbf.py | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 easybuild/toolchains/gfbf.py diff --git a/easybuild/toolchains/gfbf.py b/easybuild/toolchains/gfbf.py new file mode 100644 index 0000000000..cebe49ccf2 --- /dev/null +++ b/easybuild/toolchains/gfbf.py @@ -0,0 +1,41 @@ +## +# Copyright 2021-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gfbf compiler toolchain (includes GCC, FlexiBLAS and FFTW) + +:author: Kenneth Hoste (Ghent University) +:author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) +""" + +from easybuild.toolchains.gcc import GccToolchain +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.flexiblas import FlexiBLAS + + +class Gfbf(GccToolchain, FlexiBLAS, Fftw): + """Compiler toolchain with GCC, FlexiBLAS and FFTW.""" + NAME = 'gfbf' + SUBTOOLCHAIN = GccToolchain.NAME + OPTIONAL = True From 4fd33f39b21dfd66b4f459e4f4c1fcf052aa5549 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 May 2022 14:30:58 +0200 Subject: [PATCH 847/864] enhance test for obtain_file method to catch broken error reporting when file could not be downloaded --- test/framework/easyblock.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index dd8d6ace76..f77823fcec 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1694,6 +1694,12 @@ def test_obtain_file(self): error_regex = "Couldn't find file %s anywhere, and downloading it didn't work either" % fn self.assertErrorRegex(EasyBuildError, error_regex, eb.obtain_file, fn, urls=['file://%s' % tmpdir_subdir]) + # also test triggering error when downloading from a URL that includes URL-encoded characters + # cfr. https://github.com/easybuilders/easybuild-framework/pull/4005 + url = 'file://%s' % os.path.dirname(tmpdir_subdir) + url += '%2F' + os.path.basename(tmpdir_subdir) + self.assertErrorRegex(EasyBuildError, error_regex, eb.obtain_file, fn, urls=[url]) + # file specifications via URL also work, are downloaded to (first) sourcepath init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, sandbox_sources)]) urls = [ From 2d51b1c672ed581a12d6bc8b8b4ae4635a757044 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 25 May 2022 17:40:47 +0000 Subject: [PATCH 848/864] Set FFT(W)_LIB_DIR to imkl-FFTW's lib for MPI Fixes #3987 --- easybuild/toolchains/fft/intelfftw.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py index 4c1a29194c..58aff8e00d 100644 --- a/easybuild/toolchains/fft/intelfftw.py +++ b/easybuild/toolchains/fft/intelfftw.py @@ -98,7 +98,9 @@ def _set_fftw_variables(self): imklfftwroot = get_software_root('imkl-FFTW') if imklfftwroot: # only get cluster_interface_lib from seperate module imkl-FFTW, rest via libmkl_gf/libmkl_intel - fft_lib_dirs += [os.path.join(imklfftwroot, 'lib')] + if self.options.get('usempi', False): + fft_lib_dirs += [os.path.join(imklfftwroot, 'lib')] + self.FFT_LIB_DIR = [os.path.join(imklfftwroot, 'lib')] fftw_libs.remove(interface_lib) fftw_mt_libs.remove(interface_lib) From e5d0d8b1d7d5e78b58e86af3723eef971f94fcd1 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 25 May 2022 19:32:23 +0000 Subject: [PATCH 849/864] Support FFTW.MPI module in framework (FFT*DIR variables) --- easybuild/toolchains/fft/fftw.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easybuild/toolchains/fft/fftw.py b/easybuild/toolchains/fft/fftw.py index 3003634ac8..ee83197634 100644 --- a/easybuild/toolchains/fft/fftw.py +++ b/easybuild/toolchains/fft/fftw.py @@ -33,6 +33,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.fft import Fft +from easybuild.tools.modules import get_software_root class Fftw(Fft): @@ -56,6 +57,11 @@ def _set_fftw_variables(self): fftw_libs = ["fftw%s" % suffix] if self.options.get('usempi', False): fftw_libs.insert(0, "fftw%s_mpi" % suffix) + fftwmpiroot = get_software_root('FFTW.MPI') + if fftwmpiroot: + # get libfft%_mpi via the FFTW.MPI module + self.FFT_MODULE_NAME = ['FFTW.MPI'] + fftw_libs_mt = ["fftw%s" % suffix] if self.options.get('openmp', False): fftw_libs_mt.insert(0, "fftw%s_omp" % suffix) From af9e436d8c88f6bde8d15c6aea76781bbfac0aac Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 26 May 2022 01:30:23 +0000 Subject: [PATCH 850/864] Clean up intelfftw.py logic Instead of adding and then removing interface_lib/cluster_interface_lib, only add them if needed, so they don't need removing later. Also check for existence of mt libraries (longer list) to be more thorough. --- easybuild/toolchains/fft/intelfftw.py | 62 ++++++++++++--------------- 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py index 58aff8e00d..338c6d75f2 100644 --- a/easybuild/toolchains/fft/intelfftw.py +++ b/easybuild/toolchains/fft/intelfftw.py @@ -49,7 +49,10 @@ def _set_fftw_variables(self): if not hasattr(self, 'BLAS_LIB_DIR'): raise EasyBuildError("_set_fftw_variables: IntelFFT based on IntelMKL (no BLAS_LIB_DIR found)") + imklroot = get_software_root(self.FFT_MODULE_NAME[0]) imklver = get_software_version(self.FFT_MODULE_NAME[0]) + self.FFT_LIB_DIR = self.BLAS_LIB_DIR + self.FFT_INCLUDE_DIR = [os.path.join(d, 'fftw') for d in self.BLAS_INCLUDE_DIR] picsuff = '' if self.options.get('pic', None): @@ -69,11 +72,26 @@ def _set_fftw_variables(self): raise EasyBuildError(error_msg) interface_lib = "fftw3xc%s%s" % (compsuff, picsuff) - fftw_libs = [interface_lib] - cluster_interface_lib = None + fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR] + + def fftw_lib_exists(libname): + """Helper function to check whether FFTW library with specified name exists.""" + return any(os.path.exists(os.path.join(d, "lib%s.a" % libname)) for d in fft_lib_dirs) + + # interface libs can be optional: + # MKL >= 10.2 include fftw3xc and fftw3xf interfaces in LIBBLAS=libmkl_gf/libmkl_intel + # See https://software.intel.com/en-us/articles/intel-mkl-main-libraries-contain-fftw3-interfaces + # The cluster interface libs (libfftw3x_cdft*) can be omitted if the toolchain does not provide MPI-FFTW + # interfaces. + fftw_libs = [] + if fftw_lib_exists(interface_lib) or LooseVersion(imklver) < LooseVersion("10.2"): + fftw_libs = [interface_lib] + if self.options.get('usempi', False): # add cluster interface for recent imkl versions - if LooseVersion(imklver) >= LooseVersion('10.3'): + # only get cluster_interface_lib from seperate module imkl-FFTW, rest via libmkl_gf/libmkl_intel + imklfftwroot = get_software_root('imkl-FFTW') + if LooseVersion(imklver) >= LooseVersion('10.3') and (fftw_libs or imklfftwroot): suff = picsuff if LooseVersion(imklver) >= LooseVersion('11.0.2'): suff = bitsuff + suff @@ -81,44 +99,20 @@ def _set_fftw_variables(self): fftw_libs.append(cluster_interface_lib) fftw_libs.append("mkl_cdft_core") # add cluster dft fftw_libs.extend(self.variables['LIBBLACS'].flatten()) # add BLACS; use flatten because ListOfList - - fftw_mt_libs = fftw_libs + [x % self.BLAS_LIB_MAP for x in self.BLAS_LIB_MT] + if imklfftwroot: + fft_lib_dirs += [os.path.join(imklfftwroot, 'lib')] + self.FFT_LIB_DIR = [os.path.join(imklfftwroot, 'lib')] self.log.debug('fftw_libs %s' % fftw_libs.__repr__()) fftw_libs.extend(self.variables['LIBBLAS'].flatten()) # add BLAS libs (contains dft) self.log.debug('fftw_libs %s' % fftw_libs.__repr__()) - self.FFT_LIB_DIR = self.BLAS_LIB_DIR - self.FFT_INCLUDE_DIR = [os.path.join(d, 'fftw') for d in self.BLAS_INCLUDE_DIR] - # building the FFTW interfaces is optional, # so make sure libraries are there before FFT_LIB is set - imklroot = get_software_root(self.FFT_MODULE_NAME[0]) - fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR] - imklfftwroot = get_software_root('imkl-FFTW') - if imklfftwroot: - # only get cluster_interface_lib from seperate module imkl-FFTW, rest via libmkl_gf/libmkl_intel - if self.options.get('usempi', False): - fft_lib_dirs += [os.path.join(imklfftwroot, 'lib')] - self.FFT_LIB_DIR = [os.path.join(imklfftwroot, 'lib')] - fftw_libs.remove(interface_lib) - fftw_mt_libs.remove(interface_lib) - - def fftw_lib_exists(libname): - """Helper function to check whether FFTW library with specified name exists.""" - return any(os.path.exists(os.path.join(d, "lib%s.a" % libname)) for d in fft_lib_dirs) - - if not fftw_lib_exists(interface_lib) and LooseVersion(imklver) >= LooseVersion("10.2"): - # interface libs can be optional: - # MKL >= 10.2 include fftw3xc and fftw3xf interfaces in LIBBLAS=libmkl_gf/libmkl_intel - # See https://software.intel.com/en-us/articles/intel-mkl-main-libraries-contain-fftw3-interfaces - # The cluster interface libs (libfftw3x_cdft*) can be omitted if the toolchain does not provide MPI-FFTW - # interfaces. - fftw_libs = [lib for lib in fftw_libs if lib not in [interface_lib, cluster_interface_lib]] - fftw_mt_libs = [lib for lib in fftw_mt_libs if lib not in [interface_lib, cluster_interface_lib]] + fftw_mt_libs = fftw_libs + [x % self.BLAS_LIB_MAP for x in self.BLAS_LIB_MT] # filter out libraries from list of FFTW libraries to check for if they are not provided by Intel MKL - check_fftw_libs = [lib for lib in fftw_libs if lib not in ['dl', 'gfortran']] + check_fftw_libs = [lib for lib in fftw_mt_libs if lib not in ['dl', 'gfortran']] missing_fftw_libs = [lib for lib in check_fftw_libs if not fftw_lib_exists(lib)] if missing_fftw_libs: @@ -129,6 +123,6 @@ def fftw_lib_exists(libname): else: raise EasyBuildError(msg) else: - self.FFT_LIB = fftw_libs + self.FFT_LIB_MT = fftw_mt_libs - self.FFT_LIB_MT = fftw_mt_libs + self.FFT_LIB = fftw_libs From d3c36ba7384801092de8966b639b832e56a64ecf Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 26 May 2022 02:25:20 +0000 Subject: [PATCH 851/864] Fix test, and multithreaded FFT libs. --- easybuild/toolchains/fft/intelfftw.py | 9 ++++----- test/framework/toolchain.py | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py index 338c6d75f2..2e969241b8 100644 --- a/easybuild/toolchains/fft/intelfftw.py +++ b/easybuild/toolchains/fft/intelfftw.py @@ -103,16 +103,16 @@ def fftw_lib_exists(libname): fft_lib_dirs += [os.path.join(imklfftwroot, 'lib')] self.FFT_LIB_DIR = [os.path.join(imklfftwroot, 'lib')] + fftw_mt_libs = fftw_libs + [x % self.BLAS_LIB_MAP for x in self.BLAS_LIB_MT] + self.log.debug('fftw_libs %s' % fftw_libs.__repr__()) fftw_libs.extend(self.variables['LIBBLAS'].flatten()) # add BLAS libs (contains dft) self.log.debug('fftw_libs %s' % fftw_libs.__repr__()) # building the FFTW interfaces is optional, # so make sure libraries are there before FFT_LIB is set - fftw_mt_libs = fftw_libs + [x % self.BLAS_LIB_MAP for x in self.BLAS_LIB_MT] - # filter out libraries from list of FFTW libraries to check for if they are not provided by Intel MKL - check_fftw_libs = [lib for lib in fftw_mt_libs if lib not in ['dl', 'gfortran']] + check_fftw_libs = [lib for lib in fftw_libs + fftw_mt_libs if lib not in ['dl', 'gfortran']] missing_fftw_libs = [lib for lib in check_fftw_libs if not fftw_lib_exists(lib)] if missing_fftw_libs: @@ -123,6 +123,5 @@ def fftw_lib_exists(libname): else: raise EasyBuildError(msg) else: + self.FFT_LIB = fftw_libs self.FFT_LIB_MT = fftw_mt_libs - - self.FFT_LIB = fftw_libs diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 13e6743ba2..9f3e9667a2 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1185,6 +1185,7 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'): mkl_libs = ['mkl_cdft_core', 'mkl_blacs_intelmpi_lp64'] mkl_libs += ['mkl_intel_lp64', 'mkl_sequential', 'mkl_core', 'mkl_intel_ilp64'] + mkl_libs += ['mkl_intel_thread'] fftw_libs = ['fftw3xc_intel', 'fftw3xc_pgi'] if LooseVersion(imklver) >= LooseVersion('11'): fftw_libs.extend(['fftw3x_cdft_ilp64', 'fftw3x_cdft_lp64']) From dc48d2542f12ab87b8e6238f3bb467722dc15f54 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 26 May 2022 02:39:58 +0000 Subject: [PATCH 852/864] Add tests for FFT_LIB_DIR, with/without MPI. --- test/framework/toolchain.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 9f3e9667a2..90aeb8728b 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1115,6 +1115,9 @@ def test_fft_env_vars_intel(self): self.assertEqual(tc.get_variable('LIBFFT'), libfft) self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt) + fft_lib_dir = os.path.join(modules.get_software_root('imkl'), 'mkl/2021.4.0/lib/intel64') + self.assertEqual(tc.get_variable('FFT_LIB_DIR'), fft_lib_dir) + tc = self.get_toolchain('intel', version='2021b') tc.set_options({'usempi': True}) tc.prepare() @@ -1138,6 +1141,9 @@ def test_fft_env_vars_intel(self): libfft_mt += '-Wl,-Bdynamic -liomp5 -lpthread' self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt) + fft_lib_dir = os.path.join(modules.get_software_root('imkl-FFTW'), 'lib') + self.assertEqual(tc.get_variable('FFT_LIB_DIR'), fft_lib_dir) + def test_fosscuda(self): """Test whether fosscuda is handled properly.""" tc = self.get_toolchain("fosscuda", version="2018a") From 6ebf8f622915e7d7e5b443d89fbc8ebddea82acc Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 26 May 2022 02:46:00 +0000 Subject: [PATCH 853/864] We must also add libmkl_pgi_thread.a to test sandbox. --- test/framework/toolchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 90aeb8728b..ca7c22eb65 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1191,7 +1191,7 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'): mkl_libs = ['mkl_cdft_core', 'mkl_blacs_intelmpi_lp64'] mkl_libs += ['mkl_intel_lp64', 'mkl_sequential', 'mkl_core', 'mkl_intel_ilp64'] - mkl_libs += ['mkl_intel_thread'] + mkl_libs += ['mkl_intel_thread', 'mkl_pgi_thread'] fftw_libs = ['fftw3xc_intel', 'fftw3xc_pgi'] if LooseVersion(imklver) >= LooseVersion('11'): fftw_libs.extend(['fftw3x_cdft_ilp64', 'fftw3x_cdft_lp64']) From f5f010a62ca755b5dd670b24424c50452fc93ed4 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 26 May 2022 03:22:38 +0000 Subject: [PATCH 854/864] Write tests for FFT_LIB_DIR with foss and FFTW.MPI --- test/framework/modules/foss/2018a-FFTW.MPI | 46 ++++++++++++ test/framework/toolchain.py | 81 ++++++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 test/framework/modules/foss/2018a-FFTW.MPI diff --git a/test/framework/modules/foss/2018a-FFTW.MPI b/test/framework/modules/foss/2018a-FFTW.MPI new file mode 100644 index 0000000000..37d9ae0c14 --- /dev/null +++ b/test/framework/modules/foss/2018a-FFTW.MPI @@ -0,0 +1,46 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { GCC based compiler toolchain including + OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none) +} +} + +module-whatis {GCC based compiler toolchain including + OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none)} + +set root /prefix/software/foss/2018a + +conflict foss + +if { ![is-loaded GCC/6.4.0-2.28] } { + module load GCC/6.4.0-2.28 +} + +if { ![is-loaded OpenMPI/2.1.2-GCC-6.4.0-2.28] } { + module load OpenMPI/2.1.2-GCC-6.4.0-2.28 +} + +if { ![is-loaded OpenBLAS/0.2.20-GCC-6.4.0-2.28] } { + module load OpenBLAS/0.2.20-GCC-6.4.0-2.28 +} + +if { ![is-loaded FFTW/3.3.7] } { + module load FFTW/3.3.7 +} + +if { ![is-loaded FFTW.MPI/3.3.7 ] } { + module load FFTW.MPI/3.3.7 +} + +if { ![is-loaded ScaLAPACK/2.0.2-gompi-2018a-OpenBLAS-0.2.20] } { + module load ScaLAPACK/2.0.2-gompi-2018a-OpenBLAS-0.2.20 +} + + +setenv EBROOTFOSS "$root" +setenv EBVERSIONFOSS "2018a" +setenv EBDEVELFOSS "$root/easybuild/foss-2018a-easybuild-devel" + + +# built with EasyBuild version 1.4.0dev diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 13e6743ba2..a030202dc7 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1017,6 +1017,60 @@ def test_fft_env_vars_foss(self): self.assertEqual(tc.get_variable('LIBFFT'), '-lfftw3_mpi -lfftw3') self.assertEqual(tc.get_variable('LIBFFT_MT'), '-lfftw3 -lpthread') + self.modtool.purge() + self.setup_sandbox_for_foss_fftw(self.test_prefix) + self.modtool.prepend_module_path(self.test_prefix) + + tc = self.get_toolchain('foss', version='2018a-FFTW.MPI') + tc.prepare() + + fft_static_libs = 'libfftw3.a' + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs) + + fft_static_libs_mt = 'libfftw3.a,libpthread.a' + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS_MT'), fft_static_libs_mt) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS_MT'), fft_static_libs_mt) + + self.assertEqual(tc.get_variable('LIBFFT'), '-lfftw3') + self.assertEqual(tc.get_variable('LIBFFT_MT'), '-lfftw3 -lpthread') + + fft_lib_dir = os.path.join(modules.get_software_root('FFTW'), 'lib') + self.assertEqual(tc.get_variable('FFT_LIB_DIR'), fft_lib_dir) + + tc = self.get_toolchain('foss', version='2018a-FFTW.MPI') + tc.set_options({'openmp': True}) + tc.prepare() + + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs) + + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS_MT'), 'libfftw3_omp.a,' + fft_static_libs_mt) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS_MT'), 'libfftw3_omp.a,' + fft_static_libs_mt) + + self.assertEqual(tc.get_variable('LIBFFT'), '-lfftw3') + self.assertEqual(tc.get_variable('LIBFFT_MT'), '-lfftw3_omp -lfftw3 -lpthread') + + fft_lib_dir = os.path.join(modules.get_software_root('FFTW'), 'lib') + self.assertEqual(tc.get_variable('FFT_LIB_DIR'), fft_lib_dir) + + tc = self.get_toolchain('foss', version='2018a-FFTW.MPI') + tc.set_options({'usempi': True}) + tc.prepare() + + fft_static_libs = 'libfftw3_mpi.a,libfftw3.a' + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs) + + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS_MT'), fft_static_libs_mt) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS_MT'), fft_static_libs_mt) + + self.assertEqual(tc.get_variable('LIBFFT'), '-lfftw3_mpi -lfftw3') + self.assertEqual(tc.get_variable('LIBFFT_MT'), '-lfftw3 -lpthread') + + fft_lib_dir = os.path.join(modules.get_software_root('FFTW.MPI'), 'lib') + self.assertEqual(tc.get_variable('FFT_LIB_DIR'), fft_lib_dir) + def test_fft_env_vars_intel(self): """Test setting of $FFT* environment variables using intel toolchain.""" @@ -1168,6 +1222,33 @@ def test_fosscuda(self): # check CUDA runtime lib self.assertTrue("-lrt -lcudart" in tc.get_variable('LIBS')) + def setup_sandbox_for_foss_fftw(self, moddir, fftwver='3.3.7'): + """Set up sandbox for foss FFTW and FFTW.MPI""" + # hack to make foss FFTW lib check pass + # create dummy FFTW and FFTW.MPI modules + + fftw_module_path = os.path.join(moddir, 'FFTW', fftwver) + fftw_dir = os.path.join(self.test_prefix, 'software', 'FFTW', fftwver) + + fftw_mod_txt = '\n'.join([ + "#%Module", + "setenv EBROOTFFTW %s" % fftw_dir, + "setenv EBVERSIONFFTW %s" % fftwver, + ]) + write_file(fftw_module_path, fftw_mod_txt) + + fftw_mpi_module_path = os.path.join(moddir, 'FFTW.MPI', fftwver) + fftw_mpi_dir = os.path.join(self.test_prefix, 'software', 'FFTW.MPI', fftwver) + fftw_mpi_mod_txt = '\n'.join([ + "#%Module", + "setenv EBROOTFFTWMPI %s" % fftw_mpi_dir, + "setenv EBVERSIONFFTWMPI %s" % fftwver, + ]) + write_file(fftw_mpi_module_path, fftw_mpi_mod_txt) + + os.makedirs(os.path.join(fftw_dir, 'lib')) + os.makedirs(os.path.join(fftw_mpi_dir, 'lib')) + def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'): """Set up sandbox for Intel FFTW""" # hack to make Intel FFTW lib check pass From b34bfc1dad2bd72ae888a80809720d5b9f86eb32 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 26 May 2022 03:55:17 +0000 Subject: [PATCH 855/864] Adjust dependent nongenerated modules for test case. --- test/framework/modules.py | 2 +- test/framework/modules/FFTW.MPI/3.3.7 | 31 +++++++++++++++++++++++++++ test/framework/modules/FFTW/3.3.7 | 31 +++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 test/framework/modules/FFTW.MPI/3.3.7 create mode 100644 test/framework/modules/FFTW/3.3.7 diff --git a/test/framework/modules.py b/test/framework/modules.py index 00d6deabc2..6950e372a5 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -54,7 +54,7 @@ # number of modules included for testing purposes -TEST_MODULES_COUNT = 87 +TEST_MODULES_COUNT = 90 class ModulesTest(EnhancedTestCase): diff --git a/test/framework/modules/FFTW.MPI/3.3.7 b/test/framework/modules/FFTW.MPI/3.3.7 new file mode 100644 index 0000000000..4c2b8db563 --- /dev/null +++ b/test/framework/modules/FFTW.MPI/3.3.7 @@ -0,0 +1,31 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data. - Homepage: http://www.fftw.org +} +} + +module-whatis {FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data. - Homepage: http://www.fftw.org} + +set root /home-2/khoste/.local/easybuild/software/FFTW.MPI/3.3.7 + +conflict FFTW.MPI + +if { ![is-loaded gompi/2018a] } { + module load gompi/2018a +} + +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig + +setenv EBROOTFFTWMPI "$root" +setenv EBVERSIONFFTWMPI "3.3.7" +setenv EBDEVELFFTWMPI "$root/easybuild/FFTW.MPI-3.3.7-easybuild-devel" + + +# built with EasyBuild version 1.4.0dev diff --git a/test/framework/modules/FFTW/3.3.7 b/test/framework/modules/FFTW/3.3.7 new file mode 100644 index 0000000000..350330b116 --- /dev/null +++ b/test/framework/modules/FFTW/3.3.7 @@ -0,0 +1,31 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data. - Homepage: http://www.fftw.org +} +} + +module-whatis {FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data. - Homepage: http://www.fftw.org} + +set root /home-2/khoste/.local/easybuild/software/FFTW/3.3.7 + +conflict FFTW + +if { ![is-loaded gompi/2018a] } { + module load gompi/2018a +} + +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig + +setenv EBROOTFFTW "$root" +setenv EBVERSIONFFTW "3.3.7" +setenv EBDEVELFFTW "$root/easybuild/FFTW-3.3.7-easybuild-devel" + + +# built with EasyBuild version 1.4.0dev From f9af9032e69eac00e4a1d6ce1aaf0a60d899b0c9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 May 2022 14:44:16 +0200 Subject: [PATCH 856/864] correctly identify Apple Silicon M1 as Arm 64-bit by also considering arm64 next to aarch64 --- easybuild/tools/systemtools.py | 2 +- test/framework/systemtools.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 482e4d1364..1c6474afaf 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -320,7 +320,7 @@ def get_cpu_architecture(): :return: a value from the CPU_ARCHITECTURES list """ aarch32_regex = re.compile("arm.*") - aarch64_regex = re.compile("aarch64.*") + aarch64_regex = re.compile("(aarch64|arm64).*") power_regex = re.compile("ppc64.*") riscv32_regex = re.compile("riscv32.*") riscv64_regex = re.compile("riscv64.*") diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 940f4ae3fa..fbdbee1ce7 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -566,6 +566,7 @@ def test_cpu_architecture(self): machine_names = { 'aarch64': AARCH64, 'aarch64_be': AARCH64, + 'arm64': AARCH64, 'armv7l': AARCH32, 'ppc64': POWER, 'ppc64le': POWER, From 07e69296a17b686a19979f829677e5ab5334b711 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 May 2022 15:30:26 +0200 Subject: [PATCH 857/864] fix 'eb --show-system-info' on Apple M1 system --- easybuild/tools/systemtools.py | 20 +++++++++++++++----- test/framework/systemtools.py | 15 +++++++++------ 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 482e4d1364..0daae61624 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -94,6 +94,7 @@ # Vendor constants AMD = 'AMD' APM = 'Applied Micro' +APPLE = 'Apple' ARM = 'ARM' BROADCOM = 'Broadcom' CAVIUM = 'Cavium' @@ -123,7 +124,7 @@ CPU_ARCHITECTURES = [AARCH32, AARCH64, POWER, RISCV32, RISCV64, X86_64] CPU_FAMILIES = [AMD, ARM, INTEL, POWER, POWER_LE, RISCV] -CPU_VENDORS = [AMD, APM, ARM, BROADCOM, CAVIUM, DEC, IBM, INTEL, MARVELL, MOTOROLA, NVIDIA, QUALCOMM] +CPU_VENDORS = [AMD, APM, APPLE, ARM, BROADCOM, CAVIUM, DEC, IBM, INTEL, MARVELL, MOTOROLA, NVIDIA, QUALCOMM] # ARM implementer IDs (i.e., the hexadeximal keys) taken from ARMv8-A Architecture Reference Manual # (ARM DDI 0487A.j, Section G6.2.102, Page G6-4493) VENDOR_IDS = { @@ -384,11 +385,18 @@ def get_cpu_vendor(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.vendor" - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False) out = out.strip() if ec == 0 and out in VENDOR_IDS: vendor = VENDOR_IDS[out] _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd)) + else: + cmd = "sysctl -n machdep.cpu.brand_string" + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False) + out = out.strip().split(' ')[0] + if ec == 0 and out in CPU_VENDORS: + vendor = out + _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd)) if vendor is None: vendor = UNKNOWN @@ -533,9 +541,11 @@ def get_cpu_speed(): cmd = "sysctl -n hw.cpufrequency_max" _log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd) out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) - if ec == 0: + out = out.strip() + cpu_freq = None + if ec == 0 and out: # returns clock frequency in cycles/sec, but we want MHz - cpu_freq = float(out.strip()) // (1000 ** 2) + cpu_freq = float(out) // (1000 ** 2) else: raise SystemToolsException("Could not determine CPU clock frequency (OS: %s)." % os_type) @@ -578,7 +588,7 @@ def get_cpu_features(): for feature_set in ['extfeatures', 'features', 'leaf7_features']: cmd = "sysctl -n machdep.cpu.%s" % feature_set _log.debug("Trying to determine CPU features on Darwin via cmd '%s'", cmd) - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False) if ec == 0: cpu_feat.extend(out.strip().lower().split()) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 940f4ae3fa..69d8967bb1 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -495,7 +495,7 @@ def test_cpu_features_native(self): """Test getting CPU features.""" cpu_feat = get_cpu_features() self.assertTrue(isinstance(cpu_feat, list)) - self.assertTrue(len(cpu_feat) > 0) + self.assertTrue(len(cpu_feat) >= 0) self.assertTrue(all(isinstance(x, string_type) for x in cpu_feat)) def test_cpu_features_linux(self): @@ -1016,8 +1016,10 @@ def test_check_linked_shared_libs(self): self.assertEqual(check_linked_shared_libs(txt_path, **pattern_named_args), None) self.assertEqual(check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None) for path in (bin_ls_path, lib_path): - error_msg = "Check on linked libs should pass for %s with %s" % (path, pattern_named_args) - self.assertTrue(check_linked_shared_libs(path, **pattern_named_args), error_msg) + # path may not exist, especially for library paths obtained via 'otool -L' on macOS + if os.path.exists(path): + error_msg = "Check on linked libs should pass for %s with %s" % (path, pattern_named_args) + self.assertTrue(check_linked_shared_libs(path, **pattern_named_args), error_msg) # also test with input that should result in failing check test_pattern_named_args = [ @@ -1050,9 +1052,10 @@ def test_locate_solib(self): def test_find_library_path(self): """Test find_library_path function (Linux and Darwin only).""" - if get_os_type() == LINUX: + os_type = get_os_type() + if os_type == LINUX: libname = 'libc.so.6' - elif get_os_type() == DARWIN: + elif os_type == DARWIN: libname = 'libSystem.dylib' else: libname = None @@ -1060,7 +1063,7 @@ def test_find_library_path(self): if libname: lib_path = find_library_path(libname) self.assertEqual(os.path.basename(lib_path), libname) - self.assertTrue(os.path.exists(lib_path), "%s should exist" % libname) + self.assertTrue(os.path.exists(lib_path) or os_type == DARWIN, "%s should exist" % libname) def suite(): From 25d09fbe46d14d0ed7a11e0e4b92c23fe81c91b0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 31 May 2022 09:24:35 +0200 Subject: [PATCH 858/864] add support for customizing EasyBuild command used in jobs via --job-eb-cmd --- easybuild/tools/config.py | 4 ++++ easybuild/tools/options.py | 3 ++- easybuild/tools/parallelbuild.py | 10 ++++++++-- test/framework/parallelbuild.py | 8 +++++++- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 4ef3a4c946..1c764cd56a 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -89,6 +89,7 @@ DEFAULT_ENVVAR_USERS_MODULES = 'HOME' DEFAULT_INDEX_MAX_AGE = 7 * 24 * 60 * 60 # 1 week (in seconds) DEFAULT_JOB_BACKEND = 'GC3Pie' +DEFAULT_JOB_EB_CMD = 'eb' DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") DEFAULT_MAX_FAIL_RATIO_PERMS = 0.5 DEFAULT_MINIMAL_BUILD_ENV = 'CC:gcc,CXX:g++' @@ -328,6 +329,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_INDEX_MAX_AGE: [ 'index_max_age', ], + DEFAULT_JOB_EB_CMD: [ + 'job_eb_cmd', + ], DEFAULT_MAX_FAIL_RATIO_PERMS: [ 'max_fail_ratio_adjust_permissions', ], diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index fc9c932d90..46ec3738d5 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -61,7 +61,7 @@ from easybuild.tools.build_log import init_logging, log_start, print_msg, print_warning, raise_easybuilderror from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE, DEFAULT_ALLOW_LOADED_MODULES from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_ENV_FOR_SHEBANG, DEFAULT_ENVVAR_USERS_MODULES -from easybuild.tools.config import DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE +from easybuild.tools.config import DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE, DEFAULT_JOB_EB_CMD from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL @@ -798,6 +798,7 @@ def job_options(self): 'cores': ("Number of cores to request per job", 'int', 'store', None), 'deps-type': ("Type of dependency to set between jobs (default depends on job backend)", 'choice', 'store', None, [JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN]), + 'eb-cmd': ("EasyBuild command to use in jobs", 'str', 'store', DEFAULT_JOB_EB_CMD), 'max-jobs': ("Maximum number of concurrent jobs (queued and running, 0 = unlimited)", 'int', 'store', 0), 'max-walltime': ("Maximum walltime for jobs (in hours)", 'int', 'store', 24), 'output-dir': ("Output directory for jobs (default: current directory)", None, 'store', os.getcwd()), diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py index 6e96dc4cab..6f5b5ee0cb 100644 --- a/easybuild/tools/parallelbuild.py +++ b/easybuild/tools/parallelbuild.py @@ -141,8 +141,14 @@ def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True): # compose string with command line options, properly quoted and with '%' characters escaped opts_str = ' '.join(opts).replace('%', '%%') - command = "unset TMPDIR && cd %s && eb %%(spec)s %s %%(add_opts)s --testoutput=%%(output_dir)s" % (curdir, opts_str) - _log.info("Command template for jobs: %s" % command) + eb_cmd = build_option('job_eb_cmd') + + command = ' && '.join([ + "unset TMPDIR", + "cd %s" % curdir, + "%s %%(spec)s %s %%(add_opts)s --testoutput=%%(output_dir)s" % (eb_cmd, opts_str), + ]) + _log.info("Command template for jobs: %s", command) if testing: _log.debug("Skipping actual submission of jobs since testing mode is enabled") return command diff --git a/test/framework/parallelbuild.py b/test/framework/parallelbuild.py index 60867d01e4..1d4d2834be 100644 --- a/test/framework/parallelbuild.py +++ b/test/framework/parallelbuild.py @@ -37,7 +37,7 @@ from easybuild.framework.easyconfig.tools import process_easyconfig from easybuild.tools import config from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import get_module_syntax +from easybuild.tools.config import get_module_syntax, update_build_option from easybuild.tools.filetools import adjust_permissions, mkdir, read_file, remove_dir, which, write_file from easybuild.tools.job import pbs_python from easybuild.tools.job.pbs_python import PbsPython @@ -322,6 +322,12 @@ def test_submit_jobs(self): regex = re.compile(regex) self.assertFalse(regex.search(cmd), "Pattern '%s' should *not* be found in: %s" % (regex.pattern, cmd)) + # test again with custom EasyBuild command to use in jobs + update_build_option('job_eb_cmd', "/just/testing/bin/eb --debug") + cmd = submit_jobs([toy_ec], eb_go.generate_cmd_line(), testing=True) + regex = re.compile(r" && /just/testing/bin/eb --debug %\(spec\)s ") + self.assertTrue(regex.search(cmd), "Pattern '%s' found in: %s" % (regex.pattern, cmd)) + def test_build_easyconfigs_in_parallel_slurm(self): """Test build_easyconfigs_in_parallel(), using (mocked) Slurm as backend for --job.""" From 5266a1c64f1aeca9535505cf7c149c43bea17a53 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Tue, 31 May 2022 09:04:32 +0100 Subject: [PATCH 859/864] keep alphabetical order for imports --- easybuild/tools/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 46ec3738d5..dae1d35974 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -61,8 +61,8 @@ from easybuild.tools.build_log import init_logging, log_start, print_msg, print_warning, raise_easybuilderror from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE, DEFAULT_ALLOW_LOADED_MODULES from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_ENV_FOR_SHEBANG, DEFAULT_ENVVAR_USERS_MODULES -from easybuild.tools.config import DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE, DEFAULT_JOB_EB_CMD -from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS +from easybuild.tools.config import DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE, DEFAULT_JOB_BACKEND +from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_PR_TARGET_ACCOUNT From 97a6bc7b1844c93e252ce0baf0f587e1115b889f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 5 Jun 2022 15:48:56 +0200 Subject: [PATCH 860/864] make sure that ARCH constant has 'aarch64' (rather than 'arm64') as value on macOS ARM --- easybuild/framework/easyconfig/constants.py | 16 +++++++++++++++- test/framework/easyconfig.py | 6 ++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/constants.py b/easybuild/framework/easyconfig/constants.py index 63ce5d51d7..e97c4dc99d 100644 --- a/easybuild/framework/easyconfig/constants.py +++ b/easybuild/framework/easyconfig/constants.py @@ -42,9 +42,23 @@ EXTERNAL_MODULE_MARKER = 'EXTERNAL_MODULE' + +def _get_arch_constant(): + """ + Get value for ARCH constant. + """ + arch = platform.uname()[4] + + # macOS on Arm produces 'arm64' rather than 'aarch64' + if arch == 'arm64': + arch = 'aarch64' + + return arch + + # constants that can be used in easyconfig EASYCONFIG_CONSTANTS = { - 'ARCH': (platform.uname()[4], "CPU architecture of current system (aarch64, x86_64, ppc64le, ...)"), + 'ARCH': (_get_arch_constant(), "CPU architecture of current system (aarch64, x86_64, ppc64le, ...)"), 'EXTERNAL_MODULE': (EXTERNAL_MODULE_MARKER, "External module marker"), 'HOME': (os.path.expanduser('~'), "Home directory ($HOME)"), 'OS_TYPE': (get_os_type(), "System type (e.g. 'Linux' or 'Darwin')"), diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 479557e9c3..9d2850242d 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4597,6 +4597,12 @@ def test_count_files(self): test_ec = EasyConfig(test_ec) self.assertEqual(test_ec.count_files(), 11) + def test_ARCH(self): + """Test ARCH easyconfig constant.""" + arch = easyconfig.constants.EASYCONFIG_CONSTANTS['ARCH'][0] + known_arch_values = ('aarch64', 'ppc64le', 'riscv64', 'x86_64') + self.assertTrue(arch in known_arch_values, "Unexpected value for ARCH constant: %s" % arch) + def suite(): """ returns all the testcases in this module """ From ec0af6ed8cce170566af3bace7035eda93b9df5d Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Tue, 7 Jun 2022 12:20:00 +0200 Subject: [PATCH 861/864] prepare release notes for EasyBuild v4.5.5 + bump version to 4.5.5 --- RELEASE_NOTES | 24 ++++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 7a4bd4d131..585df2cd1d 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,30 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.5.5 (June 7th 2022) +---------------------- + +update/bugfix release + +- various enhancements, including: + - add nvompi toolchain definition (NVHPC + OpenMPI) (#3969) + - add nvpsmpi toolchain definition (NVHPC + ParaStationMPI) (#3970) + - add definition for gfbf toolchain (GCC, FlexiBLAS, FFTW) (#4006) + - Support FFTW.MPI module in framework (FFT*DIR variables) (#4012) + - add support for customizing EasyBuild command used in jobs via --job-eb-cmd (#4016) +- various bug fixes, including: + - fix copying of easyconfig file with --copy-ec without --rebuild if module is already installed (#3993) + - ignore deprecation warnings regarding cryptography in Python 3.6 + Blowfish with Python 3.10 in test suite output (#3999) + - fix typo in debug log message in easyblock.py (#4000) + - fix printing of list of attempted download URLs when url-encoded characters are used in URL (#4005) + - set $FFT(W)_LIB_DIR to imkl-FFTW's lib path in build environment if usempi toolchain option is enabled (#4011) + - correctly identify Apple Silicon M1 as Arm 64-bit by also considering arm64 next to aarch64 (#4014) + - fix 'eb --show-system-info' on Apple M1 system (#4015) +- other changes: + - change 'eb' command to import easybuild to check if it's working (#3995) + - change 'eb' command to import easybuild.framework to check if EasyBuild framework is available (#3998) + + v4.5.4 (March 31st 2022) ------------------------ diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 657ad3cb75..2815738616 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.5.5.dev0') +VERSION = LooseVersion('4.5.5') UNKNOWN = 'UNKNOWN' From 6a7d237167c740070c1d06b7c370d90aaf6e6258 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 Jun 2022 15:01:35 +0200 Subject: [PATCH 862/864] minor tweak release notes for EasyBuild v4.5.5 --- RELEASE_NOTES | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 585df2cd1d..27fde3d069 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,16 +4,14 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. -v4.5.5 (June 7th 2022) +v4.5.5 (June 8th 2022) ---------------------- update/bugfix release - various enhancements, including: - - add nvompi toolchain definition (NVHPC + OpenMPI) (#3969) - - add nvpsmpi toolchain definition (NVHPC + ParaStationMPI) (#3970) - - add definition for gfbf toolchain (GCC, FlexiBLAS, FFTW) (#4006) - - Support FFTW.MPI module in framework (FFT*DIR variables) (#4012) + - add toolchain definitions for nvompi (NVHPC + OpenMPI) (#3969), nvpsmpi (NVHPC + ParaStationMPI) (#3970), gfbf (GCC, FlexiBLAS, FFTW) (#4006) + - add support for FFTW.MPI toolchain component ($FFT*DIR variables) (#4012) - add support for customizing EasyBuild command used in jobs via --job-eb-cmd (#4016) - various bug fixes, including: - fix copying of easyconfig file with --copy-ec without --rebuild if module is already installed (#3993) @@ -24,8 +22,7 @@ update/bugfix release - correctly identify Apple Silicon M1 as Arm 64-bit by also considering arm64 next to aarch64 (#4014) - fix 'eb --show-system-info' on Apple M1 system (#4015) - other changes: - - change 'eb' command to import easybuild to check if it's working (#3995) - - change 'eb' command to import easybuild.framework to check if EasyBuild framework is available (#3998) + - change 'eb' command to import easybuild.framework to check if EasyBuild framework is available (#3995, #3998) v4.5.4 (March 31st 2022) From 42b05be0a1632d33454f44f0241d84cf68f7408d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 Jun 2022 22:30:17 +0200 Subject: [PATCH 863/864] bump version to 4.5.6dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 2815738616..0559fab2b3 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.5.5') +VERSION = LooseVersion('4.5.6.dev0') UNKNOWN = 'UNKNOWN' From 0ee0ddd5510f35fbdb7be951864918b63cb7b8c6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 15 Jun 2022 10:50:30 +0200 Subject: [PATCH 864/864] add KNOWN_ARCH_CONSTANTS constant with list of known values for ARCH constant --- easybuild/framework/easyconfig/constants.py | 6 +++++- easybuild/tools/systemtools.py | 3 +++ test/framework/easyconfig.py | 6 +++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyconfig/constants.py b/easybuild/framework/easyconfig/constants.py index e97c4dc99d..f684985b21 100644 --- a/easybuild/framework/easyconfig/constants.py +++ b/easybuild/framework/easyconfig/constants.py @@ -34,7 +34,8 @@ import platform from easybuild.base import fancylogger -from easybuild.tools.systemtools import get_os_name, get_os_type, get_os_version +from easybuild.tools.build_log import print_warning +from easybuild.tools.systemtools import KNOWN_ARCH_CONSTANTS, get_os_name, get_os_type, get_os_version _log = fancylogger.getLogger('easyconfig.constants', fname=False) @@ -53,6 +54,9 @@ def _get_arch_constant(): if arch == 'arm64': arch = 'aarch64' + if arch not in KNOWN_ARCH_CONSTANTS: + print_warning("Using unknown value for ARCH constant: %s", arch) + return arch diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index e65ff52ec8..f9e319779f 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -89,6 +89,9 @@ RISCV32 = 'RISC-V-32' RISCV64 = 'RISC-V-64' +# known values for ARCH constant (determined by _get_arch_constant in easybuild.framework.easyconfig.constants) +KNOWN_ARCH_CONSTANTS = ('aarch64', 'ppc64le', 'riscv64', 'x86_64') + ARCH_KEY_PREFIX = 'arch=' # Vendor constants diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 9d2850242d..a40c8c1df5 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -73,7 +73,8 @@ from easybuild.tools.options import parse_external_modules_metadata from easybuild.tools.py2vs3 import OrderedDict, reload from easybuild.tools.robot import resolve_dependencies -from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_shared_lib_ext +from easybuild.tools.systemtools import AARCH64, KNOWN_ARCH_CONSTANTS, POWER, X86_64 +from easybuild.tools.systemtools import get_cpu_architecture, get_shared_lib_ext from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.utilities import quote_str, quote_py_str from test.framework.utilities import find_full_path @@ -4600,8 +4601,7 @@ def test_count_files(self): def test_ARCH(self): """Test ARCH easyconfig constant.""" arch = easyconfig.constants.EASYCONFIG_CONSTANTS['ARCH'][0] - known_arch_values = ('aarch64', 'ppc64le', 'riscv64', 'x86_64') - self.assertTrue(arch in known_arch_values, "Unexpected value for ARCH constant: %s" % arch) + self.assertTrue(arch in KNOWN_ARCH_CONSTANTS, "Unexpected value for ARCH constant: %s" % arch) def suite():