From 455622dbe9ef938204071264758a127a45e184a7 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 5 Jul 2022 10:36:07 +0200 Subject: [PATCH] Allow unresolved templates in `_finalize_dependencies`, extension options & `asdict` Some dependencies may use templates referring to other dependencies which cannot yet be resolved. Hence add an opt-out to the enforcment of resolved templates via `expect_resolved=False` and use that. This allows use of `%(installdir)s` or `%(ext_name)s` and similar in the options. --- easybuild/framework/easyconfig/easyconfig.py | 34 ++++++++++++-------- easybuild/framework/extension.py | 5 ++- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 1b85729a9b..deeaa0c7bb 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1641,8 +1641,9 @@ def _finalize_dependencies(self): filter_deps_specs = self.parse_filter_deps() for key in DEPENDENCY_PARAMETERS: - # loop over a *copy* of dependency dicts (with resolved templates); - deps = self[key] + # loop over a *copy* of dependency dicts with resolved templates, + # although some templates may not resolve yet (e.g. those relying on dependencies like %(pyver)s) + deps = resolve_template(self.get_ref(key), self.template_values, expect_resolved=False) # to update the original dep dict, we need to get a reference with templating disabled... deps_ref = self.get_ref(key) @@ -1851,7 +1852,8 @@ def asdict(self): if self.enable_templating: if not self.template_values: self.generate_template_values() - value = resolve_template(value, self.template_values) + # Not all values can be resolved, e.g. %(installdir)s + value = resolve_template(value, self.template_values, expect_resolved=False) res[key] = value return res @@ -1999,10 +2001,11 @@ def get_module_path(name, generic=None, decode=True): return '.'.join(modpath + [module_name]) -def resolve_template(value, tmpl_dict): +def resolve_template(value, tmpl_dict, expect_resolved=True): """Given a value, try to susbstitute the templated strings with actual values. - value: some python object (supported are string, tuple/list, dict or some mix thereof) - tmpl_dict: template dictionary + - expect_resolved: Expects that all templates get resolved """ if isinstance(value, str): # simple escaping, making all '%foo', '%%foo', '%%%foo' post-templates values available, @@ -2055,13 +2058,15 @@ def resolve_template(value, tmpl_dict): _log.deprecated(f"Easyconfig template '{old_tmpl}' is deprecated, use '{new_tmpl}' instead", ver) except KeyError: - msg = ('Failed to resolve template value %s with dict %s. ' % (value, tmpl_dict) + - 'This might cause failures or unexpected behavior, check for correct escaping if this is intended!') - if build_option('allow_unresolved_templates'): - value = raw_value # Undo "%"-escaping - print_warning(msg) - else: - raise EasyBuildError(msg) + if expect_resolved: + msg = ('Failed to resolve template value %s with dict %s. ' % (value, tmpl_dict) + + 'This might cause failures or unexpected behavior, ' + + 'check for correct escaping if this is intended!') + if build_option('allow_unresolved_templates'): + print_warning(msg) + else: + raise EasyBuildError(msg) + value = raw_value # Undo "%"-escaping for key in tmpl_dict: if key in DEPRECATED_EASYCONFIG_TEMPLATES: @@ -2081,11 +2086,12 @@ def resolve_template(value, tmpl_dict): # self._config['x']['y'] = z # it can not be intercepted with __setitem__ because the set is done at a deeper level if isinstance(value, list): - value = [resolve_template(val, tmpl_dict) for val in value] + value = [resolve_template(val, tmpl_dict, expect_resolved) for val in value] elif isinstance(value, tuple): - value = tuple(resolve_template(list(value), tmpl_dict)) + value = tuple(resolve_template(list(value), tmpl_dict, expect_resolved)) elif isinstance(value, dict): - value = {resolve_template(k, tmpl_dict): resolve_template(v, tmpl_dict) for k, v in value.items()} + value = {resolve_template(k, tmpl_dict, expect_resolved): resolve_template(v, tmpl_dict, expect_resolved) + for k, v in value.items()} return value diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 0005f24428..0ff0bf59fb 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -128,7 +128,10 @@ def __init__(self, mself, ext, extra_params=None): self.src = resolve_template(self.ext.get('src', []), self.cfg.template_values) self.src_extract_cmd = self.ext.get('extract_cmd', None) self.patches = resolve_template(self.ext.get('patches', []), self.cfg.template_values) - self.options = resolve_template(copy.deepcopy(self.ext.get('options', {})), self.cfg.template_values) + # Some options may not be resolvable yet + self.options = resolve_template(copy.deepcopy(self.ext.get('options', {})), + self.cfg.template_values, + expect_resolved=False) if extra_params: self.cfg.extend_params(extra_params, overwrite=False)