From b13db4678176259aea7e771e72deccab7665a99b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Mon, 7 Oct 2024 17:48:16 +0200 Subject: [PATCH 1/3] Enhance generic Bundle EasyBlock to transfer module requirements of components MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit implements the `make_module_req_guess` method for the generic Bundle EasyBlock. With this, all the requirements of the components in a bundle are transferred correctly to the final module. Previously, this could lead to missing environment variables, letting the build succeed but still resulting in a broken module, for example because `PATH` is not set. Signed-off-by: Jan André Reuter --- easybuild/easyblocks/generic/bundle.py | 43 ++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index b8abecf8fc..8462ab19e7 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -74,6 +74,9 @@ def __init__(self, *args, **kwargs): # list of EasyConfig instances for components self.comp_cfgs = [] + # list of EasyBlocks for components + self.comp_blocks = [] + # list of EasyConfig instances of components for which to run sanity checks self.comp_cfgs_sanity_check = [] @@ -199,6 +202,7 @@ def __init__(self, *args, **kwargs): self.cfg.update('patches', comp_cfg['patches']) self.comp_cfgs.append(comp_cfg) + self.comp_blocks.append(comp_cfg.easyblock(comp_cfg)) self.cfg.update('checksums', checksums_patches) @@ -247,14 +251,12 @@ def build_step(self): def install_step(self): """Install components, if specified.""" comp_cnt = len(self.cfg['components']) - for idx, cfg in enumerate(self.comp_cfgs): + for idx, (cfg, comp) in enumerate(zip(self.comp_cfgs, self.comp_blocks)): print_msg("installing bundle component %s v%s (%d/%d)..." % (cfg['name'], cfg['version'], idx + 1, comp_cnt)) self.log.info("Installing component %s v%s using easyblock %s", cfg['name'], cfg['version'], cfg.easyblock) - comp = cfg.easyblock(cfg) - # correct build/install dirs comp.builddir = self.builddir comp.install_subdir, comp.installdir = self.install_subdir, self.installdir @@ -324,6 +326,41 @@ def install_step(self): # close log for this component comp.close_log() + def make_module_req_guess(self): + """ + Set module requirements from all comppnents, e.g. $PATH, etc. + During the install step, we only set these requirements temporarily. + Later on when building the module, those paths are not considered. + Therefore, iterate through all the components again and gather + the requirements. + + Do not remove duplicates or check for existance of folders, + as this is done in the generic EasyBlock while creating + the modulefile already. + """ + # Start with the paths from the generic EasyBlock. + # If not added here, they might be missing entirely and fail sanity checks. + final_reqs = super(Bundle, self).make_module_req_guess() + + for cfg, comp in zip(self.comp_cfgs, self.comp_blocks): + self.log.info("Gathering module paths for component %s v%s", cfg['name'], cfg['version']) + reqs = comp.make_module_req_guess() + + if not reqs or not isinstance(reqs, dict): + self.log.warning("Expected requirements to be a dict but is not. Therefore, this component is skipped.") + continue + + for key, value in sorted(reqs.items()): + if isinstance(reqs, string_type): + self.log.warning("Hoisting string value %s into a list before iterating over it", value) + value = [value] + if key not in final_reqs: + final_reqs[key] = value + else: + final_reqs[key] += value + + return final_reqs + def make_module_extra(self, *args, **kwargs): """Set extra stuff in module file, e.g. $EBROOT*, $EBVERSION*, etc.""" if not self.altroot and not self.altversion: From eda3ca70e272d3bc7f161f555aefde1f969943a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Fri, 8 Nov 2024 14:09:06 +0100 Subject: [PATCH 2/3] Bundle: make implementation more efficient MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of zipping bundles and configs every time, use a single instance list. Also, do not check for the type when building the module requirements and catch the exception instead. Signed-off-by: Jan André Reuter --- easybuild/easyblocks/generic/bundle.py | 35 +++++++++++--------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 8462ab19e7..387539e79a 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -71,11 +71,8 @@ def __init__(self, *args, **kwargs): self.altroot = None self.altversion = None - # list of EasyConfig instances for components - self.comp_cfgs = [] - - # list of EasyBlocks for components - self.comp_blocks = [] + # list of EasyConfig instances and their EasyBlocks for components + self.comp_instances = [] # list of EasyConfig instances of components for which to run sanity checks self.comp_cfgs_sanity_check = [] @@ -201,8 +198,7 @@ def __init__(self, *args, **kwargs): if comp_cfg['patches']: self.cfg.update('patches', comp_cfg['patches']) - self.comp_cfgs.append(comp_cfg) - self.comp_blocks.append(comp_cfg.easyblock(comp_cfg)) + self.comp_instances.append((comp_cfg, comp_cfg.easyblock(comp_cfg))) self.cfg.update('checksums', checksums_patches) @@ -221,7 +217,7 @@ def check_checksums(self): """ checksum_issues = super(Bundle, self).check_checksums() - for comp in self.comp_cfgs: + for comp, _ in self.comp_instances: checksum_issues.extend(self.check_checksums_for(comp, sub="of component %s" % comp['name'])) return checksum_issues @@ -251,7 +247,7 @@ def build_step(self): def install_step(self): """Install components, if specified.""" comp_cnt = len(self.cfg['components']) - for idx, (cfg, comp) in enumerate(zip(self.comp_cfgs, self.comp_blocks)): + for idx, (cfg, comp) in enumerate(self.comp_instances): print_msg("installing bundle component %s v%s (%d/%d)..." % (cfg['name'], cfg['version'], idx + 1, comp_cnt)) @@ -342,22 +338,19 @@ def make_module_req_guess(self): # If not added here, they might be missing entirely and fail sanity checks. final_reqs = super(Bundle, self).make_module_req_guess() - for cfg, comp in zip(self.comp_cfgs, self.comp_blocks): + for cfg, comp in self.comp_instances: self.log.info("Gathering module paths for component %s v%s", cfg['name'], cfg['version']) reqs = comp.make_module_req_guess() - if not reqs or not isinstance(reqs, dict): - self.log.warning("Expected requirements to be a dict but is not. Therefore, this component is skipped.") - continue - - for key, value in sorted(reqs.items()): - if isinstance(reqs, string_type): - self.log.warning("Hoisting string value %s into a list before iterating over it", value) - value = [value] - if key not in final_reqs: - final_reqs[key] = value - else: + try: + for key, value in sorted(reqs.items()): + if isinstance(reqs, string_type): + value = [value] + final_reqs.setdefault(key, []) final_reqs[key] += value + except AttributeError: + raise EasyBuildError("Cannot process module requirements of bundle component %s v%s", + cfg['name'], cfg['version']) return final_reqs From e69615034bd7a2a2c20e2ab63de77bb84804a11b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Fri, 8 Nov 2024 15:55:48 +0100 Subject: [PATCH 3/3] Bundle: Update authors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan André Reuter --- easybuild/easyblocks/generic/bundle.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 387539e79a..5f098f83d7 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -31,6 +31,7 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) @author: Jasper Grimm (University of York) +@author: Jan Andre Reuter (Juelich Supercomputing Centre) """ import copy import os