Merge branch 'easybuilders:develop' into fix-checksums-verification

.github/workflows/eb_command.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
+        python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
       fail-fast: false
     steps:
     - uses: actions/checkout@v3

.github/workflows/linting.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
+        python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
 
     steps:
     - uses: actions/checkout@v3

.github/workflows/unit_tests.yml

@@ -37,8 +37,6 @@ jobs:
         lc_all: [""]
         include:
           # Test different Python 3 versions with Lmod 8.x
-          - python: 3.5
-            modules_tool: ${{needs.setup.outputs.lmod8}}
           - python: 3.7
             modules_tool: ${{needs.setup.outputs.lmod8}}
           - python: 3.8

easybuild/framework/easyblock.py

@@ -1068,7 +1068,7 @@ def make_builddir(self):
             self.log.info("Overriding 'cleanupoldinstall' (to False), 'cleanupoldbuild' (to True) "
                           "and 'keeppreviousinstall' because we're building in the installation directory.")
             # force cleanup before installation
-            if build_option('module_only'):
+            if build_option('module_only') or self.cfg['module_only']:
                 self.log.debug("Disabling cleanupoldbuild because we run as module-only")
                 self.cfg['cleanupoldbuild'] = False
             else:
@@ -1139,7 +1139,7 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False):
             if self.cfg['keeppreviousinstall']:
                 self.log.info("Keeping old directory %s (hopefully you know what you are doing)", dir_name)
                 return
-            elif build_option('module_only'):
+            elif build_option('module_only') or self.cfg['module_only']:
                 self.log.info("Not touching existing directory %s in module-only mode...", dir_name)
             elif clean:
                 remove_dir(dir_name)
@@ -1365,7 +1365,7 @@ def make_module_dep(self, unload_info=None):
                     multi_dep_mod_names[dep['name']].append(dep['short_mod_name'])
 
             multi_dep_load_defaults = []
-            for depname, depmods in sorted(multi_dep_mod_names.items()):
+            for _, depmods in sorted(multi_dep_mod_names.items()):
                 stmt = self.module_generator.load_module(depmods[0], multi_dep_mods=depmods,
                                                          recursive_unload=recursive_unload,
                                                          depends_on=depends_on)
@@ -1769,10 +1769,7 @@ def make_extension_string(self, name_version_sep='-', ext_sep=', ', sort=True):
         return ext_sep.join(exts_list)
 
     def prepare_for_extensions(self):
-        """
-        Also do this before (eg to set the template)
-        """
-        pass
+        """Ran before installing extensions (eg to set templates)"""
 
     def skip_extensions(self):
         """
@@ -2114,7 +2111,7 @@ def guess_start_dir(self):
         start_dir = ''
         # do not use the specified 'start_dir' when running as --module-only as
         # the directory will not exist (extract_step is skipped)
-        if self.start_dir and not build_option('module_only'):
+        if self.start_dir and not build_option('module_only') and not self.cfg['module_only']:
             start_dir = self.start_dir
 
         if not os.path.isabs(start_dir):
@@ -2198,9 +2195,9 @@ def handle_iterate_opts(self):
                 self.log.info("Current iteration index: %s", self.iter_idx)
 
             # pop first element from all iterative easyconfig parameters as next value to use
-            for opt in self.iter_opts:
-                if len(self.iter_opts[opt]) > self.iter_idx:
-                    self.cfg[opt] = self.iter_opts[opt][self.iter_idx]
+            for opt, value in self.iter_opts.items():
+                if len(value) > self.iter_idx:
+                    self.cfg[opt] = value[self.iter_idx]
                 else:
                     self.cfg[opt] = ''  # empty list => empty option as next value
                 self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt])))
@@ -2212,12 +2209,12 @@ def post_iter_step(self):
         """Restore options that were iterated over"""
         # disable templating, since we're messing about with values in self.cfg
         with self.cfg.disable_templating():
-            for opt in self.iter_opts:
-                self.cfg[opt] = self.iter_opts[opt]
+            for opt, value in self.iter_opts.items():
+                self.cfg[opt] = value
 
                 # also need to take into account extensions, since those were iterated over as well
                 for ext in self.ext_instances:
-                    ext.cfg[opt] = self.iter_opts[opt]
+                    ext.cfg[opt] = value
 
                 self.log.debug("Restored value of '%s' that was iterated over: %s", opt, self.cfg[opt])
 
@@ -2349,7 +2346,7 @@ def check_readiness_step(self):
                 self.log.info("No module %s found. Not skipping anything." % self.full_mod_name)
 
         # remove existing module file under --force (but only if --skip is not used)
-        elif build_option('force') or build_option('rebuild'):
+        elif (build_option('force') or build_option('rebuild')) and not build_option('dump_env_script'):
             self.remove_module_file()
 
     def fetch_step(self, skip_checksums=False):
@@ -2607,7 +2604,7 @@ def patch_step(self, beginpath=None, patches=None):
             copy_patch = 'copy' in patch and 'sourcepath' not in patch
 
             self.log.debug("Source index: %s; patch level: %s; source path suffix: %s; copy patch: %s",
-                           srcind, level, srcpathsuffix, copy)
+                           srcind, level, srcpathsuffix, copy_patch)
 
             if beginpath is None:
                 try:
@@ -2751,10 +2748,7 @@ def _test_step(self):
             self.report_test_failure(err)
 
     def stage_install_step(self):
-        """
-        Install in a stage directory before actual installation.
-        """
-        pass
+        """Install in a stage directory before actual installation."""
 
     def install_step(self):
         """Install built software (abstract method)."""
@@ -3248,7 +3242,7 @@ def sanity_check_linked_shared_libs(self, subdirs=None):
         required_libs.extend(self.cfg['required_linked_shared_libs'])
 
         # early return if there are no banned/required libraries
-        if not (banned_libs + required_libs):
+        if not banned_libs + required_libs:
             self.log.info("No banned/required libraries specified")
             return []
         else:
@@ -3795,7 +3789,7 @@ def make_module_step(self, fake=False):
                 try:
                     self.make_devel_module()
                 except EasyBuildError as error:
-                    if build_option('module_only'):
+                    if build_option('module_only') or self.cfg['module_only']:
                         self.log.info("Using --module-only so can recover from error: %s", error)
                     else:
                         raise error
@@ -3903,7 +3897,7 @@ def skip_step(self, step, skippable):
         """Dedice whether or not to skip the specified step."""
         skip = False
         force = build_option('force')
-        module_only = build_option('module_only')
+        module_only = build_option('module_only') or self.cfg['module_only']
         sanity_check_only = build_option('sanity_check_only')
         skip_extensions = build_option('skip_extensions')
         skip_test_step = build_option('skip_test_step')
@@ -4463,7 +4457,7 @@ def copy_easyblocks_for_reprod(easyblock_instances, reprod_dir):
             else:
                 easyblock_paths.add(easyblock_path)
     for easyblock_path in easyblock_paths:
-        easyblock_basedir, easyblock_filename = os.path.split(easyblock_path)
+        easyblock_filename = os.path.basename(easyblock_path)
         copy_file(easyblock_path, os.path.join(reprod_easyblock_dir, easyblock_filename))
         _log.info("Dumped easyblock %s required for reproduction to %s", easyblock_filename, reprod_easyblock_dir)
 
@@ -4594,10 +4588,7 @@ def build_easyconfigs(easyconfigs, output_dir, test_results):
 
 
 class StopException(Exception):
-    """
-    StopException class definition.
-    """
-    pass
+    """Exception thrown to stop running steps"""
 
 
 def inject_checksums_to_json(ecs, checksum_type):
@@ -4645,14 +4636,14 @@ def inject_checksums_to_json(ecs, checksum_type):
 
         # actually inject new checksums or overwrite existing ones (if --force)
         existing_checksums = app.get_checksums_from_json(always_read=True)
-        for filename in checksums:
+        for filename, checksum in checksums.items():
             if filename not in existing_checksums:
-                existing_checksums[filename] = checksums[filename]
+                existing_checksums[filename] = checksum
             # don't do anything if the checksum already exist and is the same
-            elif checksums[filename] != existing_checksums[filename]:
+            elif checksum != existing_checksums[filename]:
                 if build_option('force'):
                     print_warning("Found existing checksums for %s, overwriting them (due to --force)..." % ec_fn)
-                    existing_checksums[filename] = checksums[filename]
+                    existing_checksums[filename] = checksum
                 else:
                     raise EasyBuildError("Found existing checksum for %s, use --force to overwrite them" % filename)
 

easybuild/framework/easyconfig/default.py

@@ -109,6 +109,7 @@
     'hidden': [False, "Install module file as 'hidden' by prefixing its version with '.'", BUILD],
     'installopts': ['', 'Extra options for installation', BUILD],
     'maxparallel': [None, 'Max degree of parallelism', BUILD],
+    'module_only': [False, 'Only generate module file', BUILD],
     'parallel': [None, ('Degree of parallelism for e.g. make (default: based on the number of '
                         'cores, active cpuset and restrictions in ulimit)'), BUILD],
     'patches': [[], "List of patches to apply", BUILD],

easybuild/framework/easyconfig/easyconfig.py

@@ -2026,12 +2026,14 @@ def resolve_template(value, tmpl_dict):
         # '%(name)s' -> '%(name)s'
         # '%%(name)s' -> '%%(name)s'
         if '%' in value:
+            orig_value = value
             value = re.sub(re.compile(r'(%)(?!%*\(\w+\)s)'), r'\1\1', value)
 
             try:
                 value = value % tmpl_dict
             except KeyError:
                 _log.warning("Unable to resolve template value %s with dict %s", value, tmpl_dict)
+                value = orig_value  # Undo "%"-escaping
     else:
         # this block deals with references to objects and returns other references
         # for reading this is ok, but for self['x'] = {}

easybuild/framework/extensioneasyblock.py

@@ -126,18 +126,23 @@ def _set_start_dir(self):
         elif ext_start_dir is None:
             # This may be on purpose, e.g. for Python WHL files which do not get extracted
             self.log.debug("Start dir is not set.")
-        else:
+        elif self.start_dir:
             # non-existing start dir means wrong input from user
-            warn_msg = "Provided start dir (%s) for extension %s does not exist: %s" % (self.start_dir, self.name,
-                                                                                        ext_start_dir)
+            raise EasyBuildError("Provided start dir (%s) for extension %s does not exist: %s",
+                                 self.start_dir, self.name, ext_start_dir)
+        else:
+            warn_msg = 'Failed to determine start dir for extension %s: %s' % (self.name, ext_start_dir)
             self.log.warning(warn_msg)
             print_warning(warn_msg, silent=build_option('silent'))
 
     def run(self, unpack_src=False):
         """Common operations for extensions: unpacking sources, patching, ..."""
 
         # unpack file if desired
-        if unpack_src:
+        if self.options.get('nosource', False):
+            # If no source wanted use the start_dir from the main EC
+            self.ext_dir = self.master.start_dir
+        elif unpack_src:
             targetdir = os.path.join(self.master.builddir, remove_unwanted_chars(self.name))
             self.ext_dir = extract_file(self.src, targetdir, extra_options=self.unpack_options,
                                         change_into_dir=False, cmd=self.src_extract_cmd)
@@ -146,10 +151,9 @@ def run(self, unpack_src=False):
             # because start_dir value is usually a relative path (if it is set)
             change_dir(self.ext_dir)
 
-            self._set_start_dir()
+        self._set_start_dir()
+        if self.start_dir:
             change_dir(self.start_dir)
-        else:
-            self._set_start_dir()
 
         # patch if needed
         EasyBlock.patch_step(self, beginpath=self.ext_dir)

easybuild/main.py

@@ -83,6 +83,8 @@
 from easybuild.tools.repository.repository import init_repository
 from easybuild.tools.systemtools import check_easybuild_deps
 from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state
+from easybuild.tools.version import EASYBLOCKS_VERSION, FRAMEWORK_VERSION, UNKNOWN_EASYBLOCKS_VERSION
+from easybuild.tools.version import different_major_versions
 
 
 _log = None
@@ -440,9 +442,9 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
     dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules
 
     keep_available_modules = any((
-        forced, dry_run_mode, options.extended_dry_run, any_pr_option_set, options.copy_ec, options.inject_checksums,
-        options.sanity_check_only, options.inject_checksums_to_json)
-    )
+        forced, dry_run_mode, any_pr_option_set, options.copy_ec, options.dump_env_script, options.extended_dry_run,
+        options.inject_checksums, options.inject_checksums_to_json, options.sanity_check_only
+    ))
 
     # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping
     if not keep_available_modules:
@@ -618,6 +620,15 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr
     (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate,
      from_pr_list, tweaked_ecs_paths) = cfg_settings
 
+    # compare running Framework and EasyBlocks versions
+    if EASYBLOCKS_VERSION == UNKNOWN_EASYBLOCKS_VERSION:
+        # most likely reason is running framework unit tests with no easyblocks installation
+        # so log a warning, to avoid test related issues
+        _log.warning("Unable to determine EasyBlocks version, so we'll assume it is not different from Framework")
+    elif different_major_versions(FRAMEWORK_VERSION, EASYBLOCKS_VERSION):
+        raise EasyBuildError("Framework (%s) and EasyBlock (%s) major versions are different." % (FRAMEWORK_VERSION,
+                                                                                                  EASYBLOCKS_VERSION))
+
     # load hook implementations (if any)
     hooks = load_hooks(options.hooks)
 

easybuild/tools/config.py

@@ -278,6 +278,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'debug',
         'debug_lmod',
         'dump_autopep8',
+        'dump_env_script',
         'enforce_checksums',
         'experimental',
         'extended_dry_run',

easybuild/tools/github.py

@@ -60,6 +60,7 @@
 from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters, urlopen
 from easybuild.tools.systemtools import UNKNOWN, get_tool_version
 from easybuild.tools.utilities import nub, only_if_module_is_available
+from easybuild.tools.version import FRAMEWORK_VERSION, different_major_versions
 
 
 _log = fancylogger.getLogger('github', fname=False)
@@ -587,9 +588,39 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi
         else:
             raise EasyBuildError("Couldn't find path to patched file %s", full_path)
 
+    if github_repo == GITHUB_EASYCONFIGS_REPO:
+        ver = _get_version_for_repo(os.path.join(final_path, 'setup.py'))
+    elif github_repo == GITHUB_EASYBLOCKS_REPO:
+        ver = _get_version_for_repo(os.path.join(final_path, 'easybuild', 'easyblocks', '__init__.py'))
+
+    if different_major_versions(FRAMEWORK_VERSION, ver):
+        raise EasyBuildError("Framework (%s) is a different major version than used in %s/%s PR #%s (%s)",
+                             FRAMEWORK_VERSION, github_account, github_repo, pr, ver)
+
     return files
 
 
+def _get_version_for_repo(filename):
+    """Extract version from filename."""
+    _log.debug("Extract version from %s" % filename)
+
+    try:
+        ver_line = ""
+        with open(filename) as f:
+            for line in f.readlines():
+                if line.startswith("VERSION "):
+                    ver_line = line
+                    break
+
+        # version can be a string or LooseVersion
+        res = re.search(r"""^VERSION = .*['"](.*)['"].?$""", ver_line)
+
+        _log.debug("PR target version is %s" % res.group(1))
+        return res.group(1)
+    except Exception:
+        raise EasyBuildError("Couldn't determine version of PR from %s" % filename)
+
+
 def fetch_easyblocks_from_pr(pr, path=None, github_user=None):
     """Fetch patched easyblocks for a particular PR."""
     return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYBLOCKS_REPO)

easybuild/tools/version.py

@@ -47,6 +47,7 @@
 # This causes problems further up the dependency chain...
 VERSION = LooseVersion('4.9.2.dev0')
 UNKNOWN = 'UNKNOWN'
+UNKNOWN_EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS'
 
 
 def get_git_revision():
@@ -87,7 +88,7 @@ def get_git_revision():
 try:
     from easybuild.easyblocks import VERBOSE_VERSION as EASYBLOCKS_VERSION
 except Exception:
-    EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS'  # make sure it is smaller then anything
+    EASYBLOCKS_VERSION = UNKNOWN_EASYBLOCKS_VERSION  # make sure it is smaller then anything
 
 
 def this_is_easybuild():
@@ -103,3 +104,14 @@ def this_is_easybuild():
         msg = msg.encode('ascii')
 
     return msg
+
+
+def different_major_versions(v1, v2):
+    """Compare major versions"""
+    # Deal with version instances being either strings or LooseVersion
+    if isinstance(v1, str):
+        v1 = LooseVersion(v1)
+    if isinstance(v2, str):
+        v2 = LooseVersion(v2)
+
+    return v1.version[0] != v2.version[0]