From f140a328ba6a2398931e92ce5858d36ac0668e20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Kr=C3=A1l?= Date: Fri, 19 May 2023 15:22:45 +0200 Subject: [PATCH 1/2] fix error message when checksums set to [None] --- easybuild/framework/easyblock.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index df73e28db4..33d96166e0 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -395,13 +395,13 @@ def get_checksums_from_json(self, always_read=False): :param always_read: always read the checksums.json file, even if it has been read before """ if always_read or self.json_checksums is None: - try: - path = self.obtain_file("checksums.json", no_download=True) + path = self.obtain_file("checksums.json", no_download=True, warning_only=True) + if path is not None: self.log.info("Loading checksums from file %s", path) json_txt = read_file(path) self.json_checksums = json.loads(json_txt) - # if the file can't be found, return an empty dict - except EasyBuildError: + else: + # if the file can't be found, return an empty dict self.json_checksums = {} return self.json_checksums @@ -736,7 +736,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): return exts_sources def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False, - git_config=None, no_download=False, download_instructions=None, alt_location=None): + git_config=None, no_download=False, download_instructions=None, alt_location=None, + warning_only=False): """ Locate the file with the given name - searches in different subdirectories of source path @@ -789,7 +790,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No return fullpath except IOError as err: - raise EasyBuildError("Downloading file %s from url %s to %s failed: %s", filename, url, fullpath, err) + if not warning_only: + raise EasyBuildError("Downloading file %s " + "from url %s to %s failed: %s", filename, url, fullpath, err) + else: + self.log.warning("Downloading file %s " + "from url %s to %s failed: %s", filename, url, fullpath, err) + return None else: # try and find file in various locations @@ -866,8 +873,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No self.dry_run_msg(" * %s (MISSING)", filename) return filename else: - raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... " + if not warning_only: + raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... " + "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + else: + self.log.warning("Couldn't find file %s anywhere, and downloading it is disabled... " "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + return None elif git_config: return get_source_tarball_from_git(filename, targetdir, git_config) else: @@ -959,7 +971,11 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No error_msg += "and downloading it didn't work either... " error_msg += "Paths attempted (in order): %s " % failedpaths_msg - raise EasyBuildError(error_msg, filename) + if not warning_only: + raise EasyBuildError(error_msg, filename) + else: + self.log.warning(error_msg, filename) + return None # # GETTER/SETTER UTILITY FUNCTIONS From 0981458eb701b24f06211dbea5bd60df29923853 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 9 Sep 2023 18:16:10 +0200 Subject: [PATCH 2/2] enhance test_checksum_step to check lack of error message in log when checksums.json is not found --- test/framework/easyblock.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 4f27e86fb2..6afc9db94f 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -39,6 +39,7 @@ from unittest import TextTestRunner import easybuild.tools.systemtools as st +from easybuild.base import fancylogger from easybuild.framework.easyblock import EasyBlock, get_easyblock_instance from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.easyconfig import EasyConfig @@ -2424,6 +2425,30 @@ def test_checksum_step(self): eb.fetch_sources() eb.checksum_step() + with self.mocked_stdout_stderr() as (stdout, stderr): + + # using checksum-less test easyconfig in location that does not provide checksums.json + test_ec = os.path.join(self.test_prefix, 'test-no-checksums.eb') + copy_file(toy_ec, test_ec) + write_file(test_ec, 'checksums = []', append=True) + ec = process_easyconfig(test_ec)[0] + + # enable logging to screen, so we can check whether error is logged when checksums.json is not found + fancylogger.logToScreen(enable=True, stdout=True) + + eb = get_easyblock_instance(ec) + eb.fetch_sources() + eb.checksum_step() + + fancylogger.logToScreen(enable=False, stdout=True) + stdout = self.get_stdout() + + # make sure there's no error logged for not finding checksums.json, + # see also https://github.com/easybuilders/easybuild-framework/issues/4301 + regex = re.compile("ERROR .*Couldn't find file checksums.json anywhere", re.M) + regex.search(stdout) + self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in log" % regex.pattern) + # fiddle with checksum to check whether faulty checksum is catched copy_file(toy_ec, self.test_prefix) toy_ec = os.path.join(self.test_prefix, os.path.basename(toy_ec))