Add test for error logging

The toy test file is modified with a patch to fail during compilation. The tests verify that: - the source directory is copied to the error log path, - the log files are copied to the error log path, and - a warning for the compilation failure is reported in stdout.
easybuilders · Jan 7, 2025 · 97fc225 · 97fc225
1 parent 90565c3
commit 97fc225
Show file tree

Hide file tree

Showing 5 changed files with 119 additions and 5 deletions.
diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-buggy.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-buggy.eb
@@ -0,0 +1,33 @@
+name = 'toy'
+version = '0.0'
+versionsuffix = '-buggy'
+
+homepage = 'https://easybuilders.github.io/easybuild'
+description = "Toy C program, 100% toy."
+
+toolchain = SYSTEM
+
+sources = [SOURCE_TAR_GZ]
+checksums = [[
+    'be662daa971a640e40be5c804d9d7d10',  # default (MD5)
+    '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc',  # default (SHA256)
+    ('adler32', '0x998410035'),
+    ('crc32', '0x1553842328'),
+    ('md5', 'be662daa971a640e40be5c804d9d7d10'),
+    ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
+    ('size', 273),
+]]
+patches = [
+    'toy-0.0_add-bug.patch',
+    ('toy-extra.txt', 'toy-0.0'),
+]
+
+sanity_check_paths = {
+    'files': [('bin/yot', 'bin/toy')],
+    'dirs': ['bin'],
+}
+
+postinstallcmds = ["echo TOY > %(installdir)s/README"]
+
+moduleclass = 'tools'
+# trailing comment, leave this here, it may trigger bugs with extract_comments()
diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb
@@ -18,7 +18,7 @@ checksums = [[
     ('size', 273),
 ]]
 patches = [
-    'toy-0.0_fix-silly-typo-in-printf-statement.patch',
+    'toy-0.0_add-bug.patch',
     ('toy-extra.txt', 'toy-0.0'),
 ]
 

diff --git a/test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch b/test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch
@@ -0,0 +1,10 @@
+--- a/toy-0.0.orig/toy.source	2014-03-06 18:48:16.000000000 +0100
++++ b/toy-0.0/toy.source	2020-08-18 12:19:35.000000000 +0200
+@@ -2,6 +2,6 @@
+
+ int main(int argc, char* argv[]){
+
+-    printf("I'm a toy, and proud of it.\n");
++    printf("I'm a toy, and proud of it.\n")
+     return 0;
+ }
diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
@@ -40,8 +40,10 @@
 import sys
 import tempfile
 import textwrap
+import pathlib
+import filecmp
 from easybuild.tools import LooseVersion
-from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered
+from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, TempDirectory
 from test.framework.package import mock_fpm
 from unittest import TextTestRunner
 
@@ -156,9 +158,9 @@ def check_toy(self, installpath, outtxt, name='toy', version='0.0', versionprefi
         devel_module_path = os.path.join(software_path, 'easybuild', '%s-%s-easybuild-devel' % (name, full_version))
         self.assertExists(devel_module_path)
 
-    def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True, fails=False, verbose=True,
-                       raise_error=False, test_report=None, name='toy', versionsuffix='', testing=True,
-                       raise_systemexit=False, force=True, test_report_regexs=None, debug=True):
+    def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, tmp_logdir=None, verify=True, fails=False,
+                       verbose=True, raise_error=False, test_report=None, name='toy', versionsuffix='', testing=True,
+                       raise_systemexit=False, force=True, test_report_regexs=None, debug=True, check_errorlog=None):
         """Perform a toy build."""
         if extra_args is None:
             extra_args = []
@@ -172,6 +174,7 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True
             '--sourcepath=%s' % self.test_sourcepath,
             '--buildpath=%s' % self.test_buildpath,
             '--installpath=%s' % self.test_installpath,
+            '--errorlogpath=%s' % self.test_errorlogpath,
             '--unittest-file=%s' % self.logfile,
             '--robot=%s' % os.pathsep.join([self.test_buildpath, os.path.dirname(__file__)]),
         ]
@@ -183,6 +186,8 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True
             args.append('--tmpdir=%s' % tmpdir)
         if test_report is not None:
             args.append('--dump-test-report=%s' % test_report)
+        if tmp_logdir is not None:
+            args.append('--tmp-logdir=%s' % tmp_logdir)
         args.extend(extra_args)
         myerr = None
         try:
@@ -228,6 +233,9 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True
                 msg = "Pattern %s found in full test report: %s" % (regex.pattern, test_report_txt)
                 self.assertTrue(regex.search(test_report_txt), msg)
 
+        if check_errorlog is not None:
+            check_errorlog(outtxt, tmp_logdir, self.test_buildpath, self.test_errorlogpath)
+
         return outtxt
 
     def run_test_toy_build_with_output(self, *args, **kwargs):
@@ -271,6 +279,55 @@ def test_toy_broken(self):
         # cleanup
         shutil.rmtree(tmpdir)
 
+    def assert_build_files_copied(self, buildpath, errorlogpath):
+        buildir = pathlib.Path(buildpath)
+        errorlogdir = pathlib.Path(errorlogpath)
+        iso_date_pattern = r'????????-??????'
+        for file in buildir.iterdir():
+            file_relative_path = file.relative_to(buildir)
+            file_copies = list(errorlogdir.glob(f"toy-0.0/{iso_date_pattern}/{file_relative_path}"))
+            self.assertTrue(len(file_copies) == 1, f"Unique copy of toy build file '{file}' made")
+            for file_copy in file_copies:
+                if file_copy.is_file():
+                    msg = f"File '{file}' copied succesfully"
+                    self.assertTrue(filecmp.cmp(str(file), str(file_copy), shallow=False), msg)
+
+    def assert_log_files_copied(self, tmp_logpath, errorlogpath):
+        for log_file in pathlib.Path(tmp_logpath).glob("**/*.log"):
+            file_name = log_file.name
+            saved_log_files = list(pathlib.Path(errorlogpath).glob(f"**/{file_name}"))
+            self.assertTrue(len(saved_log_files) == 1, f"Unique copy of log file '{log_file}' made")
+            for saved_log_file in saved_log_files:
+                msg = f"Log file '{log_file}' copied succesfully"
+                self.assertTrue(filecmp.cmp(str(log_file), str(saved_log_file), shallow=False), msg)
+
+    def assert_error_reported(self, outtxt, output_regexs):
+        for regex_pattern in output_regexs:
+            regex = re.compile(regex_pattern, re.M)
+            msg = "Pattern %s found in full test report: %s" % (regex.pattern, outtxt)
+            self.assertTrue(regex.search(outtxt), msg)
+
+    def check_errorlog(self, outtxt, output_regexs, tmp_logpath, buildpath, errorlogpath):
+        self.assert_build_files_copied(buildpath, errorlogpath)
+        self.assert_log_files_copied(tmp_logpath, errorlogpath)
+        self.assert_error_reported(outtxt, output_regexs)
+
+    def test_toy_broken_compilation(self):
+        """Test whether log files and compilafor broken compilations are copied to a permanent location"""
+        tmpdir = TempDirectory()
+        tmp_logdir = TempDirectory()
+        broken_compilation_ec = os.path.join(os.path.dirname(__file__),
+                                             'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-buggy.eb')
+
+        output_regexs = [r'^\s+toy\.c:5:44: error: expected ;']
+
+        def check_errorlog(outtxt, tmp_logpath, buildpath, errorlogpath):
+            self.check_errorlog(outtxt, output_regexs, tmp_logpath, buildpath, errorlogpath)
+
+        self.test_toy_build(ec_file=broken_compilation_ec, tmpdir=tmpdir.get_path(), tmp_logdir=tmp_logdir.get_path(),
+                            verify=False, fails=True, verbose=False, raise_error=False,
+                            name='toy', versionsuffix='-buggy', check_errorlog=check_errorlog)
+
     def test_toy_tweaked(self):
         """Test toy build with tweaked easyconfig, for testing extra easyconfig parameters."""
         test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs')

diff --git a/test/framework/utilities.py b/test/framework/utilities.py
@@ -121,6 +121,8 @@ def setUp(self):
         os.environ['EASYBUILD_BUILDPATH'] = self.test_buildpath
         self.test_installpath = tempfile.mkdtemp()
         os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath
+        self.test_errorlogpath = tempfile.mkdtemp()
+        os.environ['EASYBUILD_ERRORLOGPATH'] = self.test_errorlogpath
 
         # make sure that the tests only pick up easyconfigs provided with the tests
         os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(testdir, 'easyconfigs', 'test_ecs')
@@ -514,3 +516,15 @@ def find_full_path(base_path, trim=(lambda x: x)):
             break
 
     return full_path
+
+
+class TempDirectory:
+    def __init__(self):
+        self.dir = tempfile.mkdtemp()
+
+    def __del__(self):
+        if os.path.exists(self.dir):
+            shutil.rmtree(self.dir)
+
+    def get_path(self):
+        return self.dir