diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-buggy.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-buggy.eb
new file mode 100644
index 0000000000..89fe5979b6
--- /dev/null
+++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-buggy.eb
@@ -0,0 +1,33 @@
+name = 'toy'
+version = '0.0'
+versionsuffix = '-buggy'
+
+homepage = 'https://easybuilders.github.io/easybuild'
+description = "Toy C program, 100% toy."
+
+toolchain = SYSTEM
+
+sources = [SOURCE_TAR_GZ]
+checksums = [[
+    'be662daa971a640e40be5c804d9d7d10',  # default (MD5)
+    '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc',  # default (SHA256)
+    ('adler32', '0x998410035'),
+    ('crc32', '0x1553842328'),
+    ('md5', 'be662daa971a640e40be5c804d9d7d10'),
+    ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
+    ('size', 273),
+]]
+patches = [
+    'toy-0.0_add-bug.patch',
+    ('toy-extra.txt', 'toy-0.0'),
+]
+
+sanity_check_paths = {
+    'files': [('bin/yot', 'bin/toy')],
+    'dirs': ['bin'],
+}
+
+postinstallcmds = ["echo TOY > %(installdir)s/README"]
+
+moduleclass = 'tools'
+# trailing comment, leave this here, it may trigger bugs with extract_comments()
diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb
index 90cc7429d3..9184a55d23 100644
--- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb
+++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb
@@ -18,7 +18,7 @@ checksums = [[
     ('size', 273),
 ]]
 patches = [
-    'toy-0.0_fix-silly-typo-in-printf-statement.patch',
+    'toy-0.0_add-bug.patch',
     ('toy-extra.txt', 'toy-0.0'),
 ]
 
diff --git a/test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch b/test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch
new file mode 100644
index 0000000000..7512447168
--- /dev/null
+++ b/test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch
@@ -0,0 +1,10 @@
+--- a/toy-0.0.orig/toy.source	2014-03-06 18:48:16.000000000 +0100
++++ b/toy-0.0/toy.source	2020-08-18 12:19:35.000000000 +0200
+@@ -2,6 +2,6 @@
+ 
+ int main(int argc, char* argv[]){
+ 
+-    printf("I'm a toy, and proud of it.\n");
++    printf("I'm a toy, and proud of it.\n")
+     return 0;
+ }
diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
index a334eeca7a..6ee92dd774 100644
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -172,6 +172,7 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True
             '--sourcepath=%s' % self.test_sourcepath,
             '--buildpath=%s' % self.test_buildpath,
             '--installpath=%s' % self.test_installpath,
+            '--errorlogpath=%s' % self.test_errorlogpath,
             '--unittest-file=%s' % self.logfile,
             '--robot=%s' % os.pathsep.join([self.test_buildpath, os.path.dirname(__file__)]),
         ]
@@ -271,6 +272,31 @@ def test_toy_broken(self):
         # cleanup
         shutil.rmtree(tmpdir)
 
+    def test_toy_broken_compilation(self):
+        """Test whether log files and compilafor broken compilations are copied to a permanent location"""
+        tmpdir = tempfile.mkdtemp()
+        broken_compilation_ec = os.path.join(os.path.dirname(__file__),
+                                             'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-buggy.eb')
+
+        repositorypath = os.path.join(self.test_installpath, 'easyconfigs_archive')
+
+        error_regex = "Build of .* failed"
+        self.assertErrorRegex(EasyBuildError, error_regex, self.test_toy_build,
+                              ec_file=broken_compilation_ec, tmpdir=tmpdir,
+                              verify=False, fails=True, verbose=False, raise_error=True)
+
+        error_regex = "Build log and any output artifacts copied to permanent storage"
+        self.assertErrorRegex(EasyBuildError, error_regex, self.test_toy_build,
+                              ec_file=broken_compilation_ec, tmpdir=tmpdir,
+                              verify=False, fails=True, verbose=False, raise_error=True)
+
+        self.test_toy_build(ec_file=broken_compilation_ec, tmpdir=tmpdir, verify=False, fails=True,
+                            verbose=False, raise_error=False,
+                            name='toy', versionsuffix='-buggy')
+
+        # cleanup
+        shutil.rmtree(tmpdir)
+
     def test_toy_tweaked(self):
         """Test toy build with tweaked easyconfig, for testing extra easyconfig parameters."""
         test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs')
diff --git a/test/framework/utilities.py b/test/framework/utilities.py
index d07c2202b1..507e0a5369 100644
--- a/test/framework/utilities.py
+++ b/test/framework/utilities.py
@@ -121,6 +121,8 @@ def setUp(self):
         os.environ['EASYBUILD_BUILDPATH'] = self.test_buildpath
         self.test_installpath = tempfile.mkdtemp()
         os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath
+        self.test_errorlogpath = tempfile.mkdtemp()
+        os.environ['EASYBUILD_ERRORLOGPATH'] = self.test_errorlogpath
 
         # make sure that the tests only pick up easyconfigs provided with the tests
         os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(testdir, 'easyconfigs', 'test_ecs')