diff --git a/nbgrader/tests/preprocessors/test_instantiatetests.py b/nbgrader/tests/preprocessors/test_instantiatetests.py
index a7134f228..d30ccca6e 100644
--- a/nbgrader/tests/preprocessors/test_instantiatetests.py
+++ b/nbgrader/tests/preprocessors/test_instantiatetests.py
@@ -1,4 +1,5 @@
 import pytest
+import shutil
 import os
 from textwrap import dedent
 from ...preprocessors import InstantiateTests
@@ -118,8 +119,15 @@ def test_kernel_workingdir(self, preprocessor, caplog):
         resources = {
             'metadata': {'path': 'nbgrader/docs/source/user_guide/source/'}
         }
+        # make sure autotest doesn't fail prior to running the
+        # preprocessor because it can't find autotests.yml
+        # we want it to fail because it can't find a resource file
+        shutil.copyfile('autotests.yml', 'source/autotests.yml')
         with pytest.raises(Exception):
             nb, resources = preprocessor.preprocess(nb, resources)
+        # remove the temporary resource
+        os.remove('source/autotests.yml')
+
         assert "FileNotFoundError" in caplog.text
 
     # test that a warning is thrown when we set enforce_metadata = False and have an AUTOTEST directive in a