diff --git a/src/integrationTest/kotlin/com/sourcegraph/cody/util/CodyIntegrationTextFixture.kt b/src/integrationTest/kotlin/com/sourcegraph/cody/util/CodyIntegrationTextFixture.kt
index dba6a9f28b..1ed77b1545 100644
--- a/src/integrationTest/kotlin/com/sourcegraph/cody/util/CodyIntegrationTextFixture.kt
+++ b/src/integrationTest/kotlin/com/sourcegraph/cody/util/CodyIntegrationTextFixture.kt
@@ -13,6 +13,7 @@ import com.intellij.openapi.fileEditor.FileDocumentManager
 import com.intellij.openapi.project.DumbService
 import com.intellij.openapi.project.Project
 import com.intellij.testFramework.EditorTestUtil
+import com.intellij.testFramework.HeavyPlatformTestCase
 import com.intellij.testFramework.PlatformTestUtil
 import com.intellij.testFramework.fixtures.BasePlatformTestCase
 import com.intellij.testFramework.runInEdtAndWait
@@ -50,7 +51,7 @@ open class CodyIntegrationTextFixture : BasePlatformTestCase(), LensListener {
       LensesService.getInstance(project).removeListener(this)
       CodyAgentService.getInstance(myFixture.project).apply {
         try {
-          stopAgent(project)
+          stopAgent(project)?.get(ASYNC_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)
         } catch (x: Exception) {
           logger.warn("Error shutting down agent", x)
         }
@@ -77,8 +78,12 @@ open class CodyIntegrationTextFixture : BasePlatformTestCase(), LensListener {
     val workspaceRootUri = ConfigUtil.getWorkspaceRootPath(project)
 
     // We copy the test resources there manually, bypassing Gradle, which is picky.
+    // We also ensure that all files are properly refreshed to the VFS.
     val testDataPath = Paths.get(workspaceRootUri.toString(), "src/").toFile()
+    testDataPath.deleteRecursively()
+    HeavyPlatformTestCase.synchronizeTempDirVfs(testDataPath.toPath())
     testResourcesDir.copyRecursively(testDataPath, overwrite = true)
+    HeavyPlatformTestCase.synchronizeTempDirVfs(testDataPath.toPath())
 
     // This useful setting lets us tell the fixture to look where we copied them.
     myFixture.testDataPath = testDataPath.path
diff --git a/src/main/kotlin/com/sourcegraph/cody/agent/CodyAgentService.kt b/src/main/kotlin/com/sourcegraph/cody/agent/CodyAgentService.kt
index c2952245b3..c1f911aa08 100644
--- a/src/main/kotlin/com/sourcegraph/cody/agent/CodyAgentService.kt
+++ b/src/main/kotlin/com/sourcegraph/cody/agent/CodyAgentService.kt
@@ -188,11 +188,12 @@ class CodyAgentService(private val project: Project) : Disposable {
     return codyAgent
   }
 
-  fun stopAgent(project: Project?) {
+  fun stopAgent(project: Project?): CompletableFuture<Unit>? {
     try {
-      codyAgent.getNow(null)?.shutdown()
+      return codyAgent.getNow(null)?.shutdown()
     } catch (e: Exception) {
       logger.warn("Failed to stop Cody agent gracefully", e)
+      return CompletableFuture.failedFuture(e)
     } finally {
       codyAgent = CompletableFuture()
       project?.let { CodyStatusService.resetApplication(it) }