diff --git a/src/integrationTest/kotlin/com/sourcegraph/cody/AllSuites.kt b/src/integrationTest/kotlin/com/sourcegraph/cody/AllSuites.kt index 4d9f29a77f..67c8ebd3c7 100644 --- a/src/integrationTest/kotlin/com/sourcegraph/cody/AllSuites.kt +++ b/src/integrationTest/kotlin/com/sourcegraph/cody/AllSuites.kt @@ -1,55 +1,8 @@ package com.sourcegraph.cody -import com.sourcegraph.cody.agent.CodyAgentService import com.sourcegraph.cody.edit.DocumentCodeTest -import com.sourcegraph.cody.util.CodyIntegrationTextFixture import com.sourcegraph.cody.util.RepeatableSuite -import java.util.concurrent.TimeUnit -import org.junit.AfterClass import org.junit.runner.RunWith import org.junit.runners.Suite -/** - * We need a single tearDown() method running after all tests are complete, so agent can be closed - * gracefully and recordings can be saved properly. - * - * Due to the limitations of JUnit 4 this can be done only using SuiteClasses and AfterClass, and we - * are forced to use JUnit 4 because IntelliJ testing classes like `BasePlatformTestCase` are based - * on that version. JUnit 4 jar is part of the ideaIC package. We will upgrade to JUnit 5 - * automatically after the platform version bump. - * - * Multiple recording files can be used, but each should have its own suite with tearDown() method - * nad define unique CODY_RECORDING_NAME. - */ -@RunWith(RepeatableSuite::class) -@Suite.SuiteClasses(DocumentCodeTest::class) -class AllSuites { - companion object { - @AfterClass - @JvmStatic - internal fun tearDown() { - CodyAgentService.withAgent(CodyIntegrationTextFixture.myProject!!) { agent -> - val errors = agent.server.testingRequestErrors().get() - // We extract polly.js errors to notify users about the missing recordings, if any - val missingRecordings = errors.filter { it.error?.contains("`recordIfMissing` is") == true } - missingRecordings.forEach { missing -> - System.err.println( - """Recording is missing: ${missing.error} - | - |------------------------------------------------------------------------------------------ - |To fix this problem please run `./gradlew :recordingIntegrationTest`. - |You need to export access tokens first, using script from the `sourcegraph/cody` repository: - |`agent/scripts/export-cody-http-recording-tokens.sh` - |------------------------------------------------------------------------------------------ - """ - .trimMargin()) - } - - agent.server - .shutdown() - .get(CodyIntegrationTextFixture.ASYNC_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS) - agent.server.exit() - } - } - } -} +@RunWith(RepeatableSuite::class) @Suite.SuiteClasses(DocumentCodeTest::class) class AllSuites diff --git a/src/integrationTest/kotlin/com/sourcegraph/cody/edit/DocumentCodeTest.kt b/src/integrationTest/kotlin/com/sourcegraph/cody/edit/DocumentCodeTest.kt index 93b9bd89d5..a760b61e57 100644 --- a/src/integrationTest/kotlin/com/sourcegraph/cody/edit/DocumentCodeTest.kt +++ b/src/integrationTest/kotlin/com/sourcegraph/cody/edit/DocumentCodeTest.kt @@ -90,13 +90,7 @@ class DocumentCodeTest : CodyIntegrationTextFixture() { runLensAction(acceptLens!!, EditAcceptAction.ID) assertNoInlayShown() - assertThat( - myFixture.editor.document.text, - startsWith( - """/** - | * Imports the necessary Java standard library classes for the Foo class, including the ArrayList class. - | */""" - .trimMargin())) + assertThat(myFixture.editor.document.text, startsWith("/**")) } @Test diff --git a/src/integrationTest/kotlin/com/sourcegraph/cody/util/CodyIntegrationTextFixture.kt b/src/integrationTest/kotlin/com/sourcegraph/cody/util/CodyIntegrationTextFixture.kt index dba6a9f28b..da7a53b59d 100644 --- a/src/integrationTest/kotlin/com/sourcegraph/cody/util/CodyIntegrationTextFixture.kt +++ b/src/integrationTest/kotlin/com/sourcegraph/cody/util/CodyIntegrationTextFixture.kt @@ -13,6 +13,7 @@ import com.intellij.openapi.fileEditor.FileDocumentManager import com.intellij.openapi.project.DumbService import com.intellij.openapi.project.Project import com.intellij.testFramework.EditorTestUtil +import com.intellij.testFramework.HeavyPlatformTestCase import com.intellij.testFramework.PlatformTestUtil import com.intellij.testFramework.fixtures.BasePlatformTestCase import com.intellij.testFramework.runInEdtAndWait @@ -30,6 +31,7 @@ import java.nio.file.Paths import java.util.concurrent.CompletableFuture import java.util.concurrent.TimeUnit import java.util.regex.Pattern +import kotlin.io.path.pathString open class CodyIntegrationTextFixture : BasePlatformTestCase(), LensListener { private val logger = Logger.getInstance(CodyIntegrationTextFixture::class.java) @@ -48,26 +50,40 @@ open class CodyIntegrationTextFixture : BasePlatformTestCase(), LensListener { override fun tearDown() { try { LensesService.getInstance(project).removeListener(this) - CodyAgentService.getInstance(myFixture.project).apply { - try { - stopAgent(project) - } catch (x: Exception) { - logger.warn("Error shutting down agent", x) + + val recordingsFuture = CompletableFuture() + CodyAgentService.withAgent(project) { agent -> + val errors = agent.server.testingRequestErrors().get() + // We extract polly.js errors to notify users about the missing recordings, if any + val missingRecordings = errors.filter { it.error?.contains("`recordIfMissing` is") == true } + missingRecordings.forEach { missing -> + logger.error( + """Recording is missing: ${missing.error} + | + |${missing.body} + | + |------------------------------------------------------------------------------------------ + |To fix this problem please run `./gradlew :recordingIntegrationTest`. + |You need to export access tokens first, using script from the `sourcegraph/cody` repository: + |`agent/scripts/export-cody-http-recording-tokens.sh` + |------------------------------------------------------------------------------------------ + """ + .trimMargin()) } + recordingsFuture.complete(null) } + recordingsFuture.get(ASYNC_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS) + CodyAgentService.getInstance(project) + .stopAgent(project) + ?.get(ASYNC_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS) } finally { + val testDataPath = Paths.get(myFixture.testDataPath) super.tearDown() + testDataPath.toFile().deleteRecursively() } } private fun configureFixture() { - // If you don't specify this system property with this setting when running the tests, - // the tests will fail, because IntelliJ will run them from the EDT, which can't block. - // Setting this property invokes the tests from an executor pool thread, which lets us - // block/wait on potentially long-running operations during the integration test. - val policy = System.getProperty("idea.test.execution.policy") - assertTrue(policy == "com.sourcegraph.cody.test.NonEdtIdeaTestExecutionPolicy") - // This is wherever src/integrationTest/resources is on the box running the tests. val testResourcesDir = File(System.getProperty("test.resources.dir")) assertTrue(testResourcesDir.exists()) @@ -77,16 +93,17 @@ open class CodyIntegrationTextFixture : BasePlatformTestCase(), LensListener { val workspaceRootUri = ConfigUtil.getWorkspaceRootPath(project) // We copy the test resources there manually, bypassing Gradle, which is picky. - val testDataPath = Paths.get(workspaceRootUri.toString(), "src/").toFile() - testResourcesDir.copyRecursively(testDataPath, overwrite = true) + // We also ensure that all files are properly refreshed to the VFS. + val testDataPath = workspaceRootUri.resolve(name) + testResourcesDir.copyRecursively(testDataPath.toFile(), overwrite = true) + HeavyPlatformTestCase.synchronizeTempDirVfs(testDataPath) // This useful setting lets us tell the fixture to look where we copied them. - myFixture.testDataPath = testDataPath.path + myFixture.testDataPath = testDataPath.pathString // The file we pass to configureByFile must be relative to testDataPath. val projectFile = "testProjects/documentCode/src/main/java/Foo.java" - val sourcePath = Paths.get(testDataPath.path, projectFile).toString() - assertTrue(File(sourcePath).exists()) + assertTrue(testDataPath.resolve(projectFile).toFile().exists()) myFixture.configureByFile(projectFile) initCredentialsAndAgent() @@ -116,6 +133,13 @@ open class CodyIntegrationTextFixture : BasePlatformTestCase(), LensListener { } private fun checkInitialConditions() { + // If you don't specify this system property with this setting when running the tests, + // the tests will fail, because IntelliJ will run them from the EDT, which can't block. + // Setting this property invokes the tests from an executor pool thread, which lets us + // block/wait on potentially long-running operations during the integration test. + val policy = System.getProperty("idea.test.execution.policy") + assertTrue(policy == "com.sourcegraph.cody.test.NonEdtIdeaTestExecutionPolicy") + val project = myFixture.project // Check if the project is in dumb mode diff --git a/src/integrationTest/resources/recordings/integration-test_2927926756/recording.har.yaml b/src/integrationTest/resources/recordings/integration-test_2927926756/recording.har.yaml index 862208e27a..f43e01b968 100644 --- a/src/integrationTest/resources/recordings/integration-test_2927926756/recording.har.yaml +++ b/src/integrationTest/resources/recordings/integration-test_2927926756/recording.har.yaml @@ -47,7 +47,7 @@ log: cookies: [] headers: - name: date - value: Mon, 29 Jul 2024 11:48:58 GMT + value: Mon, 05 Aug 2024 10:11:46 GMT - name: content-type value: text/plain; charset=utf-8 - name: transfer-encoding @@ -55,7 +55,7 @@ log: - name: connection value: keep-alive - name: retry-after - value: "294" + value: "403" - name: access-control-allow-credentials value: "true" - name: access-control-allow-origin @@ -80,7 +80,7 @@ log: redirectURL: "" status: 200 statusText: OK - startedDateTime: 2024-07-29T11:48:58.806Z + startedDateTime: 2024-08-05T10:11:45.841Z time: 0 timings: blocked: -1 @@ -107,7 +107,7 @@ log: - name: user-agent value: JetBrains / 6.0-localbuild - name: traceparent - value: 00-2ac3b3fe53a1fbbe7b3bc804b591973a-4fe5f82fd6165da4-01 + value: 00-0ec7441b05369c730163041565b0a6c3-ec2f5176d73290f6-01 - name: connection value: keep-alive - name: host @@ -206,14 +206,14 @@ log: value: 6.0-localbuild url: https://sourcegraph.com/.api/completions/stream?api-version=1&client-name=jetbrains&client-version=6.0-localbuild response: - bodySize: 2016 + bodySize: 1837 content: mimeType: text/event-stream - size: 2016 + size: 1837 text: >+ event: completion - data: {"completion":"\n/**\n * Imports the necessary Java standard library classes for the Foo class, including the ArrayList class.\n */\n","stopReason":"stop_sequence"} + data: {"completion":"/**\n * Imports the necessary Java utility classes, including {@link java.util.List} and {@link java.util.ArrayList}.\n */","stopReason":"stop_sequence"} event: done @@ -223,7 +223,7 @@ log: cookies: [] headers: - name: date - value: Mon, 29 Jul 2024 11:49:01 GMT + value: Mon, 05 Aug 2024 10:11:48 GMT - name: content-type value: text/event-stream - name: transfer-encoding @@ -231,7 +231,7 @@ log: - name: connection value: keep-alive - name: retry-after - value: "292" + value: "401" - name: access-control-allow-credentials value: "true" - name: access-control-allow-origin @@ -254,7 +254,7 @@ log: redirectURL: "" status: 200 statusText: OK - startedDateTime: 2024-07-29T11:49:00.467Z + startedDateTime: 2024-08-05T10:11:47.322Z time: 0 timings: blocked: -1 @@ -329,7 +329,7 @@ log: cookies: [] headers: - name: date - value: Mon, 29 Jul 2024 11:48:59 GMT + value: Mon, 05 Aug 2024 10:11:46 GMT - name: content-type value: application/json - name: transfer-encoding @@ -337,7 +337,7 @@ log: - name: connection value: keep-alive - name: retry-after - value: "294" + value: "403" - name: access-control-allow-credentials value: "true" - name: access-control-allow-origin @@ -362,7 +362,7 @@ log: redirectURL: "" status: 200 statusText: OK - startedDateTime: 2024-07-29T11:48:59.179Z + startedDateTime: 2024-08-05T10:11:46.212Z time: 0 timings: blocked: -1 @@ -449,7 +449,7 @@ log: cookies: [] headers: - name: date - value: Mon, 29 Jul 2024 11:48:58 GMT + value: Mon, 05 Aug 2024 10:11:45 GMT - name: content-type value: application/json - name: transfer-encoding @@ -457,7 +457,7 @@ log: - name: connection value: keep-alive - name: retry-after - value: "295" + value: "404" - name: access-control-allow-credentials value: "true" - name: access-control-allow-origin @@ -482,7 +482,7 @@ log: redirectURL: "" status: 200 statusText: OK - startedDateTime: 2024-07-29T11:48:58.034Z + startedDateTime: 2024-08-05T10:11:44.993Z time: 0 timings: blocked: -1 @@ -557,7 +557,7 @@ log: cookies: [] headers: - name: date - value: Mon, 29 Jul 2024 11:48:58 GMT + value: Mon, 05 Aug 2024 10:11:45 GMT - name: content-type value: application/json - name: transfer-encoding @@ -565,7 +565,7 @@ log: - name: connection value: keep-alive - name: retry-after - value: "295" + value: "404" - name: access-control-allow-credentials value: "true" - name: access-control-allow-origin @@ -590,7 +590,7 @@ log: redirectURL: "" status: 200 statusText: OK - startedDateTime: 2024-07-29T11:48:58.068Z + startedDateTime: 2024-08-05T10:11:45.025Z time: 0 timings: blocked: -1 @@ -650,17 +650,22 @@ log: value: null url: https://sourcegraph.com/.api/graphql?CurrentSiteCodyLlmProvider response: - bodySize: 127 + bodySize: 124 content: encoding: base64 mimeType: application/json - size: 127 + size: 124 text: "[\"H4sIAAAAAAAAA6pWSkksSVSyqlYqzixJBdHJ+SmVPj6+zvl5aZnppUWJJZn5eSDxgqL8s\ - syU1CIlK6WyxKLM/NJipdra2loAAAAA//8=\",\"AwAdobPlQQAAAA==\"]" + syU1CIlK6W0zKLU8vyi7GKl2traWgAAAAD//wMAf38NDUMAAAA=\"]" + textDecoded: + data: + site: + codyLLMConfiguration: + provider: fireworks cookies: [] headers: - name: date - value: Mon, 29 Jul 2024 11:48:58 GMT + value: Mon, 05 Aug 2024 10:11:45 GMT - name: content-type value: application/json - name: transfer-encoding @@ -668,7 +673,7 @@ log: - name: connection value: keep-alive - name: retry-after - value: "295" + value: "404" - name: access-control-allow-credentials value: "true" - name: access-control-allow-origin @@ -693,7 +698,7 @@ log: redirectURL: "" status: 200 statusText: OK - startedDateTime: 2024-07-29T11:48:58.051Z + startedDateTime: 2024-08-05T10:11:45.009Z time: 0 timings: blocked: -1 @@ -790,7 +795,7 @@ log: cookies: [] headers: - name: date - value: Mon, 29 Jul 2024 11:48:58 GMT + value: Mon, 05 Aug 2024 10:11:45 GMT - name: content-type value: application/json - name: transfer-encoding @@ -798,7 +803,7 @@ log: - name: connection value: keep-alive - name: retry-after - value: "295" + value: "404" - name: access-control-allow-credentials value: "true" - name: access-control-allow-origin @@ -823,7 +828,7 @@ log: redirectURL: "" status: 200 statusText: OK - startedDateTime: 2024-07-29T11:48:58.085Z + startedDateTime: 2024-08-05T10:11:45.041Z time: 0 timings: blocked: -1 @@ -887,19 +892,28 @@ log: value: null url: https://sourcegraph.com/.api/graphql?CurrentUserCodySubscription response: - bodySize: 231 + bodySize: 228 content: encoding: base64 mimeType: application/json - size: 231 + size: 228 text: "[\"H4sIAAAAAAAAA1zMsQrCMBSF4Xc5c4U2FpRsRToIgqWtDm6xyRCoSbi5GUrJu4uCoI7n5\ +Os0IoV5IopERnHl2joPb1ehnSPE9nA1rtXi6w4RUg0h/F4bVEgzMpBouvPKKBCmJeO\ fK/YnOzDcoRkSqb4fHeGrNcDK+KGISFKUW/K3aaqRyFkVcmtuOFPt05/2f2vzTnnJwA\ - AAP//\",\"AwB81vXLwgAAAA==\"]" + AAP//AwB81vXLwgAAAA==\"]" + textDecoded: + data: + currentUser: + codySubscription: + applyProRateLimits: true + currentPeriodEndAt: 2024-08-14T22:11:32Z + currentPeriodStartAt: 2024-07-14T22:11:32Z + plan: PRO + status: ACTIVE cookies: [] headers: - name: date - value: Mon, 29 Jul 2024 11:48:58 GMT + value: Mon, 05 Aug 2024 10:11:45 GMT - name: content-type value: application/json - name: transfer-encoding @@ -907,7 +921,7 @@ log: - name: connection value: keep-alive - name: retry-after - value: "295" + value: "403" - name: access-control-allow-credentials value: "true" - name: access-control-allow-origin @@ -932,7 +946,7 @@ log: redirectURL: "" status: 200 statusText: OK - startedDateTime: 2024-07-29T11:48:58.336Z + startedDateTime: 2024-08-05T10:11:45.289Z time: 0 timings: blocked: -1 @@ -990,21 +1004,18 @@ log: value: null url: https://sourcegraph.com/.api/graphql?SiteProductVersion response: - bodySize: 136 + bodySize: 139 content: encoding: base64 mimeType: application/json - size: 136 - text: "[\"H4sIAAAAAAAAA6pWSkksSVSyqlYqzixJBdEFRfkppcklYalFxZn5eUpWSkYWJsZmRvFGB\ - kYmugbmukam8aZ6prrJZhaplkZJyUYWScZKtbW1AAAAAP//AwBliiQpSQAAAA==\"]" - textDecoded: - data: - site: - productVersion: 284362_2024-07-25_5.5-c68e92bc28b3 + size: 139 + text: "[\"H4sIAAAAAAAAA6pWSkksSVSyqlYqzixJBdEFRfkppcklYalFxZn5eUpWSkYWphamxvFGB\ + kYmugYWugam8aZ6prqpxsZGhkYmZpYWlqlKtbW1AAAAAP//\",\"AwDZmR/0SQAAAA==\ + \"]" cookies: [] headers: - name: date - value: Mon, 29 Jul 2024 11:48:58 GMT + value: Mon, 05 Aug 2024 10:11:45 GMT - name: content-type value: application/json - name: transfer-encoding @@ -1012,7 +1023,7 @@ log: - name: connection value: keep-alive - name: retry-after - value: "295" + value: "404" - name: access-control-allow-credentials value: "true" - name: access-control-allow-origin @@ -1037,7 +1048,7 @@ log: redirectURL: "" status: 200 statusText: OK - startedDateTime: 2024-07-29T11:48:57.998Z + startedDateTime: 2024-08-05T10:11:44.958Z time: 0 timings: blocked: -1 diff --git a/src/main/kotlin/com/sourcegraph/cody/agent/CodyAgent.kt b/src/main/kotlin/com/sourcegraph/cody/agent/CodyAgent.kt index 30cb249935..912fe4bfb6 100644 --- a/src/main/kotlin/com/sourcegraph/cody/agent/CodyAgent.kt +++ b/src/main/kotlin/com/sourcegraph/cody/agent/CodyAgent.kt @@ -40,10 +40,10 @@ private constructor( fun shutdown(): CompletableFuture { return server.shutdown().completeOnTimeout(null, 15, TimeUnit.SECONDS).handle { _, throwable -> if (throwable != null) logger.warn("Graceful shutdown of Cody agent server failed", throwable) - server.exit() - logger.info("Cody Agent shut down gracefully") listeningToJsonRpc.cancel(true) connection.close() + server.exit() + logger.info("Cody Agent shut down gracefully") } } diff --git a/src/main/kotlin/com/sourcegraph/cody/agent/CodyAgentService.kt b/src/main/kotlin/com/sourcegraph/cody/agent/CodyAgentService.kt index c2952245b3..c1f911aa08 100644 --- a/src/main/kotlin/com/sourcegraph/cody/agent/CodyAgentService.kt +++ b/src/main/kotlin/com/sourcegraph/cody/agent/CodyAgentService.kt @@ -188,11 +188,12 @@ class CodyAgentService(private val project: Project) : Disposable { return codyAgent } - fun stopAgent(project: Project?) { + fun stopAgent(project: Project?): CompletableFuture? { try { - codyAgent.getNow(null)?.shutdown() + return codyAgent.getNow(null)?.shutdown() } catch (e: Exception) { logger.warn("Failed to stop Cody agent gracefully", e) + return CompletableFuture.failedFuture(e) } finally { codyAgent = CompletableFuture() project?.let { CodyStatusService.resetApplication(it) }