Add integration tests for the chat (context files)

build.gradle.kts

@@ -227,7 +227,6 @@ fun Test.sharedIntegrationTestConfig(buildCodyDir: File, mode: String) {
 
   environment(
       "CODY_RECORDING_MODE" to mode,
-      "CODY_RECORDING_NAME" to "integration-test",
       "CODY_RECORDING_DIRECTORY" to resourcesDir.resolve("recordings").absolutePath,
       "CODY_SHIM_TESTING" to "true",
       "CODY_TEMPERATURE_ZERO" to "true",

src/integrationTest/kotlin/com/sourcegraph/cody/AllSuites.kt

@@ -1,8 +1,11 @@
 package com.sourcegraph.cody
 
+import com.sourcegraph.cody.chat.ChatTest
 import com.sourcegraph.cody.edit.DocumentCodeTest
 import com.sourcegraph.cody.util.RepeatableSuite
 import org.junit.runner.RunWith
 import org.junit.runners.Suite
 
-@RunWith(RepeatableSuite::class) @Suite.SuiteClasses(DocumentCodeTest::class) class AllSuites
+@RunWith(RepeatableSuite::class)
+@Suite.SuiteClasses(ChatTest::class, DocumentCodeTest::class)
+class AllSuites

src/integrationTest/kotlin/com/sourcegraph/cody/chat/ChatTest.kt

@@ -0,0 +1,102 @@
+package com.sourcegraph.cody.chat
+
+import com.intellij.testFramework.runInEdtAndGet
+import com.intellij.testFramework.runInEdtAndWait
+import com.sourcegraph.cody.chat.ui.ContextFileActionLink
+import com.sourcegraph.cody.context.ui.EnterpriseEnhancedContextPanel
+import com.sourcegraph.cody.history.HistoryService
+import com.sourcegraph.cody.history.state.EnhancedContextState
+import com.sourcegraph.cody.history.state.RemoteRepositoryState
+import com.sourcegraph.cody.util.CodyIntegrationTestFixture
+import com.sourcegraph.cody.util.CustomJunitClassRunner
+import com.sourcegraph.cody.util.TestingCredentials
+import java.awt.Component
+import java.awt.Container
+import java.util.concurrent.TimeUnit
+import junit.framework.TestCase
+import org.awaitility.kotlin.await
+import org.awaitility.kotlin.until
+import org.junit.Test
+import org.junit.runner.RunWith
+
+@RunWith(CustomJunitClassRunner::class)
+class ChatTest : CodyIntegrationTestFixture() {
+  override fun recordingName() = "chat"
+
+  override fun credentials() = TestingCredentials.enterprise
+
+  override fun checkSuiteSpecificInitialConditions() = Unit
+
+  @Test
+  fun testRemoteContextFileItems() {
+    val enhancedContextState =
+        EnhancedContextState().apply {
+          remoteRepositories.add(
+              RemoteRepositoryState().apply {
+                isEnabled = true
+                remoteUrl = "github.com/sourcegraph/cody"
+                codebaseName = "github.com/sourcegraph/cody"
+              })
+        }
+    HistoryService.getInstance(project).updateDefaultContextState(enhancedContextState)
+
+    val session = runInEdtAndGet { AgentChatSession.createNew(project) }
+
+    await.atMost(30, TimeUnit.SECONDS) until
+        {
+          (session.getPanel().contextView as EnterpriseEnhancedContextPanel)
+              .controller
+              .getConfiguredState()
+              .find { it.name == "github.com/sourcegraph/cody" && !it.isIgnored } != null
+        }
+
+    runInEdtAndWait { session.sendMessage("What is JSON RPC?", emptyList()) }
+
+    await.atMost(30, TimeUnit.SECONDS) until { !session.messages[0].contextFiles.isNullOrEmpty() }
+    await.atMost(30, TimeUnit.SECONDS) until { session.messages.size == 2 }
+    await.atMost(30, TimeUnit.SECONDS) until { session.messages[1].text?.isNotBlank() == true }
+
+    val linkPanels =
+        findComponentsRecursively(session.getPanel(), ContextFileActionLink::class.java)
+
+    TestCase.assertEquals(
+        listOf(
+            "cody agent/CHANGELOG.md",
+            "cody agent/README.md",
+            "cody agent/src/__tests__/chat-response-quality/README.md",
+            "cody agent/src/cli/command-jsonrpc-stdio.ts",
+            "cody agent/src/cli/command-jsonrpc-websocket.ts",
+            "cody agent/src/cli/command-root.ts",
+            "cody agent/src/cli/scip-codegen/JvmCodegen.ts",
+            "cody agent/src/cli/scip-codegen/JvmFormatter.ts",
+            "cody agent/src/jsonrpc-alias.ts",
+            "cody agent/src/local-e2e/README.md",
+            "cody lib/icons/README.md",
+            "cody vscode/src/graph/bfg/spawn-bfg.ts",
+            "cody vscode/src/jsonrpc/bfg-protocol.ts",
+            "cody vscode/src/jsonrpc/CodyJsonRpcErrorCode.ts",
+            "cody vscode/src/jsonrpc/embeddings-protocol.ts",
+            "cody vscode/src/jsonrpc/isRunningInsideAgent.ts",
+            "cody vscode/src/jsonrpc/jsonrpc.ts",
+            "cody vscode/src/jsonrpc/TextDocumentWithUri.test.ts",
+            "cody vscode/src/jsonrpc/TextDocumentWithUri.ts",
+            "cody web/lib/agent/agent.client.ts"),
+        linkPanels.map { panel -> panel.text })
+  }
+
+  private fun <A> findComponentsRecursively(parent: Component, targetClass: Class<A>): List<A> {
+    val result = mutableListOf<A>()
+
+    if (targetClass.isInstance(parent)) {
+      result.add(parent as A)
+    }
+
+    if (parent is Container) {
+      for (component in parent.components) {
+        result.addAll(findComponentsRecursively(component, targetClass))
+      }
+    }
+
+    return result
+  }
+}

src/integrationTest/kotlin/com/sourcegraph/cody/edit/DocumentCodeTest.kt

@@ -1,5 +1,10 @@
 package com.sourcegraph.cody.edit
 
+import com.intellij.openapi.actionSystem.ActionManager
+import com.intellij.openapi.actionSystem.AnActionEvent
+import com.intellij.openapi.actionSystem.DataContext
+import com.intellij.openapi.editor.Editor
+import com.intellij.openapi.editor.ex.EditorEx
 import com.sourcegraph.cody.edit.actions.DocumentCodeAction
 import com.sourcegraph.cody.edit.actions.lenses.EditAcceptAction
 import com.sourcegraph.cody.edit.actions.lenses.EditCancelAction
@@ -10,15 +15,35 @@ import com.sourcegraph.cody.edit.widget.LensIcon
 import com.sourcegraph.cody.edit.widget.LensLabel
 import com.sourcegraph.cody.edit.widget.LensSpinner
 import com.sourcegraph.cody.edit.widget.LensWidgetGroup
-import com.sourcegraph.cody.util.CodyIntegrationTextFixture
+import com.sourcegraph.cody.util.CodyIntegrationTestFixture
 import com.sourcegraph.cody.util.CustomJunitClassRunner
+import com.sourcegraph.cody.util.TestingCredentials
 import org.hamcrest.Matchers.startsWith
 import org.junit.Assert.assertThat
 import org.junit.Test
 import org.junit.runner.RunWith
 
 @RunWith(CustomJunitClassRunner::class)
-class DocumentCodeTest : CodyIntegrationTextFixture() {
+class DocumentCodeTest : CodyIntegrationTestFixture() {
+  override fun recordingName() = "documentCode"
+
+  override fun credentials() = TestingCredentials.dotcom
+
+  override fun checkSuiteSpecificInitialConditions() {
+    // Check the initial state of the action's presentation
+    val action = ActionManager.getInstance().getAction("cody.documentCodeAction")
+    val event =
+        AnActionEvent.createFromAnAction(action, null, "", createEditorContext(myFixture.editor))
+    action.update(event)
+    val presentation = event.presentation
+    assertTrue("Action should be enabled", presentation.isEnabled)
+    assertTrue("Action should be visible", presentation.isVisible)
+  }
+
+  private fun createEditorContext(editor: Editor): DataContext {
+    return (editor as? EditorEx)?.dataContext ?: DataContext.EMPTY_CONTEXT
+  }
+
   @Test
   fun testGetsWorkingGroupLens() {
     val codeLensGroup = runAndWaitForLenses(DocumentCodeAction.ID, EditCancelAction.ID)

src/integrationTest/kotlin/com/sourcegraph/cody/util/CodyIntegrationTextFixture.kt → src/integrationTest/kotlin/com/sourcegraph/cody/util/CodyIntegrationTestFixture.kt

@@ -1,16 +1,11 @@
 package com.sourcegraph.cody.util
 
 import com.intellij.ide.lightEdit.LightEdit
-import com.intellij.openapi.actionSystem.ActionManager
-import com.intellij.openapi.actionSystem.AnActionEvent
-import com.intellij.openapi.actionSystem.DataContext
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.application.WriteAction
 import com.intellij.openapi.application.runInEdt
 import com.intellij.openapi.command.WriteCommandAction
 import com.intellij.openapi.diagnostic.Logger
-import com.intellij.openapi.editor.Editor
-import com.intellij.openapi.editor.ex.EditorEx
 import com.intellij.openapi.fileEditor.FileDocumentManager
 import com.intellij.openapi.project.DumbService
 import com.intellij.openapi.project.Project
@@ -31,8 +26,8 @@ import java.util.concurrent.TimeUnit
 import java.util.regex.Pattern
 import junit.framework.TestCase
 
-open class CodyIntegrationTextFixture : BasePlatformTestCase(), LensListener {
-  private val logger = Logger.getInstance(CodyIntegrationTextFixture::class.java)
+abstract class CodyIntegrationTestFixture : BasePlatformTestCase(), LensListener {
+  private val logger = Logger.getInstance(CodyIntegrationTestFixture::class.java)
   private val lensSubscribers =
       mutableListOf<
           Pair<(List<ProtocolCodeLens>) -> Boolean, CompletableFuture<LensWidgetGroup?>>>()
@@ -61,23 +56,7 @@ open class CodyIntegrationTextFixture : BasePlatformTestCase(), LensListener {
 
       val recordingsFuture = CompletableFuture<Void>()
       CodyAgentService.withAgent(project) { agent ->
-        val errors = agent.server.testingRequestErrors().get()
-        // We extract polly.js errors to notify users about the missing recordings, if any
-        val missingRecordings = errors.filter { it.error?.contains("`recordIfMissing` is") == true }
-        missingRecordings.forEach { missing ->
-          logger.error(
-              """Recording is missing: ${missing.error}
-                |
-                |${missing.body}
-                |
-                |------------------------------------------------------------------------------------------
-                |To fix this problem please run `./gradlew :recordingIntegrationTest`.
-                |You need to export access tokens first, using script from the `sourcegraph/cody` repository:
-                |`agent/scripts/export-cody-http-recording-tokens.sh`
-                |------------------------------------------------------------------------------------------
-              """
-                  .trimMargin())
-        }
+        agent.server.testing_requestErrors(params = null).get()
         recordingsFuture.complete(null)
       }
       recordingsFuture.get(ASYNC_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)
@@ -95,23 +74,27 @@ open class CodyIntegrationTextFixture : BasePlatformTestCase(), LensListener {
   // Methods there are mostly idempotent though, so calling again for every test case should not
   // change anything.
   private fun initCredentialsAndAgent() {
-    val credentials = TestingCredentials.dotcom
+    assertNotNull(
+        "Unable to start agent in a timely fashion!",
+        CodyAgentService.getInstance(project)
+            .startAgent(project, additionalEnvs = mapOf("CODY_RECORDING_NAME" to recordingName()))
+            .completeOnTimeout(null, ASYNC_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)
+            .get())
+
+    val credentials = credentials()
     CodyPersistentAccountsHost(project)
         .addAccount(
             SourcegraphServerPath.from(credentials.serverEndpoint, ""),
             login = "test_user",
             displayName = "Test User",
             token = credentials.token ?: credentials.redactedToken,
             id = "random-unique-testing-id-1337")
-
-    assertNotNull(
-        "Unable to start agent in a timely fashion!",
-        CodyAgentService.getInstance(project)
-            .startAgent(project)
-            .completeOnTimeout(null, ASYNC_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)
-            .get())
   }
 
+  abstract fun recordingName(): String
+
+  abstract fun credentials(): TestingCredentials
+
   private fun checkInitialConditions() {
     // If you don't specify this system property with this setting when running the tests,
     // the tests will fail, because IntelliJ will run them from the EDT, which can't block.
@@ -130,19 +113,10 @@ open class CodyIntegrationTextFixture : BasePlatformTestCase(), LensListener {
     val isLightEditMode = LightEdit.owns(project)
     assertFalse("Project should not be in LightEdit mode", isLightEditMode)
 
-    // Check the initial state of the action's presentation
-    val action = ActionManager.getInstance().getAction("cody.documentCodeAction")
-    val event =
-        AnActionEvent.createFromAnAction(action, null, "", createEditorContext(myFixture.editor))
-    action.update(event)
-    val presentation = event.presentation
-    assertTrue("Action should be enabled", presentation.isEnabled)
-    assertTrue("Action should be visible", presentation.isVisible)
+    checkSuiteSpecificInitialConditions()
   }
 
-  private fun createEditorContext(editor: Editor): DataContext {
-    return (editor as? EditorEx)?.dataContext ?: DataContext.EMPTY_CONTEXT
-  }
+  abstract fun checkSuiteSpecificInitialConditions()
 
   // This provides a crude mechanism for specifying the caret position in the test file.
   private fun initCaretPosition() {