Skip to content

Commit

Permalink
Add integration tests for the chat (context files) (#1930)
Browse files Browse the repository at this point in the history
This is a follow up PR for
#1928.

## Test plan
1. Green CI
  • Loading branch information
mkondratek authored Aug 9, 2024
1 parent 1897c8b commit b4672ed
Show file tree
Hide file tree
Showing 17 changed files with 2,649 additions and 142 deletions.
1 change: 0 additions & 1 deletion build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,6 @@ fun Test.sharedIntegrationTestConfig(buildCodyDir: File, mode: String) {

environment(
"CODY_RECORDING_MODE" to mode,
"CODY_RECORDING_NAME" to "integration-test",
"CODY_RECORDING_DIRECTORY" to resourcesDir.resolve("recordings").absolutePath,
"CODY_SHIM_TESTING" to "true",
"CODY_TEMPERATURE_ZERO" to "true",
Expand Down
5 changes: 4 additions & 1 deletion src/integrationTest/kotlin/com/sourcegraph/cody/AllSuites.kt
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
package com.sourcegraph.cody

import com.sourcegraph.cody.chat.ChatTest
import com.sourcegraph.cody.edit.DocumentCodeTest
import com.sourcegraph.cody.util.RepeatableSuite
import org.junit.runner.RunWith
import org.junit.runners.Suite

@RunWith(RepeatableSuite::class) @Suite.SuiteClasses(DocumentCodeTest::class) class AllSuites
@RunWith(RepeatableSuite::class)
@Suite.SuiteClasses(ChatTest::class, DocumentCodeTest::class)
class AllSuites
102 changes: 102 additions & 0 deletions src/integrationTest/kotlin/com/sourcegraph/cody/chat/ChatTest.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package com.sourcegraph.cody.chat

import com.intellij.testFramework.runInEdtAndGet
import com.intellij.testFramework.runInEdtAndWait
import com.sourcegraph.cody.chat.ui.ContextFileActionLink
import com.sourcegraph.cody.context.ui.EnterpriseEnhancedContextPanel
import com.sourcegraph.cody.history.HistoryService
import com.sourcegraph.cody.history.state.EnhancedContextState
import com.sourcegraph.cody.history.state.RemoteRepositoryState
import com.sourcegraph.cody.util.CodyIntegrationTestFixture
import com.sourcegraph.cody.util.CustomJunitClassRunner
import com.sourcegraph.cody.util.TestingCredentials
import java.awt.Component
import java.awt.Container
import java.util.concurrent.TimeUnit
import junit.framework.TestCase
import org.awaitility.kotlin.await
import org.awaitility.kotlin.until
import org.junit.Test
import org.junit.runner.RunWith

@RunWith(CustomJunitClassRunner::class)
class ChatTest : CodyIntegrationTestFixture() {
override fun recordingName() = "chat"

override fun credentials() = TestingCredentials.enterprise

override fun checkSuiteSpecificInitialConditions() = Unit

@Test
fun testRemoteContextFileItems() {
val enhancedContextState =
EnhancedContextState().apply {
remoteRepositories.add(
RemoteRepositoryState().apply {
isEnabled = true
remoteUrl = "github.com/sourcegraph/cody"
codebaseName = "github.com/sourcegraph/cody"
})
}
HistoryService.getInstance(project).updateDefaultContextState(enhancedContextState)

val session = runInEdtAndGet { AgentChatSession.createNew(project) }

await.atMost(30, TimeUnit.SECONDS) until
{
(session.getPanel().contextView as EnterpriseEnhancedContextPanel)
.controller
.getConfiguredState()
.find { it.name == "github.com/sourcegraph/cody" && !it.isIgnored } != null
}

runInEdtAndWait { session.sendMessage("What is JSON RPC?", emptyList()) }

await.atMost(30, TimeUnit.SECONDS) until { !session.messages[0].contextFiles.isNullOrEmpty() }
await.atMost(30, TimeUnit.SECONDS) until { session.messages.size == 2 }
await.atMost(30, TimeUnit.SECONDS) until { session.messages[1].text?.isNotBlank() == true }

val linkPanels =
findComponentsRecursively(session.getPanel(), ContextFileActionLink::class.java)

TestCase.assertEquals(
listOf(
"cody agent/CHANGELOG.md",
"cody agent/README.md",
"cody agent/src/__tests__/chat-response-quality/README.md",
"cody agent/src/cli/command-jsonrpc-stdio.ts",
"cody agent/src/cli/command-jsonrpc-websocket.ts",
"cody agent/src/cli/command-root.ts",
"cody agent/src/cli/scip-codegen/JvmCodegen.ts",
"cody agent/src/cli/scip-codegen/JvmFormatter.ts",
"cody agent/src/jsonrpc-alias.ts",
"cody agent/src/local-e2e/README.md",
"cody lib/icons/README.md",
"cody vscode/src/graph/bfg/spawn-bfg.ts",
"cody vscode/src/jsonrpc/bfg-protocol.ts",
"cody vscode/src/jsonrpc/CodyJsonRpcErrorCode.ts",
"cody vscode/src/jsonrpc/embeddings-protocol.ts",
"cody vscode/src/jsonrpc/isRunningInsideAgent.ts",
"cody vscode/src/jsonrpc/jsonrpc.ts",
"cody vscode/src/jsonrpc/TextDocumentWithUri.test.ts",
"cody vscode/src/jsonrpc/TextDocumentWithUri.ts",
"cody web/lib/agent/agent.client.ts"),
linkPanels.map { panel -> panel.text })
}

private fun <A> findComponentsRecursively(parent: Component, targetClass: Class<A>): List<A> {
val result = mutableListOf<A>()

if (targetClass.isInstance(parent)) {
result.add(parent as A)
}

if (parent is Container) {
for (component in parent.components) {
result.addAll(findComponentsRecursively(component, targetClass))
}
}

return result
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
package com.sourcegraph.cody.edit

import com.intellij.openapi.actionSystem.ActionManager
import com.intellij.openapi.actionSystem.AnActionEvent
import com.intellij.openapi.actionSystem.DataContext
import com.intellij.openapi.editor.Editor
import com.intellij.openapi.editor.ex.EditorEx
import com.sourcegraph.cody.edit.actions.DocumentCodeAction
import com.sourcegraph.cody.edit.actions.lenses.EditAcceptAction
import com.sourcegraph.cody.edit.actions.lenses.EditCancelAction
Expand All @@ -10,15 +15,35 @@ import com.sourcegraph.cody.edit.widget.LensIcon
import com.sourcegraph.cody.edit.widget.LensLabel
import com.sourcegraph.cody.edit.widget.LensSpinner
import com.sourcegraph.cody.edit.widget.LensWidgetGroup
import com.sourcegraph.cody.util.CodyIntegrationTextFixture
import com.sourcegraph.cody.util.CodyIntegrationTestFixture
import com.sourcegraph.cody.util.CustomJunitClassRunner
import com.sourcegraph.cody.util.TestingCredentials
import org.hamcrest.Matchers.startsWith
import org.junit.Assert.assertThat
import org.junit.Test
import org.junit.runner.RunWith

@RunWith(CustomJunitClassRunner::class)
class DocumentCodeTest : CodyIntegrationTextFixture() {
class DocumentCodeTest : CodyIntegrationTestFixture() {
override fun recordingName() = "documentCode"

override fun credentials() = TestingCredentials.dotcom

override fun checkSuiteSpecificInitialConditions() {
// Check the initial state of the action's presentation
val action = ActionManager.getInstance().getAction("cody.documentCodeAction")
val event =
AnActionEvent.createFromAnAction(action, null, "", createEditorContext(myFixture.editor))
action.update(event)
val presentation = event.presentation
assertTrue("Action should be enabled", presentation.isEnabled)
assertTrue("Action should be visible", presentation.isVisible)
}

private fun createEditorContext(editor: Editor): DataContext {
return (editor as? EditorEx)?.dataContext ?: DataContext.EMPTY_CONTEXT
}

@Test
fun testGetsWorkingGroupLens() {
val codeLensGroup = runAndWaitForLenses(DocumentCodeAction.ID, EditCancelAction.ID)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,16 +1,11 @@
package com.sourcegraph.cody.util

import com.intellij.ide.lightEdit.LightEdit
import com.intellij.openapi.actionSystem.ActionManager
import com.intellij.openapi.actionSystem.AnActionEvent
import com.intellij.openapi.actionSystem.DataContext
import com.intellij.openapi.application.ApplicationManager
import com.intellij.openapi.application.WriteAction
import com.intellij.openapi.application.runInEdt
import com.intellij.openapi.command.WriteCommandAction
import com.intellij.openapi.diagnostic.Logger
import com.intellij.openapi.editor.Editor
import com.intellij.openapi.editor.ex.EditorEx
import com.intellij.openapi.fileEditor.FileDocumentManager
import com.intellij.openapi.project.DumbService
import com.intellij.openapi.project.Project
Expand All @@ -31,8 +26,8 @@ import java.util.concurrent.TimeUnit
import java.util.regex.Pattern
import junit.framework.TestCase

open class CodyIntegrationTextFixture : BasePlatformTestCase(), LensListener {
private val logger = Logger.getInstance(CodyIntegrationTextFixture::class.java)
abstract class CodyIntegrationTestFixture : BasePlatformTestCase(), LensListener {
private val logger = Logger.getInstance(CodyIntegrationTestFixture::class.java)
private val lensSubscribers =
mutableListOf<
Pair<(List<ProtocolCodeLens>) -> Boolean, CompletableFuture<LensWidgetGroup?>>>()
Expand Down Expand Up @@ -61,23 +56,7 @@ open class CodyIntegrationTextFixture : BasePlatformTestCase(), LensListener {

val recordingsFuture = CompletableFuture<Void>()
CodyAgentService.withAgent(project) { agent ->
val errors = agent.server.testingRequestErrors().get()
// We extract polly.js errors to notify users about the missing recordings, if any
val missingRecordings = errors.filter { it.error?.contains("`recordIfMissing` is") == true }
missingRecordings.forEach { missing ->
logger.error(
"""Recording is missing: ${missing.error}
|
|${missing.body}
|
|------------------------------------------------------------------------------------------
|To fix this problem please run `./gradlew :recordingIntegrationTest`.
|You need to export access tokens first, using script from the `sourcegraph/cody` repository:
|`agent/scripts/export-cody-http-recording-tokens.sh`
|------------------------------------------------------------------------------------------
"""
.trimMargin())
}
agent.server.testing_requestErrors(params = null).get()
recordingsFuture.complete(null)
}
recordingsFuture.get(ASYNC_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)
Expand All @@ -95,23 +74,27 @@ open class CodyIntegrationTextFixture : BasePlatformTestCase(), LensListener {
// Methods there are mostly idempotent though, so calling again for every test case should not
// change anything.
private fun initCredentialsAndAgent() {
val credentials = TestingCredentials.dotcom
assertNotNull(
"Unable to start agent in a timely fashion!",
CodyAgentService.getInstance(project)
.startAgent(project, additionalEnvs = mapOf("CODY_RECORDING_NAME" to recordingName()))
.completeOnTimeout(null, ASYNC_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)
.get())

val credentials = credentials()
CodyPersistentAccountsHost(project)
.addAccount(
SourcegraphServerPath.from(credentials.serverEndpoint, ""),
login = "test_user",
displayName = "Test User",
token = credentials.token ?: credentials.redactedToken,
id = "random-unique-testing-id-1337")

assertNotNull(
"Unable to start agent in a timely fashion!",
CodyAgentService.getInstance(project)
.startAgent(project)
.completeOnTimeout(null, ASYNC_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)
.get())
}

abstract fun recordingName(): String

abstract fun credentials(): TestingCredentials

private fun checkInitialConditions() {
// If you don't specify this system property with this setting when running the tests,
// the tests will fail, because IntelliJ will run them from the EDT, which can't block.
Expand All @@ -130,19 +113,10 @@ open class CodyIntegrationTextFixture : BasePlatformTestCase(), LensListener {
val isLightEditMode = LightEdit.owns(project)
assertFalse("Project should not be in LightEdit mode", isLightEditMode)

// Check the initial state of the action's presentation
val action = ActionManager.getInstance().getAction("cody.documentCodeAction")
val event =
AnActionEvent.createFromAnAction(action, null, "", createEditorContext(myFixture.editor))
action.update(event)
val presentation = event.presentation
assertTrue("Action should be enabled", presentation.isEnabled)
assertTrue("Action should be visible", presentation.isVisible)
checkSuiteSpecificInitialConditions()
}

private fun createEditorContext(editor: Editor): DataContext {
return (editor as? EditorEx)?.dataContext ?: DataContext.EMPTY_CONTEXT
}
abstract fun checkSuiteSpecificInitialConditions()

// This provides a crude mechanism for specifying the caret position in the test file.
private fun initCaretPosition() {
Expand Down
Loading

0 comments on commit b4672ed

Please sign in to comment.