[language-platform] adding the ability to fetch precise context

cli/src/client/context.ts

@@ -27,6 +27,7 @@ export async function createCodebaseContext(
         codebase,
         embeddingsSearch,
         new LocalKeywordContextFetcherMock(),
+        null,
         null
     )
 

lib/shared/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sourcegraph/cody-shared",
-  "version": "0.0.5",
+  "version": "0.0.0-codyintel.experimental.10",
   "description": "Cody shared library",
   "license": "Apache-2.0",
   "repository": {

lib/shared/src/chat/client.ts

@@ -3,6 +3,7 @@ import { ConfigurationWithAccessToken } from '../configuration'
 import { Editor } from '../editor'
 import { PrefilledOptions, withPreselectedOptions } from '../editor/withPreselectedOptions'
 import { SourcegraphEmbeddingsSearchClient } from '../embeddings/client'
+import { GraphContextFetcher } from '../graph-context'
 import { SourcegraphIntentDetectorClient } from '../intent-detector/client'
 import { SourcegraphBrowserCompletionsClient } from '../sourcegraph-api/completions/browserClient'
 import { CompletionsClientConfig, SourcegraphCompletionsClient } from '../sourcegraph-api/completions/client'
@@ -86,8 +87,8 @@ export async function createClient({
         }
 
         const embeddingsSearch = repoId ? new SourcegraphEmbeddingsSearchClient(graphqlClient, repoId, true) : null
-
-        const codebaseContext = new CodebaseContext(config, config.codebase, embeddingsSearch, null, null)
+        const graphContext = new GraphContextFetcher(graphqlClient, editor)
+        const codebaseContext = new CodebaseContext(config, config.codebase, embeddingsSearch, null, null, graphContext)
 
         const intentDetector = new SourcegraphIntentDetectorClient(graphqlClient, completionsClient)
 
@@ -141,8 +142,10 @@ export async function createClient({
             isMessageInProgress = true
             transcript.addInteraction(interaction)
 
-            const { prompt, contextFiles } = await transcript.getPromptForLastInteraction(getPreamble(config.codebase))
-            transcript.setUsedContextFilesForLastInteraction(contextFiles)
+            const { prompt, contextFiles, preciseContexts } = await transcript.getPromptForLastInteraction(
+                getPreamble(config.codebase)
+            )
+            transcript.setUsedContextFilesForLastInteraction(contextFiles, preciseContexts)
 
             const responsePrefix = interaction.getAssistantMessage().prefix ?? ''
             let rawText = ''

lib/shared/src/chat/recipes/chat-question.ts

@@ -20,7 +20,6 @@ export class ChatQuestion implements Recipe {
 
     public async getInteraction(humanChatInput: string, context: RecipeContext): Promise<Interaction | null> {
         const truncatedText = truncateText(humanChatInput, MAX_HUMAN_INPUT_TOKENS)
-
         return Promise.resolve(
             new Interaction(
                 { speaker: 'human', text: truncatedText, displayText: humanChatInput },
@@ -60,6 +59,9 @@ export class ChatQuestion implements Recipe {
         if (isCodebaseContextRequired) {
             const codebaseContextMessages = await codebaseContext.getContextMessages(text, numResults)
             contextMessages.push(...codebaseContextMessages)
+            // TODO: Put this in getContextMessages?
+            // Let's leave this here for now :)
+            contextMessages.push(...(await codebaseContext.getGraphContextMessages()))
         }
 
         const isEditorContextRequired = intentDetector.isEditorContextRequired(text)

lib/shared/src/chat/transcript/index.ts

@@ -1,4 +1,4 @@
-import { ContextFile, ContextMessage, OldContextMessage } from '../../codebase-context/messages'
+import { ContextFile, ContextMessage, OldContextMessage, PreciseContext } from '../../codebase-context/messages'
 import { PluginFunctionExecutionInfo } from '../../plugins/api/types'
 import { CHARS_PER_TOKEN, MAX_AVAILABLE_PROMPT_LENGTH } from '../../prompt/constants'
 import { PromptMixin } from '../../prompt/prompt-mixin'
@@ -35,6 +35,7 @@ export class Transcript {
                     context,
                     fullContext,
                     usedContextFiles,
+                    usedPreciseContext,
                     timestamp,
                     pluginExecutionInfos,
                 }) => {
@@ -59,6 +60,7 @@ export class Transcript {
                             })
                         ),
                         usedContextFiles || [],
+                        usedPreciseContext || [],
                         timestamp || new Date().toISOString(),
                         pluginExecutionInfos || []
                     )
@@ -166,9 +168,9 @@ export class Transcript {
         maxPromptLength: number = MAX_AVAILABLE_PROMPT_LENGTH,
         pluginsPrompt: Message[] = [],
         onlyHumanMessages: boolean = false
-    ): Promise<{ prompt: Message[]; contextFiles: ContextFile[] }> {
+    ): Promise<{ prompt: Message[]; contextFiles: ContextFile[]; preciseContexts: PreciseContext[] }> {
         if (this.interactions.length === 0) {
-            return { prompt: [], contextFiles: [] }
+            return { prompt: [], contextFiles: [], preciseContexts: [] }
         }
 
         const lastInteractionWithContextIndex = await this.getLastInteractionWithContextIndex()
@@ -193,11 +195,17 @@ export class Transcript {
         )
         // Return what context fits in the window
         const contextFiles: ContextFile[] = []
+        const preciseContexts: PreciseContext[] = []
         for (const msg of truncatedMessages) {
             const contextFile = (msg as ContextMessage).file
             if (contextFile) {
                 contextFiles.push(contextFile)
             }
+
+            const preciseContext = (msg as ContextMessage).preciseContext
+            if (preciseContext) {
+                preciseContexts.push(preciseContext)
+            }
         }
 
         // Filter out extraneous fields from ContextMessage instances
@@ -206,17 +214,23 @@ export class Transcript {
         return {
             prompt: [...preamble, ...truncatedMessages],
             contextFiles,
+            preciseContexts,
         }
     }
 
     public setUsedContextFilesForLastInteraction(
         contextFiles: ContextFile[],
+        preciseContexts: PreciseContext[] = [],
         pluginExecutionInfos: PluginFunctionExecutionInfo[] = []
     ): void {
         if (this.interactions.length === 0) {
             throw new Error('Cannot set context files for empty transcript')
         }
-        this.interactions[this.interactions.length - 1].setUsedContext(contextFiles, pluginExecutionInfos)
+        this.interactions[this.interactions.length - 1].setUsedContext(
+            contextFiles,
+            pluginExecutionInfos,
+            preciseContexts
+        )
     }
 
     public toChat(): ChatMessage[] {

lib/shared/src/chat/transcript/interaction.ts

@@ -1,4 +1,4 @@
-import { ContextFile, ContextMessage } from '../../codebase-context/messages'
+import { ContextFile, ContextMessage, PreciseContext } from '../../codebase-context/messages'
 import { PluginFunctionExecutionInfo } from '../../plugins/api/types'
 
 import { ChatMessage, InteractionMessage } from './messages'
@@ -8,6 +8,7 @@ export interface InteractionJSON {
     assistantMessage: InteractionMessage
     fullContext: ContextMessage[]
     usedContextFiles: ContextFile[]
+    usedPreciseContext: PreciseContext[]
     pluginExecutionInfos: PluginFunctionExecutionInfo[]
     timestamp: string
 
@@ -21,6 +22,7 @@ export class Interaction {
         private assistantMessage: InteractionMessage,
         private fullContext: Promise<ContextMessage[]>,
         private usedContextFiles: ContextFile[],
+        private usedPreciseContext: PreciseContext[] = [],
         public readonly timestamp: string = new Date().toISOString(),
         private pluginExecutionInfos: PluginFunctionExecutionInfo[] = []
     ) {}
@@ -47,9 +49,14 @@ export class Interaction {
         return contextMessages.length > 0
     }
 
-    public setUsedContext(usedContextFiles: ContextFile[], pluginExecutionInfos: PluginFunctionExecutionInfo[]): void {
+    public setUsedContext(
+        usedContextFiles: ContextFile[],
+        pluginExecutionInfos: PluginFunctionExecutionInfo[],
+        usedPreciseContext: PreciseContext[]
+    ): void {
         this.usedContextFiles = usedContextFiles
         this.pluginExecutionInfos = pluginExecutionInfos
+        this.usedPreciseContext = usedPreciseContext
     }
 
     /**
@@ -61,6 +68,7 @@ export class Interaction {
             {
                 ...this.assistantMessage,
                 contextFiles: this.usedContextFiles,
+                preciseContext: this.usedPreciseContext,
                 pluginExecutionInfos: this.pluginExecutionInfos,
             },
         ]
@@ -77,6 +85,7 @@ export class Interaction {
             assistantMessage: this.assistantMessage,
             fullContext: await this.fullContext,
             usedContextFiles: this.usedContextFiles,
+            usedPreciseContext: this.usedPreciseContext,
             pluginExecutionInfos: this.pluginExecutionInfos,
             timestamp: this.timestamp,
         }

lib/shared/src/chat/transcript/messages.ts

@@ -1,4 +1,4 @@
-import { ContextFile } from '../../codebase-context/messages'
+import { ContextFile, PreciseContext } from '../../codebase-context/messages'
 import { PluginFunctionExecutionInfo } from '../../plugins/api/types'
 import { Message } from '../../sourcegraph-api'
 
@@ -13,6 +13,7 @@ export interface ChatButton {
 export interface ChatMessage extends Message {
     displayText?: string
     contextFiles?: ContextFile[]
+    preciseContext?: PreciseContext[]
     pluginExecutionInfos?: PluginFunctionExecutionInfo[]
     buttons?: ChatButton[]
     data?: any

lib/shared/src/chat/transcript/transcript.test.ts

@@ -81,6 +81,7 @@ describe('Transcript', () => {
                     'dummy-codebase',
                     embeddings,
                     defaultKeywordContextFetcher,
+                    null,
                     null
                 ),
             })
@@ -118,6 +119,7 @@ describe('Transcript', () => {
                     'dummy-codebase',
                     embeddings,
                     defaultKeywordContextFetcher,
+                    null,
                     null
                 ),
                 firstInteraction: true,
@@ -153,6 +155,7 @@ describe('Transcript', () => {
             'dummy-codebase',
             embeddings,
             defaultKeywordContextFetcher,
+            null,
             null
         )
 
@@ -239,6 +242,7 @@ describe('Transcript', () => {
             'dummy-codebase',
             embeddings,
             defaultKeywordContextFetcher,
+            null,
             null
         )
 
@@ -316,6 +320,7 @@ describe('Transcript', () => {
             'dummy-codebase',
             embeddings,
             defaultKeywordContextFetcher,
+            null,
             null
         )
 

lib/shared/src/chat/useClient.ts

@@ -5,6 +5,7 @@ import { isErrorLike } from '../common'
 import { ConfigurationWithAccessToken } from '../configuration'
 import { Editor, NoopEditor } from '../editor'
 import { PrefilledOptions, withPreselectedOptions } from '../editor/withPreselectedOptions'
+import { GraphContextFetcher } from '../graph-context'
 import { SourcegraphIntentDetectorClient } from '../intent-detector/client'
 import { SourcegraphBrowserCompletionsClient } from '../sourcegraph-api/completions/browserClient'
 import { SourcegraphGraphQLAPIClient } from '../sourcegraph-api/graphql'
@@ -278,14 +279,15 @@ export const useClient = ({
                     repoNames.push(activeEditor.repoName)
                 }
             }
-
+            const graphContext = new GraphContextFetcher(graphqlClient, editor)
             const unifiedContextFetcherClient = new UnifiedContextFetcherClient(graphqlClient, repoIds)
             const codebaseContext = new CodebaseContext(
                 config,
                 undefined,
                 null,
                 null,
                 null,
+                graphContext,
                 unifiedContextFetcherClient
             )
 
@@ -307,10 +309,10 @@ export const useClient = ({
             setIsMessageInProgressState(true)
             onEvent?.('submit')
 
-            const { prompt, contextFiles } = await transcript.getPromptForLastInteraction(
+            const { prompt, contextFiles, preciseContexts } = await transcript.getPromptForLastInteraction(
                 getMultiRepoPreamble(repoNames)
             )
-            transcript.setUsedContextFilesForLastInteraction(contextFiles)
+            transcript.setUsedContextFilesForLastInteraction(contextFiles, preciseContexts)
 
             const responsePrefix = interaction.getAssistantMessage().prefix ?? ''
             let rawText = ''

lib/shared/src/codebase-context/index.ts

@@ -1,5 +1,6 @@
 import { Configuration } from '../configuration'
 import { EmbeddingsSearch } from '../embeddings'
+import { GraphContextFetcher } from '../graph-context'
 import { ContextResult, FilenameContextFetcher, KeywordContextFetcher } from '../local-context'
 import { isMarkdownFile, populateCodeContextTemplate, populateMarkdownContextTemplate } from '../prompt/templates'
 import { Message } from '../sourcegraph-api'
@@ -22,6 +23,7 @@ export class CodebaseContext {
         private embeddings: EmbeddingsSearch | null,
         private keywords: KeywordContextFetcher | null,
         private filenames: FilenameContextFetcher | null,
+        private graph: GraphContextFetcher | null,
         private unifiedContextFetcher?: UnifiedContextFetcher | null,
         private rerank?: (query: string, results: ContextResult[]) => Promise<ContextResult[]>
     ) {}
@@ -208,6 +210,41 @@ export class CodebaseContext {
         const results = await this.filenames.getContext(query, options.numCodeResults + options.numTextResults)
         return results
     }
+
+    public async getGraphContextMessages(): Promise<ContextMessage[]> {
+        // NOTE(auguste): I recommend checking out populateCodeContextTemplate and using
+        // that in the long-term, but this will do for now :)
+
+        if (!this.graph) {
+            return Promise.resolve([])
+        }
+
+        const contextMessages: ContextMessage[] = []
+        for (const preciseContext of await this.graph.getContext()) {
+            const fence = '```'
+            const symbolName = `${preciseContext.symbol.scipName} (${preciseContext.symbol.scipDescriptorSuffix})`
+            // TODO - check comparison value
+            const repo =
+                preciseContext.repositoryName !== this.getCodebase()
+                    ? ` in repository ${preciseContext.repositoryName}`
+                    : ''
+            const text = `The symbol ${symbolName} is defined in the file ${preciseContext.filepath}${repo} as:\n\n${fence}${preciseContext.definitionSnippet}${fence}`
+            console.log({ text }) // DEBUGGING
+
+            contextMessages.push({
+                speaker: 'human',
+                file: {
+                    repoName: preciseContext.repositoryName,
+                    fileName: preciseContext.filepath,
+                },
+                preciseContext,
+                text,
+            })
+            contextMessages.push({ speaker: 'assistant', text: 'okay' })
+        }
+
+        return contextMessages
+    }
 }
 
 function groupResultsByFile(results: EmbeddingsSearchResult[]): { file: ContextFile; results: string[] }[] {

lib/shared/src/codebase-context/messages.ts

@@ -15,8 +15,21 @@ export interface ContextFile {
     source?: ContextFileSource
 }
 
+export interface PreciseContext {
+    symbol: {
+        scipName: string
+        scipDescriptorSuffix: string
+        fuzzyName?: string
+    }
+    definitionSnippet: string
+    repositoryName: string
+    filepath: string
+    canonicalLocationURL: string
+}
+
 export interface ContextMessage extends Message {
     file?: ContextFile
+    preciseContext?: PreciseContext
 }
 
 export interface OldContextMessage extends Message {