From 87a620fb3f1aca5bdce44bdb24e13aead5c77e91 Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Tue, 25 Jul 2023 14:36:23 +0100 Subject: [PATCH 01/24] init --- lib/shared/src/chat/recipes/fixup.ts | 51 +++++++++++++++++----------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/lib/shared/src/chat/recipes/fixup.ts b/lib/shared/src/chat/recipes/fixup.ts index 1d2a9fbc3dbd..5e9591e97354 100644 --- a/lib/shared/src/chat/recipes/fixup.ts +++ b/lib/shared/src/chat/recipes/fixup.ts @@ -5,7 +5,7 @@ import { truncateText, truncateTextStart } from '../../prompt/truncation' import { BufferedBotResponseSubscriber } from '../bot-response-multiplexer' import { Interaction } from '../transcript/interaction' -import { contentSanitizer, numResults } from './helpers' +import { contentSanitizer } from './helpers' import { Recipe, RecipeContext, RecipeID } from './recipe' export class Fixup implements Recipe { @@ -32,7 +32,6 @@ export class Fixup implements Recipe { // TODO: Move prompt suffix from recipe to chat view. It has other subscribers. const promptText = Fixup.prompt .replace('{humanInput}', truncateText(humanChatInput, MAX_HUMAN_INPUT_TOKENS)) - .replace('{responseMultiplexerPrompt}', context.responseMultiplexer.prompt()) .replace('{truncateFollowingText}', truncateText(selection.followingText, quarterFileContext)) .replace('{selectedText}', selection.selectedText) .replace('{truncateTextStart}', truncateTextStart(selection.precedingText, quarterFileContext)) @@ -75,28 +74,42 @@ export class Fixup implements Recipe { // Get context from editor private async getContextMessages(text: string, codebaseContext: CodebaseContext): Promise { - const contextMessages: ContextMessage[] = await codebaseContext.getContextMessages(text, numResults) - return contextMessages + // const contextMessages: ContextMessage[] = await codebaseContext.getContextMessages(text, numResults) + return Promise.resolve([]) } + public static readonly promptPreamble = '' + // Prompt Templates public static readonly prompt = ` - This is part of the file {fileName}. The part of the file I have selected is highlighted with tags. You are helping me to work on that part as my coding assistant. - Follow the instructions in the selected part plus the additional instructions to produce a rewritten replacement for only the selected part. - Put the rewritten replacement inside tags. I only want to see the code within . - Do not move code from outside the selection into the selection in your reply. - Do not remove code inside the tags that might be being used by the code outside the tags. - It is OK to provide some commentary within the replacement . - It is not acceptable to enclose the rewritten replacement with markdowns. - Only provide me with the replacement and nothing else. - If it doesn't make sense, you do not need to provide . Instead, tell me how I can help you to understand my request. + - You are an AI programming assistant who is an expert in rewriting code to meet given instructions. + - You should think step-by-step to plan your rewritten code before producing the final output. + - You should use code above and below the selection to help you plan your rewritten code. + - You should use code below the selection to help you plan your rewritten code. + - Unless you have reason to believe otherwise, you should assume that the user wants you to edit the code in their selection. + - It is not acceptable to use Markdown in your response. You should not produce Markdown-formatted code blocks. + - Enclose your response in XML tags. Do not provide anything else. + + This is part of the file {fileName}. + + I have the following code above my selection: + + {truncateTextStart} + + + I have the following code below my selection: + + {truncateFollowingText} + - \`\`\` - {truncateTextStart}{selectedText}{truncateFollowingText} - \`\`\` + I have the following code in my selection: + + {selectedText} + - Additional Instruction: - - {humanInput} - - {responseMultiplexerPrompt} + I'd like you to rewrite it using the following instructions + + {humanInput} + ` } From 3dadcb07a09c2380ee412ef72fa739f618a19744 Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Tue, 25 Jul 2023 15:41:16 +0100 Subject: [PATCH 02/24] more --- lib/shared/src/chat/recipes/fixup.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/shared/src/chat/recipes/fixup.ts b/lib/shared/src/chat/recipes/fixup.ts index 5e9591e97354..432576a2ff67 100644 --- a/lib/shared/src/chat/recipes/fixup.ts +++ b/lib/shared/src/chat/recipes/fixup.ts @@ -78,15 +78,13 @@ export class Fixup implements Recipe { return Promise.resolve([]) } - public static readonly promptPreamble = '' - // Prompt Templates public static readonly prompt = ` - You are an AI programming assistant who is an expert in rewriting code to meet given instructions. - You should think step-by-step to plan your rewritten code before producing the final output. - You should use code above and below the selection to help you plan your rewritten code. - - You should use code below the selection to help you plan your rewritten code. - Unless you have reason to believe otherwise, you should assume that the user wants you to edit the code in their selection. + - You should ensure the rewritten code matches the indentation and whitespace of the code in the users selection. - It is not acceptable to use Markdown in your response. You should not produce Markdown-formatted code blocks. - Enclose your response in XML tags. Do not provide anything else. @@ -107,7 +105,7 @@ export class Fixup implements Recipe { {selectedText} - I'd like you to rewrite it using the following instructions + I'd like you to rewrite it using the following instructions: {humanInput} From 6de23b5ea3c530150e32b8c305bc16e41a440651 Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Tue, 25 Jul 2023 16:43:12 +0100 Subject: [PATCH 03/24] update prompt --- lib/shared/src/chat/recipes/fixup.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/shared/src/chat/recipes/fixup.ts b/lib/shared/src/chat/recipes/fixup.ts index 432576a2ff67..91829ccfbe37 100644 --- a/lib/shared/src/chat/recipes/fixup.ts +++ b/lib/shared/src/chat/recipes/fixup.ts @@ -82,32 +82,34 @@ export class Fixup implements Recipe { public static readonly prompt = ` - You are an AI programming assistant who is an expert in rewriting code to meet given instructions. - You should think step-by-step to plan your rewritten code before producing the final output. - - You should use code above and below the selection to help you plan your rewritten code. - Unless you have reason to believe otherwise, you should assume that the user wants you to edit the code in their selection. - - You should ensure the rewritten code matches the indentation and whitespace of the code in the users selection. + - You should ensure the rewritten code matches the indentation and whitespace of the code in the users' selection. - It is not acceptable to use Markdown in your response. You should not produce Markdown-formatted code blocks. + - You will be provided with code that is above the users' selection, enclosed in XML tags. You can use this code, if relevant, to help you plan your rewritten code. + - You will be provided with code that is below the users' selection, enclosed in XML tags. You can use this code, if relevant, to help you plan your rewritten code. + - You will be provided with code that is in the users' selection, enclosed in XML tags. You must use this code to help you plan your rewritten code. + - You will be provided with instructions on how to modify this code, enclosed in XML tags. You must follow these instructions carefully and to the letter. - Enclose your response in XML tags. Do not provide anything else. This is part of the file {fileName}. - I have the following code above my selection: + The user has the following code above their selection: {truncateTextStart} - I have the following code below my selection: + The user has the following code below their selection: {truncateFollowingText} - I have the following code in my selection: + The user has the following code within their selection: {selectedText} - I'd like you to rewrite it using the following instructions: + You should rewrite this code using the following instructions: {humanInput} - -` + ` } From 6dfc6bba3b09bd895b0c8d4165d6e38b3485fa5c Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Tue, 25 Jul 2023 17:05:03 +0100 Subject: [PATCH 04/24] better context --- lib/shared/src/chat/recipes/fixup.ts | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/lib/shared/src/chat/recipes/fixup.ts b/lib/shared/src/chat/recipes/fixup.ts index 91829ccfbe37..fe35c0df6ffb 100644 --- a/lib/shared/src/chat/recipes/fixup.ts +++ b/lib/shared/src/chat/recipes/fixup.ts @@ -1,11 +1,9 @@ -import { CodebaseContext } from '../../codebase-context' -import { ContextMessage } from '../../codebase-context/messages' import { MAX_CURRENT_FILE_TOKENS, MAX_HUMAN_INPUT_TOKENS } from '../../prompt/constants' import { truncateText, truncateTextStart } from '../../prompt/truncation' import { BufferedBotResponseSubscriber } from '../bot-response-multiplexer' import { Interaction } from '../transcript/interaction' -import { contentSanitizer } from './helpers' +import { contentSanitizer, getContextMessagesFromSelection } from './helpers' import { Recipe, RecipeContext, RecipeID } from './recipe' export class Fixup implements Recipe { @@ -27,14 +25,17 @@ export class Fixup implements Recipe { return null } + const truncatedPrecedingText = truncateTextStart(selection.precedingText, quarterFileContext) + const truncatedFollowingText = truncateText(selection.followingText, quarterFileContext) + // Reconstruct Cody's prompt using user's context // Replace placeholders in reverse order to avoid collisions if a placeholder occurs in the input // TODO: Move prompt suffix from recipe to chat view. It has other subscribers. const promptText = Fixup.prompt .replace('{humanInput}', truncateText(humanChatInput, MAX_HUMAN_INPUT_TOKENS)) - .replace('{truncateFollowingText}', truncateText(selection.followingText, quarterFileContext)) + .replace('{truncateTextStart}', truncatedPrecedingText) .replace('{selectedText}', selection.selectedText) - .replace('{truncateTextStart}', truncateTextStart(selection.precedingText, quarterFileContext)) + .replace('{truncateFollowingText}', truncatedFollowingText) .replace('{fileName}', selection.fileName) context.responseMultiplexer.sub( @@ -66,18 +67,18 @@ export class Fixup implements Recipe { speaker: 'assistant', prefix: 'Check your document for updates from Cody.\n', }, - this.getContextMessages(selection.selectedText, context.codebaseContext), + getContextMessagesFromSelection( + selection.selectedText, + truncatedPrecedingText, + truncatedFollowingText, + selection, + context.codebaseContext + ), [] ) ) } - // Get context from editor - private async getContextMessages(text: string, codebaseContext: CodebaseContext): Promise { - // const contextMessages: ContextMessage[] = await codebaseContext.getContextMessages(text, numResults) - return Promise.resolve([]) - } - // Prompt Templates public static readonly prompt = ` - You are an AI programming assistant who is an expert in rewriting code to meet given instructions. From b662d0b3b35e3ef03dc80eaff9afb35935a97ae5 Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Wed, 26 Jul 2023 11:13:12 +0100 Subject: [PATCH 05/24] type --- vscode/webviews/Recipes.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vscode/webviews/Recipes.tsx b/vscode/webviews/Recipes.tsx index 67576d6af8a9..194f0be7c84d 100644 --- a/vscode/webviews/Recipes.tsx +++ b/vscode/webviews/Recipes.tsx @@ -6,7 +6,9 @@ import { VSCodeWrapper } from './utils/VSCodeApi' import styles from './Recipes.module.css' -export const recipesList = { +type ClickableRecipeID = Exclude + +export const recipesList: Record = { 'explain-code-detailed': 'Explain selected code (detailed)', 'explain-code-high-level': 'Explain selected code (high level)', 'generate-unit-test': 'Generate a unit test', @@ -104,7 +106,7 @@ export const Recipes: React.FunctionComponent<{ key={key} className={styles.recipeButton} type="button" - onClick={() => onRecipeClick(key as RecipeID)} + onClick={() => onRecipeClick(key as ClickableRecipeID)} > {value} From 94bb0198dc9526ef7c4465b7e70f35630284fda4 Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Wed, 26 Jul 2023 16:50:25 +0100 Subject: [PATCH 06/24] add intent detection --- cli/src/client/index.ts | 4 +- lib/shared/src/chat/client.ts | 2 +- lib/shared/src/chat/recipes/fixup.ts | 89 ++++++++++++++++++-- lib/shared/src/chat/recipes/inline-chat.ts | 35 ++++++++ lib/shared/src/chat/useClient.ts | 2 +- lib/shared/src/intent-detector/client.ts | 95 +++++++++++++++++++++- lib/shared/src/intent-detector/index.ts | 19 +++++ lib/shared/src/test/mocks.ts | 6 +- vscode/package.json | 2 +- vscode/src/external-services.ts | 2 +- vscode/src/logged-rerank.ts | 4 +- 11 files changed, 241 insertions(+), 19 deletions(-) diff --git a/cli/src/client/index.ts b/cli/src/client/index.ts index 06c994a37c54..5c319eab2fb7 100644 --- a/cli/src/client/index.ts +++ b/cli/src/client/index.ts @@ -49,8 +49,6 @@ export async function getClient({ codebase, endpoint, context: contextType, debu process.exit(1) } - const intentDetector = new SourcegraphIntentDetectorClient(sourcegraphClient) - const completionsClient = new SourcegraphNodeCompletionsClient({ serverEndpoint: endpoint, accessToken, @@ -58,5 +56,7 @@ export async function getClient({ codebase, endpoint, context: contextType, debu customHeaders: {}, }) + const intentDetector = new SourcegraphIntentDetectorClient(sourcegraphClient, completionsClient) + return { codebaseContext, intentDetector, completionsClient } } diff --git a/lib/shared/src/chat/client.ts b/lib/shared/src/chat/client.ts index d4516b89a962..55da6fb7463d 100644 --- a/lib/shared/src/chat/client.ts +++ b/lib/shared/src/chat/client.ts @@ -76,7 +76,7 @@ export async function createClient({ const codebaseContext = new CodebaseContext(config, config.codebase, embeddingsSearch, null, null) - const intentDetector = new SourcegraphIntentDetectorClient(graphqlClient) + const intentDetector = new SourcegraphIntentDetectorClient(graphqlClient, completionsClient) const transcript = initialTranscript || new Transcript() diff --git a/lib/shared/src/chat/recipes/fixup.ts b/lib/shared/src/chat/recipes/fixup.ts index fe35c0df6ffb..6cce4aef8c18 100644 --- a/lib/shared/src/chat/recipes/fixup.ts +++ b/lib/shared/src/chat/recipes/fixup.ts @@ -1,11 +1,56 @@ +import { ContextMessage } from '../../codebase-context/messages' +import { IntentClassificationOption } from '../../intent-detector' import { MAX_CURRENT_FILE_TOKENS, MAX_HUMAN_INPUT_TOKENS } from '../../prompt/constants' import { truncateText, truncateTextStart } from '../../prompt/truncation' import { BufferedBotResponseSubscriber } from '../bot-response-multiplexer' import { Interaction } from '../transcript/interaction' -import { contentSanitizer, getContextMessagesFromSelection } from './helpers' +import { contentSanitizer, getContextMessagesFromSelection, numResults } from './helpers' import { Recipe, RecipeContext, RecipeID } from './recipe' +const FixupIntentClassification: IntentClassificationOption[] = [ + { + /** + * Context: + * - preceding text, selected text, following text + * - maximum embeddings from code files and text files + */ + id: 'explain', + description: 'Explain the selected code', + examplePrompts: ['What does this code do?', 'How does this code work?'], + }, + { + /** + * Context: + * - preceding text, selected text, following text + * - limited embeddings from code files + */ + id: 'fix', // mostly context from current file, code files + description: 'Fix a problem in the selected code', + examplePrompts: ['Update this code to use async/await', 'Fix this code'], + }, + { + /** + * Context: + * - selected text + * - limited embeddings from code files + */ + id: 'document', // only the current selection, context + description: 'Generate documentation for the selected code.', + examplePrompts: ['Add a docstring for this function', 'Write comments to explain this code'], + }, + { + /** + * Context: + * - preceding text, selected text, following text + * - limited embeddings from code files + */ + id: 'test', + description: 'Generate tests for the selected code', + examplePrompts: ['Write a test for this function', 'Add a test for this code'], + }, +] + export class Fixup implements Recipe { public id: RecipeID = 'fixup' @@ -56,6 +101,40 @@ export class Fixup implements Recipe { }) ) + let dynamicContext: Promise + const intent = await context.intentDetector.classifyIntentFromOptions(humanChatInput, FixupIntentClassification) + console.log('INLINE FIXUP INTENT', intent) + switch (intent) { + case 'explain': + dynamicContext = context.codebaseContext.getContextMessages(selection.selectedText, numResults) + break + case 'fix': + dynamicContext = getContextMessagesFromSelection( + selection.selectedText, + truncatedPrecedingText, + truncatedFollowingText, + selection, + context.codebaseContext + ) + break + case 'document': + // todo: better context gather for documenting? currently just using selection - no context + dynamicContext = Promise.resolve([]) + break + case 'test': + // todo: better context gathering for tests + dynamicContext = context.codebaseContext.getContextMessages(selection.selectedText, numResults) + break + default: + dynamicContext = getContextMessagesFromSelection( + selection.selectedText, + truncatedPrecedingText, + truncatedFollowingText, + selection, + context.codebaseContext + ) + } + return Promise.resolve( new Interaction( { @@ -67,13 +146,7 @@ export class Fixup implements Recipe { speaker: 'assistant', prefix: 'Check your document for updates from Cody.\n', }, - getContextMessagesFromSelection( - selection.selectedText, - truncatedPrecedingText, - truncatedFollowingText, - selection, - context.codebaseContext - ), + dynamicContext, [] ) ) diff --git a/lib/shared/src/chat/recipes/inline-chat.ts b/lib/shared/src/chat/recipes/inline-chat.ts index 53108bfff73b..1f582b81e4d8 100644 --- a/lib/shared/src/chat/recipes/inline-chat.ts +++ b/lib/shared/src/chat/recipes/inline-chat.ts @@ -1,6 +1,7 @@ import { CodebaseContext } from '../../codebase-context' import { ContextMessage } from '../../codebase-context/messages' import { ActiveTextEditorSelection, Editor } from '../../editor' +import { IntentClassificationOption } from '../../intent-detector' import { MAX_HUMAN_INPUT_TOKENS, MAX_RECIPE_INPUT_TOKENS, MAX_RECIPE_SURROUNDING_TOKENS } from '../../prompt/constants' import { truncateText } from '../../prompt/truncation' import { Interaction } from '../transcript/interaction' @@ -11,6 +12,30 @@ import { commandRegex } from './helpers' import { InlineTouch } from './inline-touch' import { Recipe, RecipeContext, RecipeID } from './recipe' +const InlineChatClassification: IntentClassificationOption[] = [ + { + id: 'chat', + description: 'Discuss the selected code', + examplePrompts: ['What does this do?', 'How does this work?', 'How would I improve this?'], + }, + { + id: 'fix', + description: 'Fix a problem in the selected code', + examplePrompts: [ + 'Update this code', + 'Fix this code', + 'Change this code', + 'Rewrite this code', + 'Add to this code', + ], + }, + { + id: 'touch', + description: 'Generate a new file from the selected code', + examplePrompts: ['Write a test for this code', 'Create a new file from this code'], + }, +] + export class InlineChat implements Recipe { public id: RecipeID = 'inline-chat' @@ -27,6 +52,16 @@ export class InlineChat implements Recipe { return new Fixup().getInteraction(humanChatInput.replace(commandRegex.fix, ''), context) } + const intent = await context.intentDetector.classifyIntentFromOptions(humanChatInput, InlineChatClassification) + console.log('INLINE CHAT INTENT:', intent) + // TODO: We currently check intent in two places for inline chat. Do it in one place. + // if (intent === 'touch') { + // return new InlineTouch(this.debug).getInteraction(humanChatInput.replace(commandRegex.touch, ''), context) + // } + // if (intent === 'fix') { + // return new Fixup().getInteraction(humanChatInput, context) + // } + const selection = context.editor.controllers?.inline?.selection if (!humanChatInput || !selection) { await context.editor.showWarningMessage('Failed to start Inline Chat: empty input or selection.') diff --git a/lib/shared/src/chat/useClient.ts b/lib/shared/src/chat/useClient.ts index f7f9e8198c75..3cda5f3d3fa3 100644 --- a/lib/shared/src/chat/useClient.ts +++ b/lib/shared/src/chat/useClient.ts @@ -134,7 +134,7 @@ export const useClient = ({ const completionsClient = new SourcegraphBrowserCompletionsClient(config) const chatClient = new ChatClient(completionsClient) const graphqlClient = new SourcegraphGraphQLAPIClient(config) - const intentDetector = new SourcegraphIntentDetectorClient(graphqlClient) + const intentDetector = new SourcegraphIntentDetectorClient(graphqlClient, completionsClient) return { graphqlClient, chatClient, intentDetector } }, [config]) diff --git a/lib/shared/src/intent-detector/client.ts b/lib/shared/src/intent-detector/client.ts index 1343a7c2ffc2..107b0fcce830 100644 --- a/lib/shared/src/intent-detector/client.ts +++ b/lib/shared/src/intent-detector/client.ts @@ -1,11 +1,17 @@ +import { ANSWER_TOKENS } from '../prompt/constants' +import { Message } from '../sourcegraph-api' +import { SourcegraphCompletionsClient } from '../sourcegraph-api/completions/client' import { SourcegraphGraphQLAPIClient } from '../sourcegraph-api/graphql' -import { IntentDetector } from '.' +import { IntentClassificationOption, IntentDetector } from '.' const editorRegexps = [/editor/, /(open|current|this)\s+file/, /current(ly)?\s+open/, /have\s+open/] export class SourcegraphIntentDetectorClient implements IntentDetector { - constructor(private client: SourcegraphGraphQLAPIClient) {} + constructor( + private client: SourcegraphGraphQLAPIClient, + private completionsClient: SourcegraphCompletionsClient + ) {} public isCodebaseContextRequired(input: string): Promise { return this.client.isContextRequiredForQuery(input) @@ -22,4 +28,89 @@ export class SourcegraphIntentDetectorClient implements IntentDetector { } return false } + + private buildInitialPrompt(options: IntentClassificationOption[]): string { + const functions = options + .map( + ({ id, description }) => ` +Function Id: ${id} +Function Description: ${description} +` + ) + .join('\n') + + return prompt.replace('{functions}', functions) + } + + private buildExampleTranscript(options: IntentClassificationOption[]): Message[] { + const messages = options.flatMap(({ id, examplePrompts }) => + examplePrompts.flatMap( + example => + [ + { + speaker: 'human', + text: example, + }, + { + speaker: 'assistant', + text: `${id}`, + }, + ] as const + ) + ) + + return messages + } + + public async classifyIntentFromOptions( + input: string, + options: IntentClassificationOption[] + ): Promise { + const initialPrompt = this.buildInitialPrompt(options) + const exampleTranscript = this.buildExampleTranscript(options) + + const result = await this.completionsClient.complete({ + fast: true, + temperature: 0.2, + maxTokensToSample: ANSWER_TOKENS, + topK: -1, + topP: -1, + messages: [ + { + speaker: 'human', + text: initialPrompt, + }, + { + speaker: 'assistant', + text: 'Ok.', + }, + ...exampleTranscript, + { + speaker: 'human', + text: input, + }, + { + speaker: 'assistant', + }, + ], + }) + + const responseClassification = result.completion.match(/(.*?)<\/classification>/)?.[1] + + if (!responseClassification) { + return null + } + + return options.find(option => option.id === responseClassification)?.id ?? null + } } + +const prompt = ` +You are an AI chatbot in a code editor. You are at expert at understanding the request of a software developer and selecting an available function to perform that request. +Think step-by-step to understand the request. +Only provide your response if you know the answer or can make a well-informed guess, respond with "unknown". +Enclose your response in XML tags. Do not provide anything else. + +Available functions: +{functions} +` diff --git a/lib/shared/src/intent-detector/index.ts b/lib/shared/src/intent-detector/index.ts index a8f5af296fdb..9fa191e6b95c 100644 --- a/lib/shared/src/intent-detector/index.ts +++ b/lib/shared/src/intent-detector/index.ts @@ -1,4 +1,23 @@ +export interface IntentClassificationOption { + /** + * An identifier for this intent. + * This is what will be returned by the classifier. + */ + id: string + /** + * A description for this intent. + * Be specific in order to help the LLM understand the intent. + */ + description: string + /** + * Example prompts that match this intent. + * E.g. for a documentation intent: "Add documentation for this function" + */ + examplePrompts: string[] +} + export interface IntentDetector { isCodebaseContextRequired(input: string): Promise isEditorContextRequired(input: string): boolean | Error + classifyIntentFromOptions(input: string, options: IntentClassificationOption[]): Promise } diff --git a/lib/shared/src/test/mocks.ts b/lib/shared/src/test/mocks.ts index c8bd35d0f5e3..96b03133459a 100644 --- a/lib/shared/src/test/mocks.ts +++ b/lib/shared/src/test/mocks.ts @@ -3,7 +3,7 @@ import { RecipeContext } from '../chat/recipes/recipe' import { CodebaseContext } from '../codebase-context' import { ActiveTextEditor, ActiveTextEditorSelection, ActiveTextEditorVisibleContent, Editor } from '../editor' import { EmbeddingsSearch } from '../embeddings' -import { IntentDetector } from '../intent-detector' +import { IntentClassificationOption, IntentDetector } from '../intent-detector' import { ContextResult, KeywordContextFetcher } from '../local-context' import { SourcegraphCompletionsClient } from '../sourcegraph-api/completions/client' import { CompletionParameters, CompletionResponse } from '../sourcegraph-api/completions/types' @@ -56,6 +56,10 @@ export class MockIntentDetector implements IntentDetector { public isEditorContextRequired(input: string): boolean | Error { return this.mocks.isEditorContextRequired?.(input) ?? false } + + public classifyIntentFromOptions(input: string, options: IntentClassificationOption[]): Promise { + return Promise.resolve(null) + } } export class MockKeywordContextFetcher implements KeywordContextFetcher { diff --git a/vscode/package.json b/vscode/package.json index 3b7a8427bace..663c2d273f21 100644 --- a/vscode/package.json +++ b/vscode/package.json @@ -12,7 +12,7 @@ "dev": "pnpm run build:dev && pnpm run dev:instance", "generate:completions": "OUTFILE=/tmp/run-code-completions-on-dataset.js && esbuild ./test/completions/run-code-completions-on-dataset.ts --bundle --external:vscode --outfile=$OUTFILE --format=cjs --platform=node --sourcemap=inline && node --enable-source-maps $OUTFILE", "build": "scripts/download-rg.sh && tsc --build && pnpm run esbuild --minify && vite build --mode production", - "build:dev": "scripts/download-rg.sh && tsc --build && concurrently \"pnpm run esbuild --sourcemap\" \"vite build --mode development\"", + "build:dev": "scripts/download-rg.sh && TSC_COMPILE_ON_ERROR=true tsc --build && concurrently \"pnpm run esbuild --sourcemap\" \"vite build --mode development\"", "esbuild": "esbuild ./src/extension.ts --bundle --outfile=dist/extension.js --external:vscode --format=cjs --platform=node", "lint": "pnpm run lint:js", "lint:js": "eslint --cache '**/*.[tj]s?(x)'", diff --git a/vscode/src/external-services.ts b/vscode/src/external-services.ts index 584d06087ab8..e1598cb9d767 100644 --- a/vscode/src/external-services.ts +++ b/vscode/src/external-services.ts @@ -64,7 +64,7 @@ export async function configureExternalServices( const guardrails = new SourcegraphGuardrailsClient(client) return { - intentDetector: new SourcegraphIntentDetectorClient(client), + intentDetector: new SourcegraphIntentDetectorClient(client, completions), codebaseContext, chatClient, completionsClient: completions, diff --git a/vscode/src/logged-rerank.ts b/vscode/src/logged-rerank.ts index ba3d21003c7c..58dc3ef7b798 100644 --- a/vscode/src/logged-rerank.ts +++ b/vscode/src/logged-rerank.ts @@ -15,9 +15,9 @@ export function getRerankWithLog( const reranker = new LLMReranker(chatClient) return async (userQuery: string, results: ContextResult[]): Promise => { - const start = performance.now() + const start = Date.now() const rerankedResults = await reranker.rerank(userQuery, results) - const duration = performance.now() - start + const duration = Date.now() - start debug('Reranker:rerank', JSON.stringify({ duration })) return rerankedResults } From 93e2181fa245c830155071c4bbee61421fd15a97 Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Thu, 27 Jul 2023 10:23:26 +0100 Subject: [PATCH 07/24] improved prompt quality --- lib/shared/src/chat/recipes/fixup.ts | 75 ++++++++++------------ lib/shared/src/chat/recipes/inline-chat.ts | 51 ++++++++------- lib/shared/src/intent-detector/client.ts | 11 ++-- lib/shared/src/intent-detector/index.ts | 10 ++- lib/shared/src/test/mocks.ts | 8 ++- vscode/package.json | 2 + 6 files changed, 79 insertions(+), 78 deletions(-) diff --git a/lib/shared/src/chat/recipes/fixup.ts b/lib/shared/src/chat/recipes/fixup.ts index 6cce4aef8c18..c505ddf3c5c8 100644 --- a/lib/shared/src/chat/recipes/fixup.ts +++ b/lib/shared/src/chat/recipes/fixup.ts @@ -8,49 +8,43 @@ import { Interaction } from '../transcript/interaction' import { contentSanitizer, getContextMessagesFromSelection, numResults } from './helpers' import { Recipe, RecipeContext, RecipeID } from './recipe' -const FixupIntentClassification: IntentClassificationOption[] = [ +type FixupIntent = 'add' | 'edit' | 'fix' | 'document' | 'test' +const FixupIntentClassification: IntentClassificationOption[] = [ { - /** - * Context: - * - preceding text, selected text, following text - * - maximum embeddings from code files and text files - */ - id: 'explain', - description: 'Explain the selected code', - examplePrompts: ['What does this code do?', 'How does this code work?'], + id: 'add', + description: 'Add new code to complement the selected code', + examplePrompts: ['Add a new function', 'Add a new class'], }, { - /** - * Context: - * - preceding text, selected text, following text - * - limited embeddings from code files - */ - id: 'fix', // mostly context from current file, code files + id: 'edit', + description: 'Edit the selected code', + examplePrompts: ['Edit this code', 'Change this code', 'Update this code'], + }, + { + id: 'fix', description: 'Fix a problem in the selected code', - examplePrompts: ['Update this code to use async/await', 'Fix this code'], + examplePrompts: ['Implement this TODO', 'Fix this code'], }, { - /** - * Context: - * - selected text - * - limited embeddings from code files - */ - id: 'document', // only the current selection, context + id: 'document', description: 'Generate documentation for the selected code.', examplePrompts: ['Add a docstring for this function', 'Write comments to explain this code'], }, { - /** - * Context: - * - preceding text, selected text, following text - * - limited embeddings from code files - */ id: 'test', description: 'Generate tests for the selected code', examplePrompts: ['Write a test for this function', 'Add a test for this code'], }, ] +const PromptIntentInstruction: Record = { + add: 'The user wants you to add new code to the selected code by following their instructions.', + edit: 'The user wants you to replace code inside the selected code by following their instructions.', + fix: 'The user wants you to correct a problem in the selected code by following their instructions.', + document: 'The user wants you to generate overall documentation for the selected code.', + test: 'The user wants you to generate a test or multiple tests for the selected code.', +} + export class Fixup implements Recipe { public id: RecipeID = 'fixup' @@ -70,6 +64,11 @@ export class Fixup implements Recipe { return null } + const intent = await context.intentDetector.classifyIntentFromOptions( + humanChatInput, + FixupIntentClassification, + 'fix' + ) const truncatedPrecedingText = truncateTextStart(selection.precedingText, quarterFileContext) const truncatedFollowingText = truncateText(selection.followingText, quarterFileContext) @@ -82,6 +81,7 @@ export class Fixup implements Recipe { .replace('{selectedText}', selection.selectedText) .replace('{truncateFollowingText}', truncatedFollowingText) .replace('{fileName}', selection.fileName) + .replace('{intent}', PromptIntentInstruction[intent]) context.responseMultiplexer.sub( 'selection', @@ -102,12 +102,10 @@ export class Fixup implements Recipe { ) let dynamicContext: Promise - const intent = await context.intentDetector.classifyIntentFromOptions(humanChatInput, FixupIntentClassification) console.log('INLINE FIXUP INTENT', intent) switch (intent) { - case 'explain': - dynamicContext = context.codebaseContext.getContextMessages(selection.selectedText, numResults) - break + case 'add': + case 'edit': case 'fix': dynamicContext = getContextMessagesFromSelection( selection.selectedText, @@ -118,21 +116,13 @@ export class Fixup implements Recipe { ) break case 'document': - // todo: better context gather for documenting? currently just using selection - no context + // TODO: Is this the best way to get the context for documentation? dynamicContext = Promise.resolve([]) break case 'test': - // todo: better context gathering for tests + // TODO: Better retrieval of test context. E.g. test files, dependencies, etc. dynamicContext = context.codebaseContext.getContextMessages(selection.selectedText, numResults) break - default: - dynamicContext = getContextMessagesFromSelection( - selection.selectedText, - truncatedPrecedingText, - truncatedFollowingText, - selection, - context.codebaseContext - ) } return Promise.resolve( @@ -182,7 +172,8 @@ export class Fixup implements Recipe { {selectedText} - You should rewrite this code using the following instructions: + The user wants you to {intent} + Provide your generated code using the following instructions: {humanInput} ` diff --git a/lib/shared/src/chat/recipes/inline-chat.ts b/lib/shared/src/chat/recipes/inline-chat.ts index 1f582b81e4d8..989e2bf8e6dd 100644 --- a/lib/shared/src/chat/recipes/inline-chat.ts +++ b/lib/shared/src/chat/recipes/inline-chat.ts @@ -1,7 +1,6 @@ import { CodebaseContext } from '../../codebase-context' import { ContextMessage } from '../../codebase-context/messages' import { ActiveTextEditorSelection, Editor } from '../../editor' -import { IntentClassificationOption } from '../../intent-detector' import { MAX_HUMAN_INPUT_TOKENS, MAX_RECIPE_INPUT_TOKENS, MAX_RECIPE_SURROUNDING_TOKENS } from '../../prompt/constants' import { truncateText } from '../../prompt/truncation' import { Interaction } from '../transcript/interaction' @@ -12,29 +11,29 @@ import { commandRegex } from './helpers' import { InlineTouch } from './inline-touch' import { Recipe, RecipeContext, RecipeID } from './recipe' -const InlineChatClassification: IntentClassificationOption[] = [ - { - id: 'chat', - description: 'Discuss the selected code', - examplePrompts: ['What does this do?', 'How does this work?', 'How would I improve this?'], - }, - { - id: 'fix', - description: 'Fix a problem in the selected code', - examplePrompts: [ - 'Update this code', - 'Fix this code', - 'Change this code', - 'Rewrite this code', - 'Add to this code', - ], - }, - { - id: 'touch', - description: 'Generate a new file from the selected code', - examplePrompts: ['Write a test for this code', 'Create a new file from this code'], - }, -] +// const InlineChatClassification: IntentClassificationOption[] = [ +// { +// id: 'chat', +// description: 'Discuss the selected code', +// examplePrompts: ['What does this do?', 'How does this work?', 'How would I improve this?'], +// }, +// { +// id: 'fix', +// description: 'Fix a problem in the selected code', +// examplePrompts: [ +// 'Update this code', +// 'Fix this code', +// 'Change this code', +// 'Rewrite this code', +// 'Add to this code', +// ], +// }, +// { +// id: 'touch', +// description: 'Generate a new file from the selected code', +// examplePrompts: ['Write a test for this code', 'Create a new file from this code'], +// }, +// ] export class InlineChat implements Recipe { public id: RecipeID = 'inline-chat' @@ -52,8 +51,8 @@ export class InlineChat implements Recipe { return new Fixup().getInteraction(humanChatInput.replace(commandRegex.fix, ''), context) } - const intent = await context.intentDetector.classifyIntentFromOptions(humanChatInput, InlineChatClassification) - console.log('INLINE CHAT INTENT:', intent) + // const intent = await context.intentDetector.classifyIntentFromOptions(humanChatInput, InlineChatClassification) + // console.log('INLINE CHAT INTENT:', intent) // TODO: We currently check intent in two places for inline chat. Do it in one place. // if (intent === 'touch') { // return new InlineTouch(this.debug).getInteraction(humanChatInput.replace(commandRegex.touch, ''), context) diff --git a/lib/shared/src/intent-detector/client.ts b/lib/shared/src/intent-detector/client.ts index 107b0fcce830..1d3c77a6466f 100644 --- a/lib/shared/src/intent-detector/client.ts +++ b/lib/shared/src/intent-detector/client.ts @@ -62,10 +62,11 @@ Function Description: ${description} return messages } - public async classifyIntentFromOptions( + public async classifyIntentFromOptions( input: string, - options: IntentClassificationOption[] - ): Promise { + options: IntentClassificationOption[], + fallback: Intent + ): Promise { const initialPrompt = this.buildInitialPrompt(options) const exampleTranscript = this.buildExampleTranscript(options) @@ -98,10 +99,10 @@ Function Description: ${description} const responseClassification = result.completion.match(/(.*?)<\/classification>/)?.[1] if (!responseClassification) { - return null + return fallback } - return options.find(option => option.id === responseClassification)?.id ?? null + return options.find(option => option.id === responseClassification)?.id ?? fallback } } diff --git a/lib/shared/src/intent-detector/index.ts b/lib/shared/src/intent-detector/index.ts index 9fa191e6b95c..539afb641a65 100644 --- a/lib/shared/src/intent-detector/index.ts +++ b/lib/shared/src/intent-detector/index.ts @@ -1,9 +1,9 @@ -export interface IntentClassificationOption { +export interface IntentClassificationOption { /** * An identifier for this intent. * This is what will be returned by the classifier. */ - id: string + id: Intent /** * A description for this intent. * Be specific in order to help the LLM understand the intent. @@ -19,5 +19,9 @@ export interface IntentClassificationOption { export interface IntentDetector { isCodebaseContextRequired(input: string): Promise isEditorContextRequired(input: string): boolean | Error - classifyIntentFromOptions(input: string, options: IntentClassificationOption[]): Promise + classifyIntentFromOptions( + input: string, + options: IntentClassificationOption[], + fallback: Intent + ): Promise } diff --git a/lib/shared/src/test/mocks.ts b/lib/shared/src/test/mocks.ts index 96b03133459a..23424fd8e4a6 100644 --- a/lib/shared/src/test/mocks.ts +++ b/lib/shared/src/test/mocks.ts @@ -57,8 +57,12 @@ export class MockIntentDetector implements IntentDetector { return this.mocks.isEditorContextRequired?.(input) ?? false } - public classifyIntentFromOptions(input: string, options: IntentClassificationOption[]): Promise { - return Promise.resolve(null) + public classifyIntentFromOptions( + input: string, + options: IntentClassificationOption[], + fallback: Intent + ): Promise { + return Promise.resolve(fallback) } } diff --git a/vscode/package.json b/vscode/package.json index 663c2d273f21..643a6c2370f3 100644 --- a/vscode/package.json +++ b/vscode/package.json @@ -9,7 +9,9 @@ "description": "Code AI with codebase context", "scripts": { "dev:instance": "CODY_FOCUS_ON_STARTUP=1 NODE_ENV=development code --extensionDevelopmentPath=$PWD --disable-extension=sourcegraph.cody-ai --disable-extension=github.copilot --disable-extension=github.copilot-nightly --inspect-extensions=9333 --new-window . --goto ./src/logged-rerank.ts:16:5", + "dev:instance-insiders": "CODY_FOCUS_ON_STARTUP=1 NODE_ENV=development code-insiders --extensionDevelopmentPath=$PWD --disable-extension=sourcegraph.cody-ai --disable-extension=github.copilot --disable-extension=github.copilot-nightly --inspect-extensions=9333 --new-window . --goto ./src/logged-rerank.ts:16:5", "dev": "pnpm run build:dev && pnpm run dev:instance", + "dev-insiders": "pnpm run build:dev && pnpm run dev:instance-insiders", "generate:completions": "OUTFILE=/tmp/run-code-completions-on-dataset.js && esbuild ./test/completions/run-code-completions-on-dataset.ts --bundle --external:vscode --outfile=$OUTFILE --format=cjs --platform=node --sourcemap=inline && node --enable-source-maps $OUTFILE", "build": "scripts/download-rg.sh && tsc --build && pnpm run esbuild --minify && vite build --mode production", "build:dev": "scripts/download-rg.sh && TSC_COMPILE_ON_ERROR=true tsc --build && concurrently \"pnpm run esbuild --sourcemap\" \"vite build --mode development\"", From b44b9a7e5784aa7bbc6b0b21ea52ecc30168d333 Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Thu, 27 Jul 2023 11:07:07 +0100 Subject: [PATCH 08/24] fix --- lib/shared/src/chat/recipes/fixup.ts | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/lib/shared/src/chat/recipes/fixup.ts b/lib/shared/src/chat/recipes/fixup.ts index c505ddf3c5c8..ee9e78c29dc6 100644 --- a/lib/shared/src/chat/recipes/fixup.ts +++ b/lib/shared/src/chat/recipes/fixup.ts @@ -41,7 +41,7 @@ const PromptIntentInstruction: Record = { add: 'The user wants you to add new code to the selected code by following their instructions.', edit: 'The user wants you to replace code inside the selected code by following their instructions.', fix: 'The user wants you to correct a problem in the selected code by following their instructions.', - document: 'The user wants you to generate overall documentation for the selected code.', + document: 'The user wants you to add documentation or comments to the selected code.', test: 'The user wants you to generate a test or multiple tests for the selected code.', } @@ -107,6 +107,10 @@ export class Fixup implements Recipe { case 'add': case 'edit': case 'fix': + /** + * Fetch a small window of code context for the current selection. + * Include preceding and following text as additional context + */ dynamicContext = getContextMessagesFromSelection( selection.selectedText, truncatedPrecedingText, @@ -116,8 +120,11 @@ export class Fixup implements Recipe { ) break case 'document': - // TODO: Is this the best way to get the context for documentation? - dynamicContext = Promise.resolve([]) + /** + * Fetch a small window of mixed code and text context for the current selection. + * We do not include preceding and following text as they may not be relevant here. + */ + dynamicContext = context.codebaseContext.getContextMessages(selection.selectedText, { numCodeResults: 2, numTextResults: 2 }) break case 'test': // TODO: Better retrieval of test context. E.g. test files, dependencies, etc. @@ -149,24 +156,12 @@ export class Fixup implements Recipe { - Unless you have reason to believe otherwise, you should assume that the user wants you to edit the code in their selection. - You should ensure the rewritten code matches the indentation and whitespace of the code in the users' selection. - It is not acceptable to use Markdown in your response. You should not produce Markdown-formatted code blocks. - - You will be provided with code that is above the users' selection, enclosed in XML tags. You can use this code, if relevant, to help you plan your rewritten code. - - You will be provided with code that is below the users' selection, enclosed in XML tags. You can use this code, if relevant, to help you plan your rewritten code. - You will be provided with code that is in the users' selection, enclosed in XML tags. You must use this code to help you plan your rewritten code. - You will be provided with instructions on how to modify this code, enclosed in XML tags. You must follow these instructions carefully and to the letter. - Enclose your response in XML tags. Do not provide anything else. This is part of the file {fileName}. - The user has the following code above their selection: - - {truncateTextStart} - - - The user has the following code below their selection: - - {truncateFollowingText} - - The user has the following code within their selection: {selectedText} From 70ed449fdb0c3c2874c472f5d1323d09ab4e2deb Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Thu, 27 Jul 2023 13:43:56 +0100 Subject: [PATCH 09/24] add comment --- vscode/package.json | 4 ++-- vscode/src/chat/InlineChatViewProvider.ts | 5 +++++ vscode/src/logged-rerank.ts | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/vscode/package.json b/vscode/package.json index 643a6c2370f3..006bdecb0a92 100644 --- a/vscode/package.json +++ b/vscode/package.json @@ -14,7 +14,7 @@ "dev-insiders": "pnpm run build:dev && pnpm run dev:instance-insiders", "generate:completions": "OUTFILE=/tmp/run-code-completions-on-dataset.js && esbuild ./test/completions/run-code-completions-on-dataset.ts --bundle --external:vscode --outfile=$OUTFILE --format=cjs --platform=node --sourcemap=inline && node --enable-source-maps $OUTFILE", "build": "scripts/download-rg.sh && tsc --build && pnpm run esbuild --minify && vite build --mode production", - "build:dev": "scripts/download-rg.sh && TSC_COMPILE_ON_ERROR=true tsc --build && concurrently \"pnpm run esbuild --sourcemap\" \"vite build --mode development\"", + "build:dev": "scripts/download-rg.sh && tsc --build && concurrently \"pnpm run esbuild --sourcemap\" \"vite build --mode development\"", "esbuild": "esbuild ./src/extension.ts --bundle --outfile=dist/extension.js --external:vscode --format=cjs --platform=node", "lint": "pnpm run lint:js", "lint:js": "eslint --cache '**/*.[tj]s?(x)'", @@ -1053,4 +1053,4 @@ "semver": "^7.5.4", "vite-plugin-static-copy": "^0.16.0" } -} +} \ No newline at end of file diff --git a/vscode/src/chat/InlineChatViewProvider.ts b/vscode/src/chat/InlineChatViewProvider.ts index ca97fd2ca5ac..c36cea294023 100644 --- a/vscode/src/chat/InlineChatViewProvider.ts +++ b/vscode/src/chat/InlineChatViewProvider.ts @@ -51,6 +51,11 @@ export class InlineChatViewProvider extends MessageProvider { // We need to update the comment controller to support more than one active thread at a time. void vscode.commands.executeCommand('setContext', 'cody.inline.reply.pending', true) + /** + * TODO(umpox): + * We create a new comment and trigger the inline chat recipe, but may end up closing this comment and running a fix instead + * We should detect intent here (through regex and then `classifyIntentFromOptions`) and run the correct recipe/controller instead. + */ await this.editor.controllers.inline?.chat(reply, this.thread, isFixMode) this.editor.controllers.inline?.setResponsePending(true) await this.executeRecipe('inline-chat', reply.trimStart()) diff --git a/vscode/src/logged-rerank.ts b/vscode/src/logged-rerank.ts index 58dc3ef7b798..ba3d21003c7c 100644 --- a/vscode/src/logged-rerank.ts +++ b/vscode/src/logged-rerank.ts @@ -15,9 +15,9 @@ export function getRerankWithLog( const reranker = new LLMReranker(chatClient) return async (userQuery: string, results: ContextResult[]): Promise => { - const start = Date.now() + const start = performance.now() const rerankedResults = await reranker.rerank(userQuery, results) - const duration = Date.now() - start + const duration = performance.now() - start debug('Reranker:rerank', JSON.stringify({ duration })) return rerankedResults } From 1f807db34c73e4c04e91f8776b647f59d81a285b Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Thu, 27 Jul 2023 14:37:18 +0100 Subject: [PATCH 10/24] updates --- lib/shared/src/chat/recipes/fixup.ts | 40 ++++++++---------------- lib/shared/src/intent-detector/client.ts | 1 + 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/lib/shared/src/chat/recipes/fixup.ts b/lib/shared/src/chat/recipes/fixup.ts index ee9e78c29dc6..b38cdef31881 100644 --- a/lib/shared/src/chat/recipes/fixup.ts +++ b/lib/shared/src/chat/recipes/fixup.ts @@ -5,16 +5,11 @@ import { truncateText, truncateTextStart } from '../../prompt/truncation' import { BufferedBotResponseSubscriber } from '../bot-response-multiplexer' import { Interaction } from '../transcript/interaction' -import { contentSanitizer, getContextMessagesFromSelection, numResults } from './helpers' +import { contentSanitizer, getContextMessagesFromSelection } from './helpers' import { Recipe, RecipeContext, RecipeID } from './recipe' -type FixupIntent = 'add' | 'edit' | 'fix' | 'document' | 'test' +type FixupIntent = 'edit' | 'fix' | 'document' const FixupIntentClassification: IntentClassificationOption[] = [ - { - id: 'add', - description: 'Add new code to complement the selected code', - examplePrompts: ['Add a new function', 'Add a new class'], - }, { id: 'edit', description: 'Edit the selected code', @@ -30,19 +25,12 @@ const FixupIntentClassification: IntentClassificationOption[] = [ description: 'Generate documentation for the selected code.', examplePrompts: ['Add a docstring for this function', 'Write comments to explain this code'], }, - { - id: 'test', - description: 'Generate tests for the selected code', - examplePrompts: ['Write a test for this function', 'Add a test for this code'], - }, ] const PromptIntentInstruction: Record = { - add: 'The user wants you to add new code to the selected code by following their instructions.', edit: 'The user wants you to replace code inside the selected code by following their instructions.', fix: 'The user wants you to correct a problem in the selected code by following their instructions.', document: 'The user wants you to add documentation or comments to the selected code.', - test: 'The user wants you to generate a test or multiple tests for the selected code.', } export class Fixup implements Recipe { @@ -104,12 +92,11 @@ export class Fixup implements Recipe { let dynamicContext: Promise console.log('INLINE FIXUP INTENT', intent) switch (intent) { - case 'add': case 'edit': - case 'fix': + case 'fix': // TODO(umpox): For fixing code, can we extract warnings + errors from within the selection? /** * Fetch a small window of code context for the current selection. - * Include preceding and following text as additional context + * Include preceding and following text as additional context. */ dynamicContext = getContextMessagesFromSelection( selection.selectedText, @@ -121,14 +108,13 @@ export class Fixup implements Recipe { break case 'document': /** - * Fetch a small window of mixed code and text context for the current selection. + * Fetch a small window of code context for the current selection. * We do not include preceding and following text as they may not be relevant here. */ - dynamicContext = context.codebaseContext.getContextMessages(selection.selectedText, { numCodeResults: 2, numTextResults: 2 }) - break - case 'test': - // TODO: Better retrieval of test context. E.g. test files, dependencies, etc. - dynamicContext = context.codebaseContext.getContextMessages(selection.selectedText, numResults) + dynamicContext = context.codebaseContext.getContextMessages(selection.selectedText, { + numCodeResults: 2, + numTextResults: 0, + }) break } @@ -151,10 +137,10 @@ export class Fixup implements Recipe { // Prompt Templates public static readonly prompt = ` - - You are an AI programming assistant who is an expert in rewriting code to meet given instructions. - - You should think step-by-step to plan your rewritten code before producing the final output. - - Unless you have reason to believe otherwise, you should assume that the user wants you to edit the code in their selection. + - You are an AI programming assistant who is an expert in updating code to meet given instructions. + - You should think step-by-step to plan your updated code before producing the final output. - You should ensure the rewritten code matches the indentation and whitespace of the code in the users' selection. + - Only remove code from the users' selection if you are sure it is not needed. - It is not acceptable to use Markdown in your response. You should not produce Markdown-formatted code blocks. - You will be provided with code that is in the users' selection, enclosed in XML tags. You must use this code to help you plan your rewritten code. - You will be provided with instructions on how to modify this code, enclosed in XML tags. You must follow these instructions carefully and to the letter. @@ -167,7 +153,7 @@ export class Fixup implements Recipe { {selectedText} - The user wants you to {intent} + {intent} Provide your generated code using the following instructions: {humanInput} diff --git a/lib/shared/src/intent-detector/client.ts b/lib/shared/src/intent-detector/client.ts index 1d3c77a6466f..f09086c57ab7 100644 --- a/lib/shared/src/intent-detector/client.ts +++ b/lib/shared/src/intent-detector/client.ts @@ -98,6 +98,7 @@ Function Description: ${description} const responseClassification = result.completion.match(/(.*?)<\/classification>/)?.[1] + console.log('LLM CLASSIFICATION', responseClassification) if (!responseClassification) { return fallback } From 777db2fd97635b08d5450e09a5b104453e2e3de0 Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Thu, 27 Jul 2023 15:05:00 +0100 Subject: [PATCH 11/24] improve documentation context --- lib/shared/src/chat/recipes/fixup.ts | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/shared/src/chat/recipes/fixup.ts b/lib/shared/src/chat/recipes/fixup.ts index b38cdef31881..62d8d809b954 100644 --- a/lib/shared/src/chat/recipes/fixup.ts +++ b/lib/shared/src/chat/recipes/fixup.ts @@ -1,6 +1,7 @@ -import { ContextMessage } from '../../codebase-context/messages' +import { ContextMessage, getContextMessageWithResponse } from '../../codebase-context/messages' import { IntentClassificationOption } from '../../intent-detector' import { MAX_CURRENT_FILE_TOKENS, MAX_HUMAN_INPUT_TOKENS } from '../../prompt/constants' +import { populateCodeContextTemplate } from '../../prompt/templates' import { truncateText, truncateTextStart } from '../../prompt/truncation' import { BufferedBotResponseSubscriber } from '../bot-response-multiplexer' import { Interaction } from '../transcript/interaction' @@ -96,7 +97,7 @@ export class Fixup implements Recipe { case 'fix': // TODO(umpox): For fixing code, can we extract warnings + errors from within the selection? /** * Fetch a small window of code context for the current selection. - * Include preceding and following text as additional context. + * Includes preceding and following text as additional context. */ dynamicContext = getContextMessagesFromSelection( selection.selectedText, @@ -108,13 +109,17 @@ export class Fixup implements Recipe { break case 'document': /** - * Fetch a small window of code context for the current selection. - * We do not include preceding and following text as they may not be relevant here. + * Includes code context from the current file only. + * Including context from other files is unlikely to be useful, and seems to reduce response quality. */ - dynamicContext = context.codebaseContext.getContextMessages(selection.selectedText, { - numCodeResults: 2, - numTextResults: 0, - }) + dynamicContext = Promise.resolve( + [truncatedPrecedingText, truncatedFollowingText].flatMap(text => + getContextMessageWithResponse( + populateCodeContextTemplate(text, selection.fileName, selection.repoName), + selection + ) + ) + ) break } From c90a2378e9c5fcd16934df8ff04f045f0a5225a2 Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Thu, 27 Jul 2023 15:31:07 +0100 Subject: [PATCH 12/24] tweak --- lib/shared/src/intent-detector/client.ts | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/lib/shared/src/intent-detector/client.ts b/lib/shared/src/intent-detector/client.ts index f09086c57ab7..8d0944df19df 100644 --- a/lib/shared/src/intent-detector/client.ts +++ b/lib/shared/src/intent-detector/client.ts @@ -31,12 +31,7 @@ export class SourcegraphIntentDetectorClient implements IntentDetector { private buildInitialPrompt(options: IntentClassificationOption[]): string { const functions = options - .map( - ({ id, description }) => ` -Function Id: ${id} -Function Description: ${description} -` - ) + .map(({ id, description }) => `Function ID: ${id}\nFunction Description: ${description}`) .join('\n') return prompt.replace('{functions}', functions) @@ -72,7 +67,7 @@ Function Description: ${description} const result = await this.completionsClient.complete({ fast: true, - temperature: 0.2, + temperature: 0, maxTokensToSample: ANSWER_TOKENS, topK: -1, topP: -1, @@ -97,7 +92,6 @@ Function Description: ${description} }) const responseClassification = result.completion.match(/(.*?)<\/classification>/)?.[1] - console.log('LLM CLASSIFICATION', responseClassification) if (!responseClassification) { return fallback @@ -110,7 +104,7 @@ Function Description: ${description} const prompt = ` You are an AI chatbot in a code editor. You are at expert at understanding the request of a software developer and selecting an available function to perform that request. Think step-by-step to understand the request. -Only provide your response if you know the answer or can make a well-informed guess, respond with "unknown". +Only provide your response if you know the answer or can make a well-informed guess, otherwise respond with "unknown". Enclose your response in XML tags. Do not provide anything else. Available functions: From 92099c039cf27187f200bcf72577915dc1d6f28f Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Thu, 27 Jul 2023 15:59:20 +0100 Subject: [PATCH 13/24] clean up --- lib/shared/src/chat/recipes/fixup.ts | 16 +++++----- lib/shared/src/chat/recipes/inline-chat.ts | 34 ---------------------- lib/shared/src/intent-detector/index.ts | 2 +- vscode/package.json | 2 +- 4 files changed, 9 insertions(+), 45 deletions(-) diff --git a/lib/shared/src/chat/recipes/fixup.ts b/lib/shared/src/chat/recipes/fixup.ts index 62d8d809b954..5a11f0bb29b7 100644 --- a/lib/shared/src/chat/recipes/fixup.ts +++ b/lib/shared/src/chat/recipes/fixup.ts @@ -31,7 +31,8 @@ const FixupIntentClassification: IntentClassificationOption[] = [ const PromptIntentInstruction: Record = { edit: 'The user wants you to replace code inside the selected code by following their instructions.', fix: 'The user wants you to correct a problem in the selected code by following their instructions.', - document: 'The user wants you to add documentation or comments to the selected code.', + document: + 'The user wants you to add documentation or comments to the selected code by following their instructions.', } export class Fixup implements Recipe { @@ -53,22 +54,20 @@ export class Fixup implements Recipe { return null } + const truncatedPrecedingText = truncateTextStart(selection.precedingText, quarterFileContext) + const truncatedFollowingText = truncateText(selection.followingText, quarterFileContext) const intent = await context.intentDetector.classifyIntentFromOptions( humanChatInput, FixupIntentClassification, 'fix' ) - const truncatedPrecedingText = truncateTextStart(selection.precedingText, quarterFileContext) - const truncatedFollowingText = truncateText(selection.followingText, quarterFileContext) // Reconstruct Cody's prompt using user's context // Replace placeholders in reverse order to avoid collisions if a placeholder occurs in the input // TODO: Move prompt suffix from recipe to chat view. It has other subscribers. const promptText = Fixup.prompt .replace('{humanInput}', truncateText(humanChatInput, MAX_HUMAN_INPUT_TOKENS)) - .replace('{truncateTextStart}', truncatedPrecedingText) .replace('{selectedText}', selection.selectedText) - .replace('{truncateFollowingText}', truncatedFollowingText) .replace('{fileName}', selection.fileName) .replace('{intent}', PromptIntentInstruction[intent]) @@ -91,7 +90,6 @@ export class Fixup implements Recipe { ) let dynamicContext: Promise - console.log('INLINE FIXUP INTENT', intent) switch (intent) { case 'edit': case 'fix': // TODO(umpox): For fixing code, can we extract warnings + errors from within the selection? @@ -144,11 +142,11 @@ export class Fixup implements Recipe { public static readonly prompt = ` - You are an AI programming assistant who is an expert in updating code to meet given instructions. - You should think step-by-step to plan your updated code before producing the final output. - - You should ensure the rewritten code matches the indentation and whitespace of the code in the users' selection. + - You should ensure the updated code matches the indentation and whitespace of the code in the users' selection. - Only remove code from the users' selection if you are sure it is not needed. - It is not acceptable to use Markdown in your response. You should not produce Markdown-formatted code blocks. - - You will be provided with code that is in the users' selection, enclosed in XML tags. You must use this code to help you plan your rewritten code. - - You will be provided with instructions on how to modify this code, enclosed in XML tags. You must follow these instructions carefully and to the letter. + - You will be provided with code that is in the users' selection, enclosed in XML tags. You must use this code to help you plan your updated code. + - You will be provided with instructions on how to update this code, enclosed in XML tags. You must follow these instructions carefully and to the letter. - Enclose your response in XML tags. Do not provide anything else. This is part of the file {fileName}. diff --git a/lib/shared/src/chat/recipes/inline-chat.ts b/lib/shared/src/chat/recipes/inline-chat.ts index 989e2bf8e6dd..53108bfff73b 100644 --- a/lib/shared/src/chat/recipes/inline-chat.ts +++ b/lib/shared/src/chat/recipes/inline-chat.ts @@ -11,30 +11,6 @@ import { commandRegex } from './helpers' import { InlineTouch } from './inline-touch' import { Recipe, RecipeContext, RecipeID } from './recipe' -// const InlineChatClassification: IntentClassificationOption[] = [ -// { -// id: 'chat', -// description: 'Discuss the selected code', -// examplePrompts: ['What does this do?', 'How does this work?', 'How would I improve this?'], -// }, -// { -// id: 'fix', -// description: 'Fix a problem in the selected code', -// examplePrompts: [ -// 'Update this code', -// 'Fix this code', -// 'Change this code', -// 'Rewrite this code', -// 'Add to this code', -// ], -// }, -// { -// id: 'touch', -// description: 'Generate a new file from the selected code', -// examplePrompts: ['Write a test for this code', 'Create a new file from this code'], -// }, -// ] - export class InlineChat implements Recipe { public id: RecipeID = 'inline-chat' @@ -51,16 +27,6 @@ export class InlineChat implements Recipe { return new Fixup().getInteraction(humanChatInput.replace(commandRegex.fix, ''), context) } - // const intent = await context.intentDetector.classifyIntentFromOptions(humanChatInput, InlineChatClassification) - // console.log('INLINE CHAT INTENT:', intent) - // TODO: We currently check intent in two places for inline chat. Do it in one place. - // if (intent === 'touch') { - // return new InlineTouch(this.debug).getInteraction(humanChatInput.replace(commandRegex.touch, ''), context) - // } - // if (intent === 'fix') { - // return new Fixup().getInteraction(humanChatInput, context) - // } - const selection = context.editor.controllers?.inline?.selection if (!humanChatInput || !selection) { await context.editor.showWarningMessage('Failed to start Inline Chat: empty input or selection.') diff --git a/lib/shared/src/intent-detector/index.ts b/lib/shared/src/intent-detector/index.ts index 539afb641a65..5489aa1a5b45 100644 --- a/lib/shared/src/intent-detector/index.ts +++ b/lib/shared/src/intent-detector/index.ts @@ -1,4 +1,4 @@ -export interface IntentClassificationOption { +export interface IntentClassificationOption { /** * An identifier for this intent. * This is what will be returned by the classifier. diff --git a/vscode/package.json b/vscode/package.json index 006bdecb0a92..4fa1231aab3e 100644 --- a/vscode/package.json +++ b/vscode/package.json @@ -1053,4 +1053,4 @@ "semver": "^7.5.4", "vite-plugin-static-copy": "^0.16.0" } -} \ No newline at end of file +} From bddd53958504b84485f6c7aa5f4192e2950670ad Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Fri, 28 Jul 2023 12:03:39 +0100 Subject: [PATCH 14/24] more --- lib/shared/src/chat/recipes/fixup.ts | 140 ++++++++++++++++++--------- 1 file changed, 94 insertions(+), 46 deletions(-) diff --git a/lib/shared/src/chat/recipes/fixup.ts b/lib/shared/src/chat/recipes/fixup.ts index 5a11f0bb29b7..21831838a570 100644 --- a/lib/shared/src/chat/recipes/fixup.ts +++ b/lib/shared/src/chat/recipes/fixup.ts @@ -1,4 +1,5 @@ import { ContextMessage, getContextMessageWithResponse } from '../../codebase-context/messages' +import { ActiveTextEditorSelection } from '../../editor' import { IntentClassificationOption } from '../../intent-detector' import { MAX_CURRENT_FILE_TOKENS, MAX_HUMAN_INPUT_TOKENS } from '../../prompt/constants' import { populateCodeContextTemplate } from '../../prompt/templates' @@ -9,35 +10,120 @@ import { Interaction } from '../transcript/interaction' import { contentSanitizer, getContextMessagesFromSelection } from './helpers' import { Recipe, RecipeContext, RecipeID } from './recipe' -type FixupIntent = 'edit' | 'fix' | 'document' +type FixupIntent = 'add' | 'edit' | 'delete' | 'fix' | 'test' | 'document' const FixupIntentClassification: IntentClassificationOption[] = [ + { + id: 'add', + description: 'Add to the selected code', + examplePrompts: ['Add a function that concatonates two strings', 'Add error handling'], + }, { id: 'edit', - description: 'Edit the selected code', + description: 'Edit part of the selected code', examplePrompts: ['Edit this code', 'Change this code', 'Update this code'], }, + { + id: 'delete', + description: 'Delete a part of the selection code', + examplePrompts: ['Delete these comments', 'Remove log statements'], + }, { id: 'fix', - description: 'Fix a problem in the selected code', + description: 'Fix a problem in a part of the selected code', examplePrompts: ['Implement this TODO', 'Fix this code'], }, { id: 'document', - description: 'Generate documentation for the selected code.', + description: 'Generate documentation for parts of the selected code.', examplePrompts: ['Add a docstring for this function', 'Write comments to explain this code'], }, ] const PromptIntentInstruction: Record = { - edit: 'The user wants you to replace code inside the selected code by following their instructions.', + add: 'The user wants you to add to the selected code by following their instructions.', + edit: 'The user wants you to replace parts of the selected code by following their instructions.', + delete: 'The user wants you to remove parts of the selected code by following their instructions.', fix: 'The user wants you to correct a problem in the selected code by following their instructions.', document: 'The user wants you to add documentation or comments to the selected code by following their instructions.', + test: 'The user wants you to add, update or fix a test by following their instructions', } export class Fixup implements Recipe { public id: RecipeID = 'fixup' + private async getIntent(humanChatInput: string, context: RecipeContext): Promise { + // TODO(umpox): Implement a basic intent detection check that can return before reaching for the LLM. + // E.g. Current file is a test -> Test intent. Current selection is only a comment -> Documentation. + + const intent = await context.intentDetector.classifyIntentFromOptions( + humanChatInput, + FixupIntentClassification, + 'fix' + ) + return intent + } + + private async getContextFromIntent( + intent: FixupIntent, + selection: ActiveTextEditorSelection, + quarterFileContext: number, + context: RecipeContext + ): Promise { + const truncatedPrecedingText = truncateTextStart(selection.precedingText, quarterFileContext) + const truncatedFollowingText = truncateText(selection.followingText, quarterFileContext) + switch (intent) { + /** + * Intents that are focused on producing new code. + * They have a broad set of possible instructions, so we fetch a broad amount of code context files. + * Non-code files are not considered as including Markdown syntax seems to lead to more hallucinations and poorer output quality. + * + * TODO(umpox): We fetch similar context for all three cases here. + * We should investigate how we can improve each individual case. + * E.g.: + * - fix -> Can we extract warnings + errors from within the selection? + * - add/edit -> Are these fundamentally the same? Is the primary benefit here that we can provide more specific instructions to Cody? + */ + case 'add': + case 'edit': + case 'fix': + return getContextMessagesFromSelection( + selection.selectedText, + truncatedPrecedingText, + truncatedFollowingText, + selection, + context.codebaseContext + ) + /** + * The test intent is unique in that we likely want to be much more specific in that context that we retrieve. + * TODO(umpox): How can infer the current testing dependencies, etc? + */ + case 'test': + // Currently the same as add|edit|fix + return getContextMessagesFromSelection( + selection.selectedText, + truncatedPrecedingText, + truncatedFollowingText, + selection, + context.codebaseContext + ) + /** + * Intents that are focused primarily on updating code within the current file and selection. + * Providing a much more focused context window here seems to provide better quality responses. + */ + case 'delete': + case 'document': + return Promise.resolve( + [truncatedPrecedingText, truncatedFollowingText].flatMap(text => + getContextMessageWithResponse( + populateCodeContextTemplate(text, selection.fileName, selection.repoName), + selection + ) + ) + ) + } + } + public async getInteraction(humanChatInput: string, context: RecipeContext): Promise { // TODO: Prompt the user for additional direction. const selection = context.editor.getActiveTextEditorSelection() || context.editor.controllers?.inline?.selection @@ -54,15 +140,9 @@ export class Fixup implements Recipe { return null } - const truncatedPrecedingText = truncateTextStart(selection.precedingText, quarterFileContext) - const truncatedFollowingText = truncateText(selection.followingText, quarterFileContext) - const intent = await context.intentDetector.classifyIntentFromOptions( - humanChatInput, - FixupIntentClassification, - 'fix' - ) + const intent = await this.getIntent(humanChatInput, context) - // Reconstruct Cody's prompt using user's context + // Reconstruct Cody's prompt using user's context and intent // Replace placeholders in reverse order to avoid collisions if a placeholder occurs in the input // TODO: Move prompt suffix from recipe to chat view. It has other subscribers. const promptText = Fixup.prompt @@ -89,38 +169,6 @@ export class Fixup implements Recipe { }) ) - let dynamicContext: Promise - switch (intent) { - case 'edit': - case 'fix': // TODO(umpox): For fixing code, can we extract warnings + errors from within the selection? - /** - * Fetch a small window of code context for the current selection. - * Includes preceding and following text as additional context. - */ - dynamicContext = getContextMessagesFromSelection( - selection.selectedText, - truncatedPrecedingText, - truncatedFollowingText, - selection, - context.codebaseContext - ) - break - case 'document': - /** - * Includes code context from the current file only. - * Including context from other files is unlikely to be useful, and seems to reduce response quality. - */ - dynamicContext = Promise.resolve( - [truncatedPrecedingText, truncatedFollowingText].flatMap(text => - getContextMessageWithResponse( - populateCodeContextTemplate(text, selection.fileName, selection.repoName), - selection - ) - ) - ) - break - } - return Promise.resolve( new Interaction( { @@ -132,7 +180,7 @@ export class Fixup implements Recipe { speaker: 'assistant', prefix: 'Check your document for updates from Cody.\n', }, - dynamicContext, + this.getContextFromIntent(intent, selection, quarterFileContext, context), [] ) ) From 66a8ad03b5f79de4af59077fa976fa35f6751b3a Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Fri, 28 Jul 2023 14:37:29 +0100 Subject: [PATCH 15/24] include diagnostics --- agent/src/editor.ts | 5 ++++ lib/shared/src/chat/recipes/fixup.ts | 30 +++++++++++++++++---- lib/shared/src/editor/index.ts | 16 +++++++++++ lib/shared/src/intent-detector/index.ts | 2 +- lib/shared/src/prompt/templates.ts | 13 +++++++++ lib/shared/src/test/mocks.ts | 12 ++++++++- vscode/src/editor/vscode-editor.ts | 36 +++++++++++++++++++++++++ 7 files changed, 107 insertions(+), 7 deletions(-) diff --git a/agent/src/editor.ts b/agent/src/editor.ts index 39e45b4c0230..185f64c19234 100644 --- a/agent/src/editor.ts +++ b/agent/src/editor.ts @@ -1,5 +1,6 @@ import { ActiveTextEditor, + ActiveTextEditorDiagnostic, ActiveTextEditorSelection, ActiveTextEditorViewControllers, ActiveTextEditorVisibleContent, @@ -70,6 +71,10 @@ export class AgentEditor implements Editor { return this.getActiveTextEditorSelection() } + public getActiveTextEditorDiagnosticsForSelectionOrEntireFile(): ActiveTextEditorDiagnostic[] | null { + return null + } + public getActiveTextEditorVisibleContent(): ActiveTextEditorVisibleContent | null { const document = this.activeDocument() if (document === undefined) { diff --git a/lib/shared/src/chat/recipes/fixup.ts b/lib/shared/src/chat/recipes/fixup.ts index 21831838a570..f1e1ca740828 100644 --- a/lib/shared/src/chat/recipes/fixup.ts +++ b/lib/shared/src/chat/recipes/fixup.ts @@ -2,7 +2,7 @@ import { ContextMessage, getContextMessageWithResponse } from '../../codebase-co import { ActiveTextEditorSelection } from '../../editor' import { IntentClassificationOption } from '../../intent-detector' import { MAX_CURRENT_FILE_TOKENS, MAX_HUMAN_INPUT_TOKENS } from '../../prompt/constants' -import { populateCodeContextTemplate } from '../../prompt/templates' +import { populateCodeContextTemplate, populateCurrentEditorDiagnosticsTemplate } from '../../prompt/templates' import { truncateText, truncateTextStart } from '../../prompt/truncation' import { BufferedBotResponseSubscriber } from '../bot-response-multiplexer' import { Interaction } from '../transcript/interaction' @@ -72,27 +72,47 @@ export class Fixup implements Recipe { ): Promise { const truncatedPrecedingText = truncateTextStart(selection.precedingText, quarterFileContext) const truncatedFollowingText = truncateText(selection.followingText, quarterFileContext) + switch (intent) { /** * Intents that are focused on producing new code. * They have a broad set of possible instructions, so we fetch a broad amount of code context files. * Non-code files are not considered as including Markdown syntax seems to lead to more hallucinations and poorer output quality. * - * TODO(umpox): We fetch similar context for all three cases here. + * TODO(umpox): We fetch similar context for both cases here * We should investigate how we can improve each individual case. - * E.g.: - * - fix -> Can we extract warnings + errors from within the selection? - * - add/edit -> Are these fundamentally the same? Is the primary benefit here that we can provide more specific instructions to Cody? + * Are these fundamentally the same? Is the primary benefit here that we can provide more specific instructions to Cody? */ case 'add': case 'edit': + return getContextMessagesFromSelection( + selection.selectedText, + truncatedPrecedingText, + truncatedFollowingText, + selection, + context.codebaseContext + ) + /** + * The fix intent is similar to adding or editing code, but with additional context that we can include from the editor. + */ case 'fix': + // eslint-disable-next-line no-case-declarations + const diagnostics = context.editor.getActiveTextEditorDiagnosticsForSelectionOrEntireFile() || [] return getContextMessagesFromSelection( selection.selectedText, truncatedPrecedingText, truncatedFollowingText, selection, context.codebaseContext + ).then(messages => + messages.concat( + diagnostics.flatMap(diagnostic => + getContextMessageWithResponse( + populateCurrentEditorDiagnosticsTemplate(diagnostic, selection.fileName), + selection + ) + ) + ) ) /** * The test intent is unique in that we likely want to be much more specific in that context that we retrieve. diff --git a/lib/shared/src/editor/index.ts b/lib/shared/src/editor/index.ts index 7b8eafc00c81..198a5cda2cc5 100644 --- a/lib/shared/src/editor/index.ts +++ b/lib/shared/src/editor/index.ts @@ -26,6 +26,14 @@ export interface ActiveTextEditorSelection { followingText: string } +export type ActiveTextEditorDiagnosticType = 'error' | 'warning' | 'information' | 'hint' + +export interface ActiveTextEditorDiagnostic { + range: ActiveTextEditorSelectionRange + message: string + type: ActiveTextEditorDiagnosticType +} + export interface ActiveTextEditorVisibleContent { content: string fileName: string @@ -80,6 +88,10 @@ export interface Editor< * Gets the active text editor's selection, or the entire file if the selected range is empty. */ getActiveTextEditorSelectionOrEntireFile(): ActiveTextEditorSelection | null + /** + * Get diagnostics (errors, warnings) for the active text editor's selection, or the entire file if the selected range is empty. + */ + getActiveTextEditorDiagnosticsForSelectionOrEntireFile(): ActiveTextEditorDiagnostic[] | null getActiveTextEditorVisibleContent(): ActiveTextEditorVisibleContent | null replaceSelection(fileName: string, selectedText: string, replacement: string): Promise @@ -109,6 +121,10 @@ export class NoopEditor implements Editor { return null } + public getActiveTextEditorDiagnosticsForSelectionOrEntireFile(): ActiveTextEditorDiagnostic[] | null { + return null + } + public getActiveTextEditorVisibleContent(): ActiveTextEditorVisibleContent | null { return null } diff --git a/lib/shared/src/intent-detector/index.ts b/lib/shared/src/intent-detector/index.ts index 5489aa1a5b45..539afb641a65 100644 --- a/lib/shared/src/intent-detector/index.ts +++ b/lib/shared/src/intent-detector/index.ts @@ -1,4 +1,4 @@ -export interface IntentClassificationOption { +export interface IntentClassificationOption { /** * An identifier for this intent. * This is what will be returned by the classifier. diff --git a/lib/shared/src/prompt/templates.ts b/lib/shared/src/prompt/templates.ts index 348b0eea1faa..4627ce03569a 100644 --- a/lib/shared/src/prompt/templates.ts +++ b/lib/shared/src/prompt/templates.ts @@ -1,5 +1,7 @@ import path from 'path' +import { ActiveTextEditorDiagnostic } from '../editor' + const CODE_CONTEXT_TEMPLATE = `Use following code snippet from file \`{filePath}\`: \`\`\`{language} {text} @@ -66,6 +68,17 @@ export function populateCurrentEditorSelectedContextTemplate( ) } +const DIAGNOSTICS_CONTEXT_TEMPLATE = 'Use the following {type} from file `{filePath}`:\n{message}' + +export function populateCurrentEditorDiagnosticsTemplate( + { message, type }: ActiveTextEditorDiagnostic, + filePath: string +): string { + return DIAGNOSTICS_CONTEXT_TEMPLATE.replace('{type}', type) + .replace('{filePath}', filePath) + .replace('{message}', message) +} + const COMMAND_OUTPUT_TEMPLATE = 'Here is the output returned from the terminal.\n' export function populateTerminalOutputContextTemplate(output: string): string { diff --git a/lib/shared/src/test/mocks.ts b/lib/shared/src/test/mocks.ts index 23424fd8e4a6..e9a68744df9b 100644 --- a/lib/shared/src/test/mocks.ts +++ b/lib/shared/src/test/mocks.ts @@ -1,7 +1,13 @@ import { BotResponseMultiplexer } from '../chat/bot-response-multiplexer' import { RecipeContext } from '../chat/recipes/recipe' import { CodebaseContext } from '../codebase-context' -import { ActiveTextEditor, ActiveTextEditorSelection, ActiveTextEditorVisibleContent, Editor } from '../editor' +import { + ActiveTextEditor, + ActiveTextEditorDiagnostic, + ActiveTextEditorSelection, + ActiveTextEditorVisibleContent, + Editor, +} from '../editor' import { EmbeddingsSearch } from '../embeddings' import { IntentClassificationOption, IntentDetector } from '../intent-detector' import { ContextResult, KeywordContextFetcher } from '../local-context' @@ -95,6 +101,10 @@ export class MockEditor implements Editor { return this.mocks.getActiveTextEditorSelection?.() ?? null } + public getActiveTextEditorDiagnosticsForSelectionOrEntireFile(): ActiveTextEditorDiagnostic[] | null { + return this.mocks.getActiveTextEditorDiagnosticsForSelectionOrEntireFile?.() ?? null + } + public getActiveTextEditor(): ActiveTextEditor | null { return this.mocks.getActiveTextEditor?.() ?? null } diff --git a/vscode/src/editor/vscode-editor.ts b/vscode/src/editor/vscode-editor.ts index bb8b81030c85..9d505dd4aa3b 100644 --- a/vscode/src/editor/vscode-editor.ts +++ b/vscode/src/editor/vscode-editor.ts @@ -2,6 +2,8 @@ import * as vscode from 'vscode' import { ActiveTextEditor, + ActiveTextEditorDiagnostic, + ActiveTextEditorDiagnosticType, ActiveTextEditorSelection, ActiveTextEditorViewControllers, ActiveTextEditorVisibleContent, @@ -85,6 +87,40 @@ export class VSCodeEditor implements Editor range.contains(activeEditor.selection)) + // } + + return diagnostics.map(({ message, range, severity }) => ({ + range, + message, + type: this.getActiveTextEditorDiagnosticType(severity), + })) + } + private createActiveTextEditorSelection( activeEditor: vscode.TextEditor, selection: vscode.Selection From 07b01f51ddaff6eeaad8543a95e1607827dd19db Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Fri, 28 Jul 2023 15:46:29 +0100 Subject: [PATCH 16/24] Use correct selection range --- agent/src/editor.ts | 2 +- lib/shared/src/chat/context.ts | 2 +- lib/shared/src/chat/recipes/fixup.ts | 11 ++++-- lib/shared/src/editor/index.ts | 7 ++-- lib/shared/src/test/mocks.ts | 7 ++-- .../inputContext/ChatInputContext.story.tsx | 8 ++--- .../chat/inputContext/ChatInputContext.tsx | 4 +-- vscode/src/chat/ContextProvider.ts | 2 +- vscode/src/editor/vscode-editor.ts | 34 ++++++++++--------- 9 files changed, 45 insertions(+), 32 deletions(-) diff --git a/agent/src/editor.ts b/agent/src/editor.ts index 185f64c19234..e9b3a2c46dcb 100644 --- a/agent/src/editor.ts +++ b/agent/src/editor.ts @@ -71,7 +71,7 @@ export class AgentEditor implements Editor { return this.getActiveTextEditorSelection() } - public getActiveTextEditorDiagnosticsForSelectionOrEntireFile(): ActiveTextEditorDiagnostic[] | null { + public getActiveTextEditorDiagnosticsForRange(): ActiveTextEditorDiagnostic[] | null { return null } diff --git a/lib/shared/src/chat/context.ts b/lib/shared/src/chat/context.ts index cc082847db95..3bfa5ab722e8 100644 --- a/lib/shared/src/chat/context.ts +++ b/lib/shared/src/chat/context.ts @@ -6,6 +6,6 @@ export interface ChatContextStatus { connection?: boolean codebase?: string filePath?: string - selection?: ActiveTextEditorSelectionRange + selectionRange?: ActiveTextEditorSelectionRange supportsKeyword?: boolean } diff --git a/lib/shared/src/chat/recipes/fixup.ts b/lib/shared/src/chat/recipes/fixup.ts index f1e1ca740828..f6426832b4ab 100644 --- a/lib/shared/src/chat/recipes/fixup.ts +++ b/lib/shared/src/chat/recipes/fixup.ts @@ -73,6 +73,8 @@ export class Fixup implements Recipe { const truncatedPrecedingText = truncateTextStart(selection.precedingText, quarterFileContext) const truncatedFollowingText = truncateText(selection.followingText, quarterFileContext) + // Disable no case declarations because we get better type checking with a switch case + /* eslint-disable no-case-declarations */ switch (intent) { /** * Intents that are focused on producing new code. @@ -96,8 +98,12 @@ export class Fixup implements Recipe { * The fix intent is similar to adding or editing code, but with additional context that we can include from the editor. */ case 'fix': - // eslint-disable-next-line no-case-declarations - const diagnostics = context.editor.getActiveTextEditorDiagnosticsForSelectionOrEntireFile() || [] + // Get diagnostics (errors, warnings) for the current range + const range = + context.editor.getActiveTextEditor()?.selectionRange || + context.editor.controllers?.inline?.selectionRange + const diagnostics = range ? context.editor.getActiveTextEditorDiagnosticsForRange(range) || [] : [] + return getContextMessagesFromSelection( selection.selectedText, truncatedPrecedingText, @@ -142,6 +148,7 @@ export class Fixup implements Recipe { ) ) } + /* eslint-enable no-case-declarations */ } public async getInteraction(humanChatInput: string, context: RecipeContext): Promise { diff --git a/lib/shared/src/editor/index.ts b/lib/shared/src/editor/index.ts index 198a5cda2cc5..59dbe7232bed 100644 --- a/lib/shared/src/editor/index.ts +++ b/lib/shared/src/editor/index.ts @@ -3,7 +3,7 @@ export interface ActiveTextEditor { filePath: string repoName?: string revision?: string - selection?: ActiveTextEditorSelectionRange + selectionRange?: ActiveTextEditorSelectionRange } export interface ActiveTextEditorSelectionRange { @@ -43,6 +43,7 @@ export interface ActiveTextEditorVisibleContent { export interface VsCodeInlineController { selection: ActiveTextEditorSelection | null + selectionRange: ActiveTextEditorSelectionRange | null error(): Promise } @@ -91,7 +92,7 @@ export interface Editor< /** * Get diagnostics (errors, warnings) for the active text editor's selection, or the entire file if the selected range is empty. */ - getActiveTextEditorDiagnosticsForSelectionOrEntireFile(): ActiveTextEditorDiagnostic[] | null + getActiveTextEditorDiagnosticsForRange(range: ActiveTextEditorSelectionRange): ActiveTextEditorDiagnostic[] | null getActiveTextEditorVisibleContent(): ActiveTextEditorVisibleContent | null replaceSelection(fileName: string, selectedText: string, replacement: string): Promise @@ -121,7 +122,7 @@ export class NoopEditor implements Editor { return null } - public getActiveTextEditorDiagnosticsForSelectionOrEntireFile(): ActiveTextEditorDiagnostic[] | null { + public getActiveTextEditorDiagnosticsForRange(): ActiveTextEditorDiagnostic[] | null { return null } diff --git a/lib/shared/src/test/mocks.ts b/lib/shared/src/test/mocks.ts index e9a68744df9b..65a374b6d50b 100644 --- a/lib/shared/src/test/mocks.ts +++ b/lib/shared/src/test/mocks.ts @@ -5,6 +5,7 @@ import { ActiveTextEditor, ActiveTextEditorDiagnostic, ActiveTextEditorSelection, + ActiveTextEditorSelectionRange, ActiveTextEditorVisibleContent, Editor, } from '../editor' @@ -101,8 +102,10 @@ export class MockEditor implements Editor { return this.mocks.getActiveTextEditorSelection?.() ?? null } - public getActiveTextEditorDiagnosticsForSelectionOrEntireFile(): ActiveTextEditorDiagnostic[] | null { - return this.mocks.getActiveTextEditorDiagnosticsForSelectionOrEntireFile?.() ?? null + public getActiveTextEditorDiagnosticsForRange( + range: ActiveTextEditorSelectionRange + ): ActiveTextEditorDiagnostic[] | null { + return this.mocks.getActiveTextEditorDiagnosticsForRange?.(range) ?? null } public getActiveTextEditor(): ActiveTextEditor | null { diff --git a/lib/ui/src/chat/inputContext/ChatInputContext.story.tsx b/lib/ui/src/chat/inputContext/ChatInputContext.story.tsx index f689cfa51cf7..791f592cb884 100644 --- a/lib/ui/src/chat/inputContext/ChatInputContext.story.tsx +++ b/lib/ui/src/chat/inputContext/ChatInputContext.story.tsx @@ -60,7 +60,7 @@ export const CodebaseAndFileWithSelections: StoryObj = { codebase: 'github.com/sourcegraph/about', filePath: 'path/to/file.go', mode: 'embeddings', - selection: { start: { line: 0, character: 0 }, end: { line: 0, character: 0 } }, + selectionRange: { start: { line: 0, character: 0 }, end: { line: 0, character: 0 } }, }} /> = { codebase: 'github.com/sourcegraph/about', filePath: 'path/to/file.go', mode: 'embeddings', - selection: { start: { line: 0, character: 0 }, end: { line: 1, character: 0 } }, + selectionRange: { start: { line: 0, character: 0 }, end: { line: 1, character: 0 } }, }} /> = { codebase: 'github.com/sourcegraph/about', filePath: 'path/to/file.go', mode: 'embeddings', - selection: { start: { line: 0, character: 0 }, end: { line: 3, character: 0 } }, + selectionRange: { start: { line: 0, character: 0 }, end: { line: 3, character: 0 } }, }} /> = { codebase: 'github.com/sourcegraph/about', filePath: 'path/to/file.go', mode: 'embeddings', - selection: { start: { line: 42, character: 333 }, end: { line: 420, character: 999 } }, + selectionRange: { start: { line: 42, character: 333 }, end: { line: 420, character: 999 } }, }} /> diff --git a/lib/ui/src/chat/inputContext/ChatInputContext.tsx b/lib/ui/src/chat/inputContext/ChatInputContext.tsx index 6358710226c2..e682dc92c27c 100644 --- a/lib/ui/src/chat/inputContext/ChatInputContext.tsx +++ b/lib/ui/src/chat/inputContext/ChatInputContext.tsx @@ -9,7 +9,7 @@ import { Icon } from '../../utils/Icon' import styles from './ChatInputContext.module.css' -const formatFilePath = (filePath: string, selection: ChatContextStatus['selection']): string => { +const formatFilePath = (filePath: string, selection: ChatContextStatus['selectionRange']): string => { const fileName = basename(filePath) if (!selection) { @@ -57,7 +57,7 @@ export const ChatInputContext: React.FunctionComponent<{ )} {contextStatus.filePath && (

- {formatFilePath(contextStatus.filePath, contextStatus.selection)} + {formatFilePath(contextStatus.filePath, contextStatus.selectionRange)}

)} diff --git a/vscode/src/chat/ContextProvider.ts b/vscode/src/chat/ContextProvider.ts index 5966a4ca353a..f4faa55d2138 100644 --- a/vscode/src/chat/ContextProvider.ts +++ b/vscode/src/chat/ContextProvider.ts @@ -165,7 +165,7 @@ export class ContextProvider implements vscode.Disposable { connection: this.codebaseContext.checkEmbeddingsConnection(), codebase: this.codebaseContext.getCodebase(), filePath: editorContext ? vscode.workspace.asRelativePath(editorContext.filePath) : undefined, - selection: editorContext ? editorContext.selection : undefined, + selectionRange: editorContext ? editorContext.selectionRange : undefined, supportsKeyword: true, }, }) diff --git a/vscode/src/editor/vscode-editor.ts b/vscode/src/editor/vscode-editor.ts index 9d505dd4aa3b..0c61401c51cd 100644 --- a/vscode/src/editor/vscode-editor.ts +++ b/vscode/src/editor/vscode-editor.ts @@ -5,6 +5,7 @@ import { ActiveTextEditorDiagnostic, ActiveTextEditorDiagnosticType, ActiveTextEditorSelection, + ActiveTextEditorSelectionRange, ActiveTextEditorViewControllers, ActiveTextEditorVisibleContent, Editor, @@ -22,7 +23,7 @@ export class VSCodeEditor implements Editor - ) {} + ) { } public get fileName(): string { return vscode.window.activeTextEditor?.document.fileName ?? '' @@ -51,7 +52,7 @@ export class VSCodeEditor implements Editor range.contains(activeEditor.selection)) - // } - - return diagnostics.map(({ message, range, severity }) => ({ - range, - message, - type: this.getActiveTextEditorDiagnosticType(severity), - })) + const selectionRange = new vscode.Range( + new vscode.Position(start.line, start.character), + new vscode.Position(end.line, end.character) + ) + + return diagnostics + .filter(diagnostic => selectionRange.contains(diagnostic.range)) + .map(({ message, range, severity }) => ({ + range, + message, + type: this.getActiveTextEditorDiagnosticType(severity), + })) } private createActiveTextEditorSelection( From 2537ff66e3639372d0ea3f165dbb23bac101ca2b Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Fri, 28 Jul 2023 16:22:30 +0100 Subject: [PATCH 17/24] improve context by including error/warning snippet --- agent/src/editor.ts | 2 +- lib/shared/src/chat/recipes/fixup.ts | 139 ++++++++++++----------- lib/shared/src/editor/index.ts | 3 +- lib/shared/src/intent-detector/client.ts | 29 ++--- lib/shared/src/prompt/templates.ts | 13 ++- vscode/src/editor/vscode-editor.ts | 12 +- 6 files changed, 107 insertions(+), 91 deletions(-) diff --git a/agent/src/editor.ts b/agent/src/editor.ts index e9b3a2c46dcb..ca04f4f08233 100644 --- a/agent/src/editor.ts +++ b/agent/src/editor.ts @@ -72,7 +72,7 @@ export class AgentEditor implements Editor { } public getActiveTextEditorDiagnosticsForRange(): ActiveTextEditorDiagnostic[] | null { - return null + throw new Error('Method not implemented.') } public getActiveTextEditorVisibleContent(): ActiveTextEditorVisibleContent | null { diff --git a/lib/shared/src/chat/recipes/fixup.ts b/lib/shared/src/chat/recipes/fixup.ts index f6426832b4ab..7fa675986cb9 100644 --- a/lib/shared/src/chat/recipes/fixup.ts +++ b/lib/shared/src/chat/recipes/fixup.ts @@ -52,10 +52,75 @@ const PromptIntentInstruction: Record = { export class Fixup implements Recipe { public id: RecipeID = 'fixup' - private async getIntent(humanChatInput: string, context: RecipeContext): Promise { - // TODO(umpox): Implement a basic intent detection check that can return before reaching for the LLM. - // E.g. Current file is a test -> Test intent. Current selection is only a comment -> Documentation. + public async getInteraction(humanChatInput: string, context: RecipeContext): Promise { + // TODO: Prompt the user for additional direction. + const selection = context.editor.getActiveTextEditorSelection() || context.editor.controllers?.inline?.selection + if (!selection) { + await context.editor.controllers?.inline?.error() + await context.editor.showWarningMessage('Select some code to fixup.') + return null + } + const quarterFileContext = Math.floor(MAX_CURRENT_FILE_TOKENS / 4) + if (truncateText(selection.selectedText, quarterFileContext * 2) !== selection.selectedText) { + const msg = "The amount of text selected exceeds Cody's current capacity." + await context.editor.controllers?.inline?.error() + await context.editor.showWarningMessage(msg) + return null + } + const intent = await this.getIntent(humanChatInput, context) + + // Reconstruct Cody's prompt using user's context and intent + // Replace placeholders in reverse order to avoid collisions if a placeholder occurs in the input + // TODO: Move prompt suffix from recipe to chat view. It has other subscribers. + const promptText = Fixup.prompt + .replace('{humanInput}', truncateText(humanChatInput, MAX_HUMAN_INPUT_TOKENS)) + .replace('{selectedText}', selection.selectedText) + .replace('{fileName}', selection.fileName) + .replace('{intent}', PromptIntentInstruction[intent]) + + context.responseMultiplexer.sub( + 'selection', + new BufferedBotResponseSubscriber(async content => { + if (!content) { + await context.editor.controllers?.inline?.error() + await context.editor.showWarningMessage( + 'Cody did not suggest any replacement.\nTry starting a new conversation with Cody.' + ) + return + } + await context.editor.replaceSelection( + selection.fileName, + selection.selectedText, + contentSanitizer(content) + ) + }) + ) + + return Promise.resolve( + new Interaction( + { + speaker: 'human', + text: promptText, + displayText: '**✨Fixup✨** ' + humanChatInput, + }, + { + speaker: 'assistant', + prefix: 'Check your document for updates from Cody.\n', + }, + this.getContextFromIntent(intent, selection, quarterFileContext, context), + [] + ) + ) + } + + private async getIntent(humanChatInput: string, context: RecipeContext): Promise { + /** + * TODO(umpox): We should probably find a shorter way of detecting intent when possible. + * Possible methods: + * - Input -> Match first word against update|fix|add|delete verbs + * - Context -> Infer intent from context, e.g. Current file is a test -> Test intent, Current selection is a comment symbol -> Documentation intent + */ const intent = await context.intentDetector.classifyIntentFromOptions( humanChatInput, FixupIntentClassification, @@ -98,11 +163,11 @@ export class Fixup implements Recipe { * The fix intent is similar to adding or editing code, but with additional context that we can include from the editor. */ case 'fix': - // Get diagnostics (errors, warnings) for the current range const range = context.editor.getActiveTextEditor()?.selectionRange || context.editor.controllers?.inline?.selectionRange const diagnostics = range ? context.editor.getActiveTextEditorDiagnosticsForRange(range) || [] : [] + const errorsAndWarnings = diagnostics.filter(({ type }) => type === 'error' || type === 'warning') return getContextMessagesFromSelection( selection.selectedText, @@ -112,7 +177,7 @@ export class Fixup implements Recipe { context.codebaseContext ).then(messages => messages.concat( - diagnostics.flatMap(diagnostic => + errorsAndWarnings.flatMap(diagnostic => getContextMessageWithResponse( populateCurrentEditorDiagnosticsTemplate(diagnostic, selection.fileName), selection @@ -151,68 +216,6 @@ export class Fixup implements Recipe { /* eslint-enable no-case-declarations */ } - public async getInteraction(humanChatInput: string, context: RecipeContext): Promise { - // TODO: Prompt the user for additional direction. - const selection = context.editor.getActiveTextEditorSelection() || context.editor.controllers?.inline?.selection - if (!selection) { - await context.editor.controllers?.inline?.error() - await context.editor.showWarningMessage('Select some code to fixup.') - return null - } - const quarterFileContext = Math.floor(MAX_CURRENT_FILE_TOKENS / 4) - if (truncateText(selection.selectedText, quarterFileContext * 2) !== selection.selectedText) { - const msg = "The amount of text selected exceeds Cody's current capacity." - await context.editor.controllers?.inline?.error() - await context.editor.showWarningMessage(msg) - return null - } - - const intent = await this.getIntent(humanChatInput, context) - - // Reconstruct Cody's prompt using user's context and intent - // Replace placeholders in reverse order to avoid collisions if a placeholder occurs in the input - // TODO: Move prompt suffix from recipe to chat view. It has other subscribers. - const promptText = Fixup.prompt - .replace('{humanInput}', truncateText(humanChatInput, MAX_HUMAN_INPUT_TOKENS)) - .replace('{selectedText}', selection.selectedText) - .replace('{fileName}', selection.fileName) - .replace('{intent}', PromptIntentInstruction[intent]) - - context.responseMultiplexer.sub( - 'selection', - new BufferedBotResponseSubscriber(async content => { - if (!content) { - await context.editor.controllers?.inline?.error() - await context.editor.showWarningMessage( - 'Cody did not suggest any replacement.\nTry starting a new conversation with Cody.' - ) - return - } - await context.editor.replaceSelection( - selection.fileName, - selection.selectedText, - contentSanitizer(content) - ) - }) - ) - - return Promise.resolve( - new Interaction( - { - speaker: 'human', - text: promptText, - displayText: '**✨Fixup✨** ' + humanChatInput, - }, - { - speaker: 'assistant', - prefix: 'Check your document for updates from Cody.\n', - }, - this.getContextFromIntent(intent, selection, quarterFileContext, context), - [] - ) - ) - } - // Prompt Templates public static readonly prompt = ` - You are an AI programming assistant who is an expert in updating code to meet given instructions. @@ -226,7 +229,7 @@ export class Fixup implements Recipe { This is part of the file {fileName}. - The user has the following code within their selection: + The user has the following code in their selection: {selectedText} diff --git a/lib/shared/src/editor/index.ts b/lib/shared/src/editor/index.ts index 59dbe7232bed..bbf50bce029c 100644 --- a/lib/shared/src/editor/index.ts +++ b/lib/shared/src/editor/index.ts @@ -29,9 +29,10 @@ export interface ActiveTextEditorSelection { export type ActiveTextEditorDiagnosticType = 'error' | 'warning' | 'information' | 'hint' export interface ActiveTextEditorDiagnostic { + type: ActiveTextEditorDiagnosticType range: ActiveTextEditorSelectionRange + text: string message: string - type: ActiveTextEditorDiagnosticType } export interface ActiveTextEditorVisibleContent { diff --git a/lib/shared/src/intent-detector/client.ts b/lib/shared/src/intent-detector/client.ts index 8d0944df19df..20ae34b5db3e 100644 --- a/lib/shared/src/intent-detector/client.ts +++ b/lib/shared/src/intent-detector/client.ts @@ -29,12 +29,21 @@ export class SourcegraphIntentDetectorClient implements IntentDetector { return false } - private buildInitialPrompt(options: IntentClassificationOption[]): string { + private buildInitialTranscript(options: IntentClassificationOption[]): Message[] { const functions = options .map(({ id, description }) => `Function ID: ${id}\nFunction Description: ${description}`) .join('\n') - return prompt.replace('{functions}', functions) + return [ + { + speaker: 'human', + text: prompt.replace('{functions}', functions), + }, + { + speaker: 'assistant', + text: 'Ok.', + }, + ] } private buildExampleTranscript(options: IntentClassificationOption[]): Message[] { @@ -62,8 +71,8 @@ export class SourcegraphIntentDetectorClient implements IntentDetector { options: IntentClassificationOption[], fallback: Intent ): Promise { - const initialPrompt = this.buildInitialPrompt(options) - const exampleTranscript = this.buildExampleTranscript(options) + const preamble = this.buildInitialTranscript(options) + const examples = this.buildExampleTranscript(options) const result = await this.completionsClient.complete({ fast: true, @@ -72,15 +81,8 @@ export class SourcegraphIntentDetectorClient implements IntentDetector { topK: -1, topP: -1, messages: [ - { - speaker: 'human', - text: initialPrompt, - }, - { - speaker: 'assistant', - text: 'Ok.', - }, - ...exampleTranscript, + ...preamble, + ...examples, { speaker: 'human', text: input, @@ -92,7 +94,6 @@ export class SourcegraphIntentDetectorClient implements IntentDetector { }) const responseClassification = result.completion.match(/(.*?)<\/classification>/)?.[1] - console.log('LLM CLASSIFICATION', responseClassification) if (!responseClassification) { return fallback } diff --git a/lib/shared/src/prompt/templates.ts b/lib/shared/src/prompt/templates.ts index 4627ce03569a..621c16633417 100644 --- a/lib/shared/src/prompt/templates.ts +++ b/lib/shared/src/prompt/templates.ts @@ -68,15 +68,24 @@ export function populateCurrentEditorSelectedContextTemplate( ) } -const DIAGNOSTICS_CONTEXT_TEMPLATE = 'Use the following {type} from file `{filePath}`:\n{message}' +const DIAGNOSTICS_CONTEXT_TEMPLATE = `Use the following {type} from the code snippet in the file \`{filePath}\` +{prefix}: {message} +Code snippet: +\`\`\`{language} +{code} +\`\`\`` export function populateCurrentEditorDiagnosticsTemplate( - { message, type }: ActiveTextEditorDiagnostic, + { message, type, text }: ActiveTextEditorDiagnostic, filePath: string ): string { + const language = getExtension(filePath) return DIAGNOSTICS_CONTEXT_TEMPLATE.replace('{type}', type) .replace('{filePath}', filePath) + .replace('{prefix}', type) .replace('{message}', message) + .replace('{language}', language) + .replace('{code}', text) } const COMMAND_OUTPUT_TEMPLATE = 'Here is the output returned from the terminal.\n' diff --git a/vscode/src/editor/vscode-editor.ts b/vscode/src/editor/vscode-editor.ts index 0c61401c51cd..7fea3ecc4f94 100644 --- a/vscode/src/editor/vscode-editor.ts +++ b/vscode/src/editor/vscode-editor.ts @@ -23,7 +23,7 @@ export class VSCodeEditor implements Editor - ) { } + ) {} public get fileName(): string { return vscode.window.activeTextEditor?.document.fileName ?? '' @@ -101,9 +101,10 @@ export class VSCodeEditor implements Editor selectionRange.contains(diagnostic.range)) .map(({ message, range, severity }) => ({ + type: this.getActiveTextEditorDiagnosticType(severity), range, + text: activeEditor.document.getText(range), message, - type: this.getActiveTextEditorDiagnosticType(severity), })) } From 8bfa3c38a85ea033af129a091d87c7615313cc8d Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Mon, 31 Jul 2023 11:14:43 +0100 Subject: [PATCH 18/24] improve slightly --- lib/shared/src/chat/recipes/fixup.ts | 44 +++++++++++++--------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/lib/shared/src/chat/recipes/fixup.ts b/lib/shared/src/chat/recipes/fixup.ts index 7fa675986cb9..151133603b7e 100644 --- a/lib/shared/src/chat/recipes/fixup.ts +++ b/lib/shared/src/chat/recipes/fixup.ts @@ -75,9 +75,9 @@ export class Fixup implements Recipe { // TODO: Move prompt suffix from recipe to chat view. It has other subscribers. const promptText = Fixup.prompt .replace('{humanInput}', truncateText(humanChatInput, MAX_HUMAN_INPUT_TOKENS)) + .replace('{intent}', PromptIntentInstruction[intent]) .replace('{selectedText}', selection.selectedText) .replace('{fileName}', selection.fileName) - .replace('{intent}', PromptIntentInstruction[intent]) context.responseMultiplexer.sub( 'selection', @@ -124,7 +124,7 @@ export class Fixup implements Recipe { const intent = await context.intentDetector.classifyIntentFromOptions( humanChatInput, FixupIntentClassification, - 'fix' + 'edit' ) return intent } @@ -218,25 +218,23 @@ export class Fixup implements Recipe { // Prompt Templates public static readonly prompt = ` - - You are an AI programming assistant who is an expert in updating code to meet given instructions. - - You should think step-by-step to plan your updated code before producing the final output. - - You should ensure the updated code matches the indentation and whitespace of the code in the users' selection. - - Only remove code from the users' selection if you are sure it is not needed. - - It is not acceptable to use Markdown in your response. You should not produce Markdown-formatted code blocks. - - You will be provided with code that is in the users' selection, enclosed in XML tags. You must use this code to help you plan your updated code. - - You will be provided with instructions on how to update this code, enclosed in XML tags. You must follow these instructions carefully and to the letter. - - Enclose your response in XML tags. Do not provide anything else. - - This is part of the file {fileName}. - - The user has the following code in their selection: - - {selectedText} - - - {intent} - Provide your generated code using the following instructions: - - {humanInput} - ` +- You are an AI programming assistant who is an expert in updating code to meet given instructions. +- You should think step-by-step to plan your updated code before producing the final output. +- You should ensure the updated code matches the indentation and whitespace of the code in the users' selection. +- Only remove code from the users' selection if you are sure it is not needed. +- It is not acceptable to use Markdown in your response. You should not produce Markdown-formatted code blocks. +- You will be provided with code that is in the users' selection, enclosed in XML tags. You must use this code to help you plan your updated code. +- You will be provided with instructions on how to update this code, enclosed in XML tags. You must follow these instructions carefully and to the letter. +- Enclose your response in XML tags. Do not provide anything else. + +This is part of the file {fileName}. + +The user has the following code in their selection: +{selectedText} + +{intent} +Provide your generated code using the following instructions: + +{humanInput} +` } From 14f8f5b238a643ceb798e771673027288b9d937d Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Mon, 31 Jul 2023 11:30:41 +0100 Subject: [PATCH 19/24] update --- lib/shared/src/editor/index.ts | 2 +- lib/shared/src/intent-detector/client.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/shared/src/editor/index.ts b/lib/shared/src/editor/index.ts index bbf50bce029c..3a7b1714a7eb 100644 --- a/lib/shared/src/editor/index.ts +++ b/lib/shared/src/editor/index.ts @@ -91,7 +91,7 @@ export interface Editor< */ getActiveTextEditorSelectionOrEntireFile(): ActiveTextEditorSelection | null /** - * Get diagnostics (errors, warnings) for the active text editor's selection, or the entire file if the selected range is empty. + * Get diagnostics (errors, warnings, hints) for a range within the active text editor. */ getActiveTextEditorDiagnosticsForRange(range: ActiveTextEditorSelectionRange): ActiveTextEditorDiagnostic[] | null diff --git a/lib/shared/src/intent-detector/client.ts b/lib/shared/src/intent-detector/client.ts index 20ae34b5db3e..5f7f294f5d3d 100644 --- a/lib/shared/src/intent-detector/client.ts +++ b/lib/shared/src/intent-detector/client.ts @@ -103,7 +103,7 @@ export class SourcegraphIntentDetectorClient implements IntentDetector { } const prompt = ` -You are an AI chatbot in a code editor. You are at expert at understanding the request of a software developer and selecting an available function to perform that request. +You are an AI assistant in a text editor. You are at expert at understanding the request of a software developer and selecting an available function to perform that request. Think step-by-step to understand the request. Only provide your response if you know the answer or can make a well-informed guess, otherwise respond with "unknown". Enclose your response in XML tags. Do not provide anything else. From f3d8af7926e25f403c7451ee4715885a537094b4 Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Mon, 31 Jul 2023 11:38:56 +0100 Subject: [PATCH 20/24] Add fallback --- lib/shared/src/chat/recipes/fixup.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/shared/src/chat/recipes/fixup.ts b/lib/shared/src/chat/recipes/fixup.ts index 151133603b7e..4b030de29912 100644 --- a/lib/shared/src/chat/recipes/fixup.ts +++ b/lib/shared/src/chat/recipes/fixup.ts @@ -70,11 +70,18 @@ export class Fixup implements Recipe { const intent = await this.getIntent(humanChatInput, context) + // It is possible to trigger this recipe from the sidebar without any input. + // TODO: Consider deprecating this functionality once inline fixups and non-stop fixups and consolidated. + const promptInstruction = + humanChatInput.length > 0 + ? truncateText(humanChatInput, MAX_HUMAN_INPUT_TOKENS) + : "You should infer your instructions from the users' selection" + // Reconstruct Cody's prompt using user's context and intent // Replace placeholders in reverse order to avoid collisions if a placeholder occurs in the input // TODO: Move prompt suffix from recipe to chat view. It has other subscribers. const promptText = Fixup.prompt - .replace('{humanInput}', truncateText(humanChatInput, MAX_HUMAN_INPUT_TOKENS)) + .replace('{humanInput}', promptInstruction) .replace('{intent}', PromptIntentInstruction[intent]) .replace('{selectedText}', selection.selectedText) .replace('{fileName}', selection.fileName) From 3e198e63662654ab3e3372b416b1c6657a1c4c53 Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Mon, 31 Jul 2023 15:35:32 +0100 Subject: [PATCH 21/24] Update changelog --- vscode/CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vscode/CHANGELOG.md b/vscode/CHANGELOG.md index 7faf6d1f025e..d02f917a86b3 100644 --- a/vscode/CHANGELOG.md +++ b/vscode/CHANGELOG.md @@ -8,12 +8,16 @@ Starting from `0.2.0`, Cody is using `major.EVEN_NUMBER.patch` for release versi ### Added +- Inline Fixups: Cody is now aware of errors, warnings and hints within your editor selection. [pull/376](https://github.com/sourcegraph/cody/pull/376) + ### Fixed - Bug: Chat History command shows chat view instead of history view. [pull/414](https://github.com/sourcegraph/cody/pull/414) ### Changed +- Inline Fixups: Added intent detection to improve prompt and context quality. [pull/376](https://github.com/sourcegraph/cody/pull/376) + ## [0.6.3] ### Added From 945875bcfa02607dffe0dd31e085b21e32d0a62c Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Tue, 1 Aug 2023 08:36:13 +0100 Subject: [PATCH 22/24] add fallback --- lib/shared/src/intent-detector/client.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/shared/src/intent-detector/client.ts b/lib/shared/src/intent-detector/client.ts index 5f7f294f5d3d..9f6ac7295b18 100644 --- a/lib/shared/src/intent-detector/client.ts +++ b/lib/shared/src/intent-detector/client.ts @@ -10,7 +10,7 @@ const editorRegexps = [/editor/, /(open|current|this)\s+file/, /current(ly)?\s+o export class SourcegraphIntentDetectorClient implements IntentDetector { constructor( private client: SourcegraphGraphQLAPIClient, - private completionsClient: SourcegraphCompletionsClient + private completionsClient?: SourcegraphCompletionsClient ) {} public isCodebaseContextRequired(input: string): Promise { @@ -71,6 +71,10 @@ export class SourcegraphIntentDetectorClient implements IntentDetector { options: IntentClassificationOption[], fallback: Intent ): Promise { + if (!this.completionsClient) { + return fallback; + } + const preamble = this.buildInitialTranscript(options) const examples = this.buildExampleTranscript(options) From c399307214525ce30660b878caac05be0eced32f Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Tue, 1 Aug 2023 09:09:08 +0100 Subject: [PATCH 23/24] fix test --- lib/shared/src/intent-detector/client.ts | 51 +++++++++++++++--------- vscode/test/fixtures/mock-server.ts | 5 ++- 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/lib/shared/src/intent-detector/client.ts b/lib/shared/src/intent-detector/client.ts index 9f6ac7295b18..3b3d02c05c79 100644 --- a/lib/shared/src/intent-detector/client.ts +++ b/lib/shared/src/intent-detector/client.ts @@ -71,33 +71,48 @@ export class SourcegraphIntentDetectorClient implements IntentDetector { options: IntentClassificationOption[], fallback: Intent ): Promise { - if (!this.completionsClient) { + const completionsClient = this.completionsClient + if (!completionsClient) { return fallback; } const preamble = this.buildInitialTranscript(options) const examples = this.buildExampleTranscript(options) - const result = await this.completionsClient.complete({ - fast: true, - temperature: 0, - maxTokensToSample: ANSWER_TOKENS, - topK: -1, - topP: -1, - messages: [ - ...preamble, - ...examples, - { - speaker: 'human', - text: input, + const result = await new Promise(resolve => { + let responseText = ''; + return completionsClient.stream({ + fast: true, + temperature: 0, + maxTokensToSample: ANSWER_TOKENS, + topK: -1, + topP: -1, + messages: [ + ...preamble, + ...examples, + { + speaker: 'human', + text: input, + }, + { + speaker: 'assistant', + }, + ], + }, { + onChange: (text: string) => { + responseText = text }, - { - speaker: 'assistant', + onComplete: () => { + resolve(responseText) }, - ], - }) + onError: (message: string, statusCode?: number) => { + console.error(`Error detecting intent: Status code ${statusCode}: ${message}`); + resolve(fallback) + }, + }) + }) - const responseClassification = result.completion.match(/(.*?)<\/classification>/)?.[1] + const responseClassification = result.match(/(.*?)<\/classification>/)?.[1] if (!responseClassification) { return fallback } diff --git a/vscode/test/fixtures/mock-server.ts b/vscode/test/fixtures/mock-server.ts index 766fbd615854..c2e3aca466d8 100644 --- a/vscode/test/fixtures/mock-server.ts +++ b/vscode/test/fixtures/mock-server.ts @@ -22,7 +22,8 @@ const responses = { fixup: 'Goodbye Cody', } -const FIXUP_PROMPT_TAG = '' +const FIXUP_PROMPT_TAG = '' +const NON_STOP_FIXUP_PROMPT_TAG = '' // Runs a stub Cody service for testing. export async function run(around: () => Promise): Promise { @@ -36,7 +37,7 @@ export async function run(around: () => Promise): Promise { // or have a method on the server to send a set response the next time it sees a trigger word in the request. const request = req as MockRequest const lastHumanMessageIndex = request.body.messages.length - 2 - const response = request.body.messages[lastHumanMessageIndex].text.includes(FIXUP_PROMPT_TAG) + const response = request.body.messages[lastHumanMessageIndex].text.includes(FIXUP_PROMPT_TAG) || request.body.messages[lastHumanMessageIndex].text.includes(NON_STOP_FIXUP_PROMPT_TAG) ? responses.fixup : responses.chat res.send(`event: completion\ndata: {"completion": ${JSON.stringify(response)}}\n\nevent: done\ndata: {}\n\n`) From 137b4d23781ee4f3f71d19ca0ea8f1ec196e0b91 Mon Sep 17 00:00:00 2001 From: Tom Ross Date: Tue, 1 Aug 2023 09:49:03 +0100 Subject: [PATCH 24/24] Fix error --- lib/shared/src/intent-detector/client.ts | 65 +++++++++++++----------- vscode/test/fixtures/mock-server.ts | 8 +-- vscode/webviews/Recipes.tsx | 5 +- web/src/App.tsx | 3 ++ 4 files changed, 46 insertions(+), 35 deletions(-) diff --git a/lib/shared/src/intent-detector/client.ts b/lib/shared/src/intent-detector/client.ts index 3b3d02c05c79..3543b669c999 100644 --- a/lib/shared/src/intent-detector/client.ts +++ b/lib/shared/src/intent-detector/client.ts @@ -73,44 +73,47 @@ export class SourcegraphIntentDetectorClient implements IntentDetector { ): Promise { const completionsClient = this.completionsClient if (!completionsClient) { - return fallback; + return fallback } const preamble = this.buildInitialTranscript(options) const examples = this.buildExampleTranscript(options) - const result = await new Promise(resolve => { - let responseText = ''; - return completionsClient.stream({ - fast: true, - temperature: 0, - maxTokensToSample: ANSWER_TOKENS, - topK: -1, - topP: -1, - messages: [ - ...preamble, - ...examples, - { - speaker: 'human', - text: input, + const result = await new Promise(resolve => { + let responseText = '' + return completionsClient.stream( + { + fast: true, + temperature: 0, + maxTokensToSample: ANSWER_TOKENS, + topK: -1, + topP: -1, + messages: [ + ...preamble, + ...examples, + { + speaker: 'human', + text: input, + }, + { + speaker: 'assistant', + }, + ], + }, + { + onChange: (text: string) => { + responseText = text }, - { - speaker: 'assistant', + onComplete: () => { + resolve(responseText) }, - ], - }, { - onChange: (text: string) => { - responseText = text - }, - onComplete: () => { - resolve(responseText) - }, - onError: (message: string, statusCode?: number) => { - console.error(`Error detecting intent: Status code ${statusCode}: ${message}`); - resolve(fallback) - }, - }) - }) + onError: (message: string, statusCode?: number) => { + console.error(`Error detecting intent: Status code ${statusCode}: ${message}`) + resolve(fallback) + }, + } + ) + }) const responseClassification = result.match(/(.*?)<\/classification>/)?.[1] if (!responseClassification) { diff --git a/vscode/test/fixtures/mock-server.ts b/vscode/test/fixtures/mock-server.ts index c2e3aca466d8..db2d65afa55d 100644 --- a/vscode/test/fixtures/mock-server.ts +++ b/vscode/test/fixtures/mock-server.ts @@ -37,9 +37,11 @@ export async function run(around: () => Promise): Promise { // or have a method on the server to send a set response the next time it sees a trigger word in the request. const request = req as MockRequest const lastHumanMessageIndex = request.body.messages.length - 2 - const response = request.body.messages[lastHumanMessageIndex].text.includes(FIXUP_PROMPT_TAG) || request.body.messages[lastHumanMessageIndex].text.includes(NON_STOP_FIXUP_PROMPT_TAG) - ? responses.fixup - : responses.chat + const response = + request.body.messages[lastHumanMessageIndex].text.includes(FIXUP_PROMPT_TAG) || + request.body.messages[lastHumanMessageIndex].text.includes(NON_STOP_FIXUP_PROMPT_TAG) + ? responses.fixup + : responses.chat res.send(`event: completion\ndata: {"completion": ${JSON.stringify(response)}}\n\nevent: done\ndata: {}\n\n`) }) diff --git a/vscode/webviews/Recipes.tsx b/vscode/webviews/Recipes.tsx index afb18e3ca994..4430597b5745 100644 --- a/vscode/webviews/Recipes.tsx +++ b/vscode/webviews/Recipes.tsx @@ -9,7 +9,10 @@ import { VSCodeWrapper } from './utils/VSCodeApi' import styles from './Recipes.module.css' -type ClickableRecipeID = Exclude +type ClickableRecipeID = Exclude< + RecipeID, + 'chat-question' | 'inline-touch' | 'inline-chat' | 'my-prompt' | 'next-questions' | 'non-stop' +> type RecipeListType = Record diff --git a/web/src/App.tsx b/web/src/App.tsx index 53a16c6d84ee..669b1088e3cf 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -23,6 +23,9 @@ const editor: Editor = { getActiveTextEditorSelectionOrEntireFile() { return null }, + getActiveTextEditorDiagnosticsForRange() { + return null + }, getActiveTextEditorVisibleContent() { return null },