From 7d6d5b55fe94efc8ddb13e8fe872de99b079ed54 Mon Sep 17 00:00:00 2001 From: Quinn Slack Date: Sat, 16 Mar 2024 12:07:16 -0700 Subject: [PATCH] factor out @-mention query scanning and parsing --- lib/shared/src/index.ts | 5 + lib/shared/src/mentions/query.test.ts | 102 ++++++++++++++++++ lib/shared/src/mentions/query.ts | 101 +++++++++++++++++ vscode/src/chat/context/chatContext.ts | 29 ++--- vscode/src/edit/input/constants.ts | 1 + vscode/src/edit/input/get-input.ts | 34 ++++-- vscode/src/edit/input/get-matching-context.ts | 22 ++-- .../plugins/atMentions/OptionsList.tsx | 9 +- .../plugins/atMentions/atMentions.test.tsx | 61 +---------- .../plugins/atMentions/atMentions.tsx | 63 ++--------- .../plugins/atMentions/fixtures.ts | 10 +- 11 files changed, 279 insertions(+), 158 deletions(-) create mode 100644 lib/shared/src/mentions/query.test.ts create mode 100644 lib/shared/src/mentions/query.ts diff --git a/lib/shared/src/index.ts b/lib/shared/src/index.ts index 8d5444f7514e..dc06e91962b6 100644 --- a/lib/shared/src/index.ts +++ b/lib/shared/src/index.ts @@ -199,3 +199,8 @@ export type { CurrentUserCodySubscription } from './sourcegraph-api/graphql/clie export * from './auth/types' export * from './auth/tokens' export * from './chat/sse-iterator' +export { + parseMentionQuery, + type MentionQuery, + scanForMentionTriggerInUserTextInput, +} from './mentions/query' diff --git a/lib/shared/src/mentions/query.test.ts b/lib/shared/src/mentions/query.test.ts new file mode 100644 index 000000000000..b47e09dcf106 --- /dev/null +++ b/lib/shared/src/mentions/query.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, test } from 'vitest' + +import { + type MentionQuery, + type MentionTrigger, + parseMentionQuery, + scanForMentionTriggerInUserTextInput, +} from './query' + +describe('parseMentionQuery', () => { + test('empty query for empty string', () => { + expect(parseMentionQuery('')).toEqual({ + type: 'empty', + text: '', + }) + }) + + test('file query without prefix', () => { + expect(parseMentionQuery('foo')).toEqual({ + type: 'file', + text: 'foo', + }) + }) + + test('symbol query without prefix', () => { + expect(parseMentionQuery('#bar')).toEqual({ + type: 'symbol', + text: 'bar', + }) + }) + + test('file query with @ prefix', () => { + // Note: This means that the user is literally looking for a file whose name contains `@`. + // This is a very rare case. See the docstring for `parseMentionQuery`. + expect(parseMentionQuery('@baz')).toEqual({ + type: 'file', + text: '@baz', + }) + }) +}) + +describe('scanForMentionTriggerInUserTextInput', () => { + test('null if no @-mention is found', () => + expect(scanForMentionTriggerInUserTextInput('Hello world')).toBeNull()) + + test('@-mention file', () => + expect(scanForMentionTriggerInUserTextInput('Hello @abc')).toEqual({ + leadOffset: 6, + matchingString: 'abc', + replaceableString: '@abc', + })) + + test('@-mention symbol', () => + expect(scanForMentionTriggerInUserTextInput('Hello @#abc')).toEqual({ + leadOffset: 6, + matchingString: '#abc', + replaceableString: '@#abc', + })) + + describe('special chars', () => { + test('dotfile', () => + expect(scanForMentionTriggerInUserTextInput('Hello @.abc')).toEqual({ + leadOffset: 6, + matchingString: '.abc', + replaceableString: '@.abc', + })) + + test('forward slash', () => + expect(scanForMentionTriggerInUserTextInput('Hello @a/b')).toEqual({ + leadOffset: 6, + matchingString: 'a/b', + replaceableString: '@a/b', + })) + + test('backslash', () => + expect(scanForMentionTriggerInUserTextInput('Hello @a\\b')).toEqual({ + leadOffset: 6, + matchingString: 'a\\b', + replaceableString: '@a\\b', + })) + + test('hyphen', () => + expect( + scanForMentionTriggerInUserTextInput('Hello @a-b.txt') + ).toEqual({ + leadOffset: 6, + matchingString: 'a-b.txt', + replaceableString: '@a-b.txt', + })) + }) + + test('with range', () => { + expect(scanForMentionTriggerInUserTextInput('a @b/c:')).toBeNull() + expect(scanForMentionTriggerInUserTextInput('a @b/c:1')).toBeNull() + expect(scanForMentionTriggerInUserTextInput('a @b/c:12-')).toBeNull() + expect(scanForMentionTriggerInUserTextInput('a @b/c:12-34')).toEqual({ + leadOffset: 2, + matchingString: 'b/c:12-34', + replaceableString: '@b/c:12-34', + }) + }) +}) diff --git a/lib/shared/src/mentions/query.ts b/lib/shared/src/mentions/query.ts new file mode 100644 index 000000000000..48bdc92ad86b --- /dev/null +++ b/lib/shared/src/mentions/query.ts @@ -0,0 +1,101 @@ +/** + * The parsed representation of a user's (partial or complete) input of an @-mention query. + */ +export interface MentionQuery { + /** + * The type of context item to search for. + */ + type: 'file' | 'symbol' | 'empty' + + /** + * The user's text input, to be interpreted as a fuzzy-matched query. It is stripped of any + * prefix characters that indicate the {@link MentionQuery.type}, such as `#` for symbols. + */ + text: string +} + +/** + * Parse an @-mention query. The {@link query} value is whatever the UI determines is the query + * based on the current text input; it is not the full value of a message that may or may not + * contain an @-mention. + * + * The {@link query} MUST be stripped of the trigger character (usually `@`). The only valid case + * where {@link query} may begin with `@` is if the user is searching for context items that contain + * `@`, such as if the user typed `@@foo` to mention a file that is literally named `@foo.js`. + */ +export function parseMentionQuery(query: string): MentionQuery { + if (query === '') { + return { type: 'empty', text: '' } + } + + if (query.startsWith('#')) { + return { type: 'symbol', text: query.slice(1) } + } + return { type: 'file', text: query } +} + +const PUNCTUATION = ',\\+\\*\\?\\$\\@\\|#{}\\(\\)\\^\\[\\]!%\'"~=<>:;' + +const TRIGGERS = '@' + +/** Chars we expect to see in a mention (non-space, non-punctuation). */ +const VALID_CHARS = '[^' + TRIGGERS + PUNCTUATION + '\\s]' + +const MAX_LENGTH = 250 + +const RANGE_REGEXP = '(?::\\d+-\\d+)?' + +const AT_MENTIONS_REGEXP = new RegExp( + '(?^|\\s|\\()(?' + + '[' + + TRIGGERS + + ']' + + '(?#?(?:' + + VALID_CHARS + + '){0,' + + MAX_LENGTH + + '}' + + RANGE_REGEXP + + ')' + + ')$' +) + +/** + * The location and content of a mention in free-form user text input. + */ +export interface MentionTrigger { + /** The number of characters from the start of the text to the mention trigger (`@`). */ + leadOffset: number + + /** + * The string that is used to query for the context item to mention (to be passed to + * {@link parseMentionQuery}). + */ + matchingString: string + + /** + * Equal to `@` + {@link matchingString}. The entire string that should be replaced with the + * context item when the at-mention reference is chosen. + */ + replaceableString: string +} + +/** + * Scans free-form user text input (in a chat message editor, for example) for possible mentions + * with the `@` trigger character. + * + * The {@link textBeforeCursor} is all of the text in the input field before the text insertion + * point cursor. For example, if the input field looks like `hello + * @foo█bar`, then {@link textBeforeCursor} should be `hello @foo`. + */ +export function scanForMentionTriggerInUserTextInput(textBeforeCursor: string): MentionTrigger | null { + const match = AT_MENTIONS_REGEXP.exec(textBeforeCursor) + if (match?.groups) { + return { + leadOffset: match.index + match.groups.maybeLeadingWhitespace.length, + matchingString: match.groups.matchingString, + replaceableString: match.groups.replaceableString, + } + } + return null +} diff --git a/vscode/src/chat/context/chatContext.ts b/vscode/src/chat/context/chatContext.ts index 16ced211fdf9..9bdf4856b743 100644 --- a/vscode/src/chat/context/chatContext.ts +++ b/vscode/src/chat/context/chatContext.ts @@ -1,4 +1,4 @@ -import type { ContextItem } from '@sourcegraph/cody-shared' +import { type ContextItem, type MentionQuery, parseMentionQuery } from '@sourcegraph/cody-shared' import type * as vscode from 'vscode' import { getFileContextFiles, @@ -11,27 +11,30 @@ export async function getChatContextItemsForMention( cancellationToken: vscode.CancellationToken, telemetryRecorder?: { empty: () => void - withType: (type: 'symbol' | 'file') => void + withType: (type: MentionQuery['type']) => void } ): Promise { + const mentionQuery = parseMentionQuery(query) + // Logging: log when the at-mention starts, and then log when we know the type (after the 1st // character is typed). Don't log otherwise because we would be logging prefixes of the same // query repeatedly, which is not needed. - const queryType = query.startsWith('#') ? 'symbol' : 'file' - if (query === '') { + if (mentionQuery.type === 'empty') { telemetryRecorder?.empty() } else if (query.length === 1) { - telemetryRecorder?.withType(queryType) - } - - if (query.length === 0) { - return getOpenTabsContextFile() + telemetryRecorder?.withType(mentionQuery.type) } const MAX_RESULTS = 20 - if (query.startsWith('#')) { - // It would be nice if the VS Code symbols API supports cancellation, but it doesn't - return getSymbolContextFiles(query.slice(1), MAX_RESULTS) + switch (mentionQuery.type) { + case 'empty': + return getOpenTabsContextFile() + case 'symbol': + // It would be nice if the VS Code symbols API supports cancellation, but it doesn't + return getSymbolContextFiles(mentionQuery.text, MAX_RESULTS) + case 'file': + return getFileContextFiles(mentionQuery.text, MAX_RESULTS, cancellationToken) + default: + return [] } - return getFileContextFiles(query, MAX_RESULTS, cancellationToken) } diff --git a/vscode/src/edit/input/constants.ts b/vscode/src/edit/input/constants.ts index 56e85df3150f..82511feb0360 100644 --- a/vscode/src/edit/input/constants.ts +++ b/vscode/src/edit/input/constants.ts @@ -1,5 +1,6 @@ export const FILE_HELP_LABEL = 'Search for a file to include, or type # to search symbols...' export const SYMBOL_HELP_LABEL = 'Search for a symbol to include...' +export const OTHER_MENTION_HELP_LABEL = 'Search for context to include...' export const NO_MATCHES_LABEL = 'No matches found' export const QUICK_PICK_ITEM_EMPTY_INDENT_PREFIX = '\u00A0\u00A0\u00A0\u00A0\u00A0' diff --git a/vscode/src/edit/input/get-input.ts b/vscode/src/edit/input/get-input.ts index 1bb14027d087..7770c8220321 100644 --- a/vscode/src/edit/input/get-input.ts +++ b/vscode/src/edit/input/get-input.ts @@ -3,6 +3,8 @@ import { type ContextItem, type EditModel, displayLineRange, + parseMentionQuery, + scanForMentionTriggerInUserTextInput, } from '@sourcegraph/cody-shared' import * as vscode from 'vscode' @@ -17,7 +19,12 @@ import { executeEdit } from '../execute' import type { EditIntent } from '../types' import { isGenerateIntent } from '../utils/edit-intent' import { getEditModelsForUser } from '../utils/edit-models' -import { FILE_HELP_LABEL, NO_MATCHES_LABEL, SYMBOL_HELP_LABEL } from './constants' +import { + FILE_HELP_LABEL, + NO_MATCHES_LABEL, + OTHER_MENTION_HELP_LABEL, + SYMBOL_HELP_LABEL, +} from './constants' import { CURSOR_RANGE_ITEM, EXPANDED_RANGE_ITEM, SELECTION_RANGE_ITEM } from './get-items/constants' import { getDocumentInputItems } from './get-items/document' import { DOCUMENT_ITEM, MODEL_ITEM, RANGE_ITEM, TEST_ITEM, getEditInputItems } from './get-items/edit' @@ -357,23 +364,28 @@ export const getInput = async ( return } - const isFileSearch = value.endsWith('@') - const isSymbolSearch = value.endsWith('@#') + const mentionTrigger = scanForMentionTriggerInUserTextInput(value) + const mentionQuery = mentionTrigger + ? parseMentionQuery(mentionTrigger.matchingString) + : undefined // If we have the beginning of a file or symbol match, show a helpful label - if (isFileSearch) { - input.items = [{ alwaysShow: true, label: FILE_HELP_LABEL }] - return - } - if (isSymbolSearch) { - input.items = [{ alwaysShow: true, label: SYMBOL_HELP_LABEL }] + if (mentionQuery?.text === '') { + if (mentionQuery.type === 'empty' || mentionQuery.type === 'file') { + input.items = [{ alwaysShow: true, label: FILE_HELP_LABEL }] + return + } + if (mentionQuery.type === 'symbol') { + input.items = [{ alwaysShow: true, label: SYMBOL_HELP_LABEL }] + return + } + input.items = [{ alwaysShow: true, label: OTHER_MENTION_HELP_LABEL }] return } - const matchingContext = await getMatchingContext(value) + const matchingContext = mentionQuery ? await getMatchingContext(mentionQuery) : null if (matchingContext === null) { // Nothing to match, re-render existing items - // eslint-disable-next-line no-self-assign input.items = getEditInputItems( input.value, activeRangeItem, diff --git a/vscode/src/edit/input/get-matching-context.ts b/vscode/src/edit/input/get-matching-context.ts index d548af59c1cb..67830fddb075 100644 --- a/vscode/src/edit/input/get-matching-context.ts +++ b/vscode/src/edit/input/get-matching-context.ts @@ -1,15 +1,9 @@ -import type { ContextItem } from '@sourcegraph/cody-shared' +import type { ContextItem, MentionQuery } from '@sourcegraph/cody-shared' import * as vscode from 'vscode' import { getFileContextFiles, getSymbolContextFiles } from '../../editor/utils/editor-context' import { getLabelForContextItem } from './utils' -/* Match strings that end with a '@' followed by any characters except a space */ -const MATCHING_CONTEXT_FILE_REGEX = /@(\S+)$/ - -/* Match strings that end with a '@#' followed by any characters except a space */ -const MATCHING_SYMBOL_REGEX = /@#(\S+)$/ - const MAX_FUZZY_RESULTS = 20 interface FixupMatchingContext { @@ -20,10 +14,11 @@ interface FixupMatchingContext { file: ContextItem } -export async function getMatchingContext(instruction: string): Promise { - const symbolMatch = instruction.match(MATCHING_SYMBOL_REGEX) - if (symbolMatch) { - const symbolResults = await getSymbolContextFiles(symbolMatch[1], MAX_FUZZY_RESULTS) +export async function getMatchingContext( + mentionQuery: MentionQuery +): Promise { + if (mentionQuery.type === 'symbol') { + const symbolResults = await getSymbolContextFiles(mentionQuery.text, MAX_FUZZY_RESULTS) return symbolResults.map(result => ({ key: getLabelForContextItem(result), file: result, @@ -33,11 +28,10 @@ export async function getMatchingContext(instruction: string): Promise

- {query === '' + {mentionQuery.type === 'empty' ? 'Search for a file to include, or type # for symbols...' - : query.startsWith('#') + : mentionQuery.type === 'symbol' ? options.length > 0 ? 'Search for a symbol to include...' : `No symbols found${ - query.length <= 2 + mentionQuery.text.length <= 2 ? ' (try installing language extensions and opening a file)' : '' }` diff --git a/vscode/webviews/promptEditor/plugins/atMentions/atMentions.test.tsx b/vscode/webviews/promptEditor/plugins/atMentions/atMentions.test.tsx index f7eb08d9b4d9..b09dc3e0d8f1 100644 --- a/vscode/webviews/promptEditor/plugins/atMentions/atMentions.test.tsx +++ b/vscode/webviews/promptEditor/plugins/atMentions/atMentions.test.tsx @@ -1,65 +1,6 @@ -import type { MenuTextMatch } from '@lexical/react/LexicalTypeaheadMenuPlugin' import { describe, expect, test } from 'vitest' -import { getPossibleQueryMatch, parseLineRangeInMention } from './atMentions' +import { parseLineRangeInMention } from './atMentions' -describe('getPossibleQueryMatch', () => { - test('null if no @-mention is found', () => expect(getPossibleQueryMatch('Hello world')).toBeNull()) - - test('@-mention file', () => - expect(getPossibleQueryMatch('Hello @abc')).toEqual({ - leadOffset: 6, - matchingString: 'abc', - replaceableString: '@abc', - })) - - test('@-mention symbol', () => - expect(getPossibleQueryMatch('Hello @#abc')).toEqual({ - leadOffset: 6, - matchingString: '#abc', - replaceableString: '@#abc', - })) - - describe('special chars', () => { - test('dotfile', () => - expect(getPossibleQueryMatch('Hello @.abc')).toEqual({ - leadOffset: 6, - matchingString: '.abc', - replaceableString: '@.abc', - })) - - test('forward slash', () => - expect(getPossibleQueryMatch('Hello @a/b')).toEqual({ - leadOffset: 6, - matchingString: 'a/b', - replaceableString: '@a/b', - })) - - test('backslash', () => - expect(getPossibleQueryMatch('Hello @a\\b')).toEqual({ - leadOffset: 6, - matchingString: 'a\\b', - replaceableString: '@a\\b', - })) - - test('hyphen', () => - expect(getPossibleQueryMatch('Hello @a-b.txt')).toEqual({ - leadOffset: 6, - matchingString: 'a-b.txt', - replaceableString: '@a-b.txt', - })) - }) - - test('with range', () => { - expect(getPossibleQueryMatch('a @b/c:')).toBeNull() - expect(getPossibleQueryMatch('a @b/c:1')).toBeNull() - expect(getPossibleQueryMatch('a @b/c:12-')).toBeNull() - expect(getPossibleQueryMatch('a @b/c:12-34')).toEqual({ - leadOffset: 2, - matchingString: 'b/c:12-34', - replaceableString: '@b/c:12-34', - }) - }) -}) describe('parseLineRangeInMention', () => { test('invalid line ranges', () => { expect(parseLineRangeInMention('')).toEqual({ textWithoutRange: '' }) diff --git a/vscode/webviews/promptEditor/plugins/atMentions/atMentions.tsx b/vscode/webviews/promptEditor/plugins/atMentions/atMentions.tsx index 99674c6acc1d..eda2221b6540 100644 --- a/vscode/webviews/promptEditor/plugins/atMentions/atMentions.tsx +++ b/vscode/webviews/promptEditor/plugins/atMentions/atMentions.tsx @@ -1,10 +1,6 @@ import { FloatingPortal, flip, offset, shift, useFloating } from '@floating-ui/react' import { useLexicalComposerContext } from '@lexical/react/LexicalComposerContext' -import { - LexicalTypeaheadMenuPlugin, - MenuOption, - type MenuTextMatch, -} from '@lexical/react/LexicalTypeaheadMenuPlugin' +import { LexicalTypeaheadMenuPlugin, MenuOption } from '@lexical/react/LexicalTypeaheadMenuPlugin' import { $createTextNode, COMMAND_PRIORITY_NORMAL, @@ -15,23 +11,17 @@ import { import { type FunctionComponent, useCallback, useEffect, useMemo, useState } from 'react' import styles from './atMentions.module.css' -import { type ContextItem, type RangeData, displayPath } from '@sourcegraph/cody-shared' +import { + type ContextItem, + type RangeData, + displayPath, + scanForMentionTriggerInUserTextInput, +} from '@sourcegraph/cody-shared' import classNames from 'classnames' import { $createContextItemMentionNode } from '../../nodes/ContextItemMentionNode' import { OptionsList } from './OptionsList' import { useChatContextItems } from './chatContextClient' -const PUNCTUATION = ',\\+\\*\\?\\$\\@\\|#{}\\(\\)\\^\\[\\]!%\'"~=<>:;' - -const TRIGGERS = ['@'].join('') - -// Chars we expect to see in a mention (non-space, non-punctuation). -const VALID_CHARS = '[^' + TRIGGERS + PUNCTUATION + '\\s]' - -const MAX_LENGTH = 75 - -const RANGE_REGEXP = '(?::\\d+-\\d+)?' - /** * Parses the line range (if any) at the end of a string like `foo.txt:1-2`. Because this means "all * of lines 1 and 2", the returned range actually goes to the start of line 3 to ensure all of line @@ -61,45 +51,8 @@ export function parseLineRangeInMention(text: string): { } } -const AT_MENTIONS_REGEXP = new RegExp( - '(?^|\\s|\\()(?' + - '[' + - TRIGGERS + - ']' + - '(?#?(?:' + - VALID_CHARS + - '){0,' + - MAX_LENGTH + - '}' + - RANGE_REGEXP + - ')' + - ')$' -) - const SUGGESTION_LIST_LENGTH_LIMIT = 20 -function checkForAtSignMentions(text: string, minMatchLength: number): MenuTextMatch | null { - const match = AT_MENTIONS_REGEXP.exec(text) - - if (match?.groups) { - const maybeLeadingWhitespace = match.groups.maybeLeadingWhitespace - const replaceableString = match.groups.replaceableString - const matchingString = match.groups.matchingString - if (matchingString.length >= minMatchLength) { - return { - leadOffset: match.index + maybeLeadingWhitespace.length, - matchingString, - replaceableString, - } - } - } - return null -} - -export function getPossibleQueryMatch(text: string): MenuTextMatch | null { - return checkForAtSignMentions(text, 0) -} - export class MentionTypeaheadOption extends MenuOption { public displayPath: string @@ -169,7 +122,7 @@ export default function MentionsPlugin(): JSX.Element | null { onQueryChange={onQueryChange} onSelectOption={onSelectOption} - triggerFn={getPossibleQueryMatch} + triggerFn={scanForMentionTriggerInUserTextInput} options={options} anchorClassName={styles.resetAnchor} commandPriority={ diff --git a/vscode/webviews/promptEditor/plugins/atMentions/fixtures.ts b/vscode/webviews/promptEditor/plugins/atMentions/fixtures.ts index c9c449e2dfb7..259c2f3c8a59 100644 --- a/vscode/webviews/promptEditor/plugins/atMentions/fixtures.ts +++ b/vscode/webviews/promptEditor/plugins/atMentions/fixtures.ts @@ -1,4 +1,9 @@ -import type { ContextItem, ContextItemSymbol, SymbolKind } from '@sourcegraph/cody-shared' +import { + type ContextItem, + type ContextItemSymbol, + type SymbolKind, + parseMentionQuery, +} from '@sourcegraph/cody-shared' import { URI } from 'vscode-uri' import type { ChatContextClient } from './chatContextClient' @@ -11,7 +16,8 @@ export const dummyChatContextClient: ChatContextClient = { await new Promise(resolve => setTimeout(resolve, 250)) query = query.toLowerCase() - return query.startsWith('#') + const mentionQuery = parseMentionQuery(query) + return mentionQuery.type === 'symbol' ? DUMMY_SYMBOLS.filter( f => f.symbolName.toLowerCase().includes(query.slice(1)) ||