factor out @-mention query scanning and parsing

lib/shared/src/index.ts

@@ -199,3 +199,8 @@ export type { CurrentUserCodySubscription } from './sourcegraph-api/graphql/clie
 export * from './auth/types'
 export * from './auth/tokens'
 export * from './chat/sse-iterator'
+export {
+    parseMentionQuery,
+    type MentionQuery,
+    scanForMentionTriggerInUserTextInput,
+} from './mentions/query'

lib/shared/src/mentions/query.test.ts

@@ -0,0 +1,102 @@
+import { describe, expect, test } from 'vitest'
+
+import {
+    type MentionQuery,
+    type MentionTrigger,
+    parseMentionQuery,
+    scanForMentionTriggerInUserTextInput,
+} from './query'
+
+describe('parseMentionQuery', () => {
+    test('empty query for empty string', () => {
+        expect(parseMentionQuery('')).toEqual<MentionQuery>({
+            type: 'empty',
+            text: '',
+        })
+    })
+
+    test('file query without prefix', () => {
+        expect(parseMentionQuery('foo')).toEqual<MentionQuery>({
+            type: 'file',
+            text: 'foo',
+        })
+    })
+
+    test('symbol query without prefix', () => {
+        expect(parseMentionQuery('#bar')).toEqual<MentionQuery>({
+            type: 'symbol',
+            text: 'bar',
+        })
+    })
+
+    test('file query with @ prefix', () => {
+        // Note: This means that the user is literally looking for a file whose name contains `@`.
+        // This is a very rare case. See the docstring for `parseMentionQuery`.
+        expect(parseMentionQuery('@baz')).toEqual<MentionQuery>({
+            type: 'file',
+            text: '@baz',
+        })
+    })
+})
+
+describe('scanForMentionTriggerInUserTextInput', () => {
+    test('null if no @-mention is found', () =>
+        expect(scanForMentionTriggerInUserTextInput('Hello world')).toBeNull())
+
+    test('@-mention file', () =>
+        expect(scanForMentionTriggerInUserTextInput('Hello @abc')).toEqual<MentionTrigger | null>({
+            leadOffset: 6,
+            matchingString: 'abc',
+            replaceableString: '@abc',
+        }))
+
+    test('@-mention symbol', () =>
+        expect(scanForMentionTriggerInUserTextInput('Hello @#abc')).toEqual<MentionTrigger | null>({
+            leadOffset: 6,
+            matchingString: '#abc',
+            replaceableString: '@#abc',
+        }))
+
+    describe('special chars', () => {
+        test('dotfile', () =>
+            expect(scanForMentionTriggerInUserTextInput('Hello @.abc')).toEqual<MentionTrigger | null>({
+                leadOffset: 6,
+                matchingString: '.abc',
+                replaceableString: '@.abc',
+            }))
+
+        test('forward slash', () =>
+            expect(scanForMentionTriggerInUserTextInput('Hello @a/b')).toEqual<MentionTrigger | null>({
+                leadOffset: 6,
+                matchingString: 'a/b',
+                replaceableString: '@a/b',
+            }))
+
+        test('backslash', () =>
+            expect(scanForMentionTriggerInUserTextInput('Hello @a\\b')).toEqual<MentionTrigger | null>({
+                leadOffset: 6,
+                matchingString: 'a\\b',
+                replaceableString: '@a\\b',
+            }))
+
+        test('hyphen', () =>
+            expect(
+                scanForMentionTriggerInUserTextInput('Hello @a-b.txt')
+            ).toEqual<MentionTrigger | null>({
+                leadOffset: 6,
+                matchingString: 'a-b.txt',
+                replaceableString: '@a-b.txt',
+            }))
+    })
+
+    test('with range', () => {
+        expect(scanForMentionTriggerInUserTextInput('a @b/c:')).toBeNull()
+        expect(scanForMentionTriggerInUserTextInput('a @b/c:1')).toBeNull()
+        expect(scanForMentionTriggerInUserTextInput('a @b/c:12-')).toBeNull()
+        expect(scanForMentionTriggerInUserTextInput('a @b/c:12-34')).toEqual<MentionTrigger>({
+            leadOffset: 2,
+            matchingString: 'b/c:12-34',
+            replaceableString: '@b/c:12-34',
+        })
+    })
+})

lib/shared/src/mentions/query.ts

@@ -0,0 +1,101 @@
+/**
+ * The parsed representation of a user's (partial or complete) input of an @-mention query.
+ */
+export interface MentionQuery {
+    /**
+     * The type of context item to search for.
+     */
+    type: 'file' | 'symbol' | 'empty'
+
+    /**
+     * The user's text input, to be interpreted as a fuzzy-matched query. It is stripped of any
+     * prefix characters that indicate the {@link MentionQuery.type}, such as `#` for symbols.
+     */
+    text: string
+}
+
+/**
+ * Parse an @-mention query. The {@link query} value is whatever the UI determines is the query
+ * based on the current text input; it is not the full value of a message that may or may not
+ * contain an @-mention.
+ *
+ * The {@link query} MUST be stripped of the trigger character (usually `@`). The only valid case
+ * where {@link query} may begin with `@` is if the user is searching for context items that contain
+ * `@`, such as if the user typed `@@foo` to mention a file that is literally named `@foo.js`.
+ */
+export function parseMentionQuery(query: string): MentionQuery {
+    if (query === '') {
+        return { type: 'empty', text: '' }
+    }
+
+    if (query.startsWith('#')) {
+        return { type: 'symbol', text: query.slice(1) }
+    }
+    return { type: 'file', text: query }
+}
+
+const PUNCTUATION = ',\\+\\*\\?\\$\\@\\|#{}\\(\\)\\^\\[\\]!%\'"~=<>:;'
+
+const TRIGGERS = '@'
+
+/** Chars we expect to see in a mention (non-space, non-punctuation). */
+const VALID_CHARS = '[^' + TRIGGERS + PUNCTUATION + '\\s]'
+
+const MAX_LENGTH = 250
+
+const RANGE_REGEXP = '(?::\\d+-\\d+)?'
+
+const AT_MENTIONS_REGEXP = new RegExp(
+    '(?<maybeLeadingWhitespace>^|\\s|\\()(?<replaceableString>' +
+        '[' +
+        TRIGGERS +
+        ']' +
+        '(?<matchingString>#?(?:' +
+        VALID_CHARS +
+        '){0,' +
+        MAX_LENGTH +
+        '}' +
+        RANGE_REGEXP +
+        ')' +
+        ')$'
+)
+
+/**
+ * The location and content of a mention in free-form user text input.
+ */
+export interface MentionTrigger {
+    /** The number of characters from the start of the text to the mention trigger (`@`). */
+    leadOffset: number
+
+    /**
+     * The string that is used to query for the context item to mention (to be passed to
+     * {@link parseMentionQuery}).
+     */
+    matchingString: string
+
+    /**
+     * Equal to `@` + {@link matchingString}. The entire string that should be replaced with the
+     * context item when the at-mention reference is chosen.
+     */
+    replaceableString: string
+}
+
+/**
+ * Scans free-form user text input (in a chat message editor, for example) for possible mentions
+ * with the `@` trigger character.
+ *
+ * The {@link textBeforeCursor} is all of the text in the input field before the text insertion
+ * point cursor. For example, if the input field looks like `hello
+ * @foo█bar`, then {@link textBeforeCursor} should be `hello @foo`.
+ */
+export function scanForMentionTriggerInUserTextInput(textBeforeCursor: string): MentionTrigger | null {
+    const match = AT_MENTIONS_REGEXP.exec(textBeforeCursor)
+    if (match?.groups) {
+        return {
+            leadOffset: match.index + match.groups.maybeLeadingWhitespace.length,
+            matchingString: match.groups.matchingString,
+            replaceableString: match.groups.replaceableString,
+        }
+    }
+    return null
+}

vscode/src/chat/context/chatContext.ts

@@ -1,4 +1,4 @@
-import type { ContextItem } from '@sourcegraph/cody-shared'
+import { type ContextItem, type MentionQuery, parseMentionQuery } from '@sourcegraph/cody-shared'
 import type * as vscode from 'vscode'
 import {
     getFileContextFiles,
@@ -11,27 +11,30 @@ export async function getChatContextItemsForMention(
     cancellationToken: vscode.CancellationToken,
     telemetryRecorder?: {
         empty: () => void
-        withType: (type: 'symbol' | 'file') => void
+        withType: (type: MentionQuery['type']) => void
     }
 ): Promise<ContextItem[]> {
+    const mentionQuery = parseMentionQuery(query)
+
     // Logging: log when the at-mention starts, and then log when we know the type (after the 1st
     // character is typed). Don't log otherwise because we would be logging prefixes of the same
     // query repeatedly, which is not needed.
-    const queryType = query.startsWith('#') ? 'symbol' : 'file'
-    if (query === '') {
+    if (mentionQuery.type === 'empty') {
         telemetryRecorder?.empty()
     } else if (query.length === 1) {
-        telemetryRecorder?.withType(queryType)
-    }
-
-    if (query.length === 0) {
-        return getOpenTabsContextFile()
+        telemetryRecorder?.withType(mentionQuery.type)
     }
 
     const MAX_RESULTS = 20
-    if (query.startsWith('#')) {
-        // It would be nice if the VS Code symbols API supports cancellation, but it doesn't
-        return getSymbolContextFiles(query.slice(1), MAX_RESULTS)
+    switch (mentionQuery.type) {
+        case 'empty':
+            return getOpenTabsContextFile()
+        case 'symbol':
+            // It would be nice if the VS Code symbols API supports cancellation, but it doesn't
+            return getSymbolContextFiles(mentionQuery.text, MAX_RESULTS)
+        case 'file':
+            return getFileContextFiles(mentionQuery.text, MAX_RESULTS, cancellationToken)
+        default:
+            return []
     }
-    return getFileContextFiles(query, MAX_RESULTS, cancellationToken)
 }

vscode/src/edit/input/constants.ts

@@ -1,5 +1,6 @@
 export const FILE_HELP_LABEL = 'Search for a file to include, or type # to search symbols...'
 export const SYMBOL_HELP_LABEL = 'Search for a symbol to include...'
+export const OTHER_MENTION_HELP_LABEL = 'Search for context to include...'
 export const NO_MATCHES_LABEL = 'No matches found'
 
 export const QUICK_PICK_ITEM_EMPTY_INDENT_PREFIX = '\u00A0\u00A0\u00A0\u00A0\u00A0'

vscode/src/edit/input/get-input.ts

@@ -3,6 +3,8 @@ import {
     type ContextItem,
     type EditModel,
     displayLineRange,
+    parseMentionQuery,
+    scanForMentionTriggerInUserTextInput,
 } from '@sourcegraph/cody-shared'
 import * as vscode from 'vscode'
 
@@ -17,7 +19,12 @@ import { executeEdit } from '../execute'
 import type { EditIntent } from '../types'
 import { isGenerateIntent } from '../utils/edit-intent'
 import { getEditModelsForUser } from '../utils/edit-models'
-import { FILE_HELP_LABEL, NO_MATCHES_LABEL, SYMBOL_HELP_LABEL } from './constants'
+import {
+    FILE_HELP_LABEL,
+    NO_MATCHES_LABEL,
+    OTHER_MENTION_HELP_LABEL,
+    SYMBOL_HELP_LABEL,
+} from './constants'
 import { CURSOR_RANGE_ITEM, EXPANDED_RANGE_ITEM, SELECTION_RANGE_ITEM } from './get-items/constants'
 import { getDocumentInputItems } from './get-items/document'
 import { DOCUMENT_ITEM, MODEL_ITEM, RANGE_ITEM, TEST_ITEM, getEditInputItems } from './get-items/edit'
@@ -357,23 +364,28 @@ export const getInput = async (
                     return
                 }
 
-                const isFileSearch = value.endsWith('@')
-                const isSymbolSearch = value.endsWith('@#')
+                const mentionTrigger = scanForMentionTriggerInUserTextInput(value)
+                const mentionQuery = mentionTrigger
+                    ? parseMentionQuery(mentionTrigger.matchingString)
+                    : undefined
 
                 // If we have the beginning of a file or symbol match, show a helpful label
-                if (isFileSearch) {
-                    input.items = [{ alwaysShow: true, label: FILE_HELP_LABEL }]
-                    return
-                }
-                if (isSymbolSearch) {
-                    input.items = [{ alwaysShow: true, label: SYMBOL_HELP_LABEL }]
+                if (mentionQuery?.text === '') {
+                    if (mentionQuery.type === 'empty' || mentionQuery.type === 'file') {
+                        input.items = [{ alwaysShow: true, label: FILE_HELP_LABEL }]
+                        return
+                    }
+                    if (mentionQuery.type === 'symbol') {
+                        input.items = [{ alwaysShow: true, label: SYMBOL_HELP_LABEL }]
+                        return
+                    }
+                    input.items = [{ alwaysShow: true, label: OTHER_MENTION_HELP_LABEL }]
                     return
                 }
 
-                const matchingContext = await getMatchingContext(value)
+                const matchingContext = mentionQuery ? await getMatchingContext(mentionQuery) : null
                 if (matchingContext === null) {
                     // Nothing to match, re-render existing items
-                    // eslint-disable-next-line no-self-assign
                     input.items = getEditInputItems(
                         input.value,
                         activeRangeItem,

vscode/src/edit/input/get-matching-context.ts

@@ -1,15 +1,9 @@
-import type { ContextItem } from '@sourcegraph/cody-shared'
+import type { ContextItem, MentionQuery } from '@sourcegraph/cody-shared'
 import * as vscode from 'vscode'
 
 import { getFileContextFiles, getSymbolContextFiles } from '../../editor/utils/editor-context'
 import { getLabelForContextItem } from './utils'
 
-/* Match strings that end with a '@' followed by any characters except a space */
-const MATCHING_CONTEXT_FILE_REGEX = /@(\S+)$/
-
-/* Match strings that end with a '@#' followed by any characters except a space */
-const MATCHING_SYMBOL_REGEX = /@#(\S+)$/
-
 const MAX_FUZZY_RESULTS = 20
 
 interface FixupMatchingContext {
@@ -20,10 +14,11 @@ interface FixupMatchingContext {
     file: ContextItem
 }
 
-export async function getMatchingContext(instruction: string): Promise<FixupMatchingContext[] | null> {
-    const symbolMatch = instruction.match(MATCHING_SYMBOL_REGEX)
-    if (symbolMatch) {
-        const symbolResults = await getSymbolContextFiles(symbolMatch[1], MAX_FUZZY_RESULTS)
+export async function getMatchingContext(
+    mentionQuery: MentionQuery
+): Promise<FixupMatchingContext[] | null> {
+    if (mentionQuery.type === 'symbol') {
+        const symbolResults = await getSymbolContextFiles(mentionQuery.text, MAX_FUZZY_RESULTS)
         return symbolResults.map(result => ({
             key: getLabelForContextItem(result),
             file: result,
@@ -33,11 +28,10 @@ export async function getMatchingContext(instruction: string): Promise<FixupMatc
         }))
     }
 
-    const fileMatch = instruction.match(MATCHING_CONTEXT_FILE_REGEX)
-    if (fileMatch) {
+    if (mentionQuery.type === 'file') {
         const cancellation = new vscode.CancellationTokenSource()
         const fileResults = await getFileContextFiles(
-            fileMatch[1],
+            mentionQuery.text,
             MAX_FUZZY_RESULTS,
             cancellation.token
         )