Skip to content

Commit

Permalink
speed up & respect ignores in finding workspace files for file @-ment…
Browse files Browse the repository at this point in the history
…ions (#3433)
  • Loading branch information
sqs authored Mar 18, 2024
1 parent dfbdc6c commit be985fb
Show file tree
Hide file tree
Showing 8 changed files with 157 additions and 96 deletions.
1 change: 1 addition & 0 deletions vscode/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ This is a log of all notable changes to Cody for VS Code. [Unreleased] changes a

- Document: Fixed an issue where the generated documentation would be incorrectly inserted for Python. Cody will now follow PEP 257 – Docstring Conventions. [pull/3275](https://github.com/sourcegraph/cody/pull/3275)
- Edit: Fixed incorrect decorations being shown for edits that only insert new code. [pull/3424](https://github.com/sourcegraph/cody/pull/3424)
- When `@`-mentioning files in chat and edits, the list of fuzzy-matching files is shown much faster (which is especially noticeable in large workspaces).

### Changed

Expand Down
87 changes: 32 additions & 55 deletions vscode/src/chat/chat-view/SimpleChatPanelProvider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,7 @@ import {
import type { View } from '../../../webviews/NavBar'
import { getFullConfig } from '../../configuration'
import { type RemoteSearch, RepoInclusion } from '../../context/remote-search'
import {
fillInContextItemContent,
getFileContextFiles,
getOpenTabsContextFile,
getSymbolContextFiles,
} from '../../editor/utils/editor-context'
import { fillInContextItemContent } from '../../editor/utils/editor-context'
import type { VSCodeEditor } from '../../editor/vscode-editor'
import { ContextStatusAggregator } from '../../local-context/enhanced-context-status'
import type { LocalEmbeddingsController } from '../../local-context/local-embeddings'
Expand Down Expand Up @@ -66,6 +61,7 @@ import { chatModel } from '../../models'
import { getContextWindowForModel } from '../../models/utilts'
import { recordExposedExperimentsToSpan } from '../../services/open-telemetry/utils'
import type { MessageErrorType } from '../MessageProvider'
import { getChatContextItemsForMention } from '../context/chatContext'
import type {
ChatSubmitType,
ConfigurationSubsetForWebview,
Expand Down Expand Up @@ -581,63 +577,44 @@ export class SimpleChatPanelProvider implements vscode.Disposable, ChatSession {
}

private async handleGetUserContextFilesCandidates(query: string): Promise<void> {
const source = 'chat'
if (!query.length) {
telemetryService.log('CodyVSCodeExtension:at-mention:executed', { source })
telemetryRecorder.recordEvent('cody.at-mention', 'executed', { privateMetadata: { source } })

const tabs = await getOpenTabsContextFile()
void this.postMessage({
type: 'userContextFiles',
userContextFiles: tabs,
})
return
}

// Log when query only has 1 char to avoid logging the same query repeatedly
if (query.length === 1) {
const type = query.startsWith('#') ? 'symbol' : 'file'
telemetryService.log(`CodyVSCodeExtension:at-mention:${type}:executed`, { source })
telemetryRecorder.recordEvent(`cody.at-mention.${type}`, 'executed', {
privateMetadata: { source },
})
}

// Cancel previously in-flight query.
const cancellation = new vscode.CancellationTokenSource()
this.contextFilesQueryCancellation?.cancel()
this.contextFilesQueryCancellation = cancellation

const source = 'chat'
const scopedTelemetryRecorder: Parameters<typeof getChatContextItemsForMention>[2] = {
empty: () => {
telemetryService.log('CodyVSCodeExtension:at-mention:executed', { source })
telemetryRecorder.recordEvent('cody.at-mention', 'executed', {
privateMetadata: { source },
})
},
withType: type => {
telemetryService.log(`CodyVSCodeExtension:at-mention:${type}:executed`, { source })
telemetryRecorder.recordEvent(`cody.at-mention.${type}`, 'executed', {
privateMetadata: { source },
})
},
}

try {
const MAX_RESULTS = 20
if (query.startsWith('#')) {
// It would be nice if the VS Code symbols API supports
// cancellation, but it doesn't
const symbolResults = await getSymbolContextFiles(query.slice(1), MAX_RESULTS)
// Check if cancellation was requested while getFileContextFiles
// was executing, which means a new request has already begun
// (i.e. prevent race conditions where slow old requests get
// processed after later faster requests)
if (!cancellation.token.isCancellationRequested) {
await this.postMessage({
type: 'userContextFiles',
userContextFiles: symbolResults,
})
}
} else {
const fileResults = await getFileContextFiles(query, MAX_RESULTS, cancellation.token)
// Check if cancellation was requested while getFileContextFiles
// was executing, which means a new request has already begun
// (i.e. prevent race conditions where slow old requests get
// processed after later faster requests)
if (!cancellation.token.isCancellationRequested) {
await this.postMessage({
type: 'userContextFiles',
userContextFiles: fileResults,
})
}
const items = await getChatContextItemsForMention(
query,
cancellation.token,
scopedTelemetryRecorder
)
if (cancellation.token.isCancellationRequested) {
return
}
void this.postMessage({
type: 'userContextFiles',
userContextFiles: items,
})
} catch (error) {
if (cancellation.token.isCancellationRequested) {
return
}
this.postError(new Error(`Error retrieving context files: ${error}`))
}
}
Expand Down
37 changes: 37 additions & 0 deletions vscode/src/chat/context/chatContext.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import type { ContextItem } from '@sourcegraph/cody-shared'
import type * as vscode from 'vscode'
import {
getFileContextFiles,
getOpenTabsContextFile,
getSymbolContextFiles,
} from '../../editor/utils/editor-context'

export async function getChatContextItemsForMention(
query: string,
cancellationToken: vscode.CancellationToken,
telemetryRecorder?: {
empty: () => void
withType: (type: 'symbol' | 'file') => void
}
): Promise<ContextItem[]> {
// Logging: log when the at-mention starts, and then log when we know the type (after the 1st
// character is typed). Don't log otherwise because we would be logging prefixes of the same
// query repeatedly, which is not needed.
const queryType = query.startsWith('#') ? 'symbol' : 'file'
if (query === '') {
telemetryRecorder?.empty()
} else if (query.length === 1) {
telemetryRecorder?.withType(queryType)
}

if (query.length === 0) {
return getOpenTabsContextFile()
}

const MAX_RESULTS = 20
if (query.startsWith('#')) {
// It would be nice if the VS Code symbols API supports cancellation, but it doesn't
return getSymbolContextFiles(query.slice(1), MAX_RESULTS)
}
return getFileContextFiles(query, MAX_RESULTS, cancellationToken)
}
9 changes: 0 additions & 9 deletions vscode/src/editor/utils/editor-context.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -125,15 +125,6 @@ describe('getFileContextFiles', () => {

expect(vscode.workspace.findFiles).toBeCalledTimes(1)
})

it('cancels previous requests', async () => {
vscode.workspace.findFiles = vi.fn().mockResolvedValueOnce([])
const cancellation = new vscode.CancellationTokenSource()
await getFileContextFiles('search', 5, cancellation.token)
await getFileContextFiles('search', 5, new vscode.CancellationTokenSource().token)
expect(cancellation.token.isCancellationRequested)
expect(vscode.workspace.findFiles).toBeCalledTimes(2)
})
})

describe('filterLargeFiles', () => {
Expand Down
29 changes: 8 additions & 21 deletions vscode/src/editor/utils/editor-context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,29 +24,16 @@ import {
type ContextItemWithContent,
} from '@sourcegraph/cody-shared/src/codebase-context/messages'
import { CHARS_PER_TOKEN } from '@sourcegraph/cody-shared/src/prompt/constants'
import { getOpenTabsUris, getWorkspaceSymbols } from '.'
import { getOpenTabsUris } from '.'
import { toVSCodeRange } from '../../common/range'

const findWorkspaceFiles = async (
cancellationToken: vscode.CancellationToken
): Promise<vscode.Uri[]> => {
// TODO(toolmantim): Add support for the search.exclude option, e.g.
// Object.keys(vscode.workspace.getConfiguration().get('search.exclude',
// {}))
const fileExcludesPattern =
'**/{*.env,.git/,.class,out/,dist/,build/,snap,node_modules/,__pycache__/}**'
// TODO(toolmantim): Check this performs with remote workspaces (do we need a UI spinner etc?)
return vscode.workspace.findFiles('', fileExcludesPattern, undefined, cancellationToken)
}
import { findWorkspaceFiles } from './findWorkspaceFiles'

// Some matches we don't want to ignore because they might be valid code (for example `bin/` in Dart)
// but could also be junk (`bin/` in .NET). If a file path contains a segment matching any of these
// items it will be ranked low unless the users query contains the exact segment.
const lowScoringPathSegments = ['bin']

// This is expensive for large repos (e.g. Chromium), so we only do it max once
// every 10 seconds. It also handily supports a cancellation callback to use
// with the cancellation token to discard old requests.
// This is expensive for large repos (e.g. Chromium), so we only do it max once every 10 seconds.
const throttledFindFiles = throttle(findWorkspaceFiles, 10000)

/**
Expand All @@ -63,12 +50,8 @@ export async function getFileContextFiles(
if (!query.trim()) {
return []
}
token.onCancellationRequested(() => {
throttledFindFiles.cancel()
})

const uris = await throttledFindFiles(token)

if (!uris) {
return []
}
Expand Down Expand Up @@ -133,7 +116,11 @@ export async function getSymbolContextFiles(
return []
}

const queryResults = await getWorkspaceSymbols(query) // doesn't support cancellation tokens :(
// doesn't support cancellation tokens :(
const queryResults = await vscode.commands.executeCommand<vscode.SymbolInformation[]>(
'vscode.executeWorkspaceSymbolProvider',
query
)

const relevantQueryResults = queryResults?.filter(
symbol =>
Expand Down
78 changes: 78 additions & 0 deletions vscode/src/editor/utils/findWorkspaceFiles.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
import * as vscode from 'vscode'

/**
* Find all files in all workspace folders, respecting the user's `files.exclude`, `search.exclude`,
* and other exclude settings. The intent is to match the files shown by VS Code's built-in `Go to
* File...` command.
*/
export async function findWorkspaceFiles(
cancellationToken: vscode.CancellationToken
): Promise<vscode.Uri[]> {
return (
await Promise.all(
(vscode.workspace.workspaceFolders ?? [null]).map(async workspaceFolder =>
vscode.workspace.findFiles(
workspaceFolder ? new vscode.RelativePattern(workspaceFolder, '**') : '',
await getExcludePattern(workspaceFolder),
undefined,
cancellationToken
)
)
)
).flat()
}

type IgnoreRecord = Record<string, boolean>

async function getExcludePattern(workspaceFolder: vscode.WorkspaceFolder | null): Promise<string> {
const config = vscode.workspace.getConfiguration('', workspaceFolder)
const filesExclude = config.get<IgnoreRecord>('files.exclude', {})
const searchExclude = config.get<IgnoreRecord>('search.exclude', {})

const useIgnoreFiles = config.get<boolean>('search.useIgnoreFiles')
const gitignoreExclude =
useIgnoreFiles && workspaceFolder
? await readIgnoreFile(vscode.Uri.joinPath(workspaceFolder.uri, '.gitignore'))
: {}
const ignoreExclude =
useIgnoreFiles && workspaceFolder
? await readIgnoreFile(vscode.Uri.joinPath(workspaceFolder.uri, '.ignore'))
: {}

const mergedExclude: IgnoreRecord = {
...filesExclude,
...searchExclude,
...gitignoreExclude,
...ignoreExclude,
}
const excludePatterns = Object.keys(mergedExclude).filter(key => mergedExclude[key] === true)
return `{${excludePatterns.join(',')}}`
}

async function readIgnoreFile(uri: vscode.Uri): Promise<IgnoreRecord> {
const ignore: IgnoreRecord = {}
try {
const data = await vscode.workspace.fs.readFile(uri)
for (let line of Buffer.from(data).toString('utf-8').split('\n')) {
if (line.startsWith('!')) {
continue
}

// Strip comment and trailing whitespace.
line = line.replace(/\s*(#.*)?$/, '')

if (line === '') {
continue
}

if (line.endsWith('/')) {
line = line.slice(0, -1)
}
if (!line.startsWith('/') && !line.startsWith('**/')) {
line = `**/${line}`
}
ignore[line] = true
}
} catch {}
return ignore
}
9 changes: 0 additions & 9 deletions vscode/src/editor/utils/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,6 @@ export async function getSmartSelection(
return getSelectionAroundLine(document, target)
}

/**
* Searches for workspace symbols matching the given query string.
* @param query - The search query string.
* @returns A promise resolving to the array of SymbolInformation objects representing the matched workspace symbols.
*/
export async function getWorkspaceSymbols(query = ''): Promise<vscode.SymbolInformation[]> {
return vscode.commands.executeCommand('vscode.executeWorkspaceSymbolProvider', query)
}

/**
* Returns an array of URI's for all unique open editor tabs.
*
Expand Down
3 changes: 1 addition & 2 deletions vscode/test/e2e/chat-atFile.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,7 @@ test.extend<ExpectedEvents>({
chatPanelFrame.getByRole('option', { name: withPlatformSlashes('visualize.go') })
).toBeVisible()
await chatInput.press('ArrowDown') // second item (visualize.go)
await chatInput.press('ArrowDown') // third item (.vscode/settings.json)
await chatInput.press('ArrowDown') // wraps back to first item
await chatInput.press('ArrowDown') // wraps back to first item (var.go)
await chatInput.press('ArrowDown') // second item again
await chatInput.press('Tab')
await expect(chatInput).toHaveText(
Expand Down

0 comments on commit be985fb

Please sign in to comment.