change: add latency gradually (#1269)

vscode/src/completions/inline-completion-item-provider.ts

@@ -255,9 +255,13 @@ export class InlineCompletionItemProvider implements vscode.InlineCompletionItem
             // latency so that we don't show a result before the user has paused typing for a brief
             // moment.
             if (result.source !== InlineCompletionsResultSource.LastCandidate) {
-                const minimumLatencyFlag = await Promise.resolve(minimumLatencyFlagsPromise)
+                const minimumLatencyFlag = await minimumLatencyFlagsPromise
                 if (triggerKind === TriggerKind.Automatic && minimumLatencyFlag) {
-                    const minimumLatency = getLatency(this.config.providerConfig.identifier, document.languageId)
+                    const minimumLatency = getLatency(
+                        this.config.providerConfig.identifier,
+                        document.uri.fsPath,
+                        document.languageId
+                    )
 
                     const delta = performance.now() - start
                     if (minimumLatency && delta < minimumLatency) {

vscode/src/completions/latency.test.ts

@@ -1,107 +1,202 @@
-import { afterEach, describe, expect, it } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
 import { getLatency, resetLatency } from './latency'
 
 describe('getLatency', () => {
+    beforeEach(() => {
+        vi.useFakeTimers()
+    })
     afterEach(() => {
+        vi.restoreAllMocks()
         resetLatency()
     })
 
-    it('returns gradually increasing latency for anthropic provider when language is unsupported', () => {
+    it('returns gradually increasing latency for anthropic provider when language is unsupported, up to max latency', () => {
         const provider = 'anthropic'
+        const fileName = 'foo/bar/test'
         const languageId = undefined
 
         // start with default high latency for unsupported lang with default user latency added
-        expect(getLatency(provider, languageId)).toBe(1000)
-        // gradually increasing latency
-        expect(getLatency(provider, languageId)).toBe(1200)
-        expect(getLatency(provider, languageId)).toBe(1400)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        // gradually increasing latency after 5 rejected suggestions
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        // gradually increasing latency after 5 rejected suggestions
+        expect(getLatency(provider, fileName, languageId)).toBe(1200)
+        expect(getLatency(provider, fileName, languageId)).toBe(1400)
         // after the suggestion was accepted, user latency resets to 0, using baseline only
         resetLatency()
-        expect(getLatency(provider, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
         resetLatency()
-        expect(getLatency(provider, languageId)).toBe(1000)
-        // next one increases user latency when last suggestion was not accepted
-        expect(getLatency(provider, languageId)).toBe(1200)
+        // next rejection doesn't change user latency until 5 rejected
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1200)
     })
 
-    it('returns gradually increasing latency up to max for low performance language on anthropic provider when suggestions are rejected', () => {
+    it('returns gradually increasing latency up to max for CSS on anthropic provider when suggestions are rejected', () => {
         const provider = 'anthropic'
+        const fileName = 'foo/bar/test.css'
         const languageId = 'css'
 
         // start with default high latency for low performance lang with default user latency added
-        expect(getLatency(provider, languageId)).toBe(1000)
-        // gradually increasing latency
-        expect(getLatency(provider, languageId)).toBe(1200)
-        expect(getLatency(provider, languageId)).toBe(1400)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        // start at default, but gradually increasing latency after 5 rejected suggestions
+        expect(getLatency(provider, fileName, languageId)).toBe(1200)
+        expect(getLatency(provider, fileName, languageId)).toBe(1400)
+        expect(getLatency(provider, fileName, languageId)).toBe(1600)
+        expect(getLatency(provider, fileName, languageId)).toBe(1800)
+        // max latency at 2000
+        expect(getLatency(provider, fileName, languageId)).toBe(2000)
+        expect(getLatency(provider, fileName, languageId)).toBe(2000)
+        expect(getLatency(provider, fileName, languageId)).toBe(2000)
         // after the suggestion was accepted, user latency resets to 0, using baseline only
         resetLatency()
-        expect(getLatency(provider, languageId)).toBe(1000)
-        resetLatency()
-        expect(getLatency(provider, languageId)).toBe(1000)
-        // gradually increasing latency again but max at 2000 after multiple rejections
-        expect(getLatency(provider, languageId)).toBe(1200)
-        expect(getLatency(provider, languageId)).toBe(1400)
-        expect(getLatency(provider, languageId)).toBe(1800)
-        expect(getLatency(provider, languageId)).toBe(2000)
-        expect(getLatency(provider, languageId)).toBe(2000)
-        expect(getLatency(provider, languageId)).toBe(2000)
-        // reset latency on accepted suggestion
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
         resetLatency()
-        // after the suggestion was accepted, user latency resets to 0
-        expect(getLatency(provider, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        // gradually increasing latency after 5 rejected suggestions
+        expect(getLatency(provider, fileName, languageId)).toBe(1200)
+        expect(getLatency(provider, fileName, languageId)).toBe(1400)
+        // Latency will not reset before 5 minutes
+        vi.advanceTimersByTime(3 * 60 * 1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1600)
+        expect(getLatency(provider, fileName, languageId)).toBe(1800)
+        expect(getLatency(provider, fileName, languageId)).toBe(2000)
+        expect(getLatency(provider, fileName, languageId)).toBe(2000)
+        // Latency will be reset after 5 minutes
+        vi.advanceTimersByTime(5 * 60 * 1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
         // reset latency on accepted suggestion
         resetLatency()
-        // next one increases user latency when last suggestion was not accepted
-        expect(getLatency(provider, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
     })
 
-    it('returns increasing latency on anthropic provider after rejecting suggestions', () => {
+    it('returns increasing latency after rejecting suggestions on anthropic provider', () => {
         const provider = 'anthropic'
+        const fileName = 'foo/bar/test.ts'
         const languageId = 'typescript'
 
-        // gradually increasing latency
-        expect(getLatency(provider, languageId)).toBe(0)
-        expect(getLatency(provider, languageId)).toBe(200)
-        expect(getLatency(provider, languageId)).toBe(400)
-        expect(getLatency(provider, languageId)).toBe(800)
+        // start at default, but gradually increasing latency after 5 rejected suggestions
+        expect(getLatency(provider, fileName, languageId)).toBe(0)
+        expect(getLatency(provider, fileName, languageId)).toBe(0)
+        expect(getLatency(provider, fileName, languageId)).toBe(0)
+        expect(getLatency(provider, fileName, languageId)).toBe(0)
+        expect(getLatency(provider, fileName, languageId)).toBe(0)
+        // gradually increasing latency after 5 rejected suggestions
+        expect(getLatency(provider, fileName, languageId)).toBe(200)
+        expect(getLatency(provider, fileName, languageId)).toBe(400)
+        expect(getLatency(provider, fileName, languageId)).toBe(600)
         // after the suggestion was accepted, user latency resets to 0, using baseline only
         resetLatency()
-        expect(getLatency(provider, languageId)).toBe(0)
+        expect(getLatency(provider, fileName, languageId)).toBe(0)
     })
 
-    it('returns default latency for CSS language on non-anthropic provider after accepting suggestion consistently', () => {
+    it('returns default latency for CSS after accepting suggestion and resets after 5 minutes', () => {
         const provider = 'non-anthropic'
+        const fileName = 'foo/bar/test.css'
         const languageId = 'css'
 
         // start with default baseline latency with low performance and user latency added
-        expect(getLatency(provider, languageId)).toBe(1400)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
         // reset to starting point on every accepted suggestion
         resetLatency()
-        expect(getLatency(provider, languageId)).toBe(1400)
-        resetLatency()
-        expect(getLatency(provider, languageId)).toBe(1400)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
         resetLatency()
-        expect(getLatency(provider, languageId)).toBe(1400)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        // Latency will not reset before 5 minutes
+        vi.advanceTimersByTime(3 * 60 * 1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1200)
+        // Latency will be reset after 5 minutes
+        vi.advanceTimersByTime(5 * 60 * 1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+    })
+
+    it('returns increasing latency up to max after multiple rejections for supported language on non-anthropic provider', () => {
+        const provider = 'non-anthropic'
+        const fileName = 'foo/bar/test.ts'
+        const languageId = 'typescript'
+
+        // start with default baseline latency with low performance and user latency added
+        expect(getLatency(provider, fileName, languageId)).toBe(400)
+        expect(getLatency(provider, fileName, languageId)).toBe(400)
+        expect(getLatency(provider, fileName, languageId)).toBe(400)
+        expect(getLatency(provider, fileName, languageId)).toBe(400)
+        expect(getLatency(provider, fileName, languageId)).toBe(400)
+        // latency should start increasing after 5 rejections, but max at 2000
+        expect(getLatency(provider, fileName, languageId)).toBe(600)
+        expect(getLatency(provider, fileName, languageId)).toBe(800)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1200)
+        expect(getLatency(provider, fileName, languageId)).toBe(1400)
+        // Latency will not reset before 5 minutes
+        vi.advanceTimersByTime(3 * 60 * 1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1600)
+        expect(getLatency(provider, fileName, languageId)).toBe(1800)
+        // max at 2000 after multiple rejections
+        expect(getLatency(provider, fileName, languageId)).toBe(2000)
+        expect(getLatency(provider, fileName, languageId)).toBe(2000)
+        expect(getLatency(provider, fileName, languageId)).toBe(2000)
+        // reset latency on accepted suggestion
         resetLatency()
-        expect(getLatency(provider, languageId)).toBe(1400)
+        expect(getLatency(provider, fileName, languageId)).toBe(400)
     })
 
-    it('returns increasing latency up to max latency for supported language on non-anthropic provider after rejecting multiple suggestions consistently', () => {
+    it('returns increasing latency up to max after rejecting multiple suggestions, resets after file change and accept', () => {
         const provider = 'non-anthropic'
+        const fileName = 'foo/bar/test.ts'
         const languageId = 'typescript'
 
         // start with default baseline latency with low performance and user latency added
-        expect(getLatency(provider, languageId)).toBe(400)
-        // latency should max at 2000 after multiple rejections
-        expect(getLatency(provider, languageId)).toBe(600)
-        expect(getLatency(provider, languageId)).toBe(800)
-        expect(getLatency(provider, languageId)).toBe(1200)
-        expect(getLatency(provider, languageId)).toBe(2000)
-        expect(getLatency(provider, languageId)).toBe(2000)
+        expect(getLatency(provider, fileName, languageId)).toBe(400)
+        // latency should start increasing after 5 rejections, but max at 2000
+        expect(getLatency(provider, fileName, languageId)).toBe(400)
+        expect(getLatency(provider, fileName, languageId)).toBe(400)
+        expect(getLatency(provider, fileName, languageId)).toBe(400)
+        expect(getLatency(provider, fileName, languageId)).toBe(400)
+
+        expect(getLatency(provider, fileName, languageId)).toBe(600)
+        expect(getLatency(provider, fileName, languageId)).toBe(800)
+        expect(getLatency(provider, fileName, languageId)).toBe(1000)
+        expect(getLatency(provider, fileName, languageId)).toBe(1200)
+        expect(getLatency(provider, fileName, languageId)).toBe(1400)
+        expect(getLatency(provider, fileName, languageId)).toBe(1600)
+        expect(getLatency(provider, fileName, languageId)).toBe(1800)
+        // max at 2000 after multiple rejections
+        expect(getLatency(provider, fileName, languageId)).toBe(2000)
+        expect(getLatency(provider, fileName, languageId)).toBe(2000)
+        expect(getLatency(provider, fileName, languageId)).toBe(2000)
+
+        // reset latency on file change to default
+        const newFileName = 'foo/test.ts'
+        // latency should start increasing again after 5 rejections
+        expect(getLatency(provider, newFileName, languageId)).toBe(400)
+        expect(getLatency(provider, newFileName, languageId)).toBe(400)
+        expect(getLatency(provider, newFileName, languageId)).toBe(400)
+        expect(getLatency(provider, newFileName, languageId)).toBe(400)
+        expect(getLatency(provider, newFileName, languageId)).toBe(400)
+        // Latency will not reset before 5 minutes
+        vi.advanceTimersByTime(3 * 60 * 1000)
+        expect(getLatency(provider, newFileName, languageId)).toBe(600)
         // reset latency on accepted suggestion
         resetLatency()
-        // back to starting latency after accepting a suggestion
-        expect(getLatency(provider, languageId)).toBe(400)
+        expect(getLatency(provider, newFileName, languageId)).toBe(400)
     })
 })

vscode/src/completions/latency.ts

@@ -2,37 +2,73 @@ import { logDebug } from '../log'
 
 export const defaultLatency = {
     baseline: 400,
-    user: 200, // set to 0 on reset after accepting suggestion
+    user: 200,
     lowPerformance: 1000,
     max: 2000,
 }
 
 // Languages with lower performance get additional latency to avoid spamming users with unhelpful suggestions
 const lowPerformanceLanguageIds = new Set(['css', 'html', 'scss', 'vue', 'dart', 'json', 'yaml', 'postcss'])
 
-let currentUserLatency = 0
+let userMetrics = {
+    sessionTimestamp: 0,
+    currentLatency: 0,
+    suggested: 0,
+    fsPath: '',
+}
 
 // Adjust the minimum latency based on user actions and env
-export function getLatency(provider: string, languageId?: string): number {
+// Start when the last 5 suggestions were not accepted
+// Increment latency by 200ms linearly up to max latency
+// Reset every 5 minutes, or on file change, or on accepting a suggestion
+export function getLatency(provider: string, fsPath: string, languageId?: string): number {
     let baseline = provider === 'anthropic' ? 0 : defaultLatency.baseline
-
     // set base latency based on provider and low performance languages
     if (!languageId || (languageId && lowPerformanceLanguageIds.has(languageId))) {
-        baseline += defaultLatency.lowPerformance
+        baseline = defaultLatency.lowPerformance
+    }
+
+    const timestamp = Date.now()
+    if (!userMetrics.sessionTimestamp) {
+        userMetrics.sessionTimestamp = timestamp
     }
 
-    const total = Math.max(baseline, Math.min(baseline + currentUserLatency, defaultLatency.max))
+    const elapsed = timestamp - userMetrics.sessionTimestamp
+    // reset metrics and timer after 5 minutes or file change
+    if (elapsed >= 5 * 60 * 1000 || userMetrics.fsPath !== fsPath) {
+        resetLatency()
+    }
 
-    // last suggestion was rejected when last candidated is undefined
-    currentUserLatency = currentUserLatency > 0 ? currentUserLatency * 2 : defaultLatency.user
+    userMetrics.suggested++
+    userMetrics.fsPath = fsPath
 
-    logDebug('CodyCompletionProvider:getLatency', `Applied Latency: ${total}`)
+    // Start after 5 rejected suggestions
+    if (userMetrics.suggested < 5) {
+        return baseline
+    }
+
+    const total = Math.max(baseline, Math.min(baseline + userMetrics.currentLatency, defaultLatency.max))
+
+    // Increase latency linearly up to max
+    if (userMetrics.currentLatency < defaultLatency.max) {
+        userMetrics.currentLatency += defaultLatency.user
+    }
+
+    logDebug('CodyCompletionProvider:getLatency', `Latency Applied: ${total}`)
 
     return total
 }
 
+// reset user latency and counter:
+// - on acceptance
+// - every 5 minutes
+// - on file change
 export function resetLatency(): void {
-    currentUserLatency = 0
-    // lastSuggestionId = undefined
-    logDebug('CodyCompletionProvider:resetLatency', 'User latency reset')
+    userMetrics = {
+        sessionTimestamp: 0,
+        currentLatency: 0,
+        suggested: 0,
+        fsPath: '',
+    }
+    logDebug('CodyCompletionProvider:resetLatency', 'Latency Reset')
 }