From eb48cda258aece6788fff555075d74a1a8e7345f Mon Sep 17 00:00:00 2001 From: arafatkatze Date: Thu, 5 Sep 2024 20:55:49 +0530 Subject: [PATCH] Making the func Async --- .../experimentation/FeatureFlagProvider.ts | 3 +- .../src/completions/artificial-delay.test.ts | 208 ++++++++++-------- vscode/src/completions/artificial-delay.ts | 13 +- .../inline-completion-item-provider.ts | 3 +- 4 files changed, 132 insertions(+), 95 deletions(-) diff --git a/lib/shared/src/experimentation/FeatureFlagProvider.ts b/lib/shared/src/experimentation/FeatureFlagProvider.ts index 0fa92f8d819e..ffd3d7a585b1 100644 --- a/lib/shared/src/experimentation/FeatureFlagProvider.ts +++ b/lib/shared/src/experimentation/FeatureFlagProvider.ts @@ -30,7 +30,8 @@ export enum FeatureFlag { CodyAutocompleteFIMModelExperimentVariant2 = 'cody-autocomplete-fim-model-experiment-variant-2-v2', CodyAutocompleteFIMModelExperimentVariant3 = 'cody-autocomplete-fim-model-experiment-variant-3-v2', CodyAutocompleteFIMModelExperimentVariant4 = 'cody-autocomplete-fim-model-experiment-variant-4-v2', - + CodyInLineSuffixAutocomplete = 'cody-in-line-suffix-autocomplete', + CodyDisableArtificialDelay = 'cody-disable-artificial-delay', // Enables Claude 3 if the user is in our holdout group CodyAutocompleteClaude3 = 'cody-autocomplete-claude-3', // Enables the bfg-mixed context retriever that will combine BFG with the default local editor diff --git a/vscode/src/completions/artificial-delay.test.ts b/vscode/src/completions/artificial-delay.test.ts index 94f8b824b0f3..80f99b74dab5 100644 --- a/vscode/src/completions/artificial-delay.test.ts +++ b/vscode/src/completions/artificial-delay.test.ts @@ -1,21 +1,49 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { FeatureFlagProvider } from '@sourcegraph/cody-shared' +import { FeatureFlag, featureFlagProvider } from '@sourcegraph/cody-shared' import { getArtificialDelay, lowPerformanceLanguageIds, resetArtificialDelay } from './artificial-delay' - const featureFlags = { user: true, } -describe('getArtificialDelay', () => { +describe('await getArtificialDelay', () => { + const enabledFeatureFlags = new Set() + let originalFeatureFlagProviderInstance: FeatureFlagProvider | null + beforeEach(() => { vi.useFakeTimers() + originalFeatureFlagProviderInstance = featureFlagProvider.instance + enabledFeatureFlags.clear() + featureFlagProvider.instance = { + evaluateFeatureFlag: (flag: FeatureFlag) => Promise.resolve(enabledFeatureFlags.has(flag)), + refresh: () => {}, + getFromCache: (flagName: FeatureFlag) => { + return false + }, + } as FeatureFlagProvider }) + afterEach(() => { + featureFlagProvider.instance = originalFeatureFlagProviderInstance vi.restoreAllMocks() resetArtificialDelay() }) - it('returns gradually increasing latency up to max for CSS when suggestions are rejected', () => { + it('returns no artificial delay when CodyAutocompleteTracing feature flag is enabled', async () => { + const uri = 'file://foo/bar/test.css' + enabledFeatureFlags.add(FeatureFlag.CodyDisableArtificialDelay) + + // css is a low performance language + const languageId = 'css' + expect(lowPerformanceLanguageIds.has(languageId)).toBe(true) + + // expect no artificial delay when feature flag is enabled + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(0) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(0) + }) + + it('returns gradually increasing latency up to max for CSS when suggestions are rejected', async () => { const uri = 'file://foo/bar/test.css' // css is a low performance language @@ -23,52 +51,52 @@ describe('getArtificialDelay', () => { expect(lowPerformanceLanguageIds.has(languageId)).toBe(true) // start with default high latency for low performance lang with default user latency added - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) // start at default, but gradually increasing latency after 5 rejected suggestions - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1050) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1100) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1150) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1200) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1250) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1300) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1350) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1400) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1050) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1100) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1150) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1200) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1250) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1300) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1350) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1400) // max latency at 1400 - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1400) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1400) // after the suggestion was accepted, user latency resets to 0, using baseline only resetArtificialDelay() - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) resetArtificialDelay() - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) // gradually increasing latency after 5 rejected suggestions - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1050) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1100) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1150) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1200) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1250) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1300) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1350) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1400) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1050) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1100) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1150) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1200) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1250) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1300) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1350) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1400) // Latency will not reset before 5 minutes vi.advanceTimersByTime(3 * 60 * 1000) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1400) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1400) // Latency will be reset after 5 minutes vi.advanceTimersByTime(5 * 60 * 1000) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) // reset latency on accepted suggestion resetArtificialDelay() - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) }) - it('returns increasing latency after rejecting suggestions', () => { + it('returns increasing latency after rejecting suggestions', async () => { const uri = 'file://foo/bar/test.ts' // Confirm typescript is not a low performance language @@ -76,23 +104,23 @@ describe('getArtificialDelay', () => { expect(lowPerformanceLanguageIds.has(languageId)).toBe(false) // start at default, but gradually increasing latency after 5 rejected suggestions - expect(getArtificialDelay(featureFlags, uri, languageId, 'arguments')).toBe(0) - expect(getArtificialDelay(featureFlags, uri, languageId, 'function.body')).toBe(0) + expect(await getArtificialDelay(featureFlags, uri, languageId, 'arguments')).toBe(0) + expect(await getArtificialDelay(featureFlags, uri, languageId, 'function.body')).toBe(0) // baseline latency increased to 1000 due to comment node type - expect(getArtificialDelay(featureFlags, uri, languageId, 'comment')).toBe(1000) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(0) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(0) + expect(await getArtificialDelay(featureFlags, uri, languageId, 'comment')).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(0) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(0) // gradually increasing latency after 5 rejected suggestions - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(50) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(100) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(150) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(200) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(50) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(100) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(150) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(200) // after the suggestion was accepted, user latency resets to 0, using baseline only resetArtificialDelay() - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(0) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(0) }) - it('returns default latency for CSS after accepting suggestion and resets after 5 minutes', () => { + it('returns default latency for CSS after accepting suggestion and resets after 5 minutes', async () => { const uri = 'file://foo/bar/test.css' // css is a low performance language @@ -100,64 +128,64 @@ describe('getArtificialDelay', () => { expect(lowPerformanceLanguageIds.has(languageId)).toBe(true) // start with default baseline latency for low performance lang - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) // reset to starting point on every accepted suggestion resetArtificialDelay() - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) resetArtificialDelay() - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) // Latency will not reset before 5 minutes vi.advanceTimersByTime(3 * 60 * 1000) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1050) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1050) // Latency will be reset after 5 minutes vi.advanceTimersByTime(5 * 60 * 1000) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1000) }) - it('returns increasing latency up to max after rejecting multiple suggestions, resets after file change and accept', () => { + it('returns increasing latency up to max after rejecting multiple suggestions, resets after file change and accept', async () => { const uri = 'file://foo/bar/test.ts' const languageId = 'typescript' expect(lowPerformanceLanguageIds.has(languageId)).toBe(false) // reject the first 5 suggestions, and confirm latency remains unchanged - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(0) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(0) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(0) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(0) - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(0) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(0) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(0) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(0) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(0) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(0) // latency should start increasing after 5 rejections, but max at 1400 - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(50) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(50) // line is a comment, so latency should be increased where: // base is 1000 due to line is a comment, and user latency is 400 as this is the 7th rejection - expect(getArtificialDelay(featureFlags, uri, languageId, 'comment')).toBe(1100) + expect(await getArtificialDelay(featureFlags, uri, languageId, 'comment')).toBe(1100) for (let i = 150; i <= 1400; i += 50) { - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(i) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(i) } // max at 1400 after multiple rejection - expect(getArtificialDelay(featureFlags, uri, languageId)).toBe(1400) + expect(await getArtificialDelay(featureFlags, uri, languageId)).toBe(1400) // reset latency on file change to default const newUri = 'foo/test.ts' // latency should start increasing again after 5 rejections - expect(getArtificialDelay(featureFlags, newUri, languageId)).toBe(0) - expect(getArtificialDelay(featureFlags, newUri, languageId)).toBe(0) + expect(await getArtificialDelay(featureFlags, newUri, languageId)).toBe(0) + expect(await getArtificialDelay(featureFlags, newUri, languageId)).toBe(0) // line is a comment, so latency should be increased - expect(getArtificialDelay(featureFlags, newUri, languageId, 'comment')).toBe(1000) - expect(getArtificialDelay(featureFlags, newUri, languageId)).toBe(0) - expect(getArtificialDelay(featureFlags, newUri, languageId)).toBe(0) + expect(await getArtificialDelay(featureFlags, newUri, languageId, 'comment')).toBe(1000) + expect(await getArtificialDelay(featureFlags, newUri, languageId)).toBe(0) + expect(await getArtificialDelay(featureFlags, newUri, languageId)).toBe(0) // Latency will not reset before 5 minutes vi.advanceTimersByTime(3 * 60 * 1000) - expect(getArtificialDelay(featureFlags, newUri, languageId)).toBe(50) + expect(await getArtificialDelay(featureFlags, newUri, languageId)).toBe(50) // reset latency on accepted suggestion resetArtificialDelay() - expect(getArtificialDelay(featureFlags, newUri, languageId)).toBe(0) + expect(await getArtificialDelay(featureFlags, newUri, languageId)).toBe(0) }) - it('returns default latency for low performance language only when only language flag is enabled', () => { + it('returns default latency for low performance language only when only language flag is enabled', async () => { const uri = 'file://foo/bar/test.css' const featureFlagsLangOnly = { @@ -174,17 +202,17 @@ describe('getArtificialDelay', () => { expect(lowPerformanceLanguageIds.has(languageId)).toBe(false) // latency should only change based on language id when only the language flag is enabled - expect(getArtificialDelay(featureFlagsLangOnly, uri, lowPerformLanguageId)).toBe(1000) - expect(getArtificialDelay(featureFlagsLangOnly, uri, lowPerformLanguageId)).toBe(1000) - expect(getArtificialDelay(featureFlagsLangOnly, uri, lowPerformLanguageId)).toBe(1000) - expect(getArtificialDelay(featureFlagsLangOnly, uri, lowPerformLanguageId)).toBe(1000) - expect(getArtificialDelay(featureFlagsLangOnly, uri, lowPerformLanguageId)).toBe(1000) - expect(getArtificialDelay(featureFlagsLangOnly, uri, lowPerformLanguageId)).toBe(1000) + expect(await getArtificialDelay(featureFlagsLangOnly, uri, lowPerformLanguageId)).toBe(1000) + expect(await getArtificialDelay(featureFlagsLangOnly, uri, lowPerformLanguageId)).toBe(1000) + expect(await getArtificialDelay(featureFlagsLangOnly, uri, lowPerformLanguageId)).toBe(1000) + expect(await getArtificialDelay(featureFlagsLangOnly, uri, lowPerformLanguageId)).toBe(1000) + expect(await getArtificialDelay(featureFlagsLangOnly, uri, lowPerformLanguageId)).toBe(1000) + expect(await getArtificialDelay(featureFlagsLangOnly, uri, lowPerformLanguageId)).toBe(1000) // latency back to 0 when language is no longer low-performance - expect(getArtificialDelay(featureFlagsLangOnly, goUri, languageId)).toBe(0) + expect(await getArtificialDelay(featureFlagsLangOnly, goUri, languageId)).toBe(0) }) - it('returns latency based on language only when user flag is disabled', () => { + it('returns latency based on language only when user flag is disabled', async () => { const uri = 'file://foo/bar/test.css' // css is a low performance language @@ -196,21 +224,21 @@ describe('getArtificialDelay', () => { } // latency starts with language latency - expect(getArtificialDelay(featureFlagsNoUser, uri, languageId)).toBe(1000) - expect(getArtificialDelay(featureFlagsNoUser, uri, languageId)).toBe(1000) - expect(getArtificialDelay(featureFlagsNoUser, uri, languageId)).toBe(1000) - expect(getArtificialDelay(featureFlagsNoUser, uri, languageId)).toBe(1000) - expect(getArtificialDelay(featureFlagsNoUser, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlagsNoUser, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlagsNoUser, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlagsNoUser, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlagsNoUser, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlagsNoUser, uri, languageId)).toBe(1000) // latency should remains unchanged after 5 rejections - expect(getArtificialDelay(featureFlagsNoUser, uri, languageId)).toBe(1000) - expect(getArtificialDelay(featureFlagsNoUser, uri, languageId)).toBe(1000) - expect(getArtificialDelay(featureFlagsNoUser, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlagsNoUser, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlagsNoUser, uri, languageId)).toBe(1000) + expect(await getArtificialDelay(featureFlagsNoUser, uri, languageId)).toBe(1000) // switch to a non-low-performance language - go is not a low performance language const goLanguageId = 'go' const goUri = 'foo/bar/test.go' expect(lowPerformanceLanguageIds.has(goLanguageId)).toBe(false) // reset to provider latency because language latency is ignored for non-low-performance languages - expect(getArtificialDelay(featureFlagsNoUser, goUri, goLanguageId)).toBe(0) + expect(await getArtificialDelay(featureFlagsNoUser, goUri, goLanguageId)).toBe(0) }) }) diff --git a/vscode/src/completions/artificial-delay.ts b/vscode/src/completions/artificial-delay.ts index 151e342b78c2..ed5d137b712b 100644 --- a/vscode/src/completions/artificial-delay.ts +++ b/vscode/src/completions/artificial-delay.ts @@ -1,6 +1,6 @@ +import { FeatureFlag, featureFlagProvider } from '@sourcegraph/cody-shared' import { logDebug } from '../log' import type { CompletionIntent } from '../tree-sitter/queries' - export interface LatencyFeatureFlags { user?: boolean } @@ -42,12 +42,19 @@ let userMetrics = { // Adjust the minimum latency based on user actions and env Start when the last 5 suggestions were // not accepted Increment latency by 200ms linearly up to max latency Reset every 5 minutes, or on // file change, or on accepting a suggestion -export function getArtificialDelay( +export async function getArtificialDelay( featureFlags: LatencyFeatureFlags, uri: string, languageId: string, completionIntent?: CompletionIntent -): number { +): Promise { + const codyDisableArtificialDelay = await featureFlagProvider.instance!.evaluateFeatureFlag( + FeatureFlag.CodyDisableArtificialDelay + ) + + if (codyDisableArtificialDelay) { + return 0 + } let baseline = 0 const isLowPerformanceLanguageId = lowPerformanceLanguageIds.has(languageId) diff --git a/vscode/src/completions/inline-completion-item-provider.ts b/vscode/src/completions/inline-completion-item-provider.ts index 75521b4a6bf8..baa9e918f491 100644 --- a/vscode/src/completions/inline-completion-item-provider.ts +++ b/vscode/src/completions/inline-completion-item-provider.ts @@ -7,6 +7,7 @@ import { FeatureFlag, RateLimitError, contextFiltersProvider, + featureFlagProvider, isCodyIgnoredFile, telemetryRecorder, wrapInActiveSpan, @@ -423,7 +424,7 @@ export class InlineCompletionItemProvider ), } - const artificialDelay = getArtificialDelay( + const artificialDelay = await getArtificialDelay( latencyFeatureFlags, document.uri.toString(), document.languageId,