From 58ce9b2c7213db91ebc35060495d621c7faa0da9 Mon Sep 17 00:00:00 2001 From: Valery Bugakov Date: Tue, 15 Oct 2024 10:59:21 +0800 Subject: [PATCH] Autocomplete: a/b test the fast-path (#5905) - Adds a feature flag for the fast-path A/B test. - Removes redundant `cody-autocomplete-claude-3` and `cody-autocomplete-fim-fine-tuned-model-hybrid` feature flags. - Updates the preload-completions-on-cursor-movement A/B debounce time for the second phase of the A/B test. --- .../experimentation/FeatureFlagProvider.ts | 8 ++--- .../completions/completion-provider-config.ts | 31 +++-------------- vscode/src/completions/providers/fireworks.ts | 33 ++++++++++++++++++- .../providers/shared/get-experiment-model.ts | 10 +----- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/lib/shared/src/experimentation/FeatureFlagProvider.ts b/lib/shared/src/experimentation/FeatureFlagProvider.ts index c4f795dcaabc..d6e2571f66b5 100644 --- a/lib/shared/src/experimentation/FeatureFlagProvider.ts +++ b/lib/shared/src/experimentation/FeatureFlagProvider.ts @@ -29,14 +29,15 @@ export enum FeatureFlag { // This flag is used to track the overall eligibility to use the StarCoder model. The `-hybrid` // suffix is no longer relevant CodyAutocompleteStarCoderHybrid = 'cody-autocomplete-default-starcoder-hybrid', - // Enable the FineTuned model as the default model via Fireworks - CodyAutocompleteFIMFineTunedModelHybrid = 'cody-autocomplete-fim-fine-tuned-model-hybrid', // Enable the deepseek-v2 as the default model via Fireworks CodyAutocompleteDeepseekV2LiteBase = 'cody-autocomplete-deepseek-v2-lite-base', // Data collection variants used for completions and next edit completions CodyAutocompleteDataCollectionFlag = 'cody-autocomplete-logs-collection-flag', + // Enables fast-path HTTP client for PLG-users + CodyAutocompleteFastPath = 'cody-autocomplete-fast-path', + // Enable various feature flags to experiment with FIM trained fine-tuned models via Fireworks CodyAutocompleteFIMModelExperimentBaseFeatureFlag = 'cody-autocomplete-model-v1-experiment-flag', CodyAutocompleteFIMModelExperimentControl = 'cody-autocomplete-model-v1-experiment-control', @@ -46,13 +47,10 @@ export enum FeatureFlag { CodyAutocompleteFIMModelExperimentVariant3 = 'cody-autocomplete-model-v1-experiment-variant-3', CodyAutocompleteFIMModelExperimentVariant4 = 'cody-autocomplete-model-v1-experiment-variant-4', CodyAutocompleteDisableLowPerfLangDelay = 'cody-autocomplete-disable-low-perf-lang-delay', - // Enables Claude 3 if the user is in our holdout group - CodyAutocompleteClaude3 = 'cody-autocomplete-claude-3', CodyAutocompletePreloadingExperimentBaseFeatureFlag = 'cody-autocomplete-preloading-experiment-flag', CodyAutocompletePreloadingExperimentVariant1 = 'cody-autocomplete-preloading-experiment-variant-1', CodyAutocompletePreloadingExperimentVariant2 = 'cody-autocomplete-preloading-experiment-variant-2', - CodyAutocompletePreloadingExperimentVariant3 = 'cody-autocomplete-preloading-experiment-variant-3', CodyAutocompleteContextExperimentBaseFeatureFlag = 'cody-autocomplete-context-experiment-flag', CodyAutocompleteContextExperimentVariant1 = 'cody-autocomplete-context-experiment-variant-1', diff --git a/vscode/src/completions/completion-provider-config.ts b/vscode/src/completions/completion-provider-config.ts index 637f082a49fb..c54d0afa1a33 100644 --- a/vscode/src/completions/completion-provider-config.ts +++ b/vscode/src/completions/completion-provider-config.ts @@ -34,10 +34,10 @@ class CompletionProviderConfig { FeatureFlag.CodyAutocompletePreloadingExperimentBaseFeatureFlag, FeatureFlag.CodyAutocompletePreloadingExperimentVariant1, FeatureFlag.CodyAutocompletePreloadingExperimentVariant2, - FeatureFlag.CodyAutocompletePreloadingExperimentVariant3, FeatureFlag.CodyAutocompleteDisableLowPerfLangDelay, FeatureFlag.CodyAutocompleteDataCollectionFlag, FeatureFlag.CodyAutocompleteTracing, + FeatureFlag.CodyAutocompleteFastPath, ] this.prefetchSubscription = combineLatest( ...featureFlagsUsed.map(flag => featureFlagProvider.evaluatedFeatureFlag(flag)) @@ -127,20 +127,7 @@ class CompletionProviderConfig { ) } - private getPreloadingExperimentGroup(): Observable< - 'variant1' | 'variant2' | 'variant3' | 'control' - > { - // The desired distribution: - // - Variant-1 25% - // - Variant-2 25% - // - Variant-3 25% - // - Control group 25% - // - // The rollout values to set: - // - CodyAutocompletePreloadingExperimentBaseFeatureFlag 75% - // - CodyAutocompleteVariant1 33% - // - CodyAutocompleteVariant2 100% - // - CodyAutocompleteVariant3 50% + private getPreloadingExperimentGroup(): Observable<'variant1' | 'variant2' | 'control'> { return combineLatest( featureFlagProvider.evaluatedFeatureFlag( FeatureFlag.CodyAutocompletePreloadingExperimentBaseFeatureFlag @@ -150,22 +137,16 @@ class CompletionProviderConfig { ), featureFlagProvider.evaluatedFeatureFlag( FeatureFlag.CodyAutocompletePreloadingExperimentVariant2 - ), - featureFlagProvider.evaluatedFeatureFlag( - FeatureFlag.CodyAutocompletePreloadingExperimentVariant3 ) ).pipe( - map(([isContextExperimentFlagEnabled, variant1, variant2, variant3]) => { + map(([isContextExperimentFlagEnabled, variant1, variant2]) => { if (isContextExperimentFlagEnabled) { if (variant1) { return 'variant1' } if (variant2) { - if (variant3) { - return 'variant2' - } - return 'variant3' + return 'variant2' } } @@ -190,9 +171,7 @@ class CompletionProviderConfig { case 'variant1': return 150 case 'variant2': - return 250 - case 'variant3': - return 350 + return 100 default: return 0 } diff --git a/vscode/src/completions/providers/fireworks.ts b/vscode/src/completions/providers/fireworks.ts index a10985a2641a..6edc3b5f324f 100644 --- a/vscode/src/completions/providers/fireworks.ts +++ b/vscode/src/completions/providers/fireworks.ts @@ -1,22 +1,30 @@ +import type * as vscode from 'vscode' + import { type AuthenticatedAuthStatus, type CodeCompletionsParams, type CompletionResponseGenerator, + FeatureFlag, currentAuthStatusAuthed, currentResolvedConfig, dotcomTokenToGatewayToken, + featureFlagProvider, isDotCom, isDotComAuthed, + subscriptionDisposable, tokensToChars, } from '@sourcegraph/cody-shared' + import { defaultCodeCompletionsClient } from '../default-client' import { createFastPathClient } from '../fast-path-client' import { TriggerKind } from '../get-inline-completions' + import { type GenerateCompletionsOptions, MAX_RESPONSE_TOKENS, Provider, type ProviderFactoryParams, + type ProviderOptions, } from './shared/provider' export const DEEPSEEK_CODER_V2_LITE_BASE = 'deepseek-coder-v2-lite-base' @@ -79,6 +87,22 @@ function getMaxContextTokens(model: FireworksModel): number { } class FireworksProvider extends Provider { + private disposables: vscode.Disposable[] = [] + private isFastPathEnabled = true + + constructor(public readonly options: Readonly) { + super(options) + + this.disposables.push( + subscriptionDisposable( + featureFlagProvider + .evaluatedFeatureFlag(FeatureFlag.CodyAutocompleteTracing) + .subscribe(isFastPathEnabled => { + this.isFastPathEnabled = Boolean(isFastPathEnabled) + }) + ) + ) + } public getRequestParams(options: GenerateCompletionsOptions): CodeCompletionsParams { const { multiline, docContext, document, triggerKind, snippets } = options const useMultilineModel = multiline || triggerKind !== TriggerKind.Automatic @@ -135,7 +159,7 @@ class FireworksProvider extends Provider { ? config.configuration.autocompleteExperimentalFireworksOptions?.token : undefined - if (fastPathAccessToken || localFastPathAccessToken) { + if ((fastPathAccessToken && this.isFastPathEnabled) || localFastPathAccessToken) { return createFastPathClient(requestParams, abortController, { isLocalInstance, fireworksConfig: localFastPathAccessToken @@ -165,6 +189,13 @@ class FireworksProvider extends Provider { return customHeaders } + + public dispose(): void { + for (const disposable of this.disposables) { + disposable.dispose() + } + this.disposables = [] + } } function getClientModel( diff --git a/vscode/src/completions/providers/shared/get-experiment-model.ts b/vscode/src/completions/providers/shared/get-experiment-model.ts index 068df34fb80b..ee0f78e37458 100644 --- a/vscode/src/completions/providers/shared/get-experiment-model.ts +++ b/vscode/src/completions/providers/shared/get-experiment-model.ts @@ -35,13 +35,12 @@ export function getDotComExperimentModel({ return combineLatest( featureFlagProvider.evaluatedFeatureFlag(FeatureFlag.CodyAutocompleteStarCoderHybrid), - featureFlagProvider.evaluatedFeatureFlag(FeatureFlag.CodyAutocompleteClaude3), featureFlagProvider.evaluatedFeatureFlag( FeatureFlag.CodyAutocompleteFIMModelExperimentBaseFeatureFlag ), featureFlagProvider.evaluatedFeatureFlag(FeatureFlag.CodyAutocompleteDeepseekV2LiteBase) ).pipe( - switchMap(([starCoderHybrid, claude3, fimModelExperimentFlag, deepseekV2LiteBase]) => { + switchMap(([starCoderHybrid, fimModelExperimentFlag, deepseekV2LiteBase]) => { // We run fine tuning experiment for VSC client only. // We disable for all agent clients like the JetBrains plugin. const isFinetuningExperimentDisabled = vscode.workspace @@ -67,13 +66,6 @@ export function getDotComExperimentModel({ }) } - if (claude3) { - return Observable.of({ - provider: 'anthropic', - model: 'anthropic/claude-3-haiku-20240307', - }) - } - return Observable.of(null) }), distinctUntilChanged()