Autocomplete: a/b test the fast-path (#5905)

- Adds a feature flag for the fast-path A/B test.
- Removes redundant `cody-autocomplete-claude-3` and
`cody-autocomplete-fim-fine-tuned-model-hybrid` feature flags.
- Updates the preload-completions-on-cursor-movement A/B debounce time
for the second phase of the A/B test.

lib/shared/src/experimentation/FeatureFlagProvider.ts

@@ -29,14 +29,15 @@ export enum FeatureFlag {
     // This flag is used to track the overall eligibility to use the StarCoder model. The `-hybrid`
     // suffix is no longer relevant
     CodyAutocompleteStarCoderHybrid = 'cody-autocomplete-default-starcoder-hybrid',
-    // Enable the FineTuned model as the default model via Fireworks
-    CodyAutocompleteFIMFineTunedModelHybrid = 'cody-autocomplete-fim-fine-tuned-model-hybrid',
     // Enable the deepseek-v2 as the default model via Fireworks
     CodyAutocompleteDeepseekV2LiteBase = 'cody-autocomplete-deepseek-v2-lite-base',
 
     // Data collection variants used for completions and next edit completions
     CodyAutocompleteDataCollectionFlag = 'cody-autocomplete-logs-collection-flag',
 
+    // Enables fast-path HTTP client for PLG-users
+    CodyAutocompleteFastPath = 'cody-autocomplete-fast-path',
+
     // Enable various feature flags to experiment with FIM trained fine-tuned models via Fireworks
     CodyAutocompleteFIMModelExperimentBaseFeatureFlag = 'cody-autocomplete-model-v1-experiment-flag',
     CodyAutocompleteFIMModelExperimentControl = 'cody-autocomplete-model-v1-experiment-control',
@@ -46,13 +47,10 @@ export enum FeatureFlag {
     CodyAutocompleteFIMModelExperimentVariant3 = 'cody-autocomplete-model-v1-experiment-variant-3',
     CodyAutocompleteFIMModelExperimentVariant4 = 'cody-autocomplete-model-v1-experiment-variant-4',
     CodyAutocompleteDisableLowPerfLangDelay = 'cody-autocomplete-disable-low-perf-lang-delay',
-    // Enables Claude 3 if the user is in our holdout group
-    CodyAutocompleteClaude3 = 'cody-autocomplete-claude-3',
 
     CodyAutocompletePreloadingExperimentBaseFeatureFlag = 'cody-autocomplete-preloading-experiment-flag',
     CodyAutocompletePreloadingExperimentVariant1 = 'cody-autocomplete-preloading-experiment-variant-1',
     CodyAutocompletePreloadingExperimentVariant2 = 'cody-autocomplete-preloading-experiment-variant-2',
-    CodyAutocompletePreloadingExperimentVariant3 = 'cody-autocomplete-preloading-experiment-variant-3',
 
     CodyAutocompleteContextExperimentBaseFeatureFlag = 'cody-autocomplete-context-experiment-flag',
     CodyAutocompleteContextExperimentVariant1 = 'cody-autocomplete-context-experiment-variant-1',

vscode/src/completions/completion-provider-config.ts

@@ -34,10 +34,10 @@ class CompletionProviderConfig {
             FeatureFlag.CodyAutocompletePreloadingExperimentBaseFeatureFlag,
             FeatureFlag.CodyAutocompletePreloadingExperimentVariant1,
             FeatureFlag.CodyAutocompletePreloadingExperimentVariant2,
-            FeatureFlag.CodyAutocompletePreloadingExperimentVariant3,
             FeatureFlag.CodyAutocompleteDisableLowPerfLangDelay,
             FeatureFlag.CodyAutocompleteDataCollectionFlag,
             FeatureFlag.CodyAutocompleteTracing,
+            FeatureFlag.CodyAutocompleteFastPath,
         ]
         this.prefetchSubscription = combineLatest(
             ...featureFlagsUsed.map(flag => featureFlagProvider.evaluatedFeatureFlag(flag))
@@ -127,20 +127,7 @@ class CompletionProviderConfig {
             )
     }
 
-    private getPreloadingExperimentGroup(): Observable<
-        'variant1' | 'variant2' | 'variant3' | 'control'
-    > {
-        // The desired distribution:
-        // - Variant-1 25%
-        // - Variant-2 25%
-        // - Variant-3 25%
-        // - Control group 25%
-        //
-        // The rollout values to set:
-        // - CodyAutocompletePreloadingExperimentBaseFeatureFlag 75%
-        // - CodyAutocompleteVariant1 33%
-        // - CodyAutocompleteVariant2 100%
-        // - CodyAutocompleteVariant3 50%
+    private getPreloadingExperimentGroup(): Observable<'variant1' | 'variant2' | 'control'> {
         return combineLatest(
             featureFlagProvider.evaluatedFeatureFlag(
                 FeatureFlag.CodyAutocompletePreloadingExperimentBaseFeatureFlag
@@ -150,22 +137,16 @@ class CompletionProviderConfig {
             ),
             featureFlagProvider.evaluatedFeatureFlag(
                 FeatureFlag.CodyAutocompletePreloadingExperimentVariant2
-            ),
-            featureFlagProvider.evaluatedFeatureFlag(
-                FeatureFlag.CodyAutocompletePreloadingExperimentVariant3
             )
         ).pipe(
-            map(([isContextExperimentFlagEnabled, variant1, variant2, variant3]) => {
+            map(([isContextExperimentFlagEnabled, variant1, variant2]) => {
                 if (isContextExperimentFlagEnabled) {
                     if (variant1) {
                         return 'variant1'
                     }
 
                     if (variant2) {
-                        if (variant3) {
-                            return 'variant2'
-                        }
-                        return 'variant3'
+                        return 'variant2'
                     }
                 }
 
@@ -190,9 +171,7 @@ class CompletionProviderConfig {
                             case 'variant1':
                                 return 150
                             case 'variant2':
-                                return 250
-                            case 'variant3':
-                                return 350
+                                return 100
                             default:
                                 return 0
                         }

vscode/src/completions/providers/fireworks.ts

@@ -1,22 +1,30 @@
+import type * as vscode from 'vscode'
+
 import {
     type AuthenticatedAuthStatus,
     type CodeCompletionsParams,
     type CompletionResponseGenerator,
+    FeatureFlag,
     currentAuthStatusAuthed,
     currentResolvedConfig,
     dotcomTokenToGatewayToken,
+    featureFlagProvider,
     isDotCom,
     isDotComAuthed,
+    subscriptionDisposable,
     tokensToChars,
 } from '@sourcegraph/cody-shared'
+
 import { defaultCodeCompletionsClient } from '../default-client'
 import { createFastPathClient } from '../fast-path-client'
 import { TriggerKind } from '../get-inline-completions'
+
 import {
     type GenerateCompletionsOptions,
     MAX_RESPONSE_TOKENS,
     Provider,
     type ProviderFactoryParams,
+    type ProviderOptions,
 } from './shared/provider'
 
 export const DEEPSEEK_CODER_V2_LITE_BASE = 'deepseek-coder-v2-lite-base'
@@ -79,6 +87,22 @@ function getMaxContextTokens(model: FireworksModel): number {
 }
 
 class FireworksProvider extends Provider {
+    private disposables: vscode.Disposable[] = []
+    private isFastPathEnabled = true
+
+    constructor(public readonly options: Readonly<ProviderOptions>) {
+        super(options)
+
+        this.disposables.push(
+            subscriptionDisposable(
+                featureFlagProvider
+                    .evaluatedFeatureFlag(FeatureFlag.CodyAutocompleteTracing)
+                    .subscribe(isFastPathEnabled => {
+                        this.isFastPathEnabled = Boolean(isFastPathEnabled)
+                    })
+            )
+        )
+    }
     public getRequestParams(options: GenerateCompletionsOptions): CodeCompletionsParams {
         const { multiline, docContext, document, triggerKind, snippets } = options
         const useMultilineModel = multiline || triggerKind !== TriggerKind.Automatic
@@ -135,7 +159,7 @@ class FireworksProvider extends Provider {
                     ? config.configuration.autocompleteExperimentalFireworksOptions?.token
                     : undefined
 
-            if (fastPathAccessToken || localFastPathAccessToken) {
+            if ((fastPathAccessToken && this.isFastPathEnabled) || localFastPathAccessToken) {
                 return createFastPathClient(requestParams, abortController, {
                     isLocalInstance,
                     fireworksConfig: localFastPathAccessToken
@@ -165,6 +189,13 @@ class FireworksProvider extends Provider {
 
         return customHeaders
     }
+
+    public dispose(): void {
+        for (const disposable of this.disposables) {
+            disposable.dispose()
+        }
+        this.disposables = []
+    }
 }
 
 function getClientModel(

vscode/src/completions/providers/shared/get-experiment-model.ts

@@ -35,13 +35,12 @@ export function getDotComExperimentModel({
 
     return combineLatest(
         featureFlagProvider.evaluatedFeatureFlag(FeatureFlag.CodyAutocompleteStarCoderHybrid),
-        featureFlagProvider.evaluatedFeatureFlag(FeatureFlag.CodyAutocompleteClaude3),
         featureFlagProvider.evaluatedFeatureFlag(
             FeatureFlag.CodyAutocompleteFIMModelExperimentBaseFeatureFlag
         ),
         featureFlagProvider.evaluatedFeatureFlag(FeatureFlag.CodyAutocompleteDeepseekV2LiteBase)
     ).pipe(
-        switchMap(([starCoderHybrid, claude3, fimModelExperimentFlag, deepseekV2LiteBase]) => {
+        switchMap(([starCoderHybrid, fimModelExperimentFlag, deepseekV2LiteBase]) => {
             // We run fine tuning experiment for VSC client only.
             // We disable for all agent clients like the JetBrains plugin.
             const isFinetuningExperimentDisabled = vscode.workspace
@@ -67,13 +66,6 @@ export function getDotComExperimentModel({
                 })
             }
 
-            if (claude3) {
-                return Observable.of({
-                    provider: 'anthropic',
-                    model: 'anthropic/claude-3-haiku-20240307',
-                })
-            }
-
             return Observable.of(null)
         }),
         distinctUntilChanged()