Autocomplete: add unit tests for server-side models config

agent/bindings/kotlin/lib/src/main/kotlin/com/sourcegraph/cody/agent/protocol_generated/Constants.kt

@@ -84,6 +84,7 @@ object Constants {
   const val tree = "tree"
   const val `tree-sitter` = "tree-sitter"
   const val unified = "unified"
+  const val unknown = "unknown"
   const val use = "use"
   const val user = "user"
   const val waitlist = "waitlist"

agent/bindings/kotlin/lib/src/main/kotlin/com/sourcegraph/cody/agent/protocol_generated/ModelRef.kt

@@ -1,9 +1,16 @@
 @file:Suppress("FunctionName", "ClassName", "unused", "EnumEntryName", "UnusedImport")
 package com.sourcegraph.cody.agent.protocol_generated;
 
+import com.google.gson.annotations.SerializedName;
+
 data class ModelRef(
-  val providerId: ProviderId,
-  val apiVersionId: ApiVersionId,
-  val modelId: ModelId,
-)
+  val providerId: String,
+  val apiVersionId: ApiVersionIdEnum, // Oneof: unknown
+  val modelId: String,
+) {
+
+  enum class ApiVersionIdEnum {
+    @SerializedName("unknown") Unknown,
+  }
+}
 

lib/shared/src/index.ts

@@ -1,6 +1,12 @@
 // Add anything else here that needs to be used outside of this library.
 
-export { Model, modelsService, type ServerModel, type ServerModelConfiguration } from './models'
+export {
+    Model,
+    modelsService,
+    mockModelsService,
+    type ServerModel,
+    type ServerModelConfiguration,
+} from './models'
 export {
     type EditModel,
     type EditProvider,
@@ -16,6 +22,7 @@ export {
     getModelInfo,
     isCodyProModel,
     isCustomModel,
+    toModelRefStr,
 } from './models/utils'
 export { BotResponseMultiplexer } from './chat/bot-response-multiplexer'
 export { ChatClient } from './chat/chat'

lib/shared/src/models/index.test.ts

@@ -6,9 +6,10 @@ import {
     type ModelCategory,
     type ModelTier,
     ModelsService,
-    type PerSitePreferences,
     type ServerModel,
     type ServerModelConfiguration,
+    type TestStorage,
+    mockModelsService,
 } from '../models/index'
 import { DOTCOM_URL } from '../sourcegraph-api/environments'
 import { CHAT_INPUT_TOKEN_BUDGET, CHAT_OUTPUT_TOKEN_BUDGET } from '../token/constants'
@@ -231,35 +232,13 @@ describe('Model Provider', () => {
 
         let storage: TestStorage
 
-        class TestStorage {
-            constructor(public data: Map<string, string> = new Map()) {}
-            get(key: string): string | null {
-                return this.data.get(key) ?? null
-            }
-
-            async set(key: string, value: string) {
-                await this.data.set(key, value)
-            }
-
-            async delete(key: string) {
-                this.data.delete(key)
-            }
-
-            parse(): PerSitePreferences | undefined {
-                const dumped = this.data.get('model-preferences')
-                console.log(dumped)
-                if (dumped) {
-                    return JSON.parse(dumped)
-                }
-                return undefined
-            }
-        }
-
         beforeEach(async () => {
-            storage = new TestStorage()
-            modelsService.setStorage(storage)
-            mockAuthStatus(enterpriseAuthStatus)
-            await modelsService.setServerSentModels(SERVER_MODELS)
+            const result = await mockModelsService({
+                config: SERVER_MODELS,
+                authStatus: enterpriseAuthStatus,
+            })
+            storage = result.storage
+            modelsService = result.modelsService
         })
 
         it('constructs from server models', () => {

lib/shared/src/models/index.ts

@@ -1,6 +1,8 @@
 import { type Observable, Subject } from 'observable-fns'
 import { authStatus, currentAuthStatus } from '../auth/authStatus'
+import { mockAuthStatus } from '../auth/authStatus'
 import { type AuthStatus, isCodyProUser, isEnterpriseUser } from '../auth/types'
+import { AUTH_STATUS_FIXTURE_AUTHED_DOTCOM } from '../auth/types'
 import { CodyIDE } from '../configuration'
 import { resolvedConfig } from '../configuration/resolver'
 import { fetchLocalOllamaModels } from '../llm-providers/ollama/utils'
@@ -16,8 +18,8 @@ type ModelId = string
 type ApiVersionId = string
 type ProviderId = string
 
-type ModelRefStr = `${ProviderId}::${ApiVersionId}::${ModelId}`
-interface ModelRef {
+export type ModelRefStr = `${ProviderId}::${ApiVersionId}::${ModelId}`
+export interface ModelRef {
     providerId: ProviderId
     apiVersionId: ApiVersionId
     modelId: ModelId
@@ -501,7 +503,6 @@ export class ModelsService {
      * Gets the available models of the specified usage type, with the default model first.
      *
      * @param type - The usage type of the models to retrieve.
-     * @param authStatus - The authentication status of the user.
      * @returns An array of models, with the default model first.
      */
     public getModels(type: ModelUsage): Model[] {
@@ -647,3 +648,56 @@ function capabilityToUsage(capability: ModelCapability): ModelUsage[] {
             return [ModelUsage.Chat, ModelUsage.Edit]
     }
 }
+
+interface MockModelsServiceResult {
+    storage: TestStorage
+    modelsService: ModelsService
+}
+
+export class TestStorage {
+    constructor(public data: Map<string, string> = new Map()) {}
+    get(key: string): string | null {
+        return this.data.get(key) ?? null
+    }
+
+    async set(key: string, value: string) {
+        await this.data.set(key, value)
+    }
+
+    async delete(key: string) {
+        this.data.delete(key)
+    }
+
+    parse(): PerSitePreferences | undefined {
+        const dumped = this.data.get('model-preferences')
+        if (dumped) {
+            return JSON.parse(dumped)
+        }
+        return undefined
+    }
+}
+
+interface MockModelsServiceParams {
+    config: ServerModelConfiguration
+    authStatus?: AuthStatus
+    modelsService?: ModelsService
+    storage?: TestStorage
+}
+
+export async function mockModelsService(
+    params: MockModelsServiceParams
+): Promise<MockModelsServiceResult> {
+    const {
+        storage = new TestStorage(),
+        modelsService = new ModelsService(),
+        authStatus = AUTH_STATUS_FIXTURE_AUTHED_DOTCOM,
+        config,
+    } = params
+
+    modelsService.setStorage(storage)
+    mockAuthStatus(authStatus)
+
+    await modelsService.setServerSentModels(config)
+
+    return { storage, modelsService }
+}

lib/shared/src/models/utils.ts

@@ -1,4 +1,4 @@
-import type { Model } from '.'
+import type { Model, ModelRef, ModelRefStr } from '.'
 import { ModelTag } from '..'
 
 export function getProviderName(name: string): string {
@@ -49,3 +49,8 @@ export function isCustomModel(model: Model): boolean {
 function modelHasTag(model: Model, modelTag: ModelTag): boolean {
     return model.tags.includes(modelTag)
 }
+
+export function toModelRefStr(modelRef: ModelRef): ModelRefStr {
+    const { providerId, apiVersionId, modelId } = modelRef
+    return `${providerId}::${apiVersionId}::${modelId}`
+}

lib/shared/src/sourcegraph-api/rest/client.ts

@@ -66,7 +66,7 @@ export class RestClient {
         // TODO(PRIME-322): Export the type information via NPM. For now, we just blindly
         // walk the returned object model.
         //
-        // NOTE: This API endpoint hasn't shippeted yet, and probably won't work for you.
+        // NOTE: This API endpoint hasn't shipped yet, and probably won't work for you.
         // Also, the URL definitely will change.
         const serverSideConfig = await this.getRequest<ServerModelConfiguration>(
             'getAvailableModels',

vscode/src/completions/artificial-delay.ts

@@ -82,7 +82,7 @@ export function getArtificialDelay(
     }
 
     if (total > 0) {
-        logDebug('CodyCompletionProvider:getLatency', `Delay added: ${total}`)
+        logDebug('AutocompleteProvider:getLatency', `Delay added: ${total}`)
     }
 
     return total

vscode/src/completions/create-inline-completion-item-provider.ts

@@ -50,7 +50,7 @@ export function createInlineCompletionItemProvider({
 }: InlineCompletionItemProviderArgs): Observable<void> {
     const authStatus = currentAuthStatus()
     if (!authStatus.authenticated) {
-        logDebug('CodyCompletionProvider:notSignedIn', 'You are not signed in.')
+        logDebug('AutocompleteProvider:notSignedIn', 'You are not signed in.')
 
         if (config.isRunningInsideAgent) {
             // Register an empty completion provider when running inside the

vscode/src/completions/create-multi-model-inline-completion-provider.ts

@@ -136,6 +136,7 @@ export async function createInlineCompletionItemFromMultipleProviders({
             legacyModel: currentProviderConfig.model,
             provider: currentProviderConfig.provider,
             config: newConfig,
+            source: 'local-editor-settings',
         })
 
         const triggerDelay = vscode.workspace

vscode/src/completions/get-inline-completions-tests/helpers.ts

@@ -45,6 +45,7 @@ import {
     SINGLE_LINE_STOP_SEQUENCES,
     createProvider as createAnthropicProvider,
 } from '../providers/anthropic'
+import type { AutocompleteProviderID } from '../providers/create-provider'
 import { createProvider as createFireworksProvider } from '../providers/fireworks'
 import { pressEnterAndGetIndentString } from '../providers/hot-streak'
 import { RequestManager } from '../request-manager'
@@ -175,7 +176,8 @@ export function params(
         legacyModel: configuration?.autocompleteAdvancedModel!,
         config: configWithAccessToken,
         anonymousUserID: 'anonymousUserID',
-        provider: configuration?.autocompleteAdvancedModel || 'anthropic',
+        provider: (configuration?.autocompleteAdvancedModel as AutocompleteProviderID) || 'anthropic',
+        source: 'local-editor-settings',
     })
 
     provider.client = client

vscode/src/completions/inline-completion-item-provider-e2e.test.ts

@@ -155,6 +155,7 @@ function createNetworkProvider(params: RequestParams): MockRequestProvider {
             id: 'mock-provider',
             anonymousUserID: 'anonymousUserID',
             legacyModel: 'test-model',
+            source: 'local-editor-settings',
         },
         providerOptions
     )

vscode/src/completions/inline-completion-item-provider.ts

@@ -191,9 +191,12 @@ export class InlineCompletionItemProvider
         this.smartThrottleService = new SmartThrottleService()
         this.disposables.push(this.smartThrottleService)
 
+        // TODO(valery): replace `model_configured_by_site_config` with the actual model ID received from backend.
         logDebug(
-            'CodyCompletionProvider:initialized',
-            [this.config.provider.id, this.config.provider.legacyModel].join('/')
+            'AutocompleteProvider:initialized',
+            `using "${this.config.provider.configSource}": "${this.config.provider.id}::${
+                this.config.provider.legacyModel || 'model_configured_by_site_config'
+            }"`
         )
 
         if (!this.config.noInlineAccept) {
@@ -1132,7 +1135,7 @@ function logIgnored(uri: vscode.Uri, reason: CodyIgnoreType, isManualCompletion:
     }
     lastIgnoredUriLogged = string
     logDebug(
-        'CodyCompletionProvider:ignored',
+        'AutocompleteProvider:ignored',
         'Cody is disabled in file ' + uri.toString() + ' (' + reason + ')'
     )
 }