From e5393b18b78023858c24445888e786d77ddaff36 Mon Sep 17 00:00:00 2001 From: Beatrix Date: Fri, 13 Sep 2024 00:24:20 -0700 Subject: [PATCH 1/4] feat(chat): support non-streaming requests This change adds support for non-streaming requests, particularly for new OpenAI models that do not support streaming. It also fixes a "Parse Error: JS exception" issue that occurred when handling non-streaming responses in the streaming client. - Remove HACK for non-streaming requests in nodeClient.ts - Implement a new `_fetchWithCallbacks` method in the `SourcegraphCompletionsClient` class to handle non-streaming requests - Update the `SourcegraphNodeCompletionsClient` and `SourcegraphBrowserCompletionsClient` classes to use the new `_fetchWithCallbacks` method when `params.stream` is `false` - This change ensures that non-streaming requests, such as those for the new OpenAI models that do not support streaming, are handled correctly and avoid issues like "Parse Error: JS exception" that can occur when the response is not parsed correctly in the streaming client The `SourcegraphCompletionsClient` class now has a new abstract method `_fetchWithCallbacks` that must be implemented by subclasses --- .../completions/browserClient.ts | 61 +++++++++++++ .../src/sourcegraph-api/completions/client.ts | 16 +++- vscode/CHANGELOG.md | 1 + vscode/src/completions/nodeClient.ts | 91 ++++++++++++++++--- 4 files changed, 153 insertions(+), 16 deletions(-) diff --git a/lib/shared/src/sourcegraph-api/completions/browserClient.ts b/lib/shared/src/sourcegraph-api/completions/browserClient.ts index 07a4b8ff49c0..c398e9d95d29 100644 --- a/lib/shared/src/sourcegraph-api/completions/browserClient.ts +++ b/lib/shared/src/sourcegraph-api/completions/browserClient.ts @@ -115,6 +115,67 @@ export class SourcegraphBrowserCompletionsClient extends SourcegraphCompletionsC console.error(error) }) } + + protected async _fetchWithCallbacks( + params: CompletionParameters, + requestParams: CompletionRequestParameters, + cb: CompletionCallbacks, + signal?: AbortSignal + ): Promise { + const { apiVersion } = requestParams + const serializedParams = await getSerializedParams(params) + + const url = new URL(this.completionsEndpoint) + if (apiVersion >= 1) { + url.searchParams.append('api-version', '' + apiVersion) + } + addClientInfoParams(url.searchParams) + + const headersInstance = new Headers({ + ...this.config.customHeaders, + ...requestParams.customHeaders, + } as HeadersInit) + addCustomUserAgent(headersInstance) + headersInstance.set('Content-Type', 'application/json; charset=utf-8') + if (this.config.accessToken) { + headersInstance.set('Authorization', `token ${this.config.accessToken}`) + } + + const parameters = new URLSearchParams(globalThis.location.search) + const trace = parameters.get('trace') + if (trace) { + headersInstance.set('X-Sourcegraph-Should-Trace', 'true') + } + + try { + const response = await fetch(url.toString(), { + method: 'POST', + headers: headersInstance, + body: JSON.stringify(serializedParams), + signal, + }) + + if (!response.ok) { + const errorMessage = await response.text() + throw new Error( + errorMessage.length === 0 + ? `Request failed with status code ${response.status}` + : errorMessage + ) + } + + const data = await response.json() + if (data?.completion) { + cb.onChange(data.completion) + cb.onComplete() + } else { + throw new Error('Unexpected response format') + } + } catch (error: any) { + cb.onError(error.message) + console.error(error) + } + } } if (isRunningInWebWorker) { diff --git a/lib/shared/src/sourcegraph-api/completions/client.ts b/lib/shared/src/sourcegraph-api/completions/client.ts index e9a8f393669f..8618ff31ac55 100644 --- a/lib/shared/src/sourcegraph-api/completions/client.ts +++ b/lib/shared/src/sourcegraph-api/completions/client.ts @@ -1,6 +1,5 @@ import type { Span } from '@opentelemetry/api' import type { ClientConfigurationWithAccessToken } from '../../configuration' - import { useCustomChatClient } from '../../llm-providers' import { recordErrorToSpan } from '../../tracing' import type { @@ -45,6 +44,8 @@ export type CompletionsClientConfig = Pick< export abstract class SourcegraphCompletionsClient { private errorEncountered = false + protected readonly isTemperatureZero = process.env.CODY_TEMPERATURE_ZERO === 'true' + constructor( protected config: CompletionsClientConfig, protected logger?: CompletionLogger @@ -88,6 +89,13 @@ export abstract class SourcegraphCompletionsClient { } } + protected abstract _fetchWithCallbacks( + params: CompletionParameters, + requestParams: CompletionRequestParameters, + cb: CompletionCallbacks, + signal?: AbortSignal + ): Promise + protected abstract _streamWithCallbacks( params: CompletionParameters, requestParams: CompletionRequestParameters, @@ -144,7 +152,11 @@ export abstract class SourcegraphCompletionsClient { }) if (!isNonSourcegraphProvider) { - await this._streamWithCallbacks(params, requestParams, callbacks, signal) + if (params.stream === false) { + await this._fetchWithCallbacks(params, requestParams, callbacks, signal) + } else { + await this._streamWithCallbacks(params, requestParams, callbacks, signal) + } } for (let i = 0; ; i++) { diff --git a/vscode/CHANGELOG.md b/vscode/CHANGELOG.md index ed0ede5ad0f9..1444718a35a2 100644 --- a/vscode/CHANGELOG.md +++ b/vscode/CHANGELOG.md @@ -8,6 +8,7 @@ This is a log of all notable changes to Cody for VS Code. [Unreleased] changes a - The [new OpenAI models (OpenAI O1 & OpenAI O1-mini)](https://sourcegraph.com/blog/openai-o1-for-cody) are now available to selected Cody Pro users for early access. [pull/5508](https://github.com/sourcegraph/cody/pull/5508) - Cody Pro users can join the waitlist for the new models by clicking the "Join Waitlist" button. [pull/5508](https://github.com/sourcegraph/cody/pull/5508) +- Chat: Support non-streaming requests. ### Fixed diff --git a/vscode/src/completions/nodeClient.ts b/vscode/src/completions/nodeClient.ts index 0cf5556232fc..cc1d57d4f639 100644 --- a/vscode/src/completions/nodeClient.ts +++ b/vscode/src/completions/nodeClient.ts @@ -29,8 +29,6 @@ import { } from '@sourcegraph/cody-shared' import { CompletionsResponseBuilder } from '@sourcegraph/cody-shared/src/sourcegraph-api/completions/CompletionsResponseBuilder' -const isTemperatureZero = process.env.CODY_TEMPERATURE_ZERO === 'true' - export class SourcegraphNodeCompletionsClient extends SourcegraphCompletionsClient { protected _streamWithCallbacks( params: CompletionParameters, @@ -56,7 +54,7 @@ export class SourcegraphNodeCompletionsClient extends SourcegraphCompletionsClie model: params.model, }) - if (isTemperatureZero) { + if (this.isTemperatureZero) { params = { ...params, temperature: 0, @@ -203,17 +201,6 @@ export class SourcegraphNodeCompletionsClient extends SourcegraphCompletionsClie bufferText += str bufferBin = buf - // HACK: Handles non-stream request. - // TODO: Implement a function to make and process non-stream requests. - if (params.stream === false) { - const json = JSON.parse(bufferText) - if (json?.completion) { - cb.onChange(json.completion) - cb.onComplete() - return - } - } - const parseResult = parseEvents(builder, bufferText) if (isError(parseResult)) { logError( @@ -286,6 +273,82 @@ export class SourcegraphNodeCompletionsClient extends SourcegraphCompletionsClie onAbort(signal, () => request.destroy()) }) } + + protected async _fetchWithCallbacks( + params: CompletionParameters, + requestParams: CompletionRequestParameters, + cb: CompletionCallbacks, + signal?: AbortSignal + ): Promise { + const { apiVersion } = requestParams + + const url = new URL(this.completionsEndpoint) + if (apiVersion >= 1) { + url.searchParams.append('api-version', '' + apiVersion) + } + addClientInfoParams(url.searchParams) + + return tracer.startActiveSpan(`POST ${url.toString()}`, async span => { + span.setAttributes({ + fast: params.fast, + maxTokensToSample: params.maxTokensToSample, + temperature: this.isTemperatureZero ? 0 : params.temperature, + topK: params.topK, + topP: params.topP, + model: params.model, + }) + + const serializedParams = await getSerializedParams(params) + + const log = this.logger?.startCompletion(params, url.toString()) + + try { + const response = await fetch(url.toString(), { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Accept-Encoding': 'gzip;q=0', + ...(this.config.accessToken + ? { Authorization: `token ${this.config.accessToken}` } + : null), + ...(customUserAgent ? { 'User-Agent': customUserAgent } : null), + ...this.config.customHeaders, + ...requestParams.customHeaders, + ...getTraceparentHeaders(), + }, + body: JSON.stringify(serializedParams), + signal, + }) + + if (!response.ok) { + const errorMessage = await response.text() + throw new NetworkError( + { + url: url.toString(), + status: response.status, + statusText: response.statusText, + }, + errorMessage, + getActiveTraceAndSpanId()?.traceId + ) + } + + const json = await response.json() + if (typeof json?.completion === 'string') { + cb.onChange(json.completion) + cb.onComplete() + return + } + + throw new Error('Unexpected response format') + } catch (error) { + const errorObject = error instanceof Error ? error : new Error(`${error}`) + log?.onError(errorObject.message, error) + recordErrorToSpan(span, errorObject) + cb.onError(errorObject) + } + }) + } } function getHeader(value: string | undefined | string[]): string | undefined { From d902b47daa17b72051661a92183227a4e8f9f3b3 Mon Sep 17 00:00:00 2001 From: Beatrix Date: Fri, 13 Sep 2024 00:48:45 -0700 Subject: [PATCH 2/4] clean up --- .../completions/browserClient.ts | 22 ++++--------------- .../src/sourcegraph-api/completions/client.ts | 15 +++++++++++++ vscode/src/completions/nodeClient.ts | 18 ++------------- 3 files changed, 21 insertions(+), 34 deletions(-) diff --git a/lib/shared/src/sourcegraph-api/completions/browserClient.ts b/lib/shared/src/sourcegraph-api/completions/browserClient.ts index c398e9d95d29..ba0c8900e641 100644 --- a/lib/shared/src/sourcegraph-api/completions/browserClient.ts +++ b/lib/shared/src/sourcegraph-api/completions/browserClient.ts @@ -122,31 +122,19 @@ export class SourcegraphBrowserCompletionsClient extends SourcegraphCompletionsC cb: CompletionCallbacks, signal?: AbortSignal ): Promise { - const { apiVersion } = requestParams - const serializedParams = await getSerializedParams(params) - - const url = new URL(this.completionsEndpoint) - if (apiVersion >= 1) { - url.searchParams.append('api-version', '' + apiVersion) - } - addClientInfoParams(url.searchParams) - + const { url, serializedParams } = await this.prepareRequest(params, requestParams) const headersInstance = new Headers({ + 'Content-Type': 'application/json; charset=utf-8', ...this.config.customHeaders, ...requestParams.customHeaders, - } as HeadersInit) + }) addCustomUserAgent(headersInstance) - headersInstance.set('Content-Type', 'application/json; charset=utf-8') if (this.config.accessToken) { headersInstance.set('Authorization', `token ${this.config.accessToken}`) } - - const parameters = new URLSearchParams(globalThis.location.search) - const trace = parameters.get('trace') - if (trace) { + if (new URLSearchParams(globalThis.location.search).get('trace')) { headersInstance.set('X-Sourcegraph-Should-Trace', 'true') } - try { const response = await fetch(url.toString(), { method: 'POST', @@ -154,7 +142,6 @@ export class SourcegraphBrowserCompletionsClient extends SourcegraphCompletionsC body: JSON.stringify(serializedParams), signal, }) - if (!response.ok) { const errorMessage = await response.text() throw new Error( @@ -163,7 +150,6 @@ export class SourcegraphBrowserCompletionsClient extends SourcegraphCompletionsC : errorMessage ) } - const data = await response.json() if (data?.completion) { cb.onChange(data.completion) diff --git a/lib/shared/src/sourcegraph-api/completions/client.ts b/lib/shared/src/sourcegraph-api/completions/client.ts index 8618ff31ac55..1b3c0b238c79 100644 --- a/lib/shared/src/sourcegraph-api/completions/client.ts +++ b/lib/shared/src/sourcegraph-api/completions/client.ts @@ -1,4 +1,5 @@ import type { Span } from '@opentelemetry/api' +import { addClientInfoParams, getSerializedParams } from '../..' import type { ClientConfigurationWithAccessToken } from '../../configuration' import { useCustomChatClient } from '../../llm-providers' import { recordErrorToSpan } from '../../tracing' @@ -89,6 +90,20 @@ export abstract class SourcegraphCompletionsClient { } } + protected async prepareRequest( + params: CompletionParameters, + requestParams: CompletionRequestParameters + ): Promise<{ url: URL; serializedParams: any }> { + const { apiVersion } = requestParams + const serializedParams = await getSerializedParams(params) + const url = new URL(this.completionsEndpoint) + if (apiVersion >= 1) { + url.searchParams.append('api-version', '' + apiVersion) + } + addClientInfoParams(url.searchParams) + return { url, serializedParams } + } + protected abstract _fetchWithCallbacks( params: CompletionParameters, requestParams: CompletionRequestParameters, diff --git a/vscode/src/completions/nodeClient.ts b/vscode/src/completions/nodeClient.ts index cc1d57d4f639..418eb6528bd0 100644 --- a/vscode/src/completions/nodeClient.ts +++ b/vscode/src/completions/nodeClient.ts @@ -280,14 +280,8 @@ export class SourcegraphNodeCompletionsClient extends SourcegraphCompletionsClie cb: CompletionCallbacks, signal?: AbortSignal ): Promise { - const { apiVersion } = requestParams - - const url = new URL(this.completionsEndpoint) - if (apiVersion >= 1) { - url.searchParams.append('api-version', '' + apiVersion) - } - addClientInfoParams(url.searchParams) - + const { url, serializedParams } = await this.prepareRequest(params, requestParams) + const log = this.logger?.startCompletion(params, url.toString()) return tracer.startActiveSpan(`POST ${url.toString()}`, async span => { span.setAttributes({ fast: params.fast, @@ -297,11 +291,6 @@ export class SourcegraphNodeCompletionsClient extends SourcegraphCompletionsClie topP: params.topP, model: params.model, }) - - const serializedParams = await getSerializedParams(params) - - const log = this.logger?.startCompletion(params, url.toString()) - try { const response = await fetch(url.toString(), { method: 'POST', @@ -319,7 +308,6 @@ export class SourcegraphNodeCompletionsClient extends SourcegraphCompletionsClie body: JSON.stringify(serializedParams), signal, }) - if (!response.ok) { const errorMessage = await response.text() throw new NetworkError( @@ -332,14 +320,12 @@ export class SourcegraphNodeCompletionsClient extends SourcegraphCompletionsClie getActiveTraceAndSpanId()?.traceId ) } - const json = await response.json() if (typeof json?.completion === 'string') { cb.onChange(json.completion) cb.onComplete() return } - throw new Error('Unexpected response format') } catch (error) { const errorObject = error instanceof Error ? error : new Error(`${error}`) From 15c9e66889f97be382af38aff39940d3f064a982 Mon Sep 17 00:00:00 2001 From: Beatrix <68532117+abeatrix@users.noreply.github.com> Date: Fri, 13 Sep 2024 00:56:54 -0700 Subject: [PATCH 3/4] Update lib/shared/src/sourcegraph-api/completions/client.ts Co-authored-by: Valery Bugakov --- lib/shared/src/sourcegraph-api/completions/client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/shared/src/sourcegraph-api/completions/client.ts b/lib/shared/src/sourcegraph-api/completions/client.ts index 1b3c0b238c79..a271a364a159 100644 --- a/lib/shared/src/sourcegraph-api/completions/client.ts +++ b/lib/shared/src/sourcegraph-api/completions/client.ts @@ -93,7 +93,7 @@ export abstract class SourcegraphCompletionsClient { protected async prepareRequest( params: CompletionParameters, requestParams: CompletionRequestParameters - ): Promise<{ url: URL; serializedParams: any }> { + ): Promise<{ url: URL; serializedParams: SerializedCompletionParameters }> { const { apiVersion } = requestParams const serializedParams = await getSerializedParams(params) const url = new URL(this.completionsEndpoint) From b5fbe3b80bf624ae04278af7544d4a584dac068f Mon Sep 17 00:00:00 2001 From: Beatrix Date: Fri, 13 Sep 2024 01:08:35 -0700 Subject: [PATCH 4/4] apply types and feedback --- .../src/sourcegraph-api/completions/browserClient.ts | 8 ++++---- lib/shared/src/sourcegraph-api/completions/client.ts | 1 + vscode/CHANGELOG.md | 2 +- vscode/src/completions/nodeClient.ts | 3 ++- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/shared/src/sourcegraph-api/completions/browserClient.ts b/lib/shared/src/sourcegraph-api/completions/browserClient.ts index ba0c8900e641..b4d23828e37e 100644 --- a/lib/shared/src/sourcegraph-api/completions/browserClient.ts +++ b/lib/shared/src/sourcegraph-api/completions/browserClient.ts @@ -8,7 +8,7 @@ import { addClientInfoParams } from '../client-name-version' import { CompletionsResponseBuilder } from './CompletionsResponseBuilder' import { type CompletionRequestParameters, SourcegraphCompletionsClient } from './client' import { parseCompletionJSON } from './parse' -import type { CompletionCallbacks, CompletionParameters, Event } from './types' +import type { CompletionCallbacks, CompletionParameters, CompletionResponse, Event } from './types' import { getSerializedParams } from './utils' declare const WorkerGlobalScope: never @@ -150,16 +150,16 @@ export class SourcegraphBrowserCompletionsClient extends SourcegraphCompletionsC : errorMessage ) } - const data = await response.json() + const data = (await response.json()) as CompletionResponse if (data?.completion) { cb.onChange(data.completion) cb.onComplete() } else { throw new Error('Unexpected response format') } - } catch (error: any) { - cb.onError(error.message) + } catch (error) { console.error(error) + cb.onError(error instanceof Error ? error : new Error(`${error}`)) } } } diff --git a/lib/shared/src/sourcegraph-api/completions/client.ts b/lib/shared/src/sourcegraph-api/completions/client.ts index a271a364a159..bab946e9a4eb 100644 --- a/lib/shared/src/sourcegraph-api/completions/client.ts +++ b/lib/shared/src/sourcegraph-api/completions/client.ts @@ -9,6 +9,7 @@ import type { CompletionParameters, CompletionResponse, Event, + SerializedCompletionParameters, } from './types' export interface CompletionLogger { diff --git a/vscode/CHANGELOG.md b/vscode/CHANGELOG.md index 1444718a35a2..f72de1b40b9e 100644 --- a/vscode/CHANGELOG.md +++ b/vscode/CHANGELOG.md @@ -8,7 +8,7 @@ This is a log of all notable changes to Cody for VS Code. [Unreleased] changes a - The [new OpenAI models (OpenAI O1 & OpenAI O1-mini)](https://sourcegraph.com/blog/openai-o1-for-cody) are now available to selected Cody Pro users for early access. [pull/5508](https://github.com/sourcegraph/cody/pull/5508) - Cody Pro users can join the waitlist for the new models by clicking the "Join Waitlist" button. [pull/5508](https://github.com/sourcegraph/cody/pull/5508) -- Chat: Support non-streaming requests. +- Chat: Support non-streaming requests. [pull/5565](https://github.com/sourcegraph/cody/pull/5565) ### Fixed diff --git a/vscode/src/completions/nodeClient.ts b/vscode/src/completions/nodeClient.ts index 418eb6528bd0..b8607ec2c5fa 100644 --- a/vscode/src/completions/nodeClient.ts +++ b/vscode/src/completions/nodeClient.ts @@ -10,6 +10,7 @@ import { type CompletionCallbacks, type CompletionParameters, type CompletionRequestParameters, + type CompletionResponse, NetworkError, RateLimitError, SourcegraphCompletionsClient, @@ -320,7 +321,7 @@ export class SourcegraphNodeCompletionsClient extends SourcegraphCompletionsClie getActiveTraceAndSpanId()?.traceId ) } - const json = await response.json() + const json = (await response.json()) as CompletionResponse if (typeof json?.completion === 'string') { cb.onChange(json.completion) cb.onComplete()