From b99ab7e5a94756d612735f8c5556c85f95ea0b5a Mon Sep 17 00:00:00 2001 From: sanjay920 Date: Wed, 18 Dec 2024 16:36:10 -0800 Subject: [PATCH 1/3] update vllm --- .../05-configuration/02-model-providers.md | 98 ++++++++++--------- .../model-providers/ModelProviderForm.tsx | 1 + .../components/model-providers/constants.ts | 12 +++ ui/admin/app/routes/_auth.model-providers.tsx | 1 + ui/admin/public/assets/vllm-logo.svg | 1 + 5 files changed, 69 insertions(+), 44 deletions(-) create mode 100644 ui/admin/public/assets/vllm-logo.svg diff --git a/docs/docs/05-configuration/02-model-providers.md b/docs/docs/05-configuration/02-model-providers.md index faa2d17de..9d4e46d36 100644 --- a/docs/docs/05-configuration/02-model-providers.md +++ b/docs/docs/05-configuration/02-model-providers.md @@ -4,42 +4,43 @@ Each packaged model provider has a slightly different configuration. Below is a ```json { - "id": "azure-openai-model-provider", - "created": "2024-12-02T08:33:34-05:00", - "revision": "1033", - "type": "modelprovider", - "name": "azure-openai-model-provider", - "toolReference": "github.com/otto8-ai/tools/azure-openai-model-provider", - "configured": false, - "requiredConfigurationParameters": [ - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_ENDPOINT", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_CLIENT_ID", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_CLIENT_SECRET", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_TENANT_ID", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_SUBSCRIPTION_ID", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_RESOURCE_GROUP" - ], - "missingConfigurationParameters": [ - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_ENDPOINT", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_CLIENT_ID", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_CLIENT_SECRET", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_TENANT_ID", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_SUBSCRIPTION_ID", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_RESOURCE_GROUP" - ] + "id": "azure-openai-model-provider", + "created": "2024-12-02T08:33:34-05:00", + "revision": "1033", + "type": "modelprovider", + "name": "azure-openai-model-provider", + "toolReference": "github.com/otto8-ai/tools/azure-openai-model-provider", + "configured": false, + "requiredConfigurationParameters": [ + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_ENDPOINT", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_CLIENT_ID", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_CLIENT_SECRET", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_TENANT_ID", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_SUBSCRIPTION_ID", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_RESOURCE_GROUP" + ], + "missingConfigurationParameters": [ + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_ENDPOINT", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_CLIENT_ID", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_CLIENT_SECRET", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_TENANT_ID", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_SUBSCRIPTION_ID", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_RESOURCE_GROUP" + ] } ``` To configure a model provider using the API, a `POST` request can be made to `/api/model-providers/azure-openai-model-provider/configure` with each required configuration parameter set in the body: + ```json { - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_ENDPOINT": "...", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_CLIENT_ID": "...", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_CLIENT_SECRET": "...", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_TENANT_ID": "...", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_SUBSCRIPTION_ID": "...", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_RESOURCE_GROUP": "..." + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_ENDPOINT": "...", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_CLIENT_ID": "...", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_CLIENT_SECRET": "...", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_TENANT_ID": "...", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_SUBSCRIPTION_ID": "...", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_RESOURCE_GROUP": "..." } ``` @@ -47,21 +48,21 @@ Once the model provider has been configured, then the API would return something ```json { - "id": "azure-openai-model-provider", - "created": "2024-12-02T08:33:34-05:00", - "revision": "1033", - "type": "modelprovider", - "name": "azure-openai-model-provider", - "toolReference": "github.com/otto8-ai/tools/azure-openai-model-provider", - "configured": true, - "requiredConfigurationParameters": [ - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_ENDPOINT", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_CLIENT_ID", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_CLIENT_SECRET", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_TENANT_ID", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_SUBSCRIPTION_ID", - "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_RESOURCE_GROUP" - ] + "id": "azure-openai-model-provider", + "created": "2024-12-02T08:33:34-05:00", + "revision": "1033", + "type": "modelprovider", + "name": "azure-openai-model-provider", + "toolReference": "github.com/otto8-ai/tools/azure-openai-model-provider", + "configured": true, + "requiredConfigurationParameters": [ + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_ENDPOINT", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_CLIENT_ID", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_CLIENT_SECRET", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_TENANT_ID", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_SUBSCRIPTION_ID", + "OTTO8_AZURE_OPENAI_MODEL_PROVIDER_RESOURCE_GROUP" + ] } ``` @@ -76,10 +77,12 @@ The OpenAI model provider is also special: you can start Otto8 with the `OPENAI_ ## Azure OpenAI The Azure OpenAI model provider requires the following configuration parameters: + - `OTTO8_AZURE_OPENAI_MODEL_PROVIDER_ENDPOINT`: The endpoint to use, found by clicking on the "Deployment" name from the "Deployments" page of the Azure OpenAI Studio. The provider endpoint must be in the format `https://.openai.azure.com` - if your Azure OpenAI resource doesn't have an endpoint that looks like this, you need to create one. - `OTTO8_AZURE_OPENAI_MODEL_PROVIDER_RESOURCE_GROUP`: The resource group name for the Azure OpenAI resource, found by clicking on the resource name in the top-right of the Azure OpenAI Studio. A service principal must be created with the (equivalent permissions of the) `Cognitive Services OpenAI User`. After this service principal is created, use the following parameters to configure the model provider in Otto8: + - `OTTO8_AZURE_OPENAI_MODEL_PROVIDER_CLIENT_ID`: The client ID for the app registration. - `OTTO8_AZURE_OPENAI_MODEL_PROVIDER_CLIENT_SECRET`: The client secret for the app registration. - `OTTO8_AZURE_OPENAI_MODEL_PROVIDER_TENANT_ID`: The tenant ID for the app registration. @@ -103,3 +106,10 @@ Voyage is Anthropic's recommended text-embedding provider. The Voyage model prov The Ollama model provider requires the configuration parameter `OTTO8_OLLAMA_MODEL_PROVIDER_HOST`. This host must point to a running instance of Ollama. For your reference, the default host and port for Ollama is `127.0.0.1:11434`. Otto8 doesn't set this by default. To set up and run an instance of Ollama, refer to the [Ollama GitHub readme](https://github.com/ollama/ollama/blob/main/README.md). + +## vLLM + +The vLLM model provider requires the following configuration parameters: + +- `OTTO8_VLLM_MODEL_PROVIDER_ENDPOINT`: The endpoint to use for your vLLM server (e.g., `your-vllm-server.com` or `https://your-vllm-server.com`) +- `OTTO8_VLLM_MODEL_PROVIDER_API_KEY`: The API key for authentication with your vLLM server diff --git a/ui/admin/app/components/model-providers/ModelProviderForm.tsx b/ui/admin/app/components/model-providers/ModelProviderForm.tsx index 9dbf283f8..674aea9b9 100644 --- a/ui/admin/app/components/model-providers/ModelProviderForm.tsx +++ b/ui/admin/app/components/model-providers/ModelProviderForm.tsx @@ -61,6 +61,7 @@ const translateUserFriendlyLabel = (label: string) => { "OBOT_OLLAMA_MODEL_PROVIDER", "OBOT_VOYAGE_MODEL_PROVIDER", "OBOT_GROQ_MODEL_PROVIDER", + "OBOT_VLLM_MODEL_PROVIDER", ]; return fieldsToStrip diff --git a/ui/admin/app/components/model-providers/constants.ts b/ui/admin/app/components/model-providers/constants.ts index 4fa68f4de..97b5bd6fa 100644 --- a/ui/admin/app/components/model-providers/constants.ts +++ b/ui/admin/app/components/model-providers/constants.ts @@ -1,6 +1,7 @@ export const CommonModelProviderIds = { OLLAMA: "ollama-model-provider", GROQ: "groq-model-provider", + VLLM: "vllm-model-provider", VOYAGE: "voyage-model-provider", ANTHROPIC: "anthropic-model-provider", OPENAI: "openai-model-provider", @@ -11,6 +12,7 @@ export const ModelProviderLinks = { [CommonModelProviderIds.VOYAGE]: "https://www.voyageai.com/", [CommonModelProviderIds.OLLAMA]: "https://ollama.com/", [CommonModelProviderIds.GROQ]: "https://groq.com/", + [CommonModelProviderIds.VLLM]: "https://docs.vllm.ai/", [CommonModelProviderIds.AZURE_OPENAI]: "https://azure.microsoft.com/en-us/explore/", [CommonModelProviderIds.ANTHROPIC]: "https://www.anthropic.com", @@ -39,6 +41,12 @@ export const ModelProviderRequiredTooltips: { "Api Key": "Groq API Key. Can be created and fetched from https://console.groq.com/keys", }, + [CommonModelProviderIds.VLLM]: { + Endpoint: + "Endpoint for the vLLM OpenAI service (eg. http://localhost:8000)", + "Api Key": + "VLLM API Key set when starting the vLLM server", + }, [CommonModelProviderIds.AZURE_OPENAI]: { Endpoint: "Endpoint for the Azure OpenAI service (eg. https://..api.cognitive.microsoft.com/)", @@ -79,4 +87,8 @@ export const ModelProviderSensitiveFields: Record = // Groq OBOT_GROQ_MODEL_PROVIDER_API_KEY: true, + + // VLLM + OBOT_VLLM_MODEL_PROVIDER_ENDPOINT: true, + OBOT_VLLM_MODEL_PROVIDER_API_KEY: true, }; diff --git a/ui/admin/app/routes/_auth.model-providers.tsx b/ui/admin/app/routes/_auth.model-providers.tsx index aac1313c4..77f189a3a 100644 --- a/ui/admin/app/routes/_auth.model-providers.tsx +++ b/ui/admin/app/routes/_auth.model-providers.tsx @@ -36,6 +36,7 @@ const sortModelProviders = (modelProviders: ModelProvider[]) => { CommonModelProviderIds.OLLAMA, CommonModelProviderIds.VOYAGE, CommonModelProviderIds.GROQ, + CommonModelProviderIds.VLLM, ]; const aIndex = preferredOrder.indexOf(a.id); const bIndex = preferredOrder.indexOf(b.id); diff --git a/ui/admin/public/assets/vllm-logo.svg b/ui/admin/public/assets/vllm-logo.svg new file mode 100644 index 000000000..c7e03485e --- /dev/null +++ b/ui/admin/public/assets/vllm-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file From 0452a69dcf66e795070cf8c06fd63523b4907b28 Mon Sep 17 00:00:00 2001 From: sanjay920 Date: Wed, 18 Dec 2024 16:47:54 -0800 Subject: [PATCH 2/3] update constants.ts --- ui/admin/app/components/model-providers/constants.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ui/admin/app/components/model-providers/constants.ts b/ui/admin/app/components/model-providers/constants.ts index 97b5bd6fa..864753449 100644 --- a/ui/admin/app/components/model-providers/constants.ts +++ b/ui/admin/app/components/model-providers/constants.ts @@ -44,8 +44,7 @@ export const ModelProviderRequiredTooltips: { [CommonModelProviderIds.VLLM]: { Endpoint: "Endpoint for the vLLM OpenAI service (eg. http://localhost:8000)", - "Api Key": - "VLLM API Key set when starting the vLLM server", + "Api Key": "VLLM API Key set when starting the vLLM server", }, [CommonModelProviderIds.AZURE_OPENAI]: { Endpoint: From 4b90d8ee3c5648dfb49475fb8502ead082195e16 Mon Sep 17 00:00:00 2001 From: sanjay920 Date: Thu, 19 Dec 2024 10:13:23 -0800 Subject: [PATCH 3/3] Update ui/admin/app/components/model-providers/constants.ts Co-authored-by: Donnie Adams --- ui/admin/app/components/model-providers/constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/admin/app/components/model-providers/constants.ts b/ui/admin/app/components/model-providers/constants.ts index 864753449..b04f06766 100644 --- a/ui/admin/app/components/model-providers/constants.ts +++ b/ui/admin/app/components/model-providers/constants.ts @@ -88,6 +88,6 @@ export const ModelProviderSensitiveFields: Record = OBOT_GROQ_MODEL_PROVIDER_API_KEY: true, // VLLM - OBOT_VLLM_MODEL_PROVIDER_ENDPOINT: true, + OBOT_VLLM_MODEL_PROVIDER_ENDPOINT: false, OBOT_VLLM_MODEL_PROVIDER_API_KEY: true, };