From 8c4c724e8aad9ba6fe56b81341d78d116b7133ad Mon Sep 17 00:00:00 2001 From: Diana <75819066+cloudjumpercat@users.noreply.github.com> Date: Tue, 3 Sep 2024 15:06:48 -0500 Subject: [PATCH 01/12] Add rough draft of gemini/bedrock tutorials, add ai providers to tables and such Signed-off-by: Diana <75819066+cloudjumpercat@users.noreply.github.com> --- app/_data/docs_nav_gateway_3.8.x.yml | 10 +++++ .../tables/support/gateway/third-party.yml | 4 ++ .../tables/support/gateway/versions/38.yml | 2 + .../bedrock.md | 43 +++++++++++++++++++ .../gemini.md | 39 +++++++++++++++++ app/_hub/kong-inc/ai-proxy/overview/_index.md | 15 +++++++ 6 files changed, 113 insertions(+) create mode 100644 app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/bedrock.md create mode 100644 app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini.md diff --git a/app/_data/docs_nav_gateway_3.8.x.yml b/app/_data/docs_nav_gateway_3.8.x.yml index aedfb24f795d..795db8b92d2c 100644 --- a/app/_data/docs_nav_gateway_3.8.x.yml +++ b/app/_data/docs_nav_gateway_3.8.x.yml @@ -392,6 +392,16 @@ items: url: /hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/llama2/ generate: false absolute_url: true + - text: AI Platform Integration Guides + items: + - text: Amazon Bedrock + url: /hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/bedrock/ + generate: false + absolute_url: true + - text: Gemini + url: /hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini/ + generate: false + absolute_url: true - text: AI Gateway Analytics url: /ai-gateway/ai-analytics/ generate: false diff --git a/app/_data/tables/support/gateway/third-party.yml b/app/_data/tables/support/gateway/third-party.yml index f7e1b4569bca..b6c5e04ced39 100644 --- a/app/_data/tables/support/gateway/third-party.yml +++ b/app/_data/tables/support/gateway/third-party.yml @@ -168,3 +168,7 @@ _third_party: - Raw - OLLAMA - OpenAI + bedrock: &bedrock + name: Amazon Bedrock + gemini: &gemini + name: Gemini diff --git a/app/_data/tables/support/gateway/versions/38.yml b/app/_data/tables/support/gateway/versions/38.yml index 82110a649e03..0c09b1440c75 100644 --- a/app/_data/tables/support/gateway/versions/38.yml +++ b/app/_data/tables/support/gateway/versions/38.yml @@ -105,3 +105,5 @@ third-party: - *anthropic - *mistral - *llama2 + - *bedrock + - *gemini diff --git a/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/bedrock.md b/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/bedrock.md new file mode 100644 index 000000000000..76fe4fb36e62 --- /dev/null +++ b/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/bedrock.md @@ -0,0 +1,43 @@ +--- +nav_title: Amazon Bedrock +title: Set up AI Proxy with Amazon Bedrock +--- + +## Prerequisites + +* [Kong Gateway is installed and running](/gateway/latest/get-started/) + +## Configure Amazon Bedrock + +1. From the [AWS Management Console](https://aws.amazon.com/console/), navigate to Amazon Bedrock. +1. Create an [Amazon Bedrock Model](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html). You will need to select a model type, configure model parameters, and deploy it. +1. Configure any necessary settings such as model parameters, input/output formats, and resource requirements. The configuration details can be found in the [Amazon Bedrock API Reference](https://docs.aws.amazon.com/bedrock/latest/APIReference/). +1. Navigate to the IAM (Identity and Access Management) section in the AWS Console to create or manage API keys and permissions. You will need to save your API key and endpoint URL. +1. Ensure that the IAM roles and policies associated with your Bedrock models have the correct permissions to allow access and interaction. For more information, see [Amazon's IAM User Guide](https://docs.aws.amazon.com/IAM/latest/UserGuide/). + +## Configure the AI Proxy plugin + +1. Create a service in {{site.base_gateway}} that will represent the Bedrock API: +```sh +curl -i -X POST http://localhost:8001/services \ + --data "name=bedrock-service" \ + --data "url=https://bedrock-api-url" +``` +1. Create a route that maps to the service you defined: +```sh +curl -i -X POST http://localhost:8001/routes \ + --data "paths[]=/bedrock" \ + --data "service.id=$(curl -s http://localhost:8001/services/bedrock-service | jq -r '.id')" +``` +1. Use the Kong Admin API to configure the AI Proxy Plugin to route requests to Amazon Bedrock: +```sh +curl -i -X POST http://localhost:8001/services/my-service/plugins \ + --data "name=ai-proxy" \ + --data "config.model=bedrock" \ + --data "config.api_key=YOUR_BEDROCK_API_KEY" \ + --data "config.api_url=https://bedrock-api-url" +``` +Replace the following placeholders: + +* `YOUR_BEDROCK_API_KEY` with your actual Bedrock API key. +* `https://bedrock-api-url` with the endpoint URL for Amazon Bedrock’s API. \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini.md b/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini.md new file mode 100644 index 000000000000..0310fb22325f --- /dev/null +++ b/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini.md @@ -0,0 +1,39 @@ +--- +nav_title: Gemini +title: Set up AI Proxy with Gemini +--- + +## Prerequisites +* [Kong Gateway is installed and running](/gateway/latest/get-started/) + +## Step 1: Configure Google Gemini Models +1. Set up or select the Google Gemini model you want to use. You must configure the [model parameters](https://cloud.google.com/vertex-ai/generative-ai/docs/samples/generativeaionvertexai-gemini-pro-config-example) and ensure it's ready for deployment. +1. [Create or retrieve an API key](https://ai.google.dev/gemini-api/docs/api-key) on the Google Cloud API Credentials Page to access Google’s AI services. +1. Auth? + +## Configure the AI Proxy plugin + +1. Create a service in {{site.base_gateway}} that will represent the Google Gemini API: +```sh +curl -i -X POST http://localhost:8001/services \ + --data "name=gemini-service" \ + --data "url=https://language.googleapis.com" +``` +1. Create a route that maps to the service you defined: +```sh +curl -i -X POST http://localhost:8001/routes \ + --data "paths[]=/gemini" \ + --data "service.id=$(curl -s http://localhost:8001/services/gemini-service | jq -r '.id')" +``` +1. Use the Kong Admin API to configure the AI Proxy Plugin to route requests to Google Gemini: +```sh +curl -i -X POST http://localhost:8001/services/my-service/plugins \ + --data "name=ai-proxy" \ + --data "config.model=gemini" \ + --data "config.api_key=YOUR_API_KEY" \ + --data "config.api_url=https://language.googleapis.com/v1/documents:analyzeSentiment" +``` +Replace the following placeholders: + +* `YOUR_API_KEY` with your actual Google API key. +* `https://language.googleapis.com/v1/documents:analyzeSentiment` with the endpoint URL for the Google Gemini API. \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/overview/_index.md b/app/_hub/kong-inc/ai-proxy/overview/_index.md index 1e0f8fc55a93..a471777a02fa 100644 --- a/app/_hub/kong-inc/ai-proxy/overview/_index.md +++ b/app/_hub/kong-inc/ai-proxy/overview/_index.md @@ -29,6 +29,10 @@ The following table describes which providers and requests the AI Proxy plugin s | Llama2 (raw, OLLAMA, and OpenAI formats) | ✅ | ✅ | ✅ | | Llama3 (OLLAMA and OpenAI formats) | ✅ | ✅ | ✅ | {% endif_version %} +{% if_version gte:3.8.x %} +| Amazon Bedrock | ✅ | ✅ | ✅ | +| Gemini | ✅ | ✅ | ✅ | +{% endif_version %} ## How it works @@ -77,6 +81,13 @@ The plugin's [`config.route_type`](/hub/kong-inc/ai-proxy/configuration/#config- | Mistral | User-defined | `llm/v1/chat` | User-defined | | Mistral | User-defined | `llm/v1/completions` | User-defined | +{% if_version gte:3.8.x %} +| Amazon Bedrock | Use the LLM `chat` upstream path | `llm/v1/chat` | [Use the model name for the specific LLM provider](https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html) | +| Amazon Bedrock | Use the LLM `completions` upstream path | `llm/v1/completions` | [Use the model name for the specific LLM provider](https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html) | +| Gemini | Use the LLM `chat` upstream path | `llm/v1/chat` | `gemini-1.5-flash` or `gemini-1.5-pro` | +| Gemini | Use the LLM `completions` upstream path | `llm/v1/completions` | `gemini-1.5-flash` or `gemini-1.5-pro` | +{% endif_version %} + The following upstream URL patterns are used: | Provider | URL | @@ -88,6 +99,10 @@ The following upstream URL patterns are used: | Llama2 | As defined in `config.model.options.upstream_url` | | Mistral | As defined in `config.model.options.upstream_url` | +{% if_version gte:3.8.x %} +| Amazon Bedrock | Use the specific LLM upstream URL | +| Gemini | Use the specific LLM upstream URL | +{% endif_version %} {:.important} > While only the **Llama2** and **Mistral** models are classed as self-hosted, the target URL can be overridden for any of the supported providers. From 2655f625a2caaea1c47de8aa2eebf5782c8eb1e9 Mon Sep 17 00:00:00 2001 From: Diana <75819066+cloudjumpercat@users.noreply.github.com> Date: Wed, 4 Sep 2024 11:22:08 -0500 Subject: [PATCH 02/12] Make tutorial examples better Signed-off-by: Diana <75819066+cloudjumpercat@users.noreply.github.com> --- .../bedrock.md | 21 ++++++++--------- .../gemini.md | 23 ++++++++++--------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/bedrock.md b/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/bedrock.md index 76fe4fb36e62..1271d9c67ac6 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/bedrock.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/bedrock.md @@ -5,15 +5,15 @@ title: Set up AI Proxy with Amazon Bedrock ## Prerequisites -* [Kong Gateway is installed and running](/gateway/latest/get-started/) +* [{{site.base_gateway}} is installed and running](/gateway/latest/get-started/) ## Configure Amazon Bedrock 1. From the [AWS Management Console](https://aws.amazon.com/console/), navigate to Amazon Bedrock. -1. Create an [Amazon Bedrock Model](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html). You will need to select a model type, configure model parameters, and deploy it. -1. Configure any necessary settings such as model parameters, input/output formats, and resource requirements. The configuration details can be found in the [Amazon Bedrock API Reference](https://docs.aws.amazon.com/bedrock/latest/APIReference/). +1. Create an [Amazon Bedrock Model](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html). You will need to select a model type, configure model parameters, and deploy it. +1. Configure any necessary settings such as model parameters, input/output formats, and resource requirements. The configuration details can be found in the [Amazon Bedrock API Reference](https://docs.aws.amazon.com/bedrock/latest/APIReference/). 1. Navigate to the IAM (Identity and Access Management) section in the AWS Console to create or manage API keys and permissions. You will need to save your API key and endpoint URL. -1. Ensure that the IAM roles and policies associated with your Bedrock models have the correct permissions to allow access and interaction. For more information, see [Amazon's IAM User Guide](https://docs.aws.amazon.com/IAM/latest/UserGuide/). +1. Ensure that the IAM roles and policies associated with your Bedrock models have the correct permissions to allow access and interaction. For more information, see [Amazon's IAM User Guide](https://docs.aws.amazon.com/IAM/latest/UserGuide/). ## Configure the AI Proxy plugin @@ -31,13 +31,12 @@ curl -i -X POST http://localhost:8001/routes \ ``` 1. Use the Kong Admin API to configure the AI Proxy Plugin to route requests to Amazon Bedrock: ```sh -curl -i -X POST http://localhost:8001/services/my-service/plugins \ +curl -i -X POST http://localhost:8001/services/bedrock-service/plugins \ --data "name=ai-proxy" \ - --data "config.model=bedrock" \ - --data "config.api_key=YOUR_BEDROCK_API_KEY" \ - --data "config.api_url=https://bedrock-api-url" + --data "config.model.provider=bedrock" \ + --data "config.model.options.bedrock.aws_region=us-west-2" \ + --data "config.auth.aws_access_key_id=YOUR_ACCESS_KEY_ID" \ + --data "config.auth.aws_secret_access_key=YOUR_SECRET_ACCESS_KEY" ``` -Replace the following placeholders: -* `YOUR_BEDROCK_API_KEY` with your actual Bedrock API key. -* `https://bedrock-api-url` with the endpoint URL for Amazon Bedrock’s API. \ No newline at end of file + \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini.md b/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini.md index 0310fb22325f..7160f56a7e1d 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini.md @@ -4,10 +4,10 @@ title: Set up AI Proxy with Gemini --- ## Prerequisites -* [Kong Gateway is installed and running](/gateway/latest/get-started/) +* [{{site.base_gateway}} is installed and running](/gateway/latest/get-started/) ## Step 1: Configure Google Gemini Models -1. Set up or select the Google Gemini model you want to use. You must configure the [model parameters](https://cloud.google.com/vertex-ai/generative-ai/docs/samples/generativeaionvertexai-gemini-pro-config-example) and ensure it's ready for deployment. +1. Set up or select the Google Gemini model you want to use. You must configure the [model parameters](https://cloud.google.com/vertex-ai/generative-ai/docs/samples/generativeaionvertexai-gemini-pro-config-example) and ensure it's ready for deployment. 1. [Create or retrieve an API key](https://ai.google.dev/gemini-api/docs/api-key) on the Google Cloud API Credentials Page to access Google’s AI services. 1. Auth? @@ -27,13 +27,14 @@ curl -i -X POST http://localhost:8001/routes \ ``` 1. Use the Kong Admin API to configure the AI Proxy Plugin to route requests to Google Gemini: ```sh -curl -i -X POST http://localhost:8001/services/my-service/plugins \ - --data "name=ai-proxy" \ - --data "config.model=gemini" \ - --data "config.api_key=YOUR_API_KEY" \ - --data "config.api_url=https://language.googleapis.com/v1/documents:analyzeSentiment" +curl -i -X POST http://localhost:8001/services/gemini-service/plugins \ + --data 'name=ai-proxy' \ + --data 'config.route_type=preserve' \ + --data 'config.auth.header_name=Authorization' \ + --data 'config.auth.header_value=Bearer ' \ + --data 'config.model.provider=gemini' \ + --data 'config.model.options.gemini.api_endpoint=YOUR_REGIONAL_API_ENDPOINT' \ + --data 'config.model.options.gemini.project_id=YOUR_PROJECT_ID' \ + --data 'config.model.options.gemini.location_id=YOUR_LOCATION_ID' ``` -Replace the following placeholders: - -* `YOUR_API_KEY` with your actual Google API key. -* `https://language.googleapis.com/v1/documents:analyzeSentiment` with the endpoint URL for the Google Gemini API. \ No newline at end of file + \ No newline at end of file From 6bb48f0ea119cec88e73658a3ca61e5dca50c178 Mon Sep 17 00:00:00 2001 From: Diana <75819066+cloudjumpercat@users.noreply.github.com> Date: Mon, 9 Sep 2024 14:29:23 -0500 Subject: [PATCH 03/12] Apply some tech review feedback Signed-off-by: Diana <75819066+cloudjumpercat@users.noreply.github.com> --- .../bedrock.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/bedrock.md b/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/bedrock.md index 1271d9c67ac6..f6e083320a60 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/bedrock.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/bedrock.md @@ -10,10 +10,10 @@ title: Set up AI Proxy with Amazon Bedrock ## Configure Amazon Bedrock 1. From the [AWS Management Console](https://aws.amazon.com/console/), navigate to Amazon Bedrock. -1. Create an [Amazon Bedrock Model](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html). You will need to select a model type, configure model parameters, and deploy it. +1. Create an [Amazon Bedrock Model](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html). You will need to select a model type, configure model parameters, and deploy it. For more information about the supported model types, see Amazon's [Model support by AWS Region](https://docs.aws.amazon.com/bedrock/latest/userguide/models-regions.html) documentation. 1. Configure any necessary settings such as model parameters, input/output formats, and resource requirements. The configuration details can be found in the [Amazon Bedrock API Reference](https://docs.aws.amazon.com/bedrock/latest/APIReference/). 1. Navigate to the IAM (Identity and Access Management) section in the AWS Console to create or manage API keys and permissions. You will need to save your API key and endpoint URL. -1. Ensure that the IAM roles and policies associated with your Bedrock models have the correct permissions to allow access and interaction. For more information, see [Amazon's IAM User Guide](https://docs.aws.amazon.com/IAM/latest/UserGuide/). +1. Ensure that the IAM roles and policies associated with your Bedrock models have the correct permissions to allow access and interaction. For more information, see [How Amazon Bedrock works with IAM](https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_service-with-iam.html) in Amazon's documentation. ## Configure the AI Proxy plugin From ae9a75e206a9f77c51f0304eaba9fdd3d22f4494 Mon Sep 17 00:00:00 2001 From: Diana <75819066+cloudjumpercat@users.noreply.github.com> Date: Wed, 11 Sep 2024 11:41:40 -0500 Subject: [PATCH 04/12] Fix Gemini instructions Signed-off-by: Diana <75819066+cloudjumpercat@users.noreply.github.com> --- .../gemini.md | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini.md b/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini.md index 7160f56a7e1d..83147f4393a3 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini.md @@ -5,11 +5,7 @@ title: Set up AI Proxy with Gemini ## Prerequisites * [{{site.base_gateway}} is installed and running](/gateway/latest/get-started/) - -## Step 1: Configure Google Gemini Models -1. Set up or select the Google Gemini model you want to use. You must configure the [model parameters](https://cloud.google.com/vertex-ai/generative-ai/docs/samples/generativeaionvertexai-gemini-pro-config-example) and ensure it's ready for deployment. -1. [Create or retrieve an API key](https://ai.google.dev/gemini-api/docs/api-key) on the Google Cloud API Credentials Page to access Google’s AI services. -1. Auth? +* [Create or retrieve an API key](https://ai.google.dev/gemini-api/docs/api-key) on the Google Cloud API Credentials Page to access Google’s AI services ## Configure the AI Proxy plugin @@ -17,7 +13,7 @@ title: Set up AI Proxy with Gemini ```sh curl -i -X POST http://localhost:8001/services \ --data "name=gemini-service" \ - --data "url=https://language.googleapis.com" + --data "url=https://generativelanguage.googleapis.com" ``` 1. Create a route that maps to the service you defined: ```sh @@ -28,13 +24,23 @@ curl -i -X POST http://localhost:8001/routes \ 1. Use the Kong Admin API to configure the AI Proxy Plugin to route requests to Google Gemini: ```sh curl -i -X POST http://localhost:8001/services/gemini-service/plugins \ - --data 'name=ai-proxy' \ - --data 'config.route_type=preserve' \ - --data 'config.auth.header_name=Authorization' \ - --data 'config.auth.header_value=Bearer ' \ - --data 'config.model.provider=gemini' \ - --data 'config.model.options.gemini.api_endpoint=YOUR_REGIONAL_API_ENDPOINT' \ - --data 'config.model.options.gemini.project_id=YOUR_PROJECT_ID' \ - --data 'config.model.options.gemini.location_id=YOUR_LOCATION_ID' +--data 'name=ai-proxy' \ +--data 'config.auth.param_name=key' \ +--data 'config.auth.param_value=' \ +--data 'config.auth.param_location=query' \ +--data 'config.route_type=llm/v1/chat' \ +--data 'config.model.provider=gemini' \ +--data 'config.model.name=gemini-1.5-flash' ``` - \ No newline at end of file + +Be sure to replace `GEMINI_API_TOKEN` with your API token. + +### Test the configuration + +Make an `llm/v1/chat` type request to test your new endpoint: + +```sh +curl -X POST http://localhost:8000/gemini \ + -H 'Content-Type: application/json' \ + --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' +``` \ No newline at end of file From 4e99bea34b09f2a16f96cf51ef7a8b978ac3b184 Mon Sep 17 00:00:00 2001 From: Diana <75819066+cloudjumpercat@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:13:33 -0500 Subject: [PATCH 05/12] Add Gemini/Bedrock to Proxy Advanced tables, add Gemini instructions to AI Proxy Advanced, remove draft Bedrock guide Signed-off-by: Diana <75819066+cloudjumpercat@users.noreply.github.com> --- .../gemini.md | 62 +++++++++++++++++++ .../ai-proxy-advanced/overview/_index.md | 10 ++- .../bedrock.md | 42 ------------- app/_hub/kong-inc/ai-proxy/overview/_index.md | 6 +- 4 files changed, 74 insertions(+), 46 deletions(-) create mode 100644 app/_hub/kong-inc/ai-proxy-advanced/how-to/ai-and-machine-learning-platform-integrations/gemini.md delete mode 100644 app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/bedrock.md diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/ai-and-machine-learning-platform-integrations/gemini.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/ai-and-machine-learning-platform-integrations/gemini.md new file mode 100644 index 000000000000..f1efe827930b --- /dev/null +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/ai-and-machine-learning-platform-integrations/gemini.md @@ -0,0 +1,62 @@ +--- +nav_title: Gemini +title: Set up AI Proxy Advanced with Gemini +--- + +## Prerequisites +* [{{site.base_gateway}} is installed and running](/gateway/latest/get-started/) +* [Create or retrieve an API key](https://ai.google.dev/gemini-api/docs/api-key) on the Google Cloud API Credentials Page to access Google’s AI services + +## Configure the AI Proxy Advanced plugin + +1. Create a service in {{site.base_gateway}} that will represent the Google Gemini API: +```sh +curl -i -X POST http://localhost:8001/services \ + --data "name=gemini-service" \ + --data "url=https://generativelanguage.googleapis.com" +``` +1. Create a route that maps to the service you defined: +```sh +curl -i -X POST http://localhost:8001/routes \ + --data "paths[]=/gemini" \ + --data "service.id=$(curl -s http://localhost:8001/services/gemini-service | jq -r '.id')" +``` +1. Use the Kong Admin API to configure the AI Proxy Advanced plugin to route requests to Google Gemini: +```sh +curl -i -X POST http://localhost:8001/services/gemini-service/plugins \ +--header "accept: application/json" \ +--header "Content-Type: application/json" \ +--data ' +{ + "name": "ai-proxy-advanced", + "config": { + "targets": [ + { + "route_type": "llm/v1/chat", + "auth": { + "param_name": "key", + "param_value": "", + "param_location": "query" + }, + "model": { + "provider": "gemini", + "name": "gemini-1.5-flash" + } + } + ] + } +} +' +``` + +Be sure to replace `GEMINI_API_TOKEN` with your API token. + +### Test the configuration + +Make an `llm/v1/chat` type request to test your new endpoint: + +```sh +curl -X POST http://localhost:8000/gemini \ + -H 'Content-Type: application/json' \ + --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' +``` \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy-advanced/overview/_index.md b/app/_hub/kong-inc/ai-proxy-advanced/overview/_index.md index 7b3cd1f8b1f0..370f9821b36b 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/overview/_index.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/overview/_index.md @@ -19,6 +19,8 @@ The following table describes which providers and requests the AI Proxy Advanced | Mistral (mistral.ai, OpenAI, raw, and OLLAMA formats) | ✅ | ✅ | ✅ | | Llama2 (raw, OLLAMA, and OpenAI formats) | ✅ | ✅ | ✅ | | Llama3 (OLLAMA and OpenAI formats) | ✅ | ✅ | ✅ | +| Amazon Bedrock | ✅ | ✅ | ✅ | +| Gemini | ✅ | ✅ | ✅ | ## How it works @@ -81,6 +83,11 @@ The plugin's [`config.route_type`](/hub/kong-inc/ai-proxy-advanced/configuration | Llama2 | User-defined | `llm/v1/completions` | User-defined | | Mistral | User-defined | `llm/v1/chat` | User-defined | | Mistral | User-defined | `llm/v1/completions` | User-defined | +| Amazon Bedrock | Use the LLM `chat` upstream path | `llm/v1/chat` | [Use the model name for the specific LLM provider](https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html) | +| Amazon Bedrock | Use the LLM `completions` upstream path | `llm/v1/completions` | [Use the model name for the specific LLM provider](https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html) | +| Gemini | `llm/v1/chat` | `llm/v1/chat` | `gemini-1.5-flash` or `gemini-1.5-pro` | +| Gemini | `llm/v1/completions` | `llm/v1/completions` | `gemini-1.5-flash` or `gemini-1.5-pro` | + The following upstream URL patterns are used: @@ -92,7 +99,8 @@ The following upstream URL patterns are used: | Anthropic | `https://api.anthropic.com:443/{route_type_path}` | | Llama2 | As defined in `config.model.options.upstream_url` | | Mistral | As defined in `config.model.options.upstream_url` | - +| Amazon Bedrock | Use the specific LLM upstream URL | +| Gemini | `https://generativelanguage.googleapis.com` | {:.important} > While only the **Llama2** and **Mistral** models are classed as self-hosted, the target URL can be overridden for any of the supported providers. diff --git a/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/bedrock.md b/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/bedrock.md deleted file mode 100644 index f6e083320a60..000000000000 --- a/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/bedrock.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -nav_title: Amazon Bedrock -title: Set up AI Proxy with Amazon Bedrock ---- - -## Prerequisites - -* [{{site.base_gateway}} is installed and running](/gateway/latest/get-started/) - -## Configure Amazon Bedrock - -1. From the [AWS Management Console](https://aws.amazon.com/console/), navigate to Amazon Bedrock. -1. Create an [Amazon Bedrock Model](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html). You will need to select a model type, configure model parameters, and deploy it. For more information about the supported model types, see Amazon's [Model support by AWS Region](https://docs.aws.amazon.com/bedrock/latest/userguide/models-regions.html) documentation. -1. Configure any necessary settings such as model parameters, input/output formats, and resource requirements. The configuration details can be found in the [Amazon Bedrock API Reference](https://docs.aws.amazon.com/bedrock/latest/APIReference/). -1. Navigate to the IAM (Identity and Access Management) section in the AWS Console to create or manage API keys and permissions. You will need to save your API key and endpoint URL. -1. Ensure that the IAM roles and policies associated with your Bedrock models have the correct permissions to allow access and interaction. For more information, see [How Amazon Bedrock works with IAM](https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_service-with-iam.html) in Amazon's documentation. - -## Configure the AI Proxy plugin - -1. Create a service in {{site.base_gateway}} that will represent the Bedrock API: -```sh -curl -i -X POST http://localhost:8001/services \ - --data "name=bedrock-service" \ - --data "url=https://bedrock-api-url" -``` -1. Create a route that maps to the service you defined: -```sh -curl -i -X POST http://localhost:8001/routes \ - --data "paths[]=/bedrock" \ - --data "service.id=$(curl -s http://localhost:8001/services/bedrock-service | jq -r '.id')" -``` -1. Use the Kong Admin API to configure the AI Proxy Plugin to route requests to Amazon Bedrock: -```sh -curl -i -X POST http://localhost:8001/services/bedrock-service/plugins \ - --data "name=ai-proxy" \ - --data "config.model.provider=bedrock" \ - --data "config.model.options.bedrock.aws_region=us-west-2" \ - --data "config.auth.aws_access_key_id=YOUR_ACCESS_KEY_ID" \ - --data "config.auth.aws_secret_access_key=YOUR_SECRET_ACCESS_KEY" -``` - - \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/overview/_index.md b/app/_hub/kong-inc/ai-proxy/overview/_index.md index a471777a02fa..f04ff61e5631 100644 --- a/app/_hub/kong-inc/ai-proxy/overview/_index.md +++ b/app/_hub/kong-inc/ai-proxy/overview/_index.md @@ -84,8 +84,8 @@ The plugin's [`config.route_type`](/hub/kong-inc/ai-proxy/configuration/#config- {% if_version gte:3.8.x %} | Amazon Bedrock | Use the LLM `chat` upstream path | `llm/v1/chat` | [Use the model name for the specific LLM provider](https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html) | | Amazon Bedrock | Use the LLM `completions` upstream path | `llm/v1/completions` | [Use the model name for the specific LLM provider](https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html) | -| Gemini | Use the LLM `chat` upstream path | `llm/v1/chat` | `gemini-1.5-flash` or `gemini-1.5-pro` | -| Gemini | Use the LLM `completions` upstream path | `llm/v1/completions` | `gemini-1.5-flash` or `gemini-1.5-pro` | +| Gemini | `llm/v1/chat` | `llm/v1/chat` | `gemini-1.5-flash` or `gemini-1.5-pro` | +| Gemini | `llm/v1/completions` | `llm/v1/completions` | `gemini-1.5-flash` or `gemini-1.5-pro` | {% endif_version %} The following upstream URL patterns are used: @@ -101,7 +101,7 @@ The following upstream URL patterns are used: {% if_version gte:3.8.x %} | Amazon Bedrock | Use the specific LLM upstream URL | -| Gemini | Use the specific LLM upstream URL | +| Gemini | https://generativelanguage.googleapis.com | {% endif_version %} {:.important} From 50db0197955613c53cdd1438c9bbd65f381a0829 Mon Sep 17 00:00:00 2001 From: Diana <75819066+cloudjumpercat@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:25:37 -0500 Subject: [PATCH 06/12] Fix Bedrock upstream URL format in tables --- app/_hub/kong-inc/ai-proxy-advanced/overview/_index.md | 2 +- app/_hub/kong-inc/ai-proxy/overview/_index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/_hub/kong-inc/ai-proxy-advanced/overview/_index.md b/app/_hub/kong-inc/ai-proxy-advanced/overview/_index.md index 370f9821b36b..340d9898e72b 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/overview/_index.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/overview/_index.md @@ -99,7 +99,7 @@ The following upstream URL patterns are used: | Anthropic | `https://api.anthropic.com:443/{route_type_path}` | | Llama2 | As defined in `config.model.options.upstream_url` | | Mistral | As defined in `config.model.options.upstream_url` | -| Amazon Bedrock | Use the specific LLM upstream URL | +| Amazon Bedrock | `https://bedrock-runtime.{region}.amazonaws.com` | | Gemini | `https://generativelanguage.googleapis.com` | {:.important} diff --git a/app/_hub/kong-inc/ai-proxy/overview/_index.md b/app/_hub/kong-inc/ai-proxy/overview/_index.md index f04ff61e5631..7912cad93ec8 100644 --- a/app/_hub/kong-inc/ai-proxy/overview/_index.md +++ b/app/_hub/kong-inc/ai-proxy/overview/_index.md @@ -100,7 +100,7 @@ The following upstream URL patterns are used: | Mistral | As defined in `config.model.options.upstream_url` | {% if_version gte:3.8.x %} -| Amazon Bedrock | Use the specific LLM upstream URL | +| Amazon Bedrock | `https://bedrock-runtime.{region}.amazonaws.com` | | Gemini | https://generativelanguage.googleapis.com | {% endif_version %} From 233ccb5e792bcb7976044500ffe9dfd8d9ca4a13 Mon Sep 17 00:00:00 2001 From: Diana <75819066+cloudjumpercat@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:30:16 -0500 Subject: [PATCH 07/12] Remove Bedrock guide link from Gateway nav, fix one of the gemini upstream URL formatting so it's not a link --- app/_data/docs_nav_gateway_3.8.x.yml | 4 ---- app/_hub/kong-inc/ai-proxy/overview/_index.md | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/app/_data/docs_nav_gateway_3.8.x.yml b/app/_data/docs_nav_gateway_3.8.x.yml index 795db8b92d2c..a71773119f7c 100644 --- a/app/_data/docs_nav_gateway_3.8.x.yml +++ b/app/_data/docs_nav_gateway_3.8.x.yml @@ -394,10 +394,6 @@ items: absolute_url: true - text: AI Platform Integration Guides items: - - text: Amazon Bedrock - url: /hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/bedrock/ - generate: false - absolute_url: true - text: Gemini url: /hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini/ generate: false diff --git a/app/_hub/kong-inc/ai-proxy/overview/_index.md b/app/_hub/kong-inc/ai-proxy/overview/_index.md index 7912cad93ec8..f4dee3790453 100644 --- a/app/_hub/kong-inc/ai-proxy/overview/_index.md +++ b/app/_hub/kong-inc/ai-proxy/overview/_index.md @@ -101,7 +101,7 @@ The following upstream URL patterns are used: {% if_version gte:3.8.x %} | Amazon Bedrock | `https://bedrock-runtime.{region}.amazonaws.com` | -| Gemini | https://generativelanguage.googleapis.com | +| Gemini | `https://generativelanguage.googleapis.com` | {% endif_version %} {:.important} From 78fd7d5d197a5bb8c8df9447f7c80f8e2d94ecea Mon Sep 17 00:00:00 2001 From: Diana <75819066+cloudjumpercat@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:54:28 -0500 Subject: [PATCH 08/12] Add intro paragraphs to gemini instructions Signed-off-by: Diana <75819066+cloudjumpercat@users.noreply.github.com> --- .../gemini.md | 10 ++++++++++ .../gemini.md | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/ai-and-machine-learning-platform-integrations/gemini.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/ai-and-machine-learning-platform-integrations/gemini.md index f1efe827930b..a23954a920d0 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/how-to/ai-and-machine-learning-platform-integrations/gemini.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/ai-and-machine-learning-platform-integrations/gemini.md @@ -3,6 +3,16 @@ nav_title: Gemini title: Set up AI Proxy Advanced with Gemini --- +This guide walks you through setting up the AI Proxy Advanced plugin with [Gemini](https://gemini.google.com/). + +For all providers, the Kong AI Proxy Advanced plugin attaches to route entities. + +It can be installed into one route per operation, for example: + +* OpenAI `chat` route +* Cohere `chat` route +* Cohere `completions` route + ## Prerequisites * [{{site.base_gateway}} is installed and running](/gateway/latest/get-started/) * [Create or retrieve an API key](https://ai.google.dev/gemini-api/docs/api-key) on the Google Cloud API Credentials Page to access Google’s AI services diff --git a/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini.md b/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini.md index 83147f4393a3..d029dd07593a 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini.md @@ -3,6 +3,16 @@ nav_title: Gemini title: Set up AI Proxy with Gemini --- +This guide walks you through setting up the AI Proxy plugin with [Gemini](https://gemini.google.com/). + +For all providers, the Kong AI Proxy plugin attaches to route entities. + +It can be installed into one route per operation, for example: + +* OpenAI `chat` route +* Cohere `chat` route +* Cohere `completions` route + ## Prerequisites * [{{site.base_gateway}} is installed and running](/gateway/latest/get-started/) * [Create or retrieve an API key](https://ai.google.dev/gemini-api/docs/api-key) on the Google Cloud API Credentials Page to access Google’s AI services From 1f823a9784f3dcf73402e46e7731bf3770d3eebe Mon Sep 17 00:00:00 2001 From: Diana <75819066+cloudjumpercat@users.noreply.github.com> Date: Wed, 11 Sep 2024 13:19:07 -0500 Subject: [PATCH 09/12] Change the folder name to avoid capitalizaiton issues with AI Signed-off-by: Diana <75819066+cloudjumpercat@users.noreply.github.com> --- .../gemini.md | 0 .../gemini.md | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename app/_hub/kong-inc/ai-proxy-advanced/how-to/{ai-and-machine-learning-platform-integrations => machine-learning-platform-integrations}/gemini.md (100%) rename app/_hub/kong-inc/ai-proxy/how-to/{ai-and-machine-learning-platform-integration-guides => machine-learning-platform-integration-guides}/gemini.md (100%) diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/ai-and-machine-learning-platform-integrations/gemini.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/machine-learning-platform-integrations/gemini.md similarity index 100% rename from app/_hub/kong-inc/ai-proxy-advanced/how-to/ai-and-machine-learning-platform-integrations/gemini.md rename to app/_hub/kong-inc/ai-proxy-advanced/how-to/machine-learning-platform-integrations/gemini.md diff --git a/app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini.md b/app/_hub/kong-inc/ai-proxy/how-to/machine-learning-platform-integration-guides/gemini.md similarity index 100% rename from app/_hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini.md rename to app/_hub/kong-inc/ai-proxy/how-to/machine-learning-platform-integration-guides/gemini.md From 6a342474962044f918fcb634624efe48e9c937ff Mon Sep 17 00:00:00 2001 From: Diana <75819066+cloudjumpercat@users.noreply.github.com> Date: Wed, 11 Sep 2024 13:20:32 -0500 Subject: [PATCH 10/12] Remove list in intro about one route per operation Signed-off-by: Diana <75819066+cloudjumpercat@users.noreply.github.com> --- .../how-to/machine-learning-platform-integrations/gemini.md | 6 ------ .../machine-learning-platform-integration-guides/gemini.md | 6 ------ 2 files changed, 12 deletions(-) diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/machine-learning-platform-integrations/gemini.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/machine-learning-platform-integrations/gemini.md index a23954a920d0..a6e9e3ac413c 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/how-to/machine-learning-platform-integrations/gemini.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/machine-learning-platform-integrations/gemini.md @@ -7,12 +7,6 @@ This guide walks you through setting up the AI Proxy Advanced plugin with [Gemin For all providers, the Kong AI Proxy Advanced plugin attaches to route entities. -It can be installed into one route per operation, for example: - -* OpenAI `chat` route -* Cohere `chat` route -* Cohere `completions` route - ## Prerequisites * [{{site.base_gateway}} is installed and running](/gateway/latest/get-started/) * [Create or retrieve an API key](https://ai.google.dev/gemini-api/docs/api-key) on the Google Cloud API Credentials Page to access Google’s AI services diff --git a/app/_hub/kong-inc/ai-proxy/how-to/machine-learning-platform-integration-guides/gemini.md b/app/_hub/kong-inc/ai-proxy/how-to/machine-learning-platform-integration-guides/gemini.md index d029dd07593a..3830a3cd2dff 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/machine-learning-platform-integration-guides/gemini.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/machine-learning-platform-integration-guides/gemini.md @@ -7,12 +7,6 @@ This guide walks you through setting up the AI Proxy plugin with [Gemini](https: For all providers, the Kong AI Proxy plugin attaches to route entities. -It can be installed into one route per operation, for example: - -* OpenAI `chat` route -* Cohere `chat` route -* Cohere `completions` route - ## Prerequisites * [{{site.base_gateway}} is installed and running](/gateway/latest/get-started/) * [Create or retrieve an API key](https://ai.google.dev/gemini-api/docs/api-key) on the Google Cloud API Credentials Page to access Google’s AI services From 89a2a1ff04b99af1eaf61485d404dd13cced6ec6 Mon Sep 17 00:00:00 2001 From: Diana <75819066+cloudjumpercat@users.noreply.github.com> Date: Wed, 11 Sep 2024 13:34:15 -0500 Subject: [PATCH 11/12] Fix broken link due to folder name change Co-authored-by: Angel --- app/_data/docs_nav_gateway_3.8.x.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/_data/docs_nav_gateway_3.8.x.yml b/app/_data/docs_nav_gateway_3.8.x.yml index a71773119f7c..23e083196d45 100644 --- a/app/_data/docs_nav_gateway_3.8.x.yml +++ b/app/_data/docs_nav_gateway_3.8.x.yml @@ -395,7 +395,7 @@ items: - text: AI Platform Integration Guides items: - text: Gemini - url: /hub/kong-inc/ai-proxy/how-to/ai-and-machine-learning-platform-integration-guides/gemini/ + url: /hub/kong-inc/ai-proxy/how-to/machine-learning-platform-integrations/gemini/ generate: false absolute_url: true - text: AI Gateway Analytics From 80df03b4d8dc84acc1db2a660ca435c4d77c1999 Mon Sep 17 00:00:00 2001 From: Diana <75819066+cloudjumpercat@users.noreply.github.com> Date: Wed, 11 Sep 2024 13:48:41 -0500 Subject: [PATCH 12/12] fix broken link --- app/_data/docs_nav_gateway_3.8.x.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/_data/docs_nav_gateway_3.8.x.yml b/app/_data/docs_nav_gateway_3.8.x.yml index 23e083196d45..9efc0e07fcfc 100644 --- a/app/_data/docs_nav_gateway_3.8.x.yml +++ b/app/_data/docs_nav_gateway_3.8.x.yml @@ -395,7 +395,7 @@ items: - text: AI Platform Integration Guides items: - text: Gemini - url: /hub/kong-inc/ai-proxy/how-to/machine-learning-platform-integrations/gemini/ + url: /hub/kong-inc/ai-proxy/how-to/machine-learning-platform-integration-guides/gemini/ generate: false absolute_url: true - text: AI Gateway Analytics