From e9e383455febb9236b97f68a691466a5f28626e5 Mon Sep 17 00:00:00 2001 From: Yaliang Wu Date: Tue, 17 Oct 2023 15:19:11 -0700 Subject: [PATCH] add bedrock titan embedding model blueprint (#1526) * add bedrock titan embedding model blueprint Signed-off-by: Yaliang Wu * fix create connector response Signed-off-by: Yaliang Wu * fix step number Signed-off-by: Yaliang Wu --------- Signed-off-by: Yaliang Wu --- ...k_connector_anthropic_claude_blueprint.md} | 5 +- ...ock_connector_titan_embedding_blueprint.md | 177 ++++++++++++++++++ 2 files changed, 180 insertions(+), 2 deletions(-) rename docs/remote_inference_blueprints/{bedrock_connector_blueprint.md => bedrock_connector_anthropic_claude_blueprint.md} (95%) create mode 100644 docs/remote_inference_blueprints/bedrock_connector_titan_embedding_blueprint.md diff --git a/docs/remote_inference_blueprints/bedrock_connector_blueprint.md b/docs/remote_inference_blueprints/bedrock_connector_anthropic_claude_blueprint.md similarity index 95% rename from docs/remote_inference_blueprints/bedrock_connector_blueprint.md rename to docs/remote_inference_blueprints/bedrock_connector_anthropic_claude_blueprint.md index ef5a705ccc..122afb5717 100644 --- a/docs/remote_inference_blueprints/bedrock_connector_blueprint.md +++ b/docs/remote_inference_blueprints/bedrock_connector_anthropic_claude_blueprint.md @@ -47,7 +47,8 @@ POST /_plugins/_ml/connectors/_create } ``` -If using the AWS Opensearch Service, you can provide an IAM role arn that allows access to the bedrock service: +If using the AWS Opensearch Service, you can provide an IAM role arn that allows access to the bedrock service. +Refer to this [AWS doc](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ml-amazon-connector.html) ```json POST /_plugins/_ml/connectors/_create @@ -123,7 +124,7 @@ GET /_plugins/_ml/tasks/SnDo-ooB0wiuGR4SfMNS POST /_plugins/_ml/models/S3Do-ooB0wiuGR4SfcNv/_deploy ``` -6. Test model inference +5. Test model inference ```json POST /_plugins/_ml/models/S3Do-ooB0wiuGR4SfcNv/_predict diff --git a/docs/remote_inference_blueprints/bedrock_connector_titan_embedding_blueprint.md b/docs/remote_inference_blueprints/bedrock_connector_titan_embedding_blueprint.md new file mode 100644 index 0000000000..33b7b3037b --- /dev/null +++ b/docs/remote_inference_blueprints/bedrock_connector_titan_embedding_blueprint.md @@ -0,0 +1,177 @@ +### Bedrock connector blueprint example + +1. Add connector endpoint to trusted URLs: + +```json +PUT /_cluster/settings +{ + "persistent": { + "plugins.ml_commons.trusted_connector_endpoints_regex": [ + "^https://bedrock-runtime\\..*[a-z0-9-]\\.amazonaws\\.com/.*$" + ] + } +} +``` + +2. Create connector for Amazon Bedrock: + +If you are using self-managed Opensearch, you should supply AWS credentials: + +```json +POST /_plugins/_ml/connectors/_create +{ + "name": "Amazon Bedrock Connector: embedding", + "description": "The connector to bedrock Titan embedding model", + "version": 1, + "protocol": "aws_sigv4", + "parameters": { + "region": "", + "service_name": "bedrock" + }, + "credential": { + "access_key": "", + "secret_key": "", + "session_token": "" + }, + "actions": [ + { + "action_type": "predict", + "method": "POST", + "url": "https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-embed-text-v1/invoke", + "headers": { + "content-type": "application/json", + "x-amz-content-sha256": "required" + }, + "request_body": "{ \"inputText\": \"${parameters.inputText}\" }", + "pre_process_function": "\n StringBuilder builder = new StringBuilder();\n builder.append(\"\\\"\");\n String first = params.text_docs[0];\n builder.append(first);\n builder.append(\"\\\"\");\n def parameters = \"{\" +\"\\\"inputText\\\":\" + builder + \"}\";\n return \"{\" +\"\\\"parameters\\\":\" + parameters + \"}\";", + "post_process_function": "\n def name = \"sentence_embedding\";\n def dataType = \"FLOAT32\";\n if (params.embedding == null || params.embedding.length == 0) {\n return params.message;\n }\n def shape = [params.embedding.length];\n def json = \"{\" +\n \"\\\"name\\\":\\\"\" + name + \"\\\",\" +\n \"\\\"data_type\\\":\\\"\" + dataType + \"\\\",\" +\n \"\\\"shape\\\":\" + shape + \",\" +\n \"\\\"data\\\":\" + params.embedding +\n \"}\";\n return json;\n " + } + ] +} +``` + +If using the AWS Opensearch Service, you can provide an IAM role arn that allows access to the bedrock service. +Refer to this [AWS doc](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ml-amazon-connector.html) + +```json +POST /_plugins/_ml/connectors/_create +{ + "name": "Amazon Bedrock Connector: embedding", + "description": "The connector to bedrock Titan embedding model", + "version": 1, + "protocol": "aws_sigv4", + "parameters": { + "region": "", + "service_name": "bedrock" + }, + "credential": { + "roleArn": "" + }, + "actions": [ + { + "action_type": "predict", + "method": "POST", + "url": "https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-embed-text-v1/invoke", + "headers": { + "content-type": "application/json", + "x-amz-content-sha256": "required" + }, + "request_body": "{ \"inputText\": \"${parameters.inputText}\" }", + "pre_process_function": "\n StringBuilder builder = new StringBuilder();\n builder.append(\"\\\"\");\n String first = params.text_docs[0];\n builder.append(first);\n builder.append(\"\\\"\");\n def parameters = \"{\" +\"\\\"inputText\\\":\" + builder + \"}\";\n return \"{\" +\"\\\"parameters\\\":\" + parameters + \"}\";", + "post_process_function": "\n def name = \"sentence_embedding\";\n def dataType = \"FLOAT32\";\n if (params.embedding == null || params.embedding.length == 0) {\n return params.message;\n }\n def shape = [params.embedding.length];\n def json = \"{\" +\n \"\\\"name\\\":\\\"\" + name + \"\\\",\" +\n \"\\\"data_type\\\":\\\"\" + dataType + \"\\\",\" +\n \"\\\"shape\\\":\" + shape + \",\" +\n \"\\\"data\\\":\" + params.embedding +\n \"}\";\n return json;\n " + } + ] +} +``` + +Response: +```json +{ + "connector_id": "nzh9PIsBnGXNcxYpPEcv" +} +``` + +3. Create model group: + +```json +POST /_plugins/_ml/model_groups/_register +{ + "name": "remote_model_group", + "description": "This is an example description" +} +``` + +Response: +```json +{ + "model_group_id": "rqR9PIsBQRofe4CScErR", + "status": "CREATED" +} +``` + +4. Register model to model group & deploy model: + +```json +POST /_plugins/_ml/models/_register +{ + "name": "Bedrock embedding model", + "function_name": "remote", + "model_group_id": "rqR9PIsBQRofe4CScErR", + "description": "test model", + "connector_id": "nzh9PIsBnGXNcxYpPEcv" +} +``` + +Response: +```json +{ + "task_id": "r6R9PIsBQRofe4CSlUoG", + "status": "CREATED" +} +``` +Get model id from task +```json +GET /_plugins/_ml/tasks/r6R9PIsBQRofe4CSlUoG +``` +Deploy model, in this demo the model id is `sKR9PIsBQRofe4CSlUov` +```json +POST /_plugins/_ml/models/sKR9PIsBQRofe4CSlUov/_deploy +``` + +5. Test model inference + +```json +POST /_plugins/_ml/models/sKR9PIsBQRofe4CSlUov/_predict +{ + "parameters": { + "inputs": "What is the meaning of life?" + } +} +``` + +Response: +```json +{ + "inference_results": [ + { + "output": [ + { + "name": "sentence_embedding", + "data_type": "FLOAT32", + "shape": [ + 1536 + ], + "data": [ + 0.5390625, + -0.46679688, + -0.125, + ... + ] + } + ], + "status_code": 200 + } + ] +} +``` +