From 866c44325abde26499f21ae1ac567b059c70535e Mon Sep 17 00:00:00 2001 From: Robin Xiang Date: Mon, 18 Mar 2024 17:47:32 +0800 Subject: [PATCH 1/5] fix(AI-Proxy): add a validator for checking if the ai provider supports log statistics, and fix a bug that Anthropic don't provide log statistics in `chat` route_type. FTI-5769 --- kong/llm/drivers/anthropic.lua | 6 ++++++ kong/llm/init.lua | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/kong/llm/drivers/anthropic.lua b/kong/llm/drivers/anthropic.lua index 9a88415364e6..18c3f2bce5b5 100644 --- a/kong/llm/drivers/anthropic.lua +++ b/kong/llm/drivers/anthropic.lua @@ -148,6 +148,12 @@ local transformers_from = { finish_reason = response_table.stop_reason, }, }, + usage = { + prompt_tokens = response_table.usage.input_tokens or 0, + completion_tokens = response_table.usage.output_tokens or 0, + total_tokens = response_table.usage.input_tokens and response_table.usage.output_tokens and + response_table.usage.input_tokens + response_table.usage.output_tokens or 0, + }, model = response_table.model, object = "chat.content", } diff --git a/kong/llm/init.lua b/kong/llm/init.lua index 489af760ccea..da0364d2ff8f 100644 --- a/kong/llm/init.lua +++ b/kong/llm/init.lua @@ -139,6 +139,11 @@ local logging_schema = { } } +local UNSUPPORTED_LOG_STATISTICS = { + ["llm/v1/completions"] = { "anthropic", "azure", "cohere", "llama2", }, + ["llm/v1/chat"] = { "cohere", "llama2", } +} + _M.config_schema = { type = "record", fields = { @@ -201,6 +206,21 @@ _M.config_schema = { if_match = { one_of = { "mistral", "llama2" } }, then_at_least_one_of = { "model.options.upstream_url" }, then_err = "must set %s for self-hosted providers/models" }}, + + { + custom_entity_check = { + field_sources = { "route_type", "model", "logging" }, + fn = function(entity) + if entity.logging.log_statistics and UNSUPPORTED_LOG_STATISTICS[entity.route_type][entity.model.provider] then + return nil, fmt("cannot log statistics for %s provider when route_type is %s", + entity.model.provider, entity.route_type) + + else + return true + end + end, + } + }, }, } From 8b599a961fae4a52646fab9cd7014ba02c5b4f5b Mon Sep 17 00:00:00 2001 From: Robin Xiang Date: Mon, 25 Mar 2024 17:05:41 +0800 Subject: [PATCH 2/5] 1. add changelog 2. remain completions of anthropic only in the unsupported api --- changelog/unreleased/kong/analytics-for-anthropic.yml | 4 ++++ kong/llm/init.lua | 3 +-- spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 changelog/unreleased/kong/analytics-for-anthropic.yml diff --git a/changelog/unreleased/kong/analytics-for-anthropic.yml b/changelog/unreleased/kong/analytics-for-anthropic.yml new file mode 100644 index 000000000000..549f1e2377c8 --- /dev/null +++ b/changelog/unreleased/kong/analytics-for-anthropic.yml @@ -0,0 +1,4 @@ +message: | + **AI-proxy-plugin**: Fix the bug that the route_type `/llm/v1/chat` does not include the analytics in the responses. +scope: Plugin +type: bug_fix diff --git a/kong/llm/init.lua b/kong/llm/init.lua index da0364d2ff8f..0c97626b5b19 100644 --- a/kong/llm/init.lua +++ b/kong/llm/init.lua @@ -140,8 +140,7 @@ local logging_schema = { } local UNSUPPORTED_LOG_STATISTICS = { - ["llm/v1/completions"] = { "anthropic", "azure", "cohere", "llama2", }, - ["llm/v1/chat"] = { "cohere", "llama2", } + ["llm/v1/completions"] = { "anthropic" }, } _M.config_schema = { diff --git a/spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua b/spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua index 13d9ec937e6b..4cfc69806a12 100644 --- a/spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua +++ b/spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua @@ -481,6 +481,7 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then -- check this is in the 'kong' response format assert.equals(json.error.message, "request format not recognised") end) + end) describe("anthropic llm/v1/completions", function() it("good request", function() @@ -521,6 +522,5 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then end) end) end) -end) end end From f8d6837e1f809c4ded52ffee64ad1ddc735939d7 Mon Sep 17 00:00:00 2001 From: Robin Xiang Date: Mon, 25 Mar 2024 17:26:00 +0800 Subject: [PATCH 3/5] Update analytics-for-anthropic.yml --- changelog/unreleased/kong/analytics-for-anthropic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/unreleased/kong/analytics-for-anthropic.yml b/changelog/unreleased/kong/analytics-for-anthropic.yml index 549f1e2377c8..04de991390df 100644 --- a/changelog/unreleased/kong/analytics-for-anthropic.yml +++ b/changelog/unreleased/kong/analytics-for-anthropic.yml @@ -1,4 +1,4 @@ message: | **AI-proxy-plugin**: Fix the bug that the route_type `/llm/v1/chat` does not include the analytics in the responses. scope: Plugin -type: bug_fix +type: bugfix From bc0beafa2ea25e3291581409fd010a4ce87c59c4 Mon Sep 17 00:00:00 2001 From: Robin Xiang Date: Thu, 11 Apr 2024 14:32:40 +0800 Subject: [PATCH 4/5] update default of statistics and update mock response of anthropic for /chat --- kong/llm/init.lua | 10 ++++++---- .../38-ai-proxy/03-anthropic_integration_spec.lua | 10 ++++++++-- .../ai-proxy/anthropic/llm-v1-chat/responses/good.json | 9 +++++++-- .../unit/real-responses/anthropic/llm-v1-chat.json | 6 +++++- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/kong/llm/init.lua b/kong/llm/init.lua index 0c97626b5b19..6b973ae262e0 100644 --- a/kong/llm/init.lua +++ b/kong/llm/init.lua @@ -131,7 +131,7 @@ local logging_schema = { description = "If enabled and supported by the driver, " .. "will add model usage and token metrics into the Kong log plugin(s) output.", required = true, - default = true }}, + default = false }}, { log_payloads = { type = "boolean", description = "If enabled, will log the request and response body into the Kong log plugin(s) output.", @@ -140,7 +140,7 @@ local logging_schema = { } local UNSUPPORTED_LOG_STATISTICS = { - ["llm/v1/completions"] = { "anthropic" }, + ["llm/v1/completions"] = { ["anthropic"] = true }, } _M.config_schema = { @@ -210,8 +210,10 @@ _M.config_schema = { custom_entity_check = { field_sources = { "route_type", "model", "logging" }, fn = function(entity) - if entity.logging.log_statistics and UNSUPPORTED_LOG_STATISTICS[entity.route_type][entity.model.provider] then - return nil, fmt("cannot log statistics for %s provider when route_type is %s", + -- print(cjson.encode(entity)) + if entity.logging.log_statistics and UNSUPPORTED_LOG_STATISTICS[entity.route_type] + and UNSUPPORTED_LOG_STATISTICS[entity.route_type][entity.model.provider] then + return nil, fmt("%s does not support statistics when route_type is %s", entity.model.provider, entity.route_type) else diff --git a/spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua b/spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua index 4cfc69806a12..d5b36fce7b22 100644 --- a/spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua +++ b/spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua @@ -225,6 +225,9 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then anthropic_version = "2023-06-01", }, }, + logging = { + log_statistics = false, -- anthropic does not support statistics + }, }, } -- @@ -315,6 +318,9 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then anthropic_version = "2023-06-01", }, }, + logging = { + log_statistics = false, -- anthropic does not support statistics + }, }, } -- @@ -363,7 +369,7 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then declarative_config = strategy == "off" and helpers.make_yaml_file() or nil, }, nil, nil, fixtures)) end) - + lazy_teardown(function() helpers.stop_kong() end) @@ -434,7 +440,7 @@ for _, strategy in helpers.all_strategies() do if strategy ~= "cassandra" then }, body = pl_file.read("spec/fixtures/ai-proxy/anthropic/llm-v1-chat/requests/good.json"), }) - + local body = assert.res_status(200 , r) local json = cjson.decode(body) diff --git a/spec/fixtures/ai-proxy/anthropic/llm-v1-chat/responses/good.json b/spec/fixtures/ai-proxy/anthropic/llm-v1-chat/responses/good.json index 34aafac88752..174220a4a210 100644 --- a/spec/fixtures/ai-proxy/anthropic/llm-v1-chat/responses/good.json +++ b/spec/fixtures/ai-proxy/anthropic/llm-v1-chat/responses/good.json @@ -5,6 +5,11 @@ "type": "text" } ], - "stop_reason": "stop_sequence", - "model": "claude-2.1" + "model": "claude-2.1", + "stop_reason": "end_turn", + "stop_sequence": "string", + "usage": { + "input_tokens": 0, + "output_tokens": 0 + } } diff --git a/spec/fixtures/ai-proxy/unit/real-responses/anthropic/llm-v1-chat.json b/spec/fixtures/ai-proxy/unit/real-responses/anthropic/llm-v1-chat.json index a6afa37fca8e..624214cff5d0 100644 --- a/spec/fixtures/ai-proxy/unit/real-responses/anthropic/llm-v1-chat.json +++ b/spec/fixtures/ai-proxy/unit/real-responses/anthropic/llm-v1-chat.json @@ -4,5 +4,9 @@ "type": "text" }], "stop_reason": "stop_sequence", - "model": "claude-2.1" + "model": "claude-2.1", + "usage": { + "input_tokens": 0, + "output_tokens": 0 + } } \ No newline at end of file From e4ec35b50bfb87cc042b2934751916fa5c9178fb Mon Sep 17 00:00:00 2001 From: Robin Xiang Date: Thu, 11 Apr 2024 15:05:56 +0800 Subject: [PATCH 5/5] update test case --- .../03-plugins/38-ai-proxy/00-config_spec.lua | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/spec/03-plugins/38-ai-proxy/00-config_spec.lua b/spec/03-plugins/38-ai-proxy/00-config_spec.lua index 296ecc8c47bc..bbd495918bbb 100644 --- a/spec/03-plugins/38-ai-proxy/00-config_spec.lua +++ b/spec/03-plugins/38-ai-proxy/00-config_spec.lua @@ -115,6 +115,34 @@ describe(PLUGIN_NAME .. ": (schema)", function() assert.is_falsy(ok) end) + it("do not support log statistics when /chat route_type is used for anthropic provider", function() + local config = { + route_type = "llm/v1/completions", + auth = { + header_name = "x-api-key", + header_value = "anthropic_key", + }, + model = { + name = "anthropic-chat", + provider = "anthropic", + options = { + max_tokens = 256, + temperature = 1.0, + anthropic_version = "2021-09-01", + }, + }, + logging = { + log_statistics = true, + }, + } + + local ok, err = validate(config) + assert.is_falsy(ok) + assert.not_nil(err["config"]["@entity"]) + assert.not_nil(err["config"]["@entity"][1]) + assert.not_nil(err["config"]["@entity"][1], "anthropic does not support statistics when route_type is llm/v1/completions") + end) + it("requires [azure_instance] field when azure provider is used", function() local config = { route_type = "llm/v1/chat",