fix(ai): (FTI-6403) Fixed error with Azure path override formatting

changelog/unreleased/kong/fix-ai-azure-incorrect-path-overriding.yml

@@ -0,0 +1,3 @@
+message: "**ai-proxy**: Fixed a bug in the Azure provider where `model.options.upstream_path` overrides would always return 404."
+type: bugfix
+scope: Plugin

kong/llm/drivers/azure.lua

@@ -143,7 +143,6 @@ function _M.configure_request(conf)
 
   local query_table = kong.request.get_query()
 
-  -- technically min supported version
   query_table["api-version"] = kong.request.get_query_arg("api-version")
                             or (conf.model.options and conf.model.options.azure_api_version)
 

kong/llm/drivers/shared.lua

@@ -82,6 +82,7 @@ _M._SUPPORTED_STREAMING_CONTENT_TYPES = {
   ["text/event-stream"] = true,
   ["application/vnd.amazon.eventstream"] = true,
   ["application/json"] = true,
+  ["application/stream+json"] = true,
 }
 
 _M.streaming_has_token_counts = {
@@ -93,10 +94,10 @@ _M.streaming_has_token_counts = {
 }
 
 _M.upstream_url_format = {
-  openai        = fmt("%s://api.openai.com:%s", (openai_override and "http") or "https", (openai_override) or "443"),
+  openai        = fmt("%s://api.openai.com:%s", openai_override and "http" or "https", openai_override or "443"),
   anthropic     = "https://api.anthropic.com:443",
   cohere        = "https://api.cohere.com:443",
-  azure         = "https://%s.openai.azure.com:443/openai/deployments/%s",
+  azure         = fmt("%s://%%s.openai.azure.com:%s/openai/deployments/%%s", openai_override and "http" or "https", openai_override or "443"),
   gemini        = "https://generativelanguage.googleapis.com",
   gemini_vertex = "https://%s",
   bedrock       = "https://bedrock-runtime.%s.amazonaws.com",
@@ -944,8 +945,12 @@ end
 
 function _M.override_upstream_url(parsed_url, conf)
   if conf.route_type == "preserve" then
-    parsed_url.path = conf.model.options and conf.model.options.upstream_path
-      or kong.request.get_path()
+    -- if `upstream_path` was set, already processes before,
+    -- for some provider, like azure and huggingface, the specific prefix need to prepended to the path.
+    if conf.model.options and conf.model.options.upstream_path then
+      return
+    end
+    parsed_url.path = kong.request.get_path()
   end
 end
 

spec/03-plugins/38-ai-proxy/05-azure_integration_spec.lua

@@ -15,8 +15,19 @@ for _, strategy in helpers.all_strategies() do
       -- set up azure mock fixtures
       local fixtures = {
         http_mock = {},
+        dns_mock = helpers.dns_mock.new({
+          mocks_only = true,      -- don't fallback to "real" DNS
+        }),
       }
 
+      fixtures.dns_mock:A {
+        name = "001-kong-t.openai.azure.com",
+        address = "127.0.0.1",
+      }
+
+      -- openai llm driver will always send to this port, if var is set
+      helpers.setenv("OPENAI_TEST_PORT", tostring(MOCK_PORT))
+
       fixtures.http_mock.azure = [[
         server {
             server_name azure;
@@ -129,6 +140,56 @@ for _, strategy in helpers.all_strategies() do
               }
             }
 
+            location = "/openai/deployments/azure-other-instance/other/operation" {
+              content_by_lua_block {
+                local pl_file = require "pl.file"
+                local json = require("cjson.safe")
+
+                local token = ngx.req.get_headers()["api-key"]
+                if token == "azure-key" then
+                  ngx.req.read_body()
+                  local body, err = ngx.req.get_body_data()
+                  body, err = json.decode(body)
+
+                  if err or (body.messages == ngx.null) then
+                    ngx.status = 400
+                    ngx.print(pl_file.read("spec/fixtures/ai-proxy/openai/llm-v1-chat/responses/bad_request.json"))
+                  else
+                    ngx.status = 200
+                    ngx.print(pl_file.read("spec/fixtures/ai-proxy/openai/llm-v1-chat/responses/good.json"))
+                  end
+                else
+                  ngx.status = 401
+                  ngx.print(pl_file.read("spec/fixtures/ai-proxy/openai/llm-v1-chat/responses/unauthorized.json"))
+                end
+              }
+            }
+
+            location = "/override/path/completely" {
+              content_by_lua_block {
+                local pl_file = require "pl.file"
+                local json = require("cjson.safe")
+
+                local token = ngx.req.get_headers()["api-key"]
+                if token == "azure-key" then
+                  ngx.req.read_body()
+                  local body, err = ngx.req.get_body_data()
+                  body, err = json.decode(body)
+
+                  if err or (body.messages == ngx.null) then
+                    ngx.status = 400
+                    ngx.print(pl_file.read("spec/fixtures/ai-proxy/openai/llm-v1-chat/responses/bad_request.json"))
+                  else
+                    ngx.status = 200
+                    ngx.print(pl_file.read("spec/fixtures/ai-proxy/openai/llm-v1-chat/responses/good.json"))
+                  end
+                else
+                  ngx.status = 401
+                  ngx.print(pl_file.read("spec/fixtures/ai-proxy/openai/llm-v1-chat/responses/unauthorized.json"))
+                end
+              }
+            }
+
         }
       ]]
 
@@ -387,6 +448,97 @@ for _, strategy in helpers.all_strategies() do
       }
       --
 
+      -- Override path with unique Azure operations
+      local chat_override_path_from_params = assert(bp.routes:insert {
+        service = empty_service,
+        protocols = { "http" },
+        strip_path = true,
+        paths = { "~/ai/openai/deployments/(?<azure_deployment>[^#?/]+)(?<operation_path>[^#?]+)$" }
+      })
+      bp.plugins:insert {
+        name = PLUGIN_NAME,
+        route = { id = chat_override_path_from_params.id },
+        config = {
+          route_type = "preserve",
+          auth = {
+            header_name = "api-key",
+            header_value = "azure-key",
+          },
+          model = {
+            name = "gpt-3.5-turbo",
+            provider = "azure",
+            options = {
+              max_tokens = 256,
+              temperature = 1.0,
+              azure_instance = "001-kong-t",
+              upstream_path = "$(uri_captures.operation_path)",
+              azure_deployment_id = "$(uri_captures.azure_deployment)",
+            },
+          },
+        },
+      }
+      --
+
+      -- Override path completely
+      local chat_override_path_completely = assert(bp.routes:insert {
+        service = empty_service,
+        protocols = { "http" },
+        strip_path = true,
+        paths = { "~/override/path/completely$" }
+      })
+      bp.plugins:insert {
+        name = PLUGIN_NAME,
+        route = { id = chat_override_path_completely.id },
+        config = {
+          route_type = "preserve",
+          auth = {
+            header_name = "api-key",
+            header_value = "azure-key",
+          },
+          model = {
+            name = "gpt-3.5-turbo",
+            provider = "azure",
+            options = {
+              max_tokens = 256,
+              temperature = 1.0,
+              azure_instance = "001-kong-t",
+              azure_deployment_id = "gpt-3.5-custom",
+            },
+          },
+        },
+      }
+      --
+
+      -- Override path and expect 404
+      local chat_override_path_incorrectly = assert(bp.routes:insert {
+        service = empty_service,
+        protocols = { "http" },
+        strip_path = true,
+        paths = { "~/override/path/incorrectly$" }
+      })
+      bp.plugins:insert {
+        name = PLUGIN_NAME,
+        route = { id = chat_override_path_incorrectly.id },
+        config = {
+          route_type = "preserve",
+          auth = {
+            header_name = "api-key",
+            header_value = "azure-key",
+          },
+          model = {
+            name = "gpt-3.5-turbo",
+            provider = "azure",
+            options = {
+              max_tokens = 256,
+              temperature = 1.0,
+              azure_instance = "001-kong-t",
+              azure_deployment_id = "gpt-3.5-custom",
+            },
+          },
+        },
+      }
+      --
+
 
 
       -- start kong
@@ -647,6 +799,47 @@ for _, strategy in helpers.all_strategies() do
         assert.equals("request body doesn't contain valid prompts", json.error.message)
       end)
     end)
+
+    describe("azure preserve", function()
+      it("override path from path params", function()
+        local r = client:get("/ai/openai/deployments/azure-other-instance/other/operation", {
+          headers = {
+            ["content-type"] = "application/json",
+            ["accept"] = "application/json",
+          },
+          body = pl_file.read("spec/fixtures/ai-proxy/openai/llm-v1-chat/requests/good.json"),
+        })
+
+        -- validate that the request succeeded, response status 200
+        assert.res_status(200 , r)
+      end)
+
+      it("override path completely", function()
+        local r = client:get("/override/path/completely", {
+          headers = {
+            ["content-type"] = "application/json",
+            ["accept"] = "application/json",
+          },
+          body = pl_file.read("spec/fixtures/ai-proxy/openai/llm-v1-chat/requests/good.json"),
+        })
+
+        -- validate that the request succeeded, response status 200
+        assert.res_status(200 , r)
+      end)
+
+      it("override path incorrectly", function()
+        local r = client:get("/override/path/incorrectly", {
+          headers = {
+            ["content-type"] = "application/json",
+            ["accept"] = "application/json",
+          },
+          body = pl_file.read("spec/fixtures/ai-proxy/openai/llm-v1-chat/requests/good.json"),
+        })
+
+        -- expect it to 404 from the backend
+        assert.res_status(404 , r)
+      end)
+    end)
   end)
 
 end