Skip to content

Commit

Permalink
fix(ai-proxy): disable gzip for SSE streams
Browse files Browse the repository at this point in the history
(cherry picked from commit e5ddb1d)
  • Loading branch information
fffonion authored and kikito committed Aug 20, 2024
1 parent f18e8ee commit 28d077c
Showing 1 changed file with 11 additions and 6 deletions.
17 changes: 11 additions & 6 deletions kong/llm/proxy/handler.lua
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,9 @@ local function handle_streaming_frame(conf, chunk, finished)
end

local response_frame = framebuffer:get()
if not finished and accept_gzip() then
-- TODO: disable gzip for SSE because it needs immediate flush for each chunk
-- and seems nginx doesn't support it
if not finished and accept_gzip() and not llm_state.is_streaming_mode() then
response_frame = kong_utils.deflate_gzip(response_frame)
end

Expand Down Expand Up @@ -274,12 +276,15 @@ function _M:header_filter(conf)
if llm_state.is_streaming_mode() then
-- we are going to send plaintext event-stream frames for ALL models
kong.response.set_header("Content-Type", "text/event-stream")
end

if accept_gzip() then
kong.response.set_header("Content-Encoding", "gzip")
-- TODO: disable gzip for SSE because it needs immediate flush for each chunk
-- and seems nginx doesn't support it
else
kong.response.clear_header("Content-Encoding")

if accept_gzip() then
kong.response.set_header("Content-Encoding", "gzip")
else
kong.response.clear_header("Content-Encoding")
end
end
end

Expand Down

0 comments on commit 28d077c

Please sign in to comment.