Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(observability): log a single warning when buffer is full until it recovers #13369

Merged
merged 1 commit into from
Jul 15, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 26 additions & 3 deletions kong/observability/logs.lua
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ local NGX_CTX_REQUEST_LOGS_KEY = "o11y_logs_request_scoped"
local worker_logs = table_new(INITIAL_SIZE_WORKER_LOGS, 0)
local logline_buf = string_buffer.new()

local notified_buffer_full = false


-- WARNING: avoid using `ngx.log` in this function to prevent recursive loops
local function configured_log_level()
Expand Down Expand Up @@ -124,6 +126,28 @@ local function get_request_log_buffer()
end


-- notifies the user that the log buffer is full, once (per worker)
local function notify_buffer_full_once()
if not notified_buffer_full then
notified_buffer_full = true
native_ngx_log(ngx.NOTICE,
"[observability] OpenTelemetry logs buffer is full: dropping new log entries."
)
end
end


local function notify_if_resumed()
-- if we are in a "resumed" state
if notified_buffer_full then
notified_buffer_full = false
native_ngx_log(ngx.NOTICE,
"[observability] OpenTelemetry logs buffer resumed accepting log entries."
)
end
end


function _M.maybe_push(stack_level, attributes, log_level, ...)
-- WARNING: do not yield in this function, as it is called from ngx.log

Expand All @@ -150,11 +174,10 @@ function _M.maybe_push(stack_level, attributes, log_level, ...)

-- return if log buffer is full
if #log_buffer >= max_logs then
ADD-SP marked this conversation as resolved.
Show resolved Hide resolved
native_ngx_log(ngx.NOTICE,
"[observability] OpenTelemetry logs buffer is full: dropping log entry."
)
notify_buffer_full_once()
return
end
notify_if_resumed()

local args = table_pack(...)
local log_str = concat_tostring(args)
Expand Down
Loading