feat(metrics) implement user-defined histogram bins

This feature is only available through the FFI. If a user provides a list of numbers when defining a histogram, those numbers will be used as the upper-bounds of the histogram's bins. For instance, the following code: local shm = require "resty.wasmx.shm" shm.metrics:define("h", shm.metrics.HISTOGRAM, { bins = { 1, 3, 5 } }) Creates a histogram with bins `[0, 1] (1, 3] (3, 5] (5, Inf+]`. Signed-off-by: Thibault Charbonnier <[email protected]>
Kong · Oct 23, 2024 · cca9f01 · cca9f01
1 parent bbf4891
commit cca9f01
Show file tree

Hide file tree

Showing 17 changed files with 354 additions and 64 deletions.
diff --git a/docs/METRICS.md b/docs/METRICS.md
@@ -8,7 +8,9 @@ memory necessary for your use-case.
 
 - [Types of Metrics](#types-of-metrics)
 - [Name Prefixing](#name-prefixing)
-- [Histogram Binning Strategy](#histogram-binning-strategy)
+- [Histogram Binning Strategies](#histogram-binning-strategies)
+    - [Logarithmic Binning](#logarithmic-binning)
+    - [Custom Binning](#custom-binning)
 - [Histogram Update and Expansion](#histogram-update-and-expansion)
 - [Memory Consumption](#memory-consumption)
 - [Shared Memory Allocation](#shared-memory-allocation)
@@ -46,35 +48,42 @@ increased in some cases.
 
 [Back to TOC](#table-of-contents)
 
-## Histogram Binning Strategy
+## Histogram Binning Strategies
 
-The above example demonstrates a histogram with ranges (or bins) whose
-upper-bound grows in powers of 2, i.e. `2^0`, `2^1`, and `2^2`. This is usually
-called "logarithmic binning" and is how histograms bins are represented in
-ngx_wasm_module.
+### Logarithmic Binning
 
-This binning strategy implies that when a value `v` is recorded, it is matched
-with the smallest power of two that is bigger than `v`. This value is the
-*upper-bound* of the bin associated with `v`. If the histogram contains or can
-contain such a bin, that bin's counter is incremented. If not, the bin with the
-next smallest upper-bound bigger than `v` has its counter incremented instead.
+By default, histograms use a logarithmic-binning strategy.
 
-[Back to TOC](#table-of-contents)
+As an example of logarithmic-binning, take the histogram with ranges (i.e.
+"bins") `[0, 1] (1, 2] (2, 4] (4, Inf]`: each bin's upper-bound is growing in
+powers of 2: `2^0`, `2^1`, and `2^2`. In logarithmic-binning, a value `v` being
+recorded is matched with the smallest power of two that is bigger than `v`. This
+value is the *upper-bound* of the bin associated with `v`. If the histogram
+contains or can contain such a bin, then its counter is incremented. If not, the
+bin with the next smallest upper-bound bigger than `v` has its counter
+incremented instead.
+
+In ngx_wasm_module, logarithmic-binning histograms are created with one
+initialized bin with upper-bound `2^32`. The counter for this bin is incremented
+if it is the only bin whose upper-bound is bigger than the recorded value.
 
-## Histogram Update and Expansion
+When a value `v` is recorded and its bin does not yet exist, a new bin with the
+upper-bound associated with `v` is initialized and its counter is incremented.
 
-Histograms are created with 5 bins: 1 initialized and 4 uninitialized.
+A logarithmic-binning histogram can contain up to 18 initialized bins.
+
+[Back to TOC](#table-of-contents)
 
-The bin initialized upon histogram creation has upper-bound `2^32` and its
-counter is incremented if it is the only bin whose upper-bound is bigger than
-the recorded value.
+### Custom Binning
 
-If a value `v` is recorded and its bin is not part of the initialized bins, a
-new bin with the upper-bound associated with `v` is initialized, and its counter
-is incremented.
+Histograms can also be created with a fixed set of bins with user-defined
+upper-bounds. These histograms store values exactly like the logarithmic-binning
+ones, except the number of bins and their upper-bounds are user-defined and
+pre-initialized.
 
-If the histogram is out of uninitialized bins, it can be expanded up to 18
-bins so as to accommodate the additional bins for other ranges of `v`.
+A custom-binning histogram can contain up to 18 bins (17 user-defined bins + one
+`2^32` upper-bound bin). Custom-binning histograms cannot be expanded with new
+bins after definition.
 
 [Back to TOC](#table-of-contents)
 

diff --git a/lib/resty/wasmx/shm.lua b/lib/resty/wasmx/shm.lua
@@ -58,6 +58,11 @@ ffi.cdef [[
         NGX_WA_METRIC_HISTOGRAM,
     } ngx_wa_metric_type_e;
 
+    typedef enum {
+        NGX_WA_HISTOGRAM_LOG2,
+        NGX_WA_HISTOGRAM_CUSTOM,
+    } ngx_wa_histogram_type_e;
+
     typedef struct {
         ngx_uint_t                   value;
         ngx_msec_t                   last_update;
@@ -69,6 +74,7 @@ ffi.cdef [[
     } ngx_wa_metrics_bin_t;
 
     typedef struct {
+        ngx_wa_histogram_type_e      h_type;
         uint8_t                      n_bins;
         uint64_t                     sum;
         ngx_wa_metrics_bin_t         bins[];
@@ -109,6 +115,8 @@ ffi.cdef [[
 
     ngx_int_t ngx_wa_ffi_shm_metric_define(ngx_str_t *name,
                                            ngx_wa_metric_type_e type,
+                                           uint32_t *bins,
+                                           uint16_t n_bins,
                                            uint32_t *metric_id);
     ngx_int_t ngx_wa_ffi_shm_metric_increment(uint32_t metric_id,
                                               ngx_uint_t value);
@@ -124,11 +132,13 @@ ffi.cdef [[
 
     ngx_int_t ngx_wa_ffi_shm_metrics_one_slot_size();
     ngx_int_t ngx_wa_ffi_shm_metrics_histogram_max_size();
+    ngx_int_t ngx_wa_ffi_shm_metrics_histogram_max_bins();
 ]]
 
 
 local WASM_SHM_KEY = {}
 local DEFAULT_KEYS_PAGE_SIZE = 500
+local HISTOGRAM_MAX_BINS = C.ngx_wa_ffi_shm_metrics_histogram_max_bins()
 
 
 local _M = setmetatable({}, {
@@ -182,7 +192,8 @@ local function key_iterator(ctx)
         ctx.ccur_index[0] = 0
 
         local rc = C.ngx_wa_ffi_shm_iterate_keys(ctx.shm, ctx.page_size,
-                                                 ctx.clast_index, ctx.ccur_index, ctx.ckeys)
+                                                 ctx.clast_index, ctx.ccur_index,
+                                                 ctx.ckeys)
 
         if rc == FFI_ABORT then
             -- users must manage locking themselves (e.g. break condition in the for loop)
@@ -341,7 +352,7 @@ local function shm_kv_set(zone, key, value, cas)
 end
 
 
-local function metrics_define(zone, name, metric_type)
+local function metrics_define(zone, name, metric_type, opts)
     if type(name) ~= "string" or name == "" then
         error("name must be a non-empty string", 2)
     end
@@ -351,16 +362,54 @@ local function metrics_define(zone, name, metric_type)
                     " resty.wasmx.shm.metrics.COUNTER," ..
                     " resty.wasmx.shm.metrics.GAUGE, or" ..
                     " resty.wasmx.shm.metrics.HISTOGRAM"
-
         error(err, 2)
     end
 
+    local cbins
+    local n_bins = 0
+
+    if opts ~= nil then
+        if type(opts) ~= "table" then
+            error("opts must be a table", 2)
+        end
+
+        if metric_type == _types.ffi_metric.HISTOGRAM
+           and opts.bins ~= nil
+        then
+            if type(opts.bins) ~= "table" then
+                error("opts.bins must be a table", 2)
+            end
+
+            if #opts.bins >= HISTOGRAM_MAX_BINS then
+                local err = "opts.bins cannot have more than %d numbers"
+                error(str_fmt(err, HISTOGRAM_MAX_BINS - 1), 2)
+            end
+
+            local previous = 0
+
+            for _, n in ipairs(opts.bins) do
+                if type(n) ~= "number"
+                   or n < 0 or n % 1 > 0 or n <= previous
+                then
+                    error("opts.bins must be an ascending list of " ..
+                          "positive integers", 2)
+                end
+
+                previous = n
+            end
+
+            n_bins = #opts.bins
+            cbins = ffi_new("uint32_t[?]", n_bins, opts.bins)
+        end
+    end
+
     name = "lua." .. name
 
     local cname = ffi_new("ngx_str_t", { data = name, len = #name })
-    local m_id = ffi_new("uint32_t [1]")
+    local m_id = ffi_new("uint32_t[1]")
 
-    local rc = C.ngx_wa_ffi_shm_metric_define(cname, metric_type, m_id)
+    local rc = C.ngx_wa_ffi_shm_metric_define(cname, metric_type,
+                                              cbins, n_bins, m_id)
     if rc == FFI_ERROR then
         return nil, "no memory"
     end

diff --git a/src/common/debug/ngx_wasm_debug_module.c b/src/common/debug/ngx_wasm_debug_module.c
@@ -22,12 +22,19 @@
 static ngx_int_t
 ngx_wasm_debug_init(ngx_cycle_t *cycle)
 {
-    static size_t            long_metric_name_len = NGX_MAX_ERROR_STR;
-    uint32_t                 mid;
-    ngx_str_t                metric_name;
-    u_char                   buf[long_metric_name_len];
-
-    static ngx_wasm_phase_t  ngx_wasm_debug_phases[] = {
+    static size_t                long_metric_name_len = NGX_MAX_ERROR_STR;
+    uint32_t                     mid;
+    ngx_str_t                    metric_name;
+    ngx_wa_metric_t             *m;
+    ngx_wa_metrics_histogram_t  *h, *h2;
+    uint32_t                     bins[NGX_WA_METRICS_HISTOGRAM_BINS_MAX + 1];
+    u_char                       buf[long_metric_name_len];
+    u_char                       m_buf[NGX_WA_METRICS_ONE_SLOT_SIZE];
+    u_char                       h_buf[NGX_WA_METRICS_HISTOGRAM_MAX_SIZE];
+    u_char                       h2_buf[NGX_WA_METRICS_HISTOGRAM_MAX_SIZE];
+    u_char                       zeros[NGX_WA_METRICS_HISTOGRAM_MAX_SIZE];
+
+    static ngx_wasm_phase_t      ngx_wasm_debug_phases[] = {
         { ngx_string("a_phase"), 0, 0, 0 },
         { ngx_null_string, 0, 0, 0 }
     };
@@ -60,6 +67,7 @@ ngx_wasm_debug_init(ngx_cycle_t *cycle)
         ngx_wa_metrics_define(ngx_wasmx_metrics(cycle),
                               &metric_name,
                               NGX_WA_METRIC_COUNTER,
+                              NULL, 0,
                               &mid) == NGX_BUSY
     );
 
@@ -68,6 +76,16 @@ ngx_wasm_debug_init(ngx_cycle_t *cycle)
         ngx_wa_metrics_define(ngx_wasmx_metrics(cycle),
                               &metric_name,
                               100,
+                              NULL, 0,
+                              &mid) == NGX_ABORT
+    );
+
+    /* invalid number of histogram bins */
+    ngx_wa_assert(
+        ngx_wa_metrics_define(ngx_wasmx_metrics(cycle),
+                              ngx_wa_metric_type_name(NGX_WA_METRIC_HISTOGRAM),
+                              NGX_WA_METRIC_HISTOGRAM,
+                              bins, NGX_WA_METRICS_HISTOGRAM_BINS_MAX + 1,
                               &mid) == NGX_ABORT
     );
 
@@ -77,6 +95,29 @@ ngx_wasm_debug_init(ngx_cycle_t *cycle)
                     "unknown", 8) == 0
     );
 
+    /* unknown histogram type */
+    ngx_memzero(m_buf, sizeof(m_buf));
+    ngx_memzero(h_buf, sizeof(h_buf));
+    ngx_memzero(h2_buf, sizeof(h2_buf));
+    ngx_memzero(zeros, sizeof(zeros));
+
+    m = (ngx_wa_metric_t *) m_buf;
+    h = (ngx_wa_metrics_histogram_t *) h_buf;
+    h->n_bins = NGX_WA_METRICS_HISTOGRAM_BINS_MAX;
+    h->h_type = 10;
+    h->sum = 1;
+    ngx_wa_metrics_histogram_set_buffer(m, h_buf, sizeof(h_buf));
+
+    h2 = (ngx_wa_metrics_histogram_t *) h2_buf;
+
+    ngx_wa_assert(
+        ngx_wa_metrics_histogram_record(ngx_wasmx_metrics(cycle),
+                                        m, 0, 0, 1) == NGX_ERROR
+    );
+
+    ngx_wa_metrics_histogram_get(ngx_wasmx_metrics(cycle), m, 1, h2);
+    ngx_wa_assert(ngx_memcmp(h2_buf, zeros, sizeof(zeros)) == 0);
+
     return NGX_OK;
 }
 

diff --git a/src/common/lua/ngx_wasm_lua_ffi.c b/src/common/lua/ngx_wasm_lua_ffi.c
@@ -409,12 +409,12 @@ ngx_wa_ffi_shm_kv_set(ngx_wa_shm_t *shm, ngx_str_t *k, ngx_str_t *v,
 
 ngx_int_t
 ngx_wa_ffi_shm_metric_define(ngx_str_t *name, ngx_wa_metric_type_e type,
-    uint32_t *metric_id)
+    uint32_t *bins, uint16_t n_bins, uint32_t *metric_id)
 {
     ngx_int_t          rc;
     ngx_wa_metrics_t  *metrics = ngx_wasmx_metrics((ngx_cycle_t *) ngx_cycle);
 
-    rc = ngx_wa_metrics_define(metrics, name, type, metric_id);
+    rc = ngx_wa_metrics_define(metrics, name, type, bins, n_bins, metric_id);
     if (rc != NGX_OK) {
         return rc;
     }

diff --git a/src/common/lua/ngx_wasm_lua_ffi.h b/src/common/lua/ngx_wasm_lua_ffi.h
@@ -58,7 +58,8 @@ ngx_int_t ngx_wa_ffi_shm_kv_set(ngx_wa_shm_t *shm, ngx_str_t *k,
     ngx_str_t *v, uint32_t cas, unsigned *written);
 
 ngx_int_t ngx_wa_ffi_shm_metric_define(ngx_str_t *name,
-    ngx_wa_metric_type_e type, uint32_t *metric_id);
+    ngx_wa_metric_type_e type, uint32_t *bins, uint16_t n_bins,
+    uint32_t *metric_id);
 ngx_int_t ngx_wa_ffi_shm_metric_increment(uint32_t metric_id, ngx_uint_t value);
 ngx_int_t ngx_wa_ffi_shm_metric_record(uint32_t metric_id, ngx_uint_t value);
 ngx_int_t ngx_wa_ffi_shm_metric_get(uint32_t metric_id, ngx_str_t *name,
@@ -93,4 +94,11 @@ ngx_wa_ffi_shm_metrics_histogram_max_size()
 }
 
 
+ngx_int_t
+ngx_wa_ffi_shm_metrics_histogram_max_bins()
+{
+    return NGX_WA_METRICS_HISTOGRAM_BINS_MAX;
+}
+
+
 #endif /* _NGX_WASM_LUA_FFI_H_INCLUDED_ */