feat(metrics) support for histograms with user-defined bins

This feature is only available through the FFI.

If a user provides a list of numbers when defining a histogram, those
numbers will be used as the upper-bounds of the histogram's bins.

For instance, the following code:

```
shm.metrics:define("h", shm.metrics.HISTOGRAM, { bins = { 1, 3, 5 } })
```

Creates a histogram with bins `[0, 1]`, `(1, 3]`, `(3, 5]` and
`(5, Inf+]`

lib/resty/wasmx/shm.lua

@@ -11,6 +11,7 @@ local tonumber = tonumber
 local min = math.min
 local new_tab = table.new
 local insert = table.insert
+local sort = table.sort
 local str_fmt = string.format
 local ffi_cast = ffi.cast
 local ffi_fill = ffi.fill
@@ -58,6 +59,11 @@ ffi.cdef [[
         NGX_WA_METRIC_HISTOGRAM,
     } ngx_wa_metric_type_e;
 
+    typedef enum {
+        NGX_WA_HISTOGRAM_LOG2,
+        NGX_WA_HISTOGRAM_CUSTOM,
+    } ngx_wa_histogram_type_e;
+
     typedef struct {
         ngx_uint_t                   value;
         ngx_msec_t                   last_update;
@@ -69,6 +75,7 @@ ffi.cdef [[
     } ngx_wa_metrics_bin_t;
 
     typedef struct {
+        ngx_wa_histogram_type_e      h_type;
         uint8_t                      n_bins;
         uint64_t                     sum;
         ngx_wa_metrics_bin_t         bins[];
@@ -109,6 +116,8 @@ ffi.cdef [[
 
     ngx_int_t ngx_wa_ffi_shm_metric_define(ngx_str_t *name,
                                            ngx_wa_metric_type_e type,
+                                           uint32_t *bins,
+                                           uint16_t n_bins,
                                            uint32_t *metric_id);
     ngx_int_t ngx_wa_ffi_shm_metric_increment(uint32_t metric_id,
                                               ngx_uint_t value);
@@ -124,11 +133,13 @@ ffi.cdef [[
 
     ngx_int_t ngx_wa_ffi_shm_metrics_one_slot_size();
     ngx_int_t ngx_wa_ffi_shm_metrics_histogram_max_size();
+    ngx_int_t ngx_wa_ffi_shm_metrics_histogram_max_bins();
 ]]
 
 
 local WASM_SHM_KEY = {}
 local DEFAULT_KEYS_PAGE_SIZE = 500
+local MAX_HISTOGRAM_BINS = C.ngx_wa_ffi_shm_metrics_histogram_max_bins()
 
 
 local _M = setmetatable({}, {
@@ -341,7 +352,7 @@ local function shm_kv_set(zone, key, value, cas)
 end
 
 
-local function metrics_define(zone, name, metric_type)
+local function metrics_define(zone, name, metric_type, opts)
     if type(name) ~= "string" or name == "" then
         error("name must be a non-empty string", 2)
     end
@@ -355,12 +366,36 @@ local function metrics_define(zone, name, metric_type)
         error(err, 2)
     end
 
+    local cbins = nil
+    local n_bins = 0
+
+    if opts ~= nil then
+        if type(opts) ~= "table" then
+            error("opts must be a table", 2)
+        end
+
+        if opts.bins ~= nil then
+            if type(opts.bins) ~= "table" then
+                error("opts.bins must be a table", 2)
+            end
+
+            if #opts.bins > MAX_HISTOGRAM_BINS then
+                local err = "opts.bins must have up to %d numbers"
+                error(str_fmt(err, MAX_HISTOGRAM_BINS), 2)
+            end
+
+            sort(opts.bins)
+            cbins = ffi_new("uint32_t[?]", #opts.bins, opts.bins)
+            n_bins = #opts.bins
+        end
+    end
+
     name = "lua." .. name
 
     local cname = ffi_new("ngx_str_t", { data = name, len = #name })
     local m_id = ffi_new("uint32_t [1]")
 
-    local rc = C.ngx_wa_ffi_shm_metric_define(cname, metric_type, m_id)
+    local rc = C.ngx_wa_ffi_shm_metric_define(cname, metric_type, cbins, n_bins, m_id)
     if rc == FFI_ERROR then
         return nil, "no memory"
     end

src/common/debug/ngx_wasm_debug_module.c

@@ -60,6 +60,7 @@ ngx_wasm_debug_init(ngx_cycle_t *cycle)
         ngx_wa_metrics_define(ngx_wasmx_metrics(cycle),
                               &metric_name,
                               NGX_WA_METRIC_COUNTER,
+                              NULL, 0,
                               &mid) == NGX_BUSY
     );
 
@@ -68,6 +69,7 @@ ngx_wasm_debug_init(ngx_cycle_t *cycle)
         ngx_wa_metrics_define(ngx_wasmx_metrics(cycle),
                               &metric_name,
                               100,
+                              NULL, 0,
                               &mid) == NGX_ABORT
     );
 

src/common/lua/ngx_wasm_lua_ffi.c

@@ -409,12 +409,12 @@ ngx_wa_ffi_shm_kv_set(ngx_wa_shm_t *shm, ngx_str_t *k, ngx_str_t *v,
 
 ngx_int_t
 ngx_wa_ffi_shm_metric_define(ngx_str_t *name, ngx_wa_metric_type_e type,
-    uint32_t *metric_id)
+    uint32_t *bins, uint16_t n_bins, uint32_t *metric_id)
 {
     ngx_int_t          rc;
     ngx_wa_metrics_t  *metrics = ngx_wasmx_metrics((ngx_cycle_t *) ngx_cycle);
 
-    rc = ngx_wa_metrics_define(metrics, name, type, metric_id);
+    rc = ngx_wa_metrics_define(metrics, name, type, bins, n_bins, metric_id);
     if (rc != NGX_OK) {
         return rc;
     }

src/common/lua/ngx_wasm_lua_ffi.h

@@ -58,7 +58,8 @@ ngx_int_t ngx_wa_ffi_shm_kv_set(ngx_wa_shm_t *shm, ngx_str_t *k,
     ngx_str_t *v, uint32_t cas, unsigned *written);
 
 ngx_int_t ngx_wa_ffi_shm_metric_define(ngx_str_t *name,
-    ngx_wa_metric_type_e type, uint32_t *metric_id);
+    ngx_wa_metric_type_e type, uint32_t *bins, uint16_t n_bins,
+    uint32_t *metric_id);
 ngx_int_t ngx_wa_ffi_shm_metric_increment(uint32_t metric_id, ngx_uint_t value);
 ngx_int_t ngx_wa_ffi_shm_metric_record(uint32_t metric_id, ngx_uint_t value);
 ngx_int_t ngx_wa_ffi_shm_metric_get(uint32_t metric_id, ngx_str_t *name,
@@ -93,4 +94,11 @@ ngx_wa_ffi_shm_metrics_histogram_max_size()
 }
 
 
+ngx_int_t
+ngx_wa_ffi_shm_metrics_histogram_max_bins()
+{
+    return NGX_WA_METRICS_BINS_MAX;
+}
+
+
 #endif /* _NGX_WASM_LUA_FFI_H_INCLUDED_ */

src/common/metrics/ngx_wa_histogram.c

@@ -77,7 +77,28 @@ histogram_grow(ngx_wa_metrics_t *metrics, ngx_wa_metrics_histogram_t *h,
 
 
 static ngx_wa_metrics_bin_t *
-histogram_bin(ngx_wa_metrics_t *metrics, ngx_wa_metrics_histogram_t *h,
+histogram_custom_bin(ngx_wa_metrics_t *metrics, ngx_wa_metrics_histogram_t *h,
+    ngx_uint_t n)
+{
+    size_t                 i = 0;
+    ngx_wa_metrics_bin_t  *b;
+
+    for (i = 0; i < h->n_bins; i++) {
+        b = &h->bins[i];
+
+        if (b->upper_bound >= n) {
+            return b;
+        }
+    }
+
+    ngx_wa_assert(0);
+
+    return NULL;
+}
+
+
+static ngx_wa_metrics_bin_t *
+histogram_log2_bin(ngx_wa_metrics_t *metrics, ngx_wa_metrics_histogram_t *h,
     ngx_uint_t n, ngx_wa_metrics_histogram_t **out)
 {
     size_t                 i, j = 0;
@@ -160,13 +181,19 @@ histogram_log(ngx_wa_metrics_t *metrics, ngx_wa_metric_t *m, uint32_t mid)
 
 
 ngx_int_t
-ngx_wa_metrics_histogram_add_locked(ngx_wa_metrics_t *metrics,
-    ngx_wa_metric_t *m)
+ngx_wa_metrics_histogram_add_locked(ngx_wa_metrics_t *metrics, uint32_t *bins,
+    uint16_t cn_bins, ngx_wa_metric_t *m)
 {
     size_t                        i;
-    static uint16_t               n_bins = NGX_WA_METRICS_BINS_INIT;
+    uint16_t                      j = 0, n_bins = NGX_WA_METRICS_BINS_INIT;
+    ngx_wa_histogram_type_e       h_type = NGX_WA_HISTOGRAM_LOG2;
     ngx_wa_metrics_histogram_t  **h;
 
+    if (bins) {
+        n_bins = cn_bins + 1;
+        h_type = NGX_WA_HISTOGRAM_CUSTOM;
+    }
+
     for (i = 0; i < metrics->workers; i++) {
         h = &m->slots[i].histogram;
         *h = ngx_slab_calloc_locked(metrics->shm->shpool,
@@ -177,7 +204,16 @@ ngx_wa_metrics_histogram_add_locked(ngx_wa_metrics_t *metrics,
         }
 
         (*h)->n_bins = n_bins;
-        (*h)->bins[0].upper_bound = NGX_MAX_UINT32_VALUE;
+        (*h)->h_type = h_type;
+
+        if (bins) {
+            /* user-defined set of bins */
+            for (/* void */; j < n_bins - 1; j++) {
+                (*h)->bins[j].upper_bound = bins[j];
+            }
+        }
+
+        (*h)->bins[j].upper_bound = NGX_MAX_UINT32_VALUE;
     }
 
     return NGX_OK;
@@ -199,13 +235,24 @@ ngx_int_t
 ngx_wa_metrics_histogram_record(ngx_wa_metrics_t *metrics, ngx_wa_metric_t *m,
     ngx_uint_t slot, uint32_t mid, ngx_uint_t n)
 {
-    ngx_wa_metrics_bin_t        *b;
     ngx_wa_metrics_histogram_t  *h;
+    ngx_wa_metrics_bin_t        *b;
 
     h = m->slots[slot].histogram;
     h->sum += n;
 
-    b = histogram_bin(metrics, h, n, &m->slots[slot].histogram);
+    switch (h->h_type) {
+    case NGX_WA_HISTOGRAM_LOG2:
+        b = histogram_log2_bin(metrics, h, n, &m->slots[slot].histogram);
+        break;
+    case NGX_WA_HISTOGRAM_CUSTOM:
+        b = histogram_custom_bin(metrics, h, n);
+        break;
+    default:
+        ngx_wa_assert(0);
+        return NGX_ERROR;
+    }
+
     b->count += 1;
 
 #if (NGX_DEBUG)
@@ -230,7 +277,20 @@ ngx_wa_metrics_histogram_get(ngx_wa_metrics_t *metrics, ngx_wa_metric_t *m,
 
         for (j = 0; j < h->n_bins; j++) {
             b = &h->bins[j];
-            out_b = histogram_bin(metrics, out, b->upper_bound, NULL);
+
+            switch (h->h_type) {
+            case NGX_WA_HISTOGRAM_LOG2:
+                out_b = histogram_log2_bin(metrics, out, b->upper_bound, NULL);
+                break;
+            case NGX_WA_HISTOGRAM_CUSTOM:
+                out_b = &out->bins[j];
+                out_b->upper_bound = b->upper_bound;
+                break;
+            default:
+                ngx_wa_assert(0);
+                return;
+            }
+
             out_b->count += b->count;
 
             if (b->upper_bound == NGX_MAX_UINT32_VALUE) {

src/common/metrics/ngx_wa_metrics.c

@@ -103,7 +103,9 @@ realloc_metrics(ngx_wa_metrics_t *metrics, ngx_rbtree_node_t *node,
     ngx_log_debug1(NGX_LOG_DEBUG_WASM, metrics->shm->log, 0,
                    "reallocating metric \"%V\"", &n->key.str);
 
-    if (ngx_wa_metrics_define(metrics, &n->key.str, m->type, &mid) != NGX_OK) {
+    if (ngx_wa_metrics_define(metrics, &n->key.str, m->type, NULL, 0, &mid)
+        != NGX_OK)
+    {
         ngx_wasm_log_error(NGX_LOG_ERR, metrics->shm->log, 0,
                            "failed redefining metric \"%V\"",
                            &n->key.str);
@@ -276,7 +278,7 @@ ngx_wa_metrics_shm_init(ngx_cycle_t *cycle)
  */
 ngx_int_t
 ngx_wa_metrics_define(ngx_wa_metrics_t *metrics, ngx_str_t *name,
-    ngx_wa_metric_type_e type, uint32_t *out)
+    ngx_wa_metric_type_e type, uint32_t *bins, uint16_t n_bins, uint32_t *out)
 {
     ssize_t           size = sizeof(ngx_wa_metric_t)
                              + sizeof(ngx_wa_metric_val_t) * metrics->workers;
@@ -316,7 +318,7 @@ ngx_wa_metrics_define(ngx_wa_metrics_t *metrics, ngx_str_t *name,
     m->type = type;
 
     if (type == NGX_WA_METRIC_HISTOGRAM) {
-        rc = ngx_wa_metrics_histogram_add_locked(metrics, m);
+        rc = ngx_wa_metrics_histogram_add_locked(metrics, bins, n_bins,  m);
         if (rc != NGX_OK) {
             goto error;
         }

src/common/metrics/ngx_wa_metrics.h

@@ -29,6 +29,12 @@ typedef enum {
 } ngx_wa_metric_type_e;
 
 
+typedef enum {
+    NGX_WA_HISTOGRAM_LOG2,
+    NGX_WA_HISTOGRAM_CUSTOM,
+} ngx_wa_histogram_type_e;
+
+
 typedef struct {
     ngx_uint_t                   value;
     ngx_msec_t                   last_update;
@@ -42,6 +48,7 @@ typedef struct {
 
 
 typedef struct {
+    ngx_wa_histogram_type_e      h_type;
     uint8_t                      n_bins;
     uint64_t                     sum;
     ngx_wa_metrics_bin_t         bins[];
@@ -84,7 +91,7 @@ char *ngx_wa_metrics_init_conf(ngx_conf_t *cf);
 ngx_int_t ngx_wa_metrics_shm_init(ngx_cycle_t *cycle);
 
 ngx_int_t ngx_wa_metrics_define(ngx_wa_metrics_t *metrics, ngx_str_t *name,
-    ngx_wa_metric_type_e type, uint32_t *out);
+    ngx_wa_metric_type_e type, uint32_t *bins, uint16_t n_bins, uint32_t *out);
 ngx_int_t ngx_wa_metrics_increment(ngx_wa_metrics_t *metrics,
     uint32_t metric_id, ngx_int_t val);
 ngx_int_t ngx_wa_metrics_record(ngx_wa_metrics_t *metrics, uint32_t metric_id,
@@ -93,7 +100,7 @@ ngx_int_t ngx_wa_metrics_get(ngx_wa_metrics_t *metrics, uint32_t metric_id,
     ngx_wa_metric_t *o);
 
 ngx_int_t ngx_wa_metrics_histogram_add_locked(ngx_wa_metrics_t *metrics,
-    ngx_wa_metric_t *m);
+    uint32_t *bins, uint16_t n_bins, ngx_wa_metric_t *m);
 ngx_int_t ngx_wa_metrics_histogram_record(ngx_wa_metrics_t *metrics,
     ngx_wa_metric_t *m, ngx_uint_t slot, uint32_t mid, ngx_uint_t n);
 void ngx_wa_metrics_histogram_get(ngx_wa_metrics_t *metrics, ngx_wa_metric_t *m,

src/common/proxy_wasm/ngx_proxy_wasm_host.c

@@ -1638,7 +1638,7 @@ ngx_proxy_wasm_hfuncs_define_metric(ngx_wavm_instance_t *instance,
     prefixed_name.len = ngx_sprintf(buf, "pw.%V.%V", filter_name, &name)
                         - buf;
 
-    rc = ngx_wa_metrics_define(metrics, &prefixed_name, type, id);
+    rc = ngx_wa_metrics_define(metrics, &prefixed_name, type, NULL, 0, id);
     switch (rc) {
     case NGX_ERROR:
         ngx_sprintf(trapmsg, "could not define metric \"%*s\": "