From ff2f99f4e86774e8707bb41feaa8aabfd4c49c48 Mon Sep 17 00:00:00 2001 From: Caio Ramos Casimiro Date: Fri, 5 Apr 2024 19:14:33 +0100 Subject: [PATCH] [WIP] feat(proxy-wasm/metrics) counters and gauges --- config | 6 +- docs/DIRECTIVES.md | 16 + docs/PROXY_WASM.md | 8 +- src/common/ngx_wa_metrics.c | 490 ++++++++++++++++++ src/common/ngx_wa_metrics.h | 45 ++ src/common/proxy_wasm/ngx_proxy_wasm.h | 2 - src/common/proxy_wasm/ngx_proxy_wasm_host.c | 153 +++++- src/common/shm/ngx_wasm_shm.h | 1 + src/common/shm/ngx_wasm_shm_kv.c | 23 +- src/common/shm/ngx_wasm_shm_kv.h | 20 + src/wasm/ngx_wasm.h | 12 +- src/wasm/ngx_wasm_core_module.c | 45 ++ src/wasm/ngx_wasm_directives.c | 101 +++- t/01-wasm/directives/011-metrics_directives.t | 57 ++ .../hfuncs/metrics/100-define_metric.t | 156 ++++++ .../metrics/101-define_metric_edge_cases.t | 248 +++++++++ .../hfuncs/metrics/200-increment_metric.t | 170 ++++++ .../metrics/201-increment_metric_misuse.t | 214 ++++++++ .../hfuncs/metrics/300-record_metric.t | 167 ++++++ .../hfuncs/metrics/301-record_metric_misuse.t | 214 ++++++++ t/07-metrics/001-metrics_sighup.t | 109 ++++ t/TestWasmX.pm | 18 + .../proxy-wasm-tests/hostcalls/src/filter.rs | 5 +- t/lib/proxy-wasm-tests/hostcalls/src/lib.rs | 27 +- .../hostcalls/src/tests/mod.rs | 93 ++++ .../hostcalls/src/types/mod.rs | 21 + .../hostcalls/src/types/test_http.rs | 8 + .../hostcalls/src/types/test_root.rs | 1 + util/setup_dev.sh | 24 + 29 files changed, 2394 insertions(+), 60 deletions(-) create mode 100644 src/common/ngx_wa_metrics.c create mode 100644 src/common/ngx_wa_metrics.h create mode 100644 t/01-wasm/directives/011-metrics_directives.t create mode 100644 t/03-proxy_wasm/hfuncs/metrics/100-define_metric.t create mode 100644 t/03-proxy_wasm/hfuncs/metrics/101-define_metric_edge_cases.t create mode 100644 t/03-proxy_wasm/hfuncs/metrics/200-increment_metric.t create mode 100644 t/03-proxy_wasm/hfuncs/metrics/201-increment_metric_misuse.t create mode 100644 t/03-proxy_wasm/hfuncs/metrics/300-record_metric.t create mode 100644 t/03-proxy_wasm/hfuncs/metrics/301-record_metric_misuse.t create mode 100644 t/07-metrics/001-metrics_sighup.t diff --git a/config b/config index dd1c748ee..0ff846bbb 100644 --- a/config +++ b/config @@ -141,7 +141,8 @@ NGX_WASMX_DEPS="\ $ngx_addon_dir/src/common/proxy_wasm/ngx_proxy_wasm_properties.h \ $ngx_addon_dir/src/common/shm/ngx_wasm_shm.h \ $ngx_addon_dir/src/common/shm/ngx_wasm_shm_kv.h \ - $ngx_addon_dir/src/common/shm/ngx_wasm_shm_queue.h" + $ngx_addon_dir/src/common/shm/ngx_wasm_shm_queue.h \ + $ngx_addon_dir/src/common/ngx_wa_metrics.h" NGX_WASMX_SRCS="\ $ngx_addon_dir/src/ngx_wasmx.c \ @@ -155,7 +156,8 @@ NGX_WASMX_SRCS="\ $ngx_addon_dir/src/common/proxy_wasm/ngx_proxy_wasm_util.c \ $ngx_addon_dir/src/common/shm/ngx_wasm_shm.c \ $ngx_addon_dir/src/common/shm/ngx_wasm_shm_kv.c \ - $ngx_addon_dir/src/common/shm/ngx_wasm_shm_queue.c" + $ngx_addon_dir/src/common/shm/ngx_wasm_shm_queue.c \ + $ngx_addon_dir/src/common/ngx_wa_metrics.c" # wasm diff --git a/docs/DIRECTIVES.md b/docs/DIRECTIVES.md index d328e74ec..76452326f 100644 --- a/docs/DIRECTIVES.md +++ b/docs/DIRECTIVES.md @@ -15,6 +15,7 @@ By alphabetical order: - [resolver_timeout](#resolver_timeout) - [shm_kv](#shm_kv) - [shm_queue](#shm_queue) +- [slab_size](#slab_size) - [socket_buffer_size](#socket_buffer_size) - [socket_buffer_reuse](#socket_buffer_reuse) - [socket_connect_timeout](#socket_connect_timeout) @@ -56,6 +57,8 @@ By context: - [tls_trusted_certificate](#tls_trusted_certificate) - [tls_verify_cert](#tls_verify_cert) - [tls_verify_host](#tls_verify_host) + - `metrics{}` + - [slab_size](#slab_size) - `wasmtime{}` - [flag](#flag) - `wasmer{}` @@ -506,6 +509,19 @@ policy, and writes will fail when the allocated memory slab is full. [Back to TOC](#directives) +shm_queue +--------- + +**usage** | `slab_size ;` +------------:|:---------------------------------------------------------------- +**contexts** | `metrics{}` +**default** | `128m` +**example** | `slab_size 1m;` + +Set the `size` of the shared memory zone dedicated to metrics storage. + +[Back to TOC](#directives) + socket_buffer_reuse ------------------- diff --git a/docs/PROXY_WASM.md b/docs/PROXY_WASM.md index 72495ea17..26bf76c1d 100644 --- a/docs/PROXY_WASM.md +++ b/docs/PROXY_WASM.md @@ -536,10 +536,10 @@ SDK ABI `0.2.1`) and their present status in ngx_wasm_module: `proxy_enqueue_shared_queue` | :heavy_check_mark: | No automatic eviction mechanism if the queue is full. `proxy_resolve_shared_queue` | :x: | *Stats/metrics* | | -`proxy_define_metric` | :x: | -`proxy_get_metric` | :x: | -`proxy_record_metric` | :x: | -`proxy_increment_metric` | :x: | +`proxy_define_metric` | :heavy_check_mark: | Support for histograms NYI. +`proxy_get_metric` | :heavy_check_mark: | +`proxy_record_metric` | :heavy_check_mark: | +`proxy_increment_metric` | :heavy_check_mark: | *Custom extension points* | | `proxy_call_foreign_function` | :x: | diff --git a/src/common/ngx_wa_metrics.c b/src/common/ngx_wa_metrics.c new file mode 100644 index 000000000..9289e34e5 --- /dev/null +++ b/src/common/ngx_wa_metrics.c @@ -0,0 +1,490 @@ +#ifndef DDEBUG +#define DDEBUG 0 +#endif +#include "ddebug.h" + +#include +#include +#include + + +typedef struct { + ngx_uint_t value; + ngx_msec_t last_update; +} ngx_wa_metrics_gauge_t; + + +typedef union { + ngx_uint_t counter; + ngx_wa_metrics_gauge_t gauge; +} ngx_wa_metric_val_t; + + +typedef struct { + ngx_wa_metric_type_e type; + ngx_wa_metric_val_t slots[]; +} ngx_wa_metric_t; + + +static ngx_str_t * +ngx_wa_metrics_type_name(ngx_wa_metric_type_e type) +{ + static ngx_str_t counter = ngx_string("counter"); + static ngx_str_t gauge = ngx_string("gauge"); + static ngx_str_t unknown = ngx_string("unknown"); + + switch (type) { + case NGX_WA_METRIC_COUNTER: + return &counter; + + case NGX_WA_METRIC_GAUGE: + return &gauge; + + default: + return &unknown; + } +} + + +ngx_wa_metrics_t * +ngx_wa_metrics_create(ngx_cycle_t *cycle, ngx_wa_metrics_conf_t *metrics_conf) +{ + ngx_cycle_t *old_cycle = cycle->old_cycle; + static ngx_str_t shm_name = ngx_string("metrics"); + ngx_core_conf_t *ccf, *old_ccf; + ngx_wa_metrics_t *metrics, *old_metrics; + + old_ccf = NULL; + old_metrics = NULL; + ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module); + + if (old_cycle->conf_ctx) { + /* reconfiguring */ + old_metrics = ngx_wasm_core_metrics(old_cycle); + old_ccf = (ngx_core_conf_t *) ngx_get_conf(old_cycle->conf_ctx, + ngx_core_module); + } + + metrics = ngx_pcalloc(cycle->pool, sizeof(ngx_wa_metrics_t)); + if (metrics == NULL) { + ngx_wasm_log_error(NGX_LOG_ERR, cycle->log, 0, + "failed allocating metrics structure"); + + return NULL; + } + + metrics->config = metrics_conf; + metrics->config->workers = ccf->worker_processes; + metrics->config->slab_size = 0; + metrics->config->noreuse = false; + + /* TODO: add an `eviction` field to ngx_wa_metrics_conf_t; add an `eviction` + * directive to `metrics` block that updates the `eviction` field. Check + * for the field's value here to decide on reusing shared memory when + * reconfiguring Nginx. + * + * If eviction != NGX_WASM_SHM_EVICTION_NONE, it means the user wants + * less recently used metrics to be evicted when out of space for a new + * metric. + * + * `noreuse = true` means the shared memory space occupied by the metrics + * will be destroyed and recreated during reconfiguration. + * + * By default, if no eviction is set for metrics, their values will be + * stored in slots, 1 slot for each worker. Each Nginx worker only ever + * updates the portion of a metric's value stored in its own slot. + * + * Fot example, a counter defined in a system running 2 workers has its + * value stored as an uint array of two positions, e.g. `[0, 0]`. If worker + * `w0` updates the counter its value becomes `[1, 0]`; if worker `w1` does + * the same the value becomes `[1, 1]`. The counter value can be simply + * computed by summing the integers in the array. + * + * Since two workers never write to the same memory address, no + * locking mechanism needs to be employed to synchronize write/read + * operations. + * + * This imples, however, that if Nginx is reconfigured with an increased + * amount of `worker_processes`, metrics previously defined need to be + * reallocated to assure the slots array has a position for each of the + * additional workers. + * + * The lock-less approach is, however, incompatible with the eviction + * strategies implemented by `ngx_wasm_shm`. This is because retrieving + * an item from the kv store configured with eviction (LRU/SLRU) implies + * updating the queue of recently used items. Since two workers might + * potentially update the LRU/SLRU queues concurrently, this operation needs + * to be sinchronized by acquiring locks. + */ + if (old_metrics && ccf->worker_processes > old_ccf->worker_processes) { + metrics->config->noreuse = true; + } + + metrics->shm = ngx_pcalloc(cycle->pool, sizeof(ngx_wasm_shm_t)); + if (metrics->shm == NULL) { + ngx_wasm_log_error(NGX_LOG_ERR, cycle->log, 0, + "failed allocating metrics shm structure"); + ngx_pfree(cycle->pool, metrics); + + return NULL; + } + + metrics->shm->name = shm_name; + metrics->shm->type = NGX_WASM_SHM_TYPE_METRICS; + metrics->shm->eviction = NGX_WASM_SHM_EVICTION_NONE; + metrics->shm->data = NULL; + + return metrics; +} + + +ngx_int_t +ngx_wa_metrics_init_conf(ngx_wa_metrics_t *metrics, ngx_conf_t *cf) +{ + if (metrics->config->slab_size == 0) { + metrics->config->slab_size = 5242880; /* 5m */ + } + + metrics->shm_zone = ngx_shared_memory_add(cf, &metrics->shm->name, + metrics->config->slab_size, + &ngx_wasmx_module); + if (metrics->shm_zone == NULL) { + return NGX_ERROR; + } + + metrics->shm_zone->data = metrics->shm; + metrics->shm_zone->init = ngx_wasm_shm_init_zone; + metrics->shm_zone->noreuse = metrics->config->noreuse; + + return NGX_OK; +} + + +static ngx_int_t +ngx_wa_metric_realloc(ngx_wa_metrics_t *metrics, ngx_rbtree_node_t *node, + ngx_rbtree_node_t *sentinel) +{ + ngx_int_t written; + ngx_wasm_shm_kv_node_t *n; + + if (node != sentinel) { + n = (ngx_wasm_shm_kv_node_t *) node; + + ngx_log_debug1(NGX_LOG_DEBUG_WASM, metrics->shm->log, 0, + "reallocating metric \"%V\"", &n->key.str); + + if (ngx_wasm_shm_kv_set_locked(metrics->shm, &n->key.str, &n->value, 0, + &written) != NGX_OK) + { + ngx_wasm_log_error(NGX_LOG_ERR, metrics->shm->log, 0, + "failed reallocating metric \"%V\"", + &n->key.str); + + return NGX_ERROR; + } + + if (node->left + && ngx_wa_metric_realloc(metrics, node->left, sentinel) != NGX_OK) + { + return NGX_ERROR; + } + + if (node->right + && ngx_wa_metric_realloc(metrics, node->right, sentinel) != NGX_OK) + { + return NGX_ERROR; + } + } + + return NGX_OK; +} + + +ngx_int_t +ngx_wa_metrics_init(ngx_wa_metrics_t *metrics, ngx_cycle_t *cycle) +{ + ngx_int_t rc; + ngx_rbtree_node_t *root, *sentinel; + ngx_wasm_shm_kv_t *old_shm_kv = NULL; + ngx_wa_metrics_t *old_metrics = NULL; + + if (cycle->old_cycle->conf_ctx) { + old_metrics = ngx_wasm_core_metrics(cycle->old_cycle); + old_shm_kv = ngx_wasm_shm_get_kv(old_metrics->shm); + } + + metrics->shm->log = cycle->log; + + if (old_shm_kv && !metrics->config->noreuse) { + /* reusing old kv store */ + + metrics->shm->data = old_shm_kv; + return NGX_OK; + } + + /* creating new kv store */ + + rc = ngx_wasm_shm_kv_init(metrics->shm); + if (rc != NGX_OK) { + return rc; + } + + if (old_shm_kv && metrics->config->noreuse) { + /* reallocating metrics due to increased number of workers */ + + root = old_shm_kv->rbtree.root; + sentinel = old_shm_kv->rbtree.sentinel; + + return ngx_wa_metric_realloc(metrics, root, sentinel); + } + + return NGX_OK; +} + + +ngx_int_t +ngx_wa_metrics_add(ngx_wa_metrics_t *metrics, ngx_str_t *name, + ngx_wa_metric_type_e type, uint32_t *out) +{ + uint32_t cas, metric_id; + ngx_uint_t i; + ngx_int_t rc, written; + ngx_str_t val; + ngx_str_t *val_ptr; + ngx_wa_metric_t *metric; + ssize_t size = sizeof(ngx_wa_metric_t) + + sizeof(ngx_wa_metric_val_t) * + metrics->config->workers; + u_char buf[size]; + + rc = NGX_OK; + metric_id = ngx_crc32_long(name->data, name->len); + + ngx_wasm_shm_lock(metrics->shm); + + rc = ngx_wasm_shm_kv_get_locked(metrics->shm, NULL, &metric_id, &val_ptr, + &cas); + if (rc == NGX_OK) { + goto done; + } + + ngx_wasm_log_error(NGX_LOG_INFO, metrics->shm->log, 0, + "metrics: defining %V \"%V\" with id %z", + ngx_wa_metrics_type_name(type), name, metric_id); + + metric = (ngx_wa_metric_t *) buf; + metric->type = type; + + for (i = 0; i < metrics->config->workers; i++) { + switch (type) { + case NGX_WA_METRIC_COUNTER: + metric->slots[i].counter = 0; + break; + + case NGX_WA_METRIC_GAUGE: + metric->slots[i].gauge.value = 0; + metric->slots[i].gauge.last_update = 0; + break; + + default: + rc = NGX_ERROR; + goto error; + } + } + + val.len = size; + val.data = buf; + + rc = ngx_wasm_shm_kv_set_locked(metrics->shm, name, &val, 0, &written); + + if (rc != NGX_OK) { + goto error; + } + +done: + + *out = metric_id; + +error: + + ngx_wasm_shm_unlock(metrics->shm); + + return rc; +} + + +ngx_int_t +ngx_wa_metrics_get(ngx_wa_metrics_t *metrics, uint32_t metric_id, + ngx_uint_t *out) +{ + uint32_t cas; + ngx_uint_t i, val, slots; + ngx_int_t rc; + ngx_msec_t l; + ngx_str_t *n; + ngx_wa_metric_t *metric; + + rc = ngx_wasm_shm_kv_get_locked(metrics->shm, NULL, &metric_id, &n, &cas); + if (rc != NGX_OK) { + return rc; + } + + val = 0; + l = 0; + slots = metrics->config->workers; + +#if 0 + if (metrics->shm->eviction != NGX_WASM_EVICTION_NONE) { + slots = 1; + } +#endif + + metric = (ngx_wa_metric_t *) n->data; + + switch (metric->type) { + case NGX_WA_METRIC_COUNTER: + for (i = 0; i < slots; i++) { + val += metric->slots[i].counter; + + dd("wasm metric %d slot %ld has value %ld", + metric_id, i, metric->slots[i].counter.value); + } + break; + + case NGX_WA_METRIC_GAUGE: + val = metric->slots[0].gauge.value; + l = metric->slots[0].gauge.last_update; + + dd("wasm metric %d slot %ld has value %ld updated %ld", + metric_id, (ngx_uint_t) 0, metric->slots[0].gauge.value, + metric->slots[0].gauge.last_update); + + for (i = 1; i < slots; i++) { + if (metric->slots[i].gauge.last_update > l) { + val = metric->slots[i].gauge.value; + l = metric->slots[i].gauge.last_update; + } + + dd("wasm metric %d slot %ld has value %ld updated %ld", + metric_id, i, metric->slots[i].gauge.value, + metric->slots[i].gauge.last_update); + } + break; + + default: + ngx_wa_assert(0); + return NGX_ERROR; + } + + ngx_log_debug2(NGX_LOG_DEBUG_WASM, metrics->shm->log, 0, + "wasm retrieving metric \"%z\" as %d", metric_id, val); + + *out = val; + + return NGX_OK; +} + + +ngx_int_t +ngx_wa_metrics_increment(ngx_wa_metrics_t *metrics, uint32_t metric_id, + ngx_int_t n) +{ + uint32_t cas; + ngx_int_t rc = NGX_OK; + ngx_uint_t slot = ngx_worker; + ngx_str_t *val; + ngx_wa_metric_t *metric; + +#if 0 + if (metrics->shm->eviction != NGX_WASM_EVICTION_NONE) { + slot = 0; + ngx_wasm_shm_lock(metrics->shm); + } +#endif + + rc = ngx_wasm_shm_kv_get_locked(metrics->shm, NULL, &metric_id, &val, &cas); + if (rc != NGX_OK) { + ngx_log_debug1(NGX_LOG_DEBUG_WASM, metrics->shm->log, 0, + "wasm metric \"%z\" not found", metric_id); + goto error; + } + + metric = (ngx_wa_metric_t *) val->data; + + if (metric->type != NGX_WA_METRIC_COUNTER) { + ngx_wasm_log_error(NGX_LOG_ERR, metrics->shm->log, 0, + "attempt to call increment_metric on a %V; " + "operation not supported", + ngx_wa_metrics_type_name(metric->type)); + rc = NGX_ERROR; + goto error; + } + + ngx_log_debug2(NGX_LOG_DEBUG_WASM, metrics->shm->log, 0, + "wasm updating metric \"%z\" with %d", metric_id, n); + + metric->slots[slot].counter += n; + +error: + +#if 0 + if (metrics->shm->eviction != NGX_WASM_EVICTION_NONE) { + ngx_wasm_shm_unlock(metrics->shm); + } +#endif + + return rc; +} + + +ngx_int_t +ngx_wa_metrics_record(ngx_wa_metrics_t *metrics, uint32_t metric_id, + ngx_int_t n) +{ + uint32_t cas; + ngx_int_t rc = NGX_OK; + ngx_uint_t slot = ngx_worker; + ngx_str_t *val; + ngx_wa_metric_t *metric; + +#if 0 + if (metrics->shm->eviction != NGX_WASM_EVICTION_NONE) { + slot = 0; + ngx_wasm_shm_lock(metrics->shm); + } +#endif + + rc = ngx_wasm_shm_kv_get_locked(metrics->shm, NULL, &metric_id, &val, &cas); + if (rc != NGX_OK) { + ngx_wasm_log_error(NGX_LOG_ERR, metrics->shm->log, 0, + "metrics: \"%V\" not found", metric_id); + + goto error; + } + + metric = (ngx_wa_metric_t *) val->data; + + if (metric->type == NGX_WA_METRIC_COUNTER) { + ngx_wasm_log_error(NGX_LOG_ERR, metrics->shm->log, 0, + "attempt to call record_metric on a counter; " + "operation not supported"); + rc = NGX_ERROR; + goto error; + } + + ngx_log_debug2(NGX_LOG_DEBUG_WASM, metrics->shm->log, 0, + "wasm recording metric \"%z\" with %d", metric_id, n); + + metric->slots[slot].gauge.value = n; + metric->slots[slot].gauge.last_update = ngx_current_msec; + +error: + +#if 0 + if (metrics->shm->eviction != NGX_WASM_EVICTION_NONE) { + ngx_wasm_shm_unlock(metrics->shm); + } +#endif + + return rc; +} diff --git a/src/common/ngx_wa_metrics.h b/src/common/ngx_wa_metrics.h new file mode 100644 index 000000000..ffdc91eda --- /dev/null +++ b/src/common/ngx_wa_metrics.h @@ -0,0 +1,45 @@ +#ifndef _NGX_WA_METRICS_H_INCLUDED_ +#define _NGX_WA_METRICS_H_INCLUDED_ + + +#include +#include +#include + + +typedef enum { + NGX_WA_METRIC_COUNTER, + NGX_WA_METRIC_GAUGE, +} ngx_wa_metric_type_e; + + +typedef struct { + size_t slab_size; + ngx_uint_t workers; + bool noreuse; +} ngx_wa_metrics_conf_t; + + +typedef struct { + ngx_shm_zone_t *shm_zone; + ngx_wasm_shm_t *shm; + ngx_wa_metrics_conf_t *config; +} ngx_wa_metrics_t; + + +ngx_wa_metrics_t *ngx_wasm_core_metrics(ngx_cycle_t *cycle); + +ngx_wa_metrics_t *ngx_wa_metrics_create(ngx_cycle_t *cycle, + ngx_wa_metrics_conf_t *metrics_conf); +ngx_int_t ngx_wa_metrics_init_conf(ngx_wa_metrics_t *metrics, ngx_conf_t *cf); +ngx_int_t ngx_wa_metrics_init(ngx_wa_metrics_t *metrics, ngx_cycle_t *cycle); +ngx_int_t ngx_wa_metrics_add(ngx_wa_metrics_t *metrics, ngx_str_t *name, + ngx_wa_metric_type_e type, uint32_t *out); +ngx_int_t ngx_wa_metrics_get(ngx_wa_metrics_t *metrics, uint32_t metric_id, + ngx_uint_t *out); +ngx_int_t ngx_wa_metrics_increment(ngx_wa_metrics_t *metrics, + uint32_t metric_id, ngx_int_t val); +ngx_int_t ngx_wa_metrics_record(ngx_wa_metrics_t *metrics, uint32_t metric_id, + ngx_int_t val); + +#endif /* _NGX_WA_METRICS_H_INCLUDED_ */ diff --git a/src/common/proxy_wasm/ngx_proxy_wasm.h b/src/common/proxy_wasm/ngx_proxy_wasm.h index 2daf69092..408054d62 100644 --- a/src/common/proxy_wasm/ngx_proxy_wasm.h +++ b/src/common/proxy_wasm/ngx_proxy_wasm.h @@ -141,13 +141,11 @@ typedef enum { } ngx_proxy_wasm_map_type_e; -#if 0 typedef enum { NGX_PROXY_WASM_METRIC_COUNTER = 0, NGX_PROXY_WASM_METRIC_GAUGE = 1, NGX_PROXY_WASM_METRIC_HISTOGRAM = 2, } ngx_proxy_wasm_metric_type_e; -#endif typedef struct ngx_proxy_wasm_ctx_s ngx_proxy_wasm_ctx_t; diff --git a/src/common/proxy_wasm/ngx_proxy_wasm_host.c b/src/common/proxy_wasm/ngx_proxy_wasm_host.c index 573812917..56476b06e 100644 --- a/src/common/proxy_wasm/ngx_proxy_wasm_host.c +++ b/src/common/proxy_wasm/ngx_proxy_wasm_host.c @@ -10,11 +10,15 @@ #include #include #include +#include #ifdef NGX_WASM_HTTP #include #endif +#define NGX_MAX_PROXY_WASM_METRIC_NAME_LEN 256 + + #ifdef NGX_WASM_HTTP static ngx_int_t ngx_proxy_wasm_hfuncs_resume_http_request( ngx_wavm_instance_t *instance, wasm_val_t args[], wasm_val_t rets[]); @@ -1554,7 +1558,146 @@ ngx_proxy_wasm_hfuncs_dequeue_shared_queue(ngx_wavm_instance_t *instance, /* stats/metrics */ -/* NYI */ + + +static ngx_int_t +ngx_proxy_wasm_hfuncs_define_metric(ngx_wavm_instance_t *instance, + wasm_val_t args[], wasm_val_t rets[]) +{ + uint32_t *id; + ngx_str_t name, prefixed_name, *filter_name; + static ngx_str_t prefix = ngx_string("pw"); + ngx_cycle_t *cycle = (ngx_cycle_t *) ngx_cycle; + ngx_wa_metrics_t *metrics = ngx_wasm_core_metrics(cycle); + ngx_wa_metric_type_e metric_type; + ngx_proxy_wasm_exec_t *pwexec; + ngx_proxy_wasm_metric_type_e pw_metric_type; + u_char buf[NGX_MAX_PROXY_WASM_METRIC_NAME_LEN]; + + pwexec = ngx_proxy_wasm_instance2pwexec(instance); + filter_name = pwexec->filter->name; + + pw_metric_type = args[0].of.i32; + switch(pw_metric_type) { + case NGX_PROXY_WASM_METRIC_COUNTER: + metric_type = NGX_WA_METRIC_COUNTER; + break; + + case NGX_PROXY_WASM_METRIC_GAUGE: + metric_type = NGX_WA_METRIC_GAUGE; + break; + + default: + return ngx_proxy_wasm_result_trap(pwexec, "unknown metric type", + rets, + NGX_WAVM_ERROR); + } + + name.len = args[2].of.i32; + prefixed_name.len = prefix.len + filter_name-> len + name.len + 2; + if (prefixed_name.len > NGX_MAX_PROXY_WASM_METRIC_NAME_LEN) { + return ngx_proxy_wasm_result_trap(pwexec, "metric name too long", + rets, + NGX_WAVM_ERROR); + } + + name.data = NGX_WAVM_HOST_LIFT_SLICE(instance, args[1].of.i32, name.len); + prefixed_name.data = buf; + + ngx_sprintf(buf, "%*s.%*s.%*s", prefix.len, prefix.data, + filter_name->len, filter_name->data, + name.len, name.data); + + id = NGX_WAVM_HOST_LIFT_SLICE(instance, args[3].of.i32, sizeof(uint32_t)); + + if (ngx_wa_metrics_add(metrics, &prefixed_name, metric_type, id) \ + != NGX_OK) + { + /* TODO: format with key */ + return ngx_proxy_wasm_result_trap(pwexec, "could not define metric", + rets, + NGX_WAVM_ERROR); + } + + return ngx_proxy_wasm_result_ok(rets); +} + + +static ngx_int_t +ngx_proxy_wasm_hfuncs_increment_metric(ngx_wavm_instance_t *instance, + wasm_val_t args[], wasm_val_t rets[]) +{ + uint32_t metric_id; + ngx_int_t rc, offset; + ngx_cycle_t *cycle = (ngx_cycle_t *) ngx_cycle; + ngx_wa_metrics_t *metrics = ngx_wasm_core_metrics(cycle); + ngx_proxy_wasm_exec_t *pwexec = ngx_proxy_wasm_instance2pwexec(instance); + + metric_id = args[0].of.i32; + offset = args[1].of.i64; + + rc = ngx_wa_metrics_increment(metrics, metric_id, offset); + + if (rc == NGX_ERROR) { + /* TODO: format with key */ + return ngx_proxy_wasm_result_trap(pwexec, "could not increment metric", + rets, NGX_WAVM_ERROR); + } + + return ngx_proxy_wasm_result_ok(rets); +} + + +static ngx_int_t +ngx_proxy_wasm_hfuncs_record_metric(ngx_wavm_instance_t *instance, + wasm_val_t args[], wasm_val_t rets[]) +{ + uint32_t metric_id; + ngx_int_t rc, offset; + ngx_cycle_t *cycle = (ngx_cycle_t *) ngx_cycle; + ngx_wa_metrics_t *metrics = ngx_wasm_core_metrics(cycle); + ngx_proxy_wasm_exec_t *pwexec = ngx_proxy_wasm_instance2pwexec(instance); + + metric_id = args[0].of.i32; + offset = args[1].of.i64; + + rc = ngx_wa_metrics_record(metrics, metric_id, offset); + + if (rc == NGX_ERROR) { + /* TODO: format with key */ + return ngx_proxy_wasm_result_trap(pwexec, + "could not record metric value", rets, + NGX_WAVM_ERROR); + } + + return ngx_proxy_wasm_result_ok(rets); +} + + +static ngx_int_t +ngx_proxy_wasm_hfuncs_get_metric(ngx_wavm_instance_t *instance, + wasm_val_t args[], wasm_val_t rets[]) +{ + uint32_t metric_id; + ngx_int_t rc; + ngx_uint_t *ret_value; + ngx_cycle_t *cycle = (ngx_cycle_t *) ngx_cycle; + ngx_wa_metrics_t *metrics = ngx_wasm_core_metrics(cycle); + ngx_proxy_wasm_exec_t *pwexec = ngx_proxy_wasm_instance2pwexec(instance); + + metric_id = args[0].of.i32; + ret_value = NGX_WAVM_HOST_LIFT(instance, args[1].of.i32, ngx_uint_t); + + rc = ngx_wa_metrics_get(metrics, metric_id, ret_value); + + if (rc == NGX_ERROR) { + /* TODO: format with key */ + return ngx_proxy_wasm_result_trap(pwexec, "could not retrieve metric", + rets, NGX_WAVM_ERROR); + } + + return ngx_proxy_wasm_result_ok(rets); +} /* custom extension points */ @@ -1936,7 +2079,7 @@ static ngx_wavm_host_func_def_t ngx_proxy_wasm_hfuncs[] = { ngx_wavm_arity_i32x4, ngx_wavm_arity_i32 }, { ngx_string("proxy_define_metric"), /* 0.2.0 && 0.2.1 */ - &ngx_proxy_wasm_hfuncs_nop, /* NYI */ + &ngx_proxy_wasm_hfuncs_define_metric, ngx_wavm_arity_i32x4, ngx_wavm_arity_i32 }, { ngx_string("proxy_get_metric_value"), /* vNEXT */ @@ -1944,7 +2087,7 @@ static ngx_wavm_host_func_def_t ngx_proxy_wasm_hfuncs[] = { ngx_wavm_arity_i32x2, ngx_wavm_arity_i32 }, { ngx_string("proxy_get_metric"), /* 0.2.0 && 0.2.1 */ - &ngx_proxy_wasm_hfuncs_nop, /* NYI */ + &ngx_proxy_wasm_hfuncs_get_metric, ngx_wavm_arity_i32x2, ngx_wavm_arity_i32 }, { ngx_string("proxy_set_metric_value"), /* vNEXT */ @@ -1952,7 +2095,7 @@ static ngx_wavm_host_func_def_t ngx_proxy_wasm_hfuncs[] = { ngx_wavm_arity_i32_i64, ngx_wavm_arity_i32 }, { ngx_string("proxy_record_metric"), /* 0.2.0 && 0.2.1 */ - &ngx_proxy_wasm_hfuncs_nop, /* NYI */ + &ngx_proxy_wasm_hfuncs_record_metric, ngx_wavm_arity_i32_i64, ngx_wavm_arity_i32 }, { ngx_string("proxy_increment_metric_value"), /* vNEXT */ @@ -1960,7 +2103,7 @@ static ngx_wavm_host_func_def_t ngx_proxy_wasm_hfuncs[] = { ngx_wavm_arity_i32_i64, ngx_wavm_arity_i32 }, { ngx_string("proxy_increment_metric"), /* 0.2.0 && 0.2.1 */ - &ngx_proxy_wasm_hfuncs_nop, /* NYI */ + &ngx_proxy_wasm_hfuncs_increment_metric, ngx_wavm_arity_i32_i64, ngx_wavm_arity_i32 }, { ngx_string("proxy_delete_metric"), /* vNEXT */ diff --git a/src/common/shm/ngx_wasm_shm.h b/src/common/shm/ngx_wasm_shm.h index 1ae08b43e..f3a885138 100644 --- a/src/common/shm/ngx_wasm_shm.h +++ b/src/common/shm/ngx_wasm_shm.h @@ -11,6 +11,7 @@ typedef enum { NGX_WASM_SHM_TYPE_KV, NGX_WASM_SHM_TYPE_QUEUE, + NGX_WASM_SHM_TYPE_METRICS, } ngx_wasm_shm_type_e; diff --git a/src/common/shm/ngx_wasm_shm_kv.c b/src/common/shm/ngx_wasm_shm_kv.c index 265023d83..94e77cb95 100644 --- a/src/common/shm/ngx_wasm_shm_kv.c +++ b/src/common/shm/ngx_wasm_shm_kv.c @@ -15,28 +15,11 @@ #define NGX_WASM_SLRU_NQUEUES(pool) (NGX_WASM_SLAB_SLOTS(pool) + 1) -typedef struct { - ngx_rbtree_t rbtree; - ngx_rbtree_node_t sentinel; - union { - ngx_queue_t lru_queue; - ngx_queue_t slru_queues[0]; - } eviction; -} ngx_wasm_shm_kv_t; - - -typedef struct { - ngx_str_node_t key; - ngx_str_t value; - uint32_t cas; - ngx_queue_t queue; -} ngx_wasm_shm_kv_node_t; - - -static ngx_inline ngx_wasm_shm_kv_t * +ngx_wasm_shm_kv_t * ngx_wasm_shm_get_kv(ngx_wasm_shm_t *shm) { - ngx_wa_assert(shm->type == NGX_WASM_SHM_TYPE_KV); + ngx_wa_assert(shm->type == NGX_WASM_SHM_TYPE_KV || \ + shm->type == NGX_WASM_SHM_TYPE_METRICS); return shm->data; } diff --git a/src/common/shm/ngx_wasm_shm_kv.h b/src/common/shm/ngx_wasm_shm_kv.h index fe280034f..d711b6b57 100644 --- a/src/common/shm/ngx_wasm_shm_kv.h +++ b/src/common/shm/ngx_wasm_shm_kv.h @@ -5,6 +5,16 @@ #include +typedef struct { + ngx_rbtree_t rbtree; + ngx_rbtree_node_t sentinel; + union { + ngx_queue_t lru_queue; + ngx_queue_t slru_queues[0]; + } eviction; +} ngx_wasm_shm_kv_t; + + typedef struct { ngx_str_t namespace; ngx_str_t key; @@ -13,6 +23,16 @@ typedef struct { } ngx_wasm_shm_kv_key_t; +typedef struct { + ngx_str_node_t key; + ngx_str_t value; + uint32_t cas; + ngx_queue_t queue; +} ngx_wasm_shm_kv_node_t; + + +ngx_wasm_shm_kv_t * ngx_wasm_shm_get_kv(ngx_wasm_shm_t *shm); + ngx_int_t ngx_wasm_shm_kv_init(ngx_wasm_shm_t *shm); ngx_int_t ngx_wasm_shm_kv_get_locked(ngx_wasm_shm_t *shm, ngx_str_t *key, uint32_t *key_hash, ngx_str_t **value_out, uint32_t *cas); diff --git a/src/wasm/ngx_wasm.h b/src/wasm/ngx_wasm.h index d832f026a..df065f598 100644 --- a/src/wasm/ngx_wasm.h +++ b/src/wasm/ngx_wasm.h @@ -5,6 +5,7 @@ #include #include #include +#include #if (NGX_SSL) #include #endif @@ -18,6 +19,7 @@ #define NGX_WASMTIME_CONF 0x00001000 #define NGX_WASMER_CONF 0x00002000 #define NGX_V8_CONF 0x00004000 +#define NGX_METRICS_CONF 0x00006000 #define NGX_WASM_DONE_PHASE 15 #define NGX_WASM_BACKGROUND_PHASE 16 @@ -83,6 +85,8 @@ typedef struct { ngx_wavm_t *vm; ngx_wavm_conf_t vm_conf; ngx_array_t shms; /* element: ngx_wasm_shm_mapping_t */ + ngx_wa_metrics_t *metrics; + ngx_wa_metrics_conf_t metrics_conf; #if (NGX_SSL) ngx_wasm_ssl_conf_t ssl_conf; #endif @@ -125,12 +129,16 @@ void ngx_wasm_log_error(ngx_uint_t level, ngx_log_t *log, ngx_err_t err, void swap_modules_if_needed(ngx_conf_t *cf, const char *m1, const char *m2); #endif -/* directives */ +/* blocks */ char *ngx_wasm_core_wasmtime_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); char *ngx_wasm_core_wasmer_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); char *ngx_wasm_core_v8_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +char *ngx_wasm_core_metrics_block(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); + +/* directives */ char *ngx_wasm_core_flag_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); char *ngx_wasm_core_module_directive(ngx_conf_t *cf, ngx_command_t *cmd, @@ -139,6 +147,8 @@ char *ngx_wasm_core_shm_kv_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); char *ngx_wasm_core_shm_queue_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +char *ngx_wasm_core_metrics_slab_size_directive(ngx_conf_t *cf, + ngx_command_t *cmd, void *conf); char *ngx_wasm_core_resolver_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); char *ngx_wasm_core_pwm_lua_resolver_directive(ngx_conf_t *cf, diff --git a/src/wasm/ngx_wasm_core_module.c b/src/wasm/ngx_wasm_core_module.c index d68c00e27..e9f8952c9 100644 --- a/src/wasm/ngx_wasm_core_module.c +++ b/src/wasm/ngx_wasm_core_module.c @@ -5,6 +5,7 @@ #include #include +#include static void *ngx_wasm_core_create_conf(ngx_conf_t *cf); @@ -43,6 +44,13 @@ static ngx_command_t ngx_wasm_core_commands[] = { 0, NULL }, + { ngx_string("metrics"), + NGX_WASM_CONF|NGX_CONF_BLOCK|NGX_CONF_NOARGS, + ngx_wasm_core_metrics_block, + NGX_WA_WASM_CONF_OFFSET, + 0, + NULL }, + /* directives */ { ngx_string("flag"), @@ -82,6 +90,13 @@ static ngx_command_t ngx_wasm_core_commands[] = { 0, NULL }, + { ngx_string("slab_size"), + NGX_METRICS_CONF|NGX_CONF_TAKE1, + ngx_wasm_core_metrics_slab_size_directive, + NGX_WA_WASM_CONF_OFFSET, + 0, + NULL }, + { ngx_string("shm_queue"), NGX_WASM_CONF|NGX_CONF_TAKE23|NGX_CONF_TAKE4, ngx_wasm_core_shm_queue_directive, @@ -239,6 +254,18 @@ ngx_wasm_core_shms(ngx_cycle_t *cycle) } +ngx_inline ngx_wa_metrics_t * +ngx_wasm_core_metrics(ngx_cycle_t *cycle) +{ + ngx_wasm_core_conf_t *wcf; + + wcf = ngx_wasm_core_cycle_get_conf(cycle); + ngx_wa_assert(wcf); + + return wcf->metrics; +} + + static void ngx_wasm_core_cleanup_pool(void *data) { @@ -316,6 +343,11 @@ ngx_wasm_core_create_conf(ngx_conf_t *cf) return NULL; } + wcf->metrics = ngx_wa_metrics_create(cycle, &wcf->metrics_conf); + if (wcf->metrics == NULL) { + return NULL; + } + cln = ngx_pool_cleanup_add(cycle->pool, 0); if (cln == NULL) { return NULL; @@ -376,6 +408,9 @@ static char * ngx_wasm_core_init_conf(ngx_conf_t *cf, void *conf) { ngx_wasm_core_conf_t *wcf = conf; + ngx_wa_metrics_t *metrics; + + metrics = wcf->metrics; #if (NGX_SSL) if (wcf->ssl_conf.verify_cert == NGX_CONF_UNSET) { @@ -428,6 +463,10 @@ ngx_wasm_core_init_conf(ngx_conf_t *cf, void *conf) wcf->pwm_lua_resolver = 0; } + if (metrics == NULL || ngx_wa_metrics_init_conf(metrics, cf) != NGX_OK) { + return NGX_CONF_ERROR; + } + return NGX_CONF_OK; } @@ -436,6 +475,7 @@ static ngx_int_t ngx_wasm_core_init(ngx_cycle_t *cycle) { ngx_wavm_t *vm; + ngx_wa_metrics_t *metrics; ngx_wasm_core_conf_t *wcf; ngx_log_debug0(NGX_LOG_DEBUG_WASM, cycle->log, 0, @@ -447,6 +487,7 @@ ngx_wasm_core_init(ngx_cycle_t *cycle) } vm = wcf->vm; + metrics = wcf->metrics; if (vm && ngx_wavm_init(vm) != NGX_OK) { return NGX_ERROR; @@ -456,6 +497,10 @@ ngx_wasm_core_init(ngx_cycle_t *cycle) return NGX_ERROR; } + if (metrics && ngx_wa_metrics_init(metrics, cycle) != NGX_OK) { + return NGX_ERROR; + } + #if (NGX_SSL) if (ngx_wasm_core_init_ssl(cycle) != NGX_OK) { ngx_wavm_destroy(vm); diff --git a/src/wasm/ngx_wasm_directives.c b/src/wasm/ngx_wasm_directives.c index 5437eae08..f3d7a4fc4 100644 --- a/src/wasm/ngx_wasm_directives.c +++ b/src/wasm/ngx_wasm_directives.c @@ -7,6 +7,10 @@ #include +static ngx_int_t ngx_wasm_core_shm_validate_size(ngx_conf_t *cf, ssize_t size, + ngx_str_t *value); + + static char * ngx_wasm_core_runtime_block(ngx_conf_t *cf, ngx_uint_t cmd_type) { @@ -45,6 +49,23 @@ ngx_wasm_core_v8_block(ngx_conf_t *cf, ngx_command_t *cmd, void *dummy) } +char * +ngx_wasm_core_metrics_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + char *rv; + ngx_conf_t save = *cf; + + cf->cmd_type = NGX_METRICS_CONF; + cf->module_type = NGX_WASM_MODULE; + + rv = ngx_conf_parse(cf, NULL); + + *cf = save; + + return rv; +} + + static ngx_int_t ngx_wasm_core_current_runtime_flag(ngx_conf_t *cf) { @@ -109,6 +130,35 @@ ngx_wasm_core_flag_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } +static ngx_int_t +ngx_wasm_core_shm_validate_size(ngx_conf_t *cf, ssize_t size, ngx_str_t *value) +{ + const ssize_t min_size = 3 * ngx_pagesize; + + if (size == NGX_ERROR) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "[wasm] invalid shm size \"%V\"", value); + return NGX_ERROR; + } + + if (size < min_size) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "[wasm] shm size of %d bytes is too small, " + "minimum required is %d bytes", size, min_size); + return NGX_ERROR; + } + + if ((size & (ngx_pagesize - 1)) != 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "[wasm] shm size of %d bytes is not page-aligned, " + "must be a multiple of %d", size, ngx_pagesize); + return NGX_ERROR; + } + + return NGX_OK; +} + + static char * ngx_wasm_core_shm_generic_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf, ngx_wasm_shm_type_e type) @@ -120,7 +170,6 @@ ngx_wasm_core_shm_generic_directive(ngx_conf_t *cf, ngx_command_t *cmd, ngx_wasm_shm_mapping_t *mapping; ngx_wasm_shm_t *shm; ngx_wasm_shm_eviction_e eviction; - const ssize_t min_size = 3 * ngx_pagesize; value = cf->args->elts; name = &value[1]; @@ -134,23 +183,7 @@ ngx_wasm_core_shm_generic_directive(ngx_conf_t *cf, ngx_command_t *cmd, return NGX_CONF_ERROR; } - if (size == NGX_ERROR) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "[wasm] invalid shm size \"%V\"", &value[2]); - return NGX_CONF_ERROR; - } - - if (size < min_size) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "[wasm] shm size of %d bytes is too small, " - "minimum required is %d bytes", size, min_size); - return NGX_CONF_ERROR; - } - - if ((size & (ngx_pagesize - 1)) != 0) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "[wasm] shm size of %d bytes is not page-aligned, " - "must be a multiple of %d", size, ngx_pagesize); + if (ngx_wasm_core_shm_validate_size(cf, size, &value[2]) != NGX_OK) { return NGX_CONF_ERROR; } @@ -295,6 +328,38 @@ ngx_wasm_core_shm_queue_directive(ngx_conf_t *cf, ngx_command_t *cmd, } +char * +ngx_wasm_core_metrics_slab_size_directive(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf) +{ + ssize_t size; + ngx_str_t *value; + ngx_wasm_core_conf_t *wcf = conf; + + if (cf->cmd_type != NGX_METRICS_CONF) { + return NGX_CONF_ERROR; + } + + if (wcf->metrics_conf.slab_size > 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "[wasm] slab_size already defined"); + + return NGX_CONF_ERROR; + } + + value = cf->args->elts; + size = ngx_parse_size(&value[1]); + + if (ngx_wasm_core_shm_validate_size(cf, size, &value[1]) != NGX_OK) { + return NGX_CONF_ERROR; + } + + wcf->metrics_conf.slab_size = size; + + return NGX_CONF_OK; +} + + char * ngx_wasm_core_resolver_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { diff --git a/t/01-wasm/directives/011-metrics_directives.t b/t/01-wasm/directives/011-metrics_directives.t new file mode 100644 index 000000000..aab82b64b --- /dev/null +++ b/t/01-wasm/directives/011-metrics_directives.t @@ -0,0 +1,57 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +plan_tests(5); +run_tests(); + +__DATA__ + +=== TEST 1: slab_size directive - sanity +--- main_config + wasm { + metrics { + slab_size 12k; + } + } +--- no_error_log +[error] +[crit] +[emerg] +[stub] + + + +=== TEST 2: slab_size directive - too small +--- main_config + wasm { + metrics { + slab_size 1k; + } + } +--- error_log eval +qr/\[emerg\] .*? \[wasm\] shm size of \d+ bytes is too small, minimum required is 12288 bytes/ +--- no_error_log +[error] +[crit] +[stub] +--- must_die + + + +=== TEST 3: slab_size directive - invalid size +--- main_config + wasm { + metrics { + slab_size 1x; + } + } +--- error_log eval +qr/\[emerg\] .*? \[wasm\] invalid shm size "1x"/ +--- no_error_log +[error] +[crit] +[stub] +--- must_die diff --git a/t/03-proxy_wasm/hfuncs/metrics/100-define_metric.t b/t/03-proxy_wasm/hfuncs/metrics/100-define_metric.t new file mode 100644 index 000000000..4180441f0 --- /dev/null +++ b/t/03-proxy_wasm/hfuncs/metrics/100-define_metric.t @@ -0,0 +1,156 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +our $workers = 4; +our $metrics = join(',', qw( + c1 + g1 +)); + +workers($workers); +if ($workers > 1) { + master_on(); +} + +plan_tests(6); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm metrics - define_metric() - on_configure +Hostcalls filter prefixes the name of a metric with the phase in which it's +defined. A metric c1 defined within on_configure ends up named Configure_c1. + +--- valgrind +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + metrics=$::metrics'; + echo ok; + } +} +--- grep_error_log eval: qr/defined metric \w+ as \d+ at \w+/ +--- grep_error_log_out eval +qr/defined metric Configure_c1 as \d+ at Configure +defined metric Configure_g1 as \d+ at Configure\n/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 2: proxy_wasm metrics - define_metric() - on_tick +--- valgrind +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_tick=define_metrics \ + tick_period=500 \ + metrics=$::metrics'; + echo ok; + } +} +--- grep_error_log eval: qr/defined metric \w+ as \d+ at \w+/ +--- grep_error_log_out eval +qr/defined metric Tick_c1 as \d+ at Tick +defined metric Tick_g1 as \d+ at Tick\n/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 3: proxy_wasm metrics - define_metric() - on: request_headers, request_body, response_headers, response_body +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +my $phases = CORE::join(',', qw( + request_headers + request_body + response_headers + response_body +)); + +qq{ + location /t { + proxy_wasm hostcalls 'on=$phases \ + test=/t/metrics/define \ + metrics=$::metrics'; + echo ok; + } +} +--- request +POST /t +hello +--- grep_error_log eval: qr/defined metric \w+ as \d+ at \w+/ +--- grep_error_log_out eval +my $checks; +my @phases = qw( + RequestHeaders + RequestBody + ResponseHeaders + ResponseBody + ResponseBody +); + +foreach my $p (@phases) { + my $prefixed_c1 = $p . "_c1"; + my $prefixed_g1 = $p . "_g1"; + $checks .= " +?defined metric $prefixed_c1 as [0-9]+ at $p +defined metric $prefixed_g1 as [0-9]+ at $p\n"; +} + +qr/$checks/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 4: proxy_wasm metrics - define_metric() - on_http_call_response +--- valgrind +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- http_config eval +--- config eval +qq{ + listen unix:$ENV{TEST_NGINX_UNIX_SOCKET}; + + location /dispatched { + return 200 "Hello back"; + } + + location /t { + proxy_wasm hostcalls 'test=/t/dispatch_http_call \ + host=unix:$ENV{TEST_NGINX_UNIX_SOCKET} \ + path=/dispatched \ + on_http_call_response=define_metrics \ + metrics=$::metrics'; + echo ok; + } +} +--- grep_error_log eval: qr/defined metric \w+ as \d+ at \w+/ +--- grep_error_log_out eval +qr/defined metric HTTPCallResponse_c1 as \d+ at HTTPCallResponse +defined metric HTTPCallResponse_g1 as \d+ at HTTPCallResponse\n/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] diff --git a/t/03-proxy_wasm/hfuncs/metrics/101-define_metric_edge_cases.t b/t/03-proxy_wasm/hfuncs/metrics/101-define_metric_edge_cases.t new file mode 100644 index 000000000..020454720 --- /dev/null +++ b/t/03-proxy_wasm/hfuncs/metrics/101-define_metric_edge_cases.t @@ -0,0 +1,248 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +our @m = (); +foreach my $n ( 0 .. 16) { + push(@m, "c$n"); +} + +our $metrics = join(',', @m); + +plan_tests(4); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm metrics - define_metric() - on_configure, no memory +When ran with Valgrind, Nginx dies returning 2. + +--- main_config eval +qq{ + wasm { + module hostcalls $ENV{TEST_NGINX_CRATES_DIR}/hostcalls.wasm; + + metrics { + slab_size 12k; + } + } +} +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + metrics=$::metrics'; + } +} +--- error_log eval +[ + qr/\[crit\] .+ \[wasm\] "metrics" shm store: no memory; cannot allocate pair with key size \d+ and value size \d+/, + qr/host trap \(internal error\): could not define metric .+/, + qr/\[emerg\] .+ \[proxy-wasm\] failed initializing "hostcalls" filter \(on_configure failure\)/, +] +--- must_die: 0 + + + +=== TEST 2: proxy_wasm metrics - define_metric() - on_tick, no memory +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- main_config eval +qq{ + wasm { + module hostcalls $ENV{TEST_NGINX_CRATES_DIR}/hostcalls.wasm; + + metrics { + slab_size 12k; + } + } +} +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_tick=define_metrics \ + tick_period=100 \ + metrics=$::metrics'; + echo ok; + } +} +--- error_log eval +[ + qr/\[crit\] .+ \[wasm\] "metrics" shm store: no memory; cannot allocate pair with key size \d+ and value size \d+/, + qr/.+on_tick.+/, + qr/host trap \(internal error\): could not define metric .+/, +] + + + +=== TEST 3: proxy_wasm metrics - define_metric() - on: request_headers, no memory +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- main_config eval +qq{ + wasm { + module hostcalls $ENV{TEST_NGINX_CRATES_DIR}/hostcalls.wasm; + + metrics { + slab_size 12k; + } + } +} +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on=request_headers \ + test=/t/metrics/define \ + metrics=$::metrics'; + echo ok; + } +} +--- error_code: 500 +--- error_log eval +[ + qr/\[crit\] .+ \[wasm\] "metrics" shm store: no memory; cannot allocate pair with key size \d+ and value size \d+/, + qr/.+on_request_headers.+/, + qr/host trap \(internal error\): could not define metric .+/, +] + + + +=== TEST 4: proxy_wasm metrics - define_metric() - on: request_body +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- main_config eval +qq{ + wasm { + module hostcalls $ENV{TEST_NGINX_CRATES_DIR}/hostcalls.wasm; + + metrics { + slab_size 12k; + } + } +} +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on=request_body \ + test=/t/metrics/define \ + metrics=$::metrics'; + echo ok; + } +} +--- request +POST /t +hello +--- error_code: 500 +--- error_log eval +[ + qr/\[crit\] .+ \[wasm\] "metrics" shm store: no memory; cannot allocate pair with key size \d+ and value size \d+/, + qr/.+on_request_body.+/, + qr/host trap \(internal error\): could not define metric .+/, +] + + + +=== TEST 5: proxy_wasm metrics - define_metric() - on: response_headers, no memory +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- main_config eval +qq{ + wasm { + module hostcalls $ENV{TEST_NGINX_CRATES_DIR}/hostcalls.wasm; + + metrics { + slab_size 12k; + } + } +} +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on=response_headers \ + test=/t/metrics/define \ + metrics=$::metrics'; + echo ok; + } +} +--- error_code: 500 +--- error_log eval +[ + qr/\[crit\] .+ \[wasm\] "metrics" shm store: no memory; cannot allocate pair with key size \d+ and value size \d+/, + qr/.+on_response_headers.+/, + qr/host trap \(internal error\): could not define metric .+/, +] + + + +=== TEST 6: proxy_wasm metrics - define_metric() - on: response_body, no memory +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- main_config eval +qq{ + wasm { + module hostcalls $ENV{TEST_NGINX_CRATES_DIR}/hostcalls.wasm; + + metrics { + slab_size 12k; + } + } +} +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on=response_body \ + test=/t/metrics/define \ + metrics=$::metrics'; + echo ok; + } +} +--- error_code: 200 +--- error_log eval +[ + qr/\[crit\] .+ \[wasm\] "metrics" shm store: no memory; cannot allocate pair with key size \d+ and value size \d+/, + qr/.+on_response_body.+/, + qr/host trap \(internal error\): could not define metric .+/, +] + + + +=== TEST 7: proxy_wasm metrics - define_metric() - on_http_call_response, no memory +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- main_config eval +qq{ + wasm { + module hostcalls $ENV{TEST_NGINX_CRATES_DIR}/hostcalls.wasm; + + metrics { + slab_size 12k; + } + } +} +--- http_config eval +--- config eval +qq{ + listen unix:$ENV{TEST_NGINX_UNIX_SOCKET}; + + location /dispatched { + return 200 "Hello back"; + } + + location /t { + proxy_wasm hostcalls 'test=/t/dispatch_http_call \ + host=unix:$ENV{TEST_NGINX_UNIX_SOCKET} \ + path=/dispatched \ + on_http_call_response=define_metrics \ + metrics=$::metrics'; + echo ok; + } +} +--- error_code: 500 +--- error_log eval +[ + qr/\[crit\] .+ \[wasm\] "metrics" shm store: no memory; cannot allocate pair with key size \d+ and value size \d+/, + qr/.+on_http_call_response.+/, + qr/host trap \(internal error\): could not define metric .+/, +] diff --git a/t/03-proxy_wasm/hfuncs/metrics/200-increment_metric.t b/t/03-proxy_wasm/hfuncs/metrics/200-increment_metric.t new file mode 100644 index 000000000..dc96e0c06 --- /dev/null +++ b/t/03-proxy_wasm/hfuncs/metrics/200-increment_metric.t @@ -0,0 +1,170 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +skip_hup(); + +our $workers = 4; +our $counters = join(',', qw( + c1 + c2 +)); + +workers($workers); +if ($workers > 1) { + master_on(); +} + +plan_tests(6); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm metrics shm - increment_metric - on_configure +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'metrics=$::counters \ + on_configure=define_and_increment_counters'; + echo ok; + } +} +--- grep_error_log eval: qr/(incremented \w+|\w+: \d+) at \w+/ +--- grep_error_log_out eval +my $check; +$check .= "incremented Configure_c1 at Configure(\n|\n.+\n)"; +$check .= "incremented Configure_c2 at Configure(\n|\n.+\n)"; +$check .= "Configure_c1: $::workers at Configure(\n|\n.+\n)"; +$check .= "Configure_c2: $::workers at Configure(\n|\n.+\n)"; + +qr/$check/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 2: proxy_wasm metrics - increment_metric() - on_tick +--- valgrind +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on_tick=increment_counters \ + tick_period=500 \ + on_worker=0 \ + metrics=$::counters'; + echo ok; + } +} +--- grep_error_log eval: qr/(incremented \w+|\w+: \d+) at \w+/ +--- grep_error_log_out eval +qr/incremented Configure_c1 at Tick +incremented Configure_c2 at Tick +Configure_c1: 1 at Tick +Configure_c2: 1 at Tick\n/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 3: proxy_wasm metrics - increment_metric() - on: request_headers, request_body, response_headers, response_body +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +my $phases = CORE::join(',', qw( + request_headers + request_body + response_headers + response_body +)); + +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on=$phases \ + test=/t/metrics/increment_counters \ + metrics=$::counters'; + echo ok; + } +} +--- request +POST /t +hello +--- grep_error_log eval: qr/(incremented \w+|\w+: \d+) at \w+/ +--- grep_error_log_out eval +my $checks; +my $i = 0; +my @phases = qw( + RequestHeaders + RequestBody + ResponseHeaders + ResponseBody + ResponseBody +); + +foreach my $p (@phases) { + $i++; + $checks .= " +?incremented Configure_c1 at $p +incremented Configure_c2 at $p +Configure_c1: $i at $p +Configure_c2: $i at $p\n"; +} + +qr/$checks/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 4: proxy_wasm metrics - increment_metric() - on_http_call_response +--- valgrind +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- http_config eval +--- config eval +qq{ + listen unix:$ENV{TEST_NGINX_UNIX_SOCKET}; + + location /dispatched { + return 200 "Hello back"; + } + + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + test=/t/dispatch_http_call \ + host=unix:$ENV{TEST_NGINX_UNIX_SOCKET} \ + path=/dispatched \ + on_http_call_response=increment_counters \ + metrics=$::counters'; + echo ok; + } +} +--- grep_error_log eval: qr/(incremented \w+|\w+: \d+) at \w+/ +--- grep_error_log_out eval +qr/incremented Configure_c1 at HTTPCallResponse +incremented Configure_c2 at HTTPCallResponse +Configure_c1: 1 at HTTPCallResponse +Configure_c2: 1 at HTTPCallResponse\n/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] diff --git a/t/03-proxy_wasm/hfuncs/metrics/201-increment_metric_misuse.t b/t/03-proxy_wasm/hfuncs/metrics/201-increment_metric_misuse.t new file mode 100644 index 000000000..d3a611530 --- /dev/null +++ b/t/03-proxy_wasm/hfuncs/metrics/201-increment_metric_misuse.t @@ -0,0 +1,214 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +our $gauges = join(',', qw( + g1 + g2 +)); + +plan_tests(7); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm metrics shm - increment_metric - on_configure, misuse - gauge +When ran with Valgrind, Nginx dies returning 2. + +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'metrics=$::gauges \ + on_configure=define_and_increment_counters'; + } +} +--- error_log eval +[ + qr/\[error\] .+ \[wasm\] attempt to call increment_metric on a gauge; operation not supported/, + qr/host trap \(internal error\): could not increment metric .+/, + qr/\[emerg\] .+ \[proxy-wasm\] failed initializing "hostcalls" filter \(on_configure failure\)/, +] +--- must_die: 0 +--- no_error_log +[crit] +[alert] +[stub] + + + +=== TEST 2: proxy_wasm metrics - increment_metric() - on_tick, misuse - gauge +--- valgrind +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on_tick=increment_counters \ + tick_period=100 \ + metrics=$::gauges'; + echo ok; + } +} +--- error_log eval +[ + qr/\[error\] .+ \[wasm\] attempt to call increment_metric on a gauge; operation not supported/, + qr/.+on_tick.+/, + qr/host trap \(internal error\): could not increment metric .+/, +] +--- no_error_log +[crit] +[emerg] +[alert] + + + +=== TEST 3: proxy_wasm metrics - increment_metric() - request_headers, misuse - gauge +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on=request_headers \ + test=/t/metrics/increment_counters \ + metrics=$::gauges'; + echo ok; + } +} +--- error_code: 500 +--- error_log eval +[ + qr/\[error\] .+ \[wasm\] attempt to call increment_metric on a gauge; operation not supported/, + qr/.+on_request_headers.+/, + qr/host trap \(internal error\): could not increment metric .+/, +] +--- no_error_log +[crit] +[emerg] +[alert] + + + +=== TEST 4: proxy_wasm metrics - increment_metric() - request_body, misuse - gauge +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on=request_body \ + test=/t/metrics/increment_counters \ + metrics=$::gauges'; + echo ok; + } +} +--- request +POST /t +hello +--- error_code: 500 +--- error_log eval +[ + qr/\[error\] .+ \[wasm\] attempt to call increment_metric on a gauge; operation not supported/, + qr/.+on_request_body.+/, + qr/host trap \(internal error\): could not increment metric .+/, +] +--- no_error_log +[crit] +[emerg] +[alert] + + + +=== TEST 5: proxy_wasm metrics - increment_metric() - response_headers, misuse - gauge +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on=response_headers \ + test=/t/metrics/increment_counters \ + metrics=$::gauges'; + echo ok; + } +} +--- error_code: 500 +--- error_log eval +[ + qr/\[error\] .+ \[wasm\] attempt to call increment_metric on a gauge; operation not supported/, + qr/.+on_response_headers.+/, + qr/host trap \(internal error\): could not increment metric .+/, +] +--- no_error_log +[crit] +[emerg] +[alert] + + + +=== TEST 6: proxy_wasm metrics - increment_metric() - response_body, misuse - gauge +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on=response_body \ + test=/t/metrics/increment_counters \ + metrics=$::gauges'; + echo ok; + } +} +--- error_code: 200 +--- error_log eval +[ + qr/\[error\] .+ \[wasm\] attempt to call increment_metric on a gauge; operation not supported/, + qr/.+on_response_body.+/, + qr/host trap \(internal error\): could not increment metric .+/, +] +--- no_error_log +[crit] +[emerg] +[alert] + + + +=== TEST 7: proxy_wasm metrics - increment_metric() - on_http_call_response, misuse - gauge +--- valgrind +--- wasm_modules: hostcalls +--- config eval +qq{ + listen unix:$ENV{TEST_NGINX_UNIX_SOCKET}; + + location /dispatched { + return 200 "Hello back"; + } + + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + test=/t/dispatch_http_call \ + host=unix:$ENV{TEST_NGINX_UNIX_SOCKET} \ + path=/dispatched \ + on_http_call_response=increment_counters \ + metrics=$::gauges'; + } +} +--- error_code: 500 +--- error_log eval +[ + qr/\[error\] .+ \[wasm\] attempt to call increment_metric on a gauge; operation not supported/, + qr/.+on_http_call_response.+/, + qr/host trap \(internal error\): could not increment metric .+/, +] +--- no_error_log +[crit] +[emerg] +[alert] diff --git a/t/03-proxy_wasm/hfuncs/metrics/300-record_metric.t b/t/03-proxy_wasm/hfuncs/metrics/300-record_metric.t new file mode 100644 index 000000000..0fe0e91e9 --- /dev/null +++ b/t/03-proxy_wasm/hfuncs/metrics/300-record_metric.t @@ -0,0 +1,167 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +skip_hup(); + +our $workers = 4; +our $gauges = join(',', qw( + g1 + g2 +)); + +workers($workers); +if ($workers > 1) { + master_on(); +} + +plan_tests(6); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm metrics shm - record_metric, gauges - on_configure +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'metrics=$::gauges \ + on_configure=define_and_toggle_gauges'; + echo ok; + } +} +--- grep_error_log eval: qr/(toggled \w+|\w+: \d+) at \w+/ +--- grep_error_log_out eval +qr/toggled Configure_g1 at Configure +toggled Configure_g2 at Configure +Configure_g1: 1 at Configure +Configure_g2: 1 at Configure\n/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 2: proxy_wasm metrics - record_metric(), gauges - on_tick +--- valgrind +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on_tick=toggle_gauges \ + on_worker=0 \ + tick_period=100 \ + metrics=$::gauges'; + echo ok; + } +} +--- grep_error_log eval: qr/(toggled \w+|\w+: \d+) at \w+/ +--- grep_error_log_out eval +qr/toggled Configure_g1 at Tick +toggled Configure_g2 at Tick +Configure_g1: 1 at Tick +Configure_g2: 1 at Tick\n/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 3: proxy_wasm metrics - record_metric(), gauges - on: request_headers, request_body, response_headers, response_body +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +my $phases = CORE::join(',', qw( + request_headers + request_body + response_headers + response_body +)); + +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on=$phases \ + test=/t/metrics/toggle_gauges \ + metrics=$::gauges'; + echo ok; + } +} +--- request +POST /t +hello +--- grep_error_log eval: qr/(toggled \w+|\w+: \d+) at \w+/ +--- grep_error_log_out eval +my $i = 0; +my $checks; +my @phases = qw( + RequestHeaders + RequestBody + ResponseHeaders + ResponseBody + ResponseBody +); + +foreach my $phase (@phases) { + $i = $i ? 0 : 1; + $checks .= " +?toggled Configure_g1 at $phase +toggled Configure_g2 at $phase +Configure_g1: $i at $phase +Configure_g2: $i at $phase\n"; +} + +qr/$checks/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 4: proxy_wasm metrics - record_metric(), gauges - on_http_call_response +--- valgrind +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- http_config eval +--- config eval +qq{ + listen unix:$ENV{TEST_NGINX_UNIX_SOCKET}; + + location /dispatched { + return 200 "Hello back"; + } + + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + test=/t/dispatch_http_call \ + host=unix:$ENV{TEST_NGINX_UNIX_SOCKET} \ + path=/dispatched \ + on_http_call_response=toggle_gauges \ + metrics=$::gauges'; + echo ok; + } +} +--- grep_error_log eval: qr/(toggled \w+|\w+: \d+) at \w+/ +--- grep_error_log_out eval +qr/toggled Configure_g1 at HTTPCallResponse +toggled Configure_g2 at HTTPCallResponse +Configure_g1: 1 at HTTPCallResponse +Configure_g2: 1 at HTTPCallResponse\n/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] diff --git a/t/03-proxy_wasm/hfuncs/metrics/301-record_metric_misuse.t b/t/03-proxy_wasm/hfuncs/metrics/301-record_metric_misuse.t new file mode 100644 index 000000000..fcf8776c1 --- /dev/null +++ b/t/03-proxy_wasm/hfuncs/metrics/301-record_metric_misuse.t @@ -0,0 +1,214 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +our $counters = join(',', qw( + c1 + c2 +)); + +plan_tests(7); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm metrics shm - record_metric - on_configure, misuse +When ran with Valgrind, Nginx dies returning 2. + +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'metrics=$::counters \ + on_configure=define_and_toggle_gauges'; + } +} +--- error_log eval +[ + qr/\[error\] .+ \[wasm\] attempt to call record_metric on a counter; operation not supported/, + qr/host trap \(internal error\): could not record metric .+/, + qr/\[emerg\] .+ \[proxy-wasm\] failed initializing "hostcalls" filter \(on_configure failure\)/, +] +--- must_die: 0 +--- no_error_log +[crit] +[alert] +[stub] + + + +=== TEST 2: proxy_wasm metrics - record_metric() - on_tick, misuse +--- valgrind +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on_tick=toggle_gauges \ + tick_period=100 \ + metrics=$::counters'; + echo ok; + } +} +--- error_log eval +[ + qr/\[error\] .+ \[wasm\] attempt to call record_metric on a counter; operation not supported/, + qr/.+on_tick.+/, + qr/host trap \(internal error\): could not record metric .+/, +] +--- no_error_log +[crit] +[emerg] +[alert] + + + +=== TEST 3: proxy_wasm metrics - record_metric() - request_headers, misuse +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on=request_headers \ + test=/t/metrics/toggle_gauges \ + metrics=$::counters'; + echo ok; + } +} +--- error_code: 500 +--- error_log eval +[ + qr/\[error\] .+ \[wasm\] attempt to call record_metric on a counter; operation not supported/, + qr/.+on_request_headers.+/, + qr/host trap \(internal error\): could not record metric .+/, +] +--- no_error_log +[crit] +[emerg] +[alert] + + + +=== TEST 4: proxy_wasm metrics - record_metric() - request_body, misuse +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on=request_body \ + test=/t/metrics/toggle_gauges \ + metrics=$::counters'; + echo ok; + } +} +--- request +POST /t +hello +--- error_code: 500 +--- error_log eval +[ + qr/\[error\] .+ \[wasm\] attempt to call record_metric on a counter; operation not supported/, + qr/.+on_request_body.+/, + qr/host trap \(internal error\): could not record metric .+/, +] +--- no_error_log +[crit] +[emerg] +[alert] + + + +=== TEST 5: proxy_wasm metrics - record_metric() - response_headers, misuse +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on=response_headers \ + test=/t/metrics/toggle_gauges \ + metrics=$::counters'; + echo ok; + } +} +--- error_code: 500 +--- error_log eval +[ + qr/\[error\] .+ \[wasm\] attempt to call record_metric on a counter; operation not supported/, + qr/.+on_response_headers.+/, + qr/host trap \(internal error\): could not record metric .+/, +] +--- no_error_log +[crit] +[emerg] +[alert] + + + +=== TEST 6: proxy_wasm metrics - record_metric() - response_body, misuse +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on=response_body \ + test=/t/metrics/toggle_gauges \ + metrics=$::counters'; + echo ok; + } +} +--- error_code: 200 +--- error_log eval +[ + qr/\[error\] .+ \[wasm\] attempt to call record_metric on a counter; operation not supported/, + qr/.+on_response_body.+/, + qr/host trap \(internal error\): could not record metric .+/, +] +--- no_error_log +[crit] +[emerg] +[alert] + + + +=== TEST 7: proxy_wasm metrics - record_metric() - on_http_call_response, misuse +--- valgrind +--- wasm_modules: hostcalls +--- config eval +qq{ + listen unix:$ENV{TEST_NGINX_UNIX_SOCKET}; + + location /dispatched { + return 200 "Hello back"; + } + + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + test=/t/dispatch_http_call \ + host=unix:$ENV{TEST_NGINX_UNIX_SOCKET} \ + path=/dispatched \ + on_http_call_response=toggle_gauges \ + metrics=$::counters'; + } +} +--- error_code: 500 +--- error_log eval +[ + qr/\[error\] .+ \[wasm\] attempt to call record_metric on a counter; operation not supported/, + qr/.+on_http_call_response.+/, + qr/host trap \(internal error\): could not record metric .+/, +] +--- no_error_log +[crit] +[emerg] +[alert] diff --git a/t/07-metrics/001-metrics_sighup.t b/t/07-metrics/001-metrics_sighup.t new file mode 100644 index 000000000..5cf61ebbe --- /dev/null +++ b/t/07-metrics/001-metrics_sighup.t @@ -0,0 +1,109 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: +use strict; +use lib '.'; +use t::TestWasmX; + +skip_no_hup(); + +our $workers = 2; +our $counters = join(',', qw( + c1 + c2 +)); + +workers($workers); +if ($workers > 1) { + master_on(); +} + +no_shuffle(); +plan_tests(8); +run_tests(); + +__DATA__ + +=== TEST 1: SIGHUP metrics - define and increment counters +Evaluating counters values in error_log rather than in response headers as some +worker(s) might not have done their increment by the time response_headers phase +is invoked. + +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_increment_counters \ + metrics=$::counters'; + echo ok; + } +} +--- grep_error_log eval: qr/Configure_c\d: \d+/ +--- grep_error_log_out eval +my $total = $::workers; + +qr/Configure_c1: $total.* +Configure_c2: $total.*/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] +[stub] + + + +=== TEST 2: SIGHUP metrics - shm preserved, no realloc +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_increment_counters \ + on=response_headers \ + test=/t/metrics/get \ + metrics=$::counters'; + echo ok; + } +} +--- response_headers +Configure-c1: 4 +Configure-c2: 4 +--- no_error_log +reallocating metric +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 3: SIGHUP metrics - increased worker_processes - shm preserved, realloc +--- workers: 4 +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_increment_counters \ + on=response_headers \ + test=/t/metrics/get \ + metrics=$::counters'; + echo ok; + } +} +--- response_headers +Configure-c1: 8 +Configure-c2: 8 +--- grep_error_log eval: qr/reallocating metric .+/ +--- grep_error_log_out eval +qr/reallocating metric .*c1.* +reallocating metric .*c2.*/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] +[stub] diff --git a/t/TestWasmX.pm b/t/TestWasmX.pm index c2c3df560..a8d207146 100644 --- a/t/TestWasmX.pm +++ b/t/TestWasmX.pm @@ -29,6 +29,7 @@ our @EXPORT = qw( load_nginx_modules plan_tests skip_hup + skip_no_hup skip_no_ssl skip_no_ipc skip_no_debug @@ -59,6 +60,12 @@ sub skip_hup { } } +sub skip_no_hup { + if ($ENV{TEST_NGINX_USE_HUP} == 0) { + plan(skip_all => "skip in default mode"); + } +} + sub skip_no_ssl { if ($nginxV !~ m/built with \S+SSL/) { plan(skip_all => "SSL support required (NGX_BUILD_SSL=1)"); @@ -201,6 +208,17 @@ add_block_preprocessor(sub { $wasm_config = $wasm_config . (join "\n", @arr); } + # --- metrics + + my $metrics = $block->metrics; + + if (defined $metrics) { + $wasm_config = $wasm_config . + " metrics {\n" . + " slab_size $metrics" . ";\n" . + " }\n"; + } + # --- shm_queue my $shm_queue = $block->shm_queue; diff --git a/t/lib/proxy-wasm-tests/hostcalls/src/filter.rs b/t/lib/proxy-wasm-tests/hostcalls/src/filter.rs index bea8a9391..a5b9e6487 100644 --- a/t/lib/proxy-wasm-tests/hostcalls/src/filter.rs +++ b/t/lib/proxy-wasm-tests/hostcalls/src/filter.rs @@ -1,4 +1,4 @@ -use crate::{test_http::*, types::*}; +use crate::{test_http::*, types::*, tests::*}; use http::StatusCode; use log::*; use proxy_wasm::{traits::*, types::*}; @@ -102,6 +102,9 @@ impl Context for TestHttp { Some(format!("called {} times", self.n_sync_calls + 1).as_str()), ); } + "define_metrics" => test_define_metrics(self, TestPhase::HTTPCallResponse), + "increment_counters" => test_increment_counters(self, TestPhase::HTTPCallResponse), + "toggle_gauges" => test_toggle_gauges(self, TestPhase::HTTPCallResponse), _ => {} } diff --git a/t/lib/proxy-wasm-tests/hostcalls/src/lib.rs b/t/lib/proxy-wasm-tests/hostcalls/src/lib.rs index 783432b53..c0f3f8456 100644 --- a/t/lib/proxy-wasm-tests/hostcalls/src/lib.rs +++ b/t/lib/proxy-wasm-tests/hostcalls/src/lib.rs @@ -7,13 +7,14 @@ mod types; use crate::{tests::*, types::test_http::*, types::test_root::*, types::*}; use log::*; use proxy_wasm::{traits::*, types::*}; -use std::{collections::HashMap, time::Duration}; +use std::{collections::HashMap, collections::BTreeMap, time::Duration}; proxy_wasm::main! {{ proxy_wasm::set_log_level(LogLevel::Trace); proxy_wasm::set_root_context(|_| -> Box { Box::new(TestRoot { config: HashMap::new(), + metrics: BTreeMap::new(), n_sync_calls: 0, }) }); @@ -63,12 +64,19 @@ impl RootContext for TestRoot { )); } - if let Some(on_configure) = self.get_config("on_configure") { - match on_configure { - "do_trap" => panic!("trap on_configure"), - "do_return_false" => return false, - _ => (), - } + match self.get_config("on_configure").unwrap_or("") { + "do_trap" => panic!("trap on_configure"), + "do_return_false" => return false, + "define_metrics" => test_define_metrics(self, TestPhase::Configure), + "define_and_increment_counters" => { + test_define_metrics(self, TestPhase::Configure); + test_increment_counters(self, TestPhase::Configure); + }, + "define_and_toggle_gauges" => { + test_define_metrics(self, TestPhase::Configure); + test_toggle_gauges(self, TestPhase::Configure); + }, + _ => (), } true @@ -86,6 +94,10 @@ impl RootContext for TestRoot { match self.get_config("on_tick").unwrap_or("") { "log_property" => test_log_property(self), + "define_metrics" => test_define_metrics(self, TestPhase::Tick), + "increment_counters" => test_increment_counters(self, TestPhase::Tick), + "toggle_gauges" => test_toggle_gauges(self, TestPhase::Tick), + "log_metrics" => test_log_metrics(self, TestPhase::Tick), "set_property" => test_set_property(self), "dispatch" => { let n_sync_calls = self @@ -172,6 +184,7 @@ impl RootContext for TestRoot { Some(Box::new(TestHttp { config: self.config.clone(), on_phases: phases, + metrics: self.metrics.clone(), n_sync_calls: 0, ncalls: 0, })) diff --git a/t/lib/proxy-wasm-tests/hostcalls/src/tests/mod.rs b/t/lib/proxy-wasm-tests/hostcalls/src/tests/mod.rs index f503a4951..75bdebbde 100644 --- a/t/lib/proxy-wasm-tests/hostcalls/src/tests/mod.rs +++ b/t/lib/proxy-wasm-tests/hostcalls/src/tests/mod.rs @@ -123,6 +123,13 @@ pub(crate) fn test_log_properties(ctx: &(dyn TestContext + 'static), phase: Test } } +pub(crate) fn test_log_metrics(ctx: &(dyn TestContext + 'static), phase: TestPhase) { + for (n, id) in ctx.get_metrics_mapping() { + let value = get_metric(*id).unwrap(); + info!("{}: {} at {:?}", n, value, phase) + } +} + fn show_property(ctx: &(dyn TestContext + 'static), path: &[&str]) -> String { if let Some(p) = ctx.get_property(path.to_vec()) { if let Ok(value) = std::str::from_utf8(&p) { @@ -400,6 +407,92 @@ pub(crate) fn test_set_shared_data_by_len(ctx: &mut TestHttp) { test_set_shared_data(ctx); } +fn should_run_on_current_worker(ctx: &(dyn TestContext + 'static)) -> bool { + match ctx.get_config("on_worker") { + Some(on_worker) => { + let w_id_bytes = ctx.get_property(vec![("worker_id")]) + .expect("could not get worker_id"); + let w_id = std::str::from_utf8(&w_id_bytes) + .expect("bad worker_id value"); + + on_worker == w_id + }, + None => true, + } +} + +pub(crate) fn test_define_metrics(ctx: &mut (dyn TestContext + 'static), phase: TestPhase) { + if !should_run_on_current_worker(ctx) { + return; + } + + let metrics_config = ctx + .get_config("metrics") + .map(|x| x.to_string()) + .expect("missing metrics parameter"); + + for metric in metrics_config.split(",") { + let metric_type = match metric.chars().nth(0) { + Some('c') => MetricType::Counter, + Some('g') => MetricType::Gauge, + _ => panic!("unexpected metric type"), + }; + let name = format!("{:?}_{}", phase, metric); + let m_id = define_metric(metric_type, &name) + .expect("cannot define new metric"); + + info!("defined metric {} as {:?} at {:?}", &name, m_id, phase); + + ctx.save_metric_mapping(name.as_str(), m_id); + } +} + +pub(crate) fn test_increment_counters(ctx: &(dyn TestContext + 'static), phase: TestPhase) { + if !should_run_on_current_worker(ctx) { + return; + } + + let n_increments = ctx + .get_config("n_increments") + .map_or(1, |x| x.parse::().expect("bad n_increments value")); + + for (n, id) in ctx.get_metrics_mapping() { + for _ in 0..n_increments { + increment_metric(*id, 1).unwrap(); + } + + info!("incremented {} at {:?}", n, phase); + } + + test_log_metrics(ctx, phase); +} + +pub(crate) fn test_toggle_gauges(ctx: &(dyn TestContext + 'static), phase: TestPhase) { + if !should_run_on_current_worker(ctx) { + return; + } + + for (n, id) in ctx.get_metrics_mapping() { + let new_value = match get_metric(*id).unwrap() { + 0 => 1, + _ => 0, + }; + + record_metric(*id, new_value).unwrap(); + info!("toggled {} at {:?}", n, phase); + } + + test_log_metrics(ctx, phase); +} + +pub(crate) fn test_get_metrics(ctx: &TestHttp) { + for (n, id) in ctx.get_metrics_mapping() { + let name = n.replace("_", "-"); + let value = get_metric(*id).unwrap().to_string(); + ctx.add_http_response_header(name.as_str(), value.as_str()); + } +} + pub(crate) fn test_shared_queue_enqueue(ctx: &TestHttp) { let queue_id: u32 = ctx .config diff --git a/t/lib/proxy-wasm-tests/hostcalls/src/types/mod.rs b/t/lib/proxy-wasm-tests/hostcalls/src/types/mod.rs index c4f07751e..fc042ecc1 100644 --- a/t/lib/proxy-wasm-tests/hostcalls/src/types/mod.rs +++ b/t/lib/proxy-wasm-tests/hostcalls/src/types/mod.rs @@ -6,16 +6,21 @@ use crate::*; #[derive(Debug, Eq, PartialEq, enum_utils::FromStr)] #[enumeration(rename_all = "snake_case")] pub enum TestPhase { + Configure, + Tick, RequestHeaders, RequestBody, ResponseHeaders, ResponseBody, ResponseTrailers, + HTTPCallResponse, Log, } pub trait TestContext { fn get_config(&self, name: &str) -> Option<&str>; + fn get_metrics_mapping(&self) -> &BTreeMap; + fn save_metric_mapping(&mut self, name: &str, metric_id: u32) -> Option; } impl Context for dyn TestContext {} @@ -24,10 +29,26 @@ impl TestContext for TestRoot { fn get_config(&self, name: &str) -> Option<&str> { self.config.get(name).map(|s| s.as_str()) } + + fn get_metrics_mapping(&self) -> &BTreeMap { + &self.metrics + } + + fn save_metric_mapping(&mut self, name: &str, metric_id: u32) -> Option { + self.metrics.insert(name.to_string(), metric_id) + } } impl TestContext for TestHttp { fn get_config(&self, name: &str) -> Option<&str> { self.config.get(name).map(|s| s.as_str()) } + + fn get_metrics_mapping(&self) -> &BTreeMap { + &self.metrics + } + + fn save_metric_mapping(&mut self, name: &str, metric_id: u32) -> Option { + self.metrics.insert(name.to_string(), metric_id) + } } diff --git a/t/lib/proxy-wasm-tests/hostcalls/src/types/test_http.rs b/t/lib/proxy-wasm-tests/hostcalls/src/types/test_http.rs index 0fda3ce29..c3dcbadcf 100644 --- a/t/lib/proxy-wasm-tests/hostcalls/src/types/test_http.rs +++ b/t/lib/proxy-wasm-tests/hostcalls/src/types/test_http.rs @@ -6,6 +6,7 @@ use proxy_wasm::{traits::*, types::*}; pub struct TestHttp { pub on_phases: Vec, pub config: HashMap, + pub metrics: BTreeMap, pub n_sync_calls: usize, pub ncalls: usize, } @@ -65,6 +66,7 @@ impl TestHttp { "/t/log/response_body" => test_log_response_body(self), "/t/log/property" => test_log_property(self), "/t/log/properties" => test_log_properties(self, cur_phase), + "/t/log/metrics" => test_log_metrics(self, cur_phase), /* send_local_response */ "/t/send_local_response/status/204" => test_send_status(self, 204), @@ -115,6 +117,12 @@ impl TestHttp { "/t/shm/enqueue" => test_shared_queue_enqueue(self), "/t/shm/dequeue" => test_shared_queue_dequeue(self), + /* metrics */ + "/t/metrics/define" => test_define_metrics(self, cur_phase), + "/t/metrics/increment_counters" => test_increment_counters(self, cur_phase), + "/t/metrics/toggle_gauges" => test_toggle_gauges(self, cur_phase), + "/t/metrics/get" => test_get_metrics(self), + /* errors */ "/t/trap" => panic!("custom trap"), "/t/error/get_response_body" => { diff --git a/t/lib/proxy-wasm-tests/hostcalls/src/types/test_root.rs b/t/lib/proxy-wasm-tests/hostcalls/src/types/test_root.rs index 95ad06857..6300c47a2 100644 --- a/t/lib/proxy-wasm-tests/hostcalls/src/types/test_root.rs +++ b/t/lib/proxy-wasm-tests/hostcalls/src/types/test_root.rs @@ -2,5 +2,6 @@ use crate::*; pub struct TestRoot { pub config: HashMap, + pub metrics: BTreeMap, pub n_sync_calls: usize, } diff --git a/util/setup_dev.sh b/util/setup_dev.sh index 02f19fbc2..4b3944401 100755 --- a/util/setup_dev.sh +++ b/util/setup_dev.sh @@ -136,6 +136,30 @@ EOF use_http2 use_http3 env_to_nginx +EOF + patch --forward --ignore-whitespace lib/perl5/Test/Nginx/Util.pm <<'EOF' + @@ -977,6 +977,11 @@ + my $post_main_config = $block->post_main_config; + my $err_log_file = $block->error_log_file; + my $server_name = $block->server_name; + + my $workers = $block->workers; + + + + if (!$workers) { + + $workers = $Workers; + + } + + if ($UseHup) { + master_on(); # config reload is buggy when master is off + @@ -1054,7 +1059,7 @@ + bail_out "Can't open $ConfFile for writing: $!\n"; + print $out "daemon $DaemonEnabled;" if ($DaemonEnabled eq 'off'); + print $out <<_EOC_; + -worker_processes $Workers; + +worker_processes $workers; + master_process $MasterProcessEnabled; + error_log $err_log_file $LogLevel; + pid $PidFile; + EOF set -e popd