diff --git a/config b/config index dd1c748ee..0ff846bbb 100644 --- a/config +++ b/config @@ -141,7 +141,8 @@ NGX_WASMX_DEPS="\ $ngx_addon_dir/src/common/proxy_wasm/ngx_proxy_wasm_properties.h \ $ngx_addon_dir/src/common/shm/ngx_wasm_shm.h \ $ngx_addon_dir/src/common/shm/ngx_wasm_shm_kv.h \ - $ngx_addon_dir/src/common/shm/ngx_wasm_shm_queue.h" + $ngx_addon_dir/src/common/shm/ngx_wasm_shm_queue.h \ + $ngx_addon_dir/src/common/ngx_wa_metrics.h" NGX_WASMX_SRCS="\ $ngx_addon_dir/src/ngx_wasmx.c \ @@ -155,7 +156,8 @@ NGX_WASMX_SRCS="\ $ngx_addon_dir/src/common/proxy_wasm/ngx_proxy_wasm_util.c \ $ngx_addon_dir/src/common/shm/ngx_wasm_shm.c \ $ngx_addon_dir/src/common/shm/ngx_wasm_shm_kv.c \ - $ngx_addon_dir/src/common/shm/ngx_wasm_shm_queue.c" + $ngx_addon_dir/src/common/shm/ngx_wasm_shm_queue.c \ + $ngx_addon_dir/src/common/ngx_wa_metrics.c" # wasm diff --git a/docs/DIRECTIVES.md b/docs/DIRECTIVES.md index 950d17f56..97ae4d9fb 100644 --- a/docs/DIRECTIVES.md +++ b/docs/DIRECTIVES.md @@ -16,6 +16,7 @@ By alphabetical order: - [resolver_timeout](#resolver_timeout) - [shm_kv](#shm_kv) - [shm_queue](#shm_queue) +- [slab_size](#slab_size) - [socket_buffer_size](#socket_buffer_size) - [socket_buffer_reuse](#socket_buffer_reuse) - [socket_connect_timeout](#socket_connect_timeout) @@ -57,6 +58,8 @@ By context: - [tls_trusted_certificate](#tls_trusted_certificate) - [tls_verify_cert](#tls_verify_cert) - [tls_verify_host](#tls_verify_host) + - `metrics{}` + - [slab_size](#slab_size) - `wasmtime{}` - [cache_config](#cache-config) - [flag](#flag) @@ -525,6 +528,19 @@ policy, and writes will fail when the allocated memory slab is full. [Back to TOC](#directives) +shm_queue +--------- + +**usage** | `slab_size ;` +------------:|:---------------------------------------------------------------- +**contexts** | `metrics{}` +**default** | `12m` +**example** | `slab_size 5m;` + +Set the `size` of the shared memory slab dedicated to metrics storage. + +[Back to TOC](#directives) + socket_buffer_reuse ------------------- diff --git a/docs/PROXY_WASM.md b/docs/PROXY_WASM.md index 72495ea17..26bf76c1d 100644 --- a/docs/PROXY_WASM.md +++ b/docs/PROXY_WASM.md @@ -536,10 +536,10 @@ SDK ABI `0.2.1`) and their present status in ngx_wasm_module: `proxy_enqueue_shared_queue` | :heavy_check_mark: | No automatic eviction mechanism if the queue is full. `proxy_resolve_shared_queue` | :x: | *Stats/metrics* | | -`proxy_define_metric` | :x: | -`proxy_get_metric` | :x: | -`proxy_record_metric` | :x: | -`proxy_increment_metric` | :x: | +`proxy_define_metric` | :heavy_check_mark: | Support for histograms NYI. +`proxy_get_metric` | :heavy_check_mark: | +`proxy_record_metric` | :heavy_check_mark: | +`proxy_increment_metric` | :heavy_check_mark: | *Custom extension points* | | `proxy_call_foreign_function` | :x: | diff --git a/src/common/ngx_wa_metrics.c b/src/common/ngx_wa_metrics.c new file mode 100644 index 000000000..fcb59fdee --- /dev/null +++ b/src/common/ngx_wa_metrics.c @@ -0,0 +1,456 @@ +#ifndef DDEBUG +#define DDEBUG 0 +#endif +#include "ddebug.h" + +#include +#include +#include + + +typedef struct { + ngx_uint_t value; + ngx_msec_t last_update; +} ngx_wa_metrics_gauge_t; + + +typedef union { + ngx_uint_t counter; + ngx_wa_metrics_gauge_t gauge; +} ngx_wa_metric_val_t; + + +typedef struct { + ngx_wa_metric_type_e type; + ngx_wa_metric_val_t slots[]; +} ngx_wa_metric_t; + + +static ngx_str_t * +ngx_wa_metrics_type_name(ngx_wa_metric_type_e type) +{ + static ngx_str_t counter = ngx_string("counter"); + static ngx_str_t gauge = ngx_string("gauge"); + static ngx_str_t unknown = ngx_string("unknown"); + + switch (type) { + case NGX_WA_METRIC_COUNTER: + return &counter; + + case NGX_WA_METRIC_GAUGE: + return &gauge; + + default: + return &unknown; + } +} + + +static ngx_int_t +ngx_wa_metric_realloc(ngx_wa_metrics_t *metrics, ngx_rbtree_node_t *node, + ngx_rbtree_node_t *sentinel) +{ + uint32_t mid; + ngx_int_t rc; + ngx_uint_t val; + ngx_wasm_shm_kv_node_t *n = (ngx_wasm_shm_kv_node_t *) node; + ngx_wa_metric_t *m = (ngx_wa_metric_t *) n->value.data; + + if (node == sentinel) { + return NGX_OK; + } + + ngx_log_debug1(NGX_LOG_DEBUG_WASM, metrics->shm->log, 0, + "reallocating metric \"%V\"", &n->key.str); + + if (ngx_wa_metrics_add(metrics, &n->key.str, m->type, &mid) != NGX_OK) { + ngx_wasm_log_error(NGX_LOG_ERR, metrics->shm->log, 0, + "failed redefining metric \"%V\"", &n->key.str); + + return NGX_ERROR; + } + + if (ngx_wa_metrics_get(metrics->old_metrics, mid, &val) != NGX_OK) { + ngx_wasm_log_error(NGX_LOG_ERR, metrics->shm->log, 0, + "failed retrieving metric \"%V\" from old metrics", + &n->key.str); + + return NGX_ERROR; + } + + switch (m->type) { + case NGX_WA_METRIC_COUNTER: + rc = ngx_wa_metrics_increment(metrics, mid, val); + break; + + case NGX_WA_METRIC_GAUGE: + rc = ngx_wa_metrics_record(metrics, mid, val); + break; + + default: + ngx_wa_assert(0); + return NGX_ERROR; + } + + if (rc != NGX_OK) { + ngx_wasm_log_error(NGX_LOG_ERR, metrics->shm->log, 0, + "failed updating metric \"%V\"", &n->key.str); + return NGX_ERROR; + } + + if (node->left + && ngx_wa_metric_realloc(metrics, node->left, sentinel) != NGX_OK) + { + return NGX_ERROR; + } + + if (node->right + && ngx_wa_metric_realloc(metrics, node->right, sentinel) != NGX_OK) + { + return NGX_ERROR; + } + + return NGX_OK; +} + + +ngx_wa_metrics_t * +ngx_wa_metrics_create(ngx_cycle_t *cycle) +{ + static ngx_str_t shm_name = ngx_string("metrics"); + ngx_wa_metrics_t *metrics; + + metrics = ngx_pcalloc(cycle->pool, sizeof(ngx_wa_metrics_t)); + if (metrics == NULL) { + ngx_wasm_log_error(NGX_LOG_ERR, cycle->log, 0, + "failed allocating metrics structure"); + + return NULL; + } + + metrics->workers = 1; + metrics->old_metrics = ngx_wasmx_metrics(); + metrics->config.slab_size = NGX_CONF_UNSET_SIZE; + metrics->config.max_metric_name_length = NGX_CONF_UNSET_SIZE; + + metrics->shm = ngx_pcalloc(cycle->pool, sizeof(ngx_wasm_shm_t)); + if (metrics->shm == NULL) { + ngx_wasm_log_error(NGX_LOG_ERR, cycle->log, 0, + "failed allocating metrics shm structure"); + ngx_pfree(cycle->pool, metrics); + + return NULL; + } + + /* cycle->log is set to `&cycle->new_log` later in ngx_init_cycle */ + metrics->shm->log = &cycle->new_log; + metrics->shm->name = shm_name; + metrics->shm->type = NGX_WASM_SHM_TYPE_METRICS; + metrics->shm->eviction = NGX_WASM_SHM_EVICTION_NONE; + metrics->shm->data = NULL; + + return metrics; +} + + +ngx_int_t +ngx_wa_metrics_init_conf(ngx_wa_metrics_t *metrics, ngx_conf_t *cf) +{ + ngx_cycle_t *cycle = cf->cycle; + ngx_core_conf_t *ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, + ngx_core_module); + ngx_wa_metrics_t *old_metrics = metrics->old_metrics;; + + if (metrics->config.slab_size == NGX_CONF_UNSET_SIZE) { + metrics->config.slab_size = 5242880; /* 5m */ + } + + if (metrics->config.max_metric_name_length == NGX_CONF_UNSET_SIZE) { + metrics->config.max_metric_name_length = 64; + } + + metrics->workers = ccf->worker_processes; + metrics->shm_zone = ngx_shared_memory_add(cf, &metrics->shm->name, + metrics->config.slab_size, + &ngx_wasmx_module); + if (metrics->shm_zone == NULL) { + return NGX_ERROR; + } + + metrics->shm_zone->data = metrics->shm; + metrics->shm_zone->init = ngx_wasm_shm_init_zone; + metrics->shm_zone->noreuse = false; /* reuse shm zone by default */ + + if (old_metrics + && (metrics->workers != old_metrics->workers + || metrics->config.slab_size != old_metrics->config.slab_size)) + { + metrics->shm_zone->noreuse = true; + } + + return NGX_OK; +} + + +ngx_int_t +ngx_wa_metrics_init(ngx_wa_metrics_t *metrics, ngx_cycle_t *cycle) +{ + ngx_int_t rc; + ngx_rbtree_node_t *root, *sentinel; + ngx_wasm_shm_kv_t *old_shm_kv; + + if (metrics->old_metrics && !metrics->shm_zone->noreuse) { + /* reuse old kv store */ + metrics->shm->data = metrics->old_metrics->shm->data; + + return NGX_OK; + } + + rc = ngx_wasm_shm_kv_init(metrics->shm); + if (rc != NGX_OK) { + return rc; + } + + if (metrics->old_metrics && metrics->shm_zone->noreuse) { + /* reallocate metrics due to new number of workers or shm slab resize */ + old_shm_kv = ngx_wasm_shm_get_kv(metrics->old_metrics->shm); + root = old_shm_kv->rbtree.root; + sentinel = old_shm_kv->rbtree.sentinel; + + return ngx_wa_metric_realloc(metrics, root, sentinel); + } + + return NGX_OK; +} + + +ngx_int_t +ngx_wa_metrics_add(ngx_wa_metrics_t *metrics, ngx_str_t *name, + ngx_wa_metric_type_e type, uint32_t *out) +{ + uint32_t cas, mid; + ngx_int_t rc, written; + ngx_str_t *p, val; + ngx_wa_metric_t *metric; + ssize_t size = sizeof(ngx_wa_metric_t) + + sizeof(ngx_wa_metric_val_t) * metrics->workers; + u_char buf[size]; + + mid = ngx_crc32_long(name->data, name->len); + + if (type != NGX_WA_METRIC_COUNTER && type != NGX_WA_METRIC_GAUGE) { + dd("invalid metric type"); + return NGX_ERROR; + } + + if (name->len > metrics->config.max_metric_name_length) { + dd("invalid metric name length"); + return NGX_ERROR; + } + + ngx_wasm_shm_lock(metrics->shm); + + rc = ngx_wasm_shm_kv_get_locked(metrics->shm, NULL, &mid, &p, &cas); + if (rc == NGX_OK) { + goto done; + } + + ngx_memzero(buf, size); + metric = (ngx_wa_metric_t *) buf; + metric->type = type; + + val.len = size; + val.data = buf; + + rc = ngx_wasm_shm_kv_set_locked(metrics->shm, name, &val, 0, &written); + + if (rc != NGX_OK) { + dd("failed setting shm kv value"); + goto error; + } + +done: + + *out = mid; + +error: + + ngx_wasm_shm_unlock(metrics->shm); + + if (rc == NGX_OK) { + ngx_wasm_log_error(NGX_LOG_INFO, metrics->shm->log, 0, + "defined %V \"%V\" with id %uD", + ngx_wa_metrics_type_name(type), name, mid); + } + + return rc; +} + + +ngx_int_t +ngx_wa_metrics_get(ngx_wa_metrics_t *metrics, uint32_t metric_id, + ngx_uint_t *out) +{ + uint32_t cas; + ngx_int_t rc; + ngx_str_t *n; + ngx_msec_t l; + ngx_uint_t i, val, slots; + ngx_wa_metric_t *metric; + + rc = ngx_wasm_shm_kv_get_locked(metrics->shm, NULL, &metric_id, &n, &cas); + if (rc != NGX_OK) { + return rc; + } + + val = 0; + slots = metrics->workers; + +#if 0 + if (metrics->shm->eviction != NGX_WASM_EVICTION_NONE) { + slots = 1; + } +#endif + + metric = (ngx_wa_metric_t *) n->data; + + switch (metric->type) { + case NGX_WA_METRIC_COUNTER: + for (i = 0; i < slots; i++) { + val += metric->slots[i].counter; + } + break; + + case NGX_WA_METRIC_GAUGE: + val = metric->slots[0].gauge.value; + l = metric->slots[0].gauge.last_update; + + for (i = 1; i < slots; i++) { + if (metric->slots[i].gauge.last_update > l) { + val = metric->slots[i].gauge.value; + l = metric->slots[i].gauge.last_update; + } + } + break; + + default: + ngx_wa_assert(0); + return NGX_ERROR; + } + + ngx_log_debug2(NGX_LOG_DEBUG_WASM, metrics->shm->log, 0, + "wasm retrieving metric \"%z\" as %d", metric_id, val); + + *out = val; + + return NGX_OK; +} + + +ngx_int_t +ngx_wa_metrics_increment(ngx_wa_metrics_t *metrics, uint32_t metric_id, + ngx_int_t n) +{ + uint32_t cas; + ngx_int_t rc = NGX_OK; + ngx_str_t *val; + ngx_uint_t slot; + ngx_wa_metric_t *metric; + + slot = (ngx_process == NGX_PROCESS_WORKER) ? ngx_worker : 0; + +#if 0 + if (metrics->shm->eviction != NGX_WASM_EVICTION_NONE) { + slot = 0; + ngx_wasm_shm_lock(metrics->shm); + } +#endif + + rc = ngx_wasm_shm_kv_get_locked(metrics->shm, NULL, &metric_id, &val, &cas); + if (rc != NGX_OK) { + ngx_wasm_log_error(NGX_LOG_ERR, metrics->shm->log, 0, + "metric \"%uD\" not found", metric_id); + goto error; + } + + metric = (ngx_wa_metric_t *) val->data; + + if (metric->type != NGX_WA_METRIC_COUNTER) { + ngx_wasm_log_error(NGX_LOG_ERR, metrics->shm->log, 0, + "attempt to call increment_metric on a %V; " + "operation not supported", + ngx_wa_metrics_type_name(metric->type)); + rc = NGX_ERROR; + goto error; + } + + ngx_log_debug2(NGX_LOG_DEBUG_WASM, metrics->shm->log, 0, + "wasm updating metric \"%uD\" with %d", metric_id, n); + + metric->slots[slot].counter += n; + +error: + +#if 0 + if (metrics->shm->eviction != NGX_WASM_EVICTION_NONE) { + ngx_wasm_shm_unlock(metrics->shm); + } +#endif + + return rc; +} + + +ngx_int_t +ngx_wa_metrics_record(ngx_wa_metrics_t *metrics, uint32_t metric_id, + ngx_int_t n) +{ + uint32_t cas; + ngx_int_t rc = NGX_OK; + ngx_str_t *val; + ngx_uint_t slot; + ngx_wa_metric_t *metric; + + slot = (ngx_process == NGX_PROCESS_WORKER) ? ngx_worker : 0; + +#if 0 + if (metrics->shm->eviction != NGX_WASM_EVICTION_NONE) { + slot = 0; + ngx_wasm_shm_lock(metrics->shm); + } +#endif + + rc = ngx_wasm_shm_kv_get_locked(metrics->shm, NULL, &metric_id, &val, &cas); + if (rc != NGX_OK) { + ngx_wasm_log_error(NGX_LOG_ERR, metrics->shm->log, 0, + "metric \"%uD\" not found", metric_id); + + goto error; + } + + metric = (ngx_wa_metric_t *) val->data; + + if (metric->type == NGX_WA_METRIC_COUNTER) { + ngx_wasm_log_error(NGX_LOG_ERR, metrics->shm->log, 0, + "attempt to call record_metric on a counter; " + "operation not supported"); + rc = NGX_ERROR; + goto error; + } + + ngx_log_debug2(NGX_LOG_DEBUG_WASM, metrics->shm->log, 0, + "wasm recording metric \"%z\" with %d", metric_id, n); + + metric->slots[slot].gauge.value = n; + metric->slots[slot].gauge.last_update = ngx_current_msec; + +error: + +#if 0 + if (metrics->shm->eviction != NGX_WASM_EVICTION_NONE) { + ngx_wasm_shm_unlock(metrics->shm); + } +#endif + + return rc; +} diff --git a/src/common/ngx_wa_metrics.h b/src/common/ngx_wa_metrics.h new file mode 100644 index 000000000..ff6923fdf --- /dev/null +++ b/src/common/ngx_wa_metrics.h @@ -0,0 +1,49 @@ +#ifndef _NGX_WA_METRICS_H_INCLUDED_ +#define _NGX_WA_METRICS_H_INCLUDED_ + + +#include +#include +#include + + +typedef struct ngx_wa_metrics_s ngx_wa_metrics_t; + + +typedef enum { + NGX_WA_METRIC_COUNTER, + NGX_WA_METRIC_GAUGE, +} ngx_wa_metric_type_e; + + +typedef struct { + size_t slab_size; + size_t max_metric_name_length; +} ngx_wa_metrics_conf_t; + + +struct ngx_wa_metrics_s { + ngx_uint_t workers; + ngx_shm_zone_t *shm_zone; + ngx_wasm_shm_t *shm; + ngx_wa_metrics_t *old_metrics; + + ngx_wa_metrics_conf_t config; +}; + + +ngx_wa_metrics_t *ngx_wasmx_metrics(); + +ngx_wa_metrics_t *ngx_wa_metrics_create(ngx_cycle_t *cycle); +ngx_int_t ngx_wa_metrics_init_conf(ngx_wa_metrics_t *metrics, ngx_conf_t *cf); +ngx_int_t ngx_wa_metrics_init(ngx_wa_metrics_t *metrics, ngx_cycle_t *cycle); +ngx_int_t ngx_wa_metrics_add(ngx_wa_metrics_t *metrics, ngx_str_t *name, + ngx_wa_metric_type_e type, uint32_t *out); +ngx_int_t ngx_wa_metrics_get(ngx_wa_metrics_t *metrics, uint32_t metric_id, + ngx_uint_t *out); +ngx_int_t ngx_wa_metrics_increment(ngx_wa_metrics_t *metrics, + uint32_t metric_id, ngx_int_t val); +ngx_int_t ngx_wa_metrics_record(ngx_wa_metrics_t *metrics, uint32_t metric_id, + ngx_int_t val); + +#endif /* _NGX_WA_METRICS_H_INCLUDED_ */ diff --git a/src/common/proxy_wasm/ngx_proxy_wasm.h b/src/common/proxy_wasm/ngx_proxy_wasm.h index 2daf69092..408054d62 100644 --- a/src/common/proxy_wasm/ngx_proxy_wasm.h +++ b/src/common/proxy_wasm/ngx_proxy_wasm.h @@ -141,13 +141,11 @@ typedef enum { } ngx_proxy_wasm_map_type_e; -#if 0 typedef enum { NGX_PROXY_WASM_METRIC_COUNTER = 0, NGX_PROXY_WASM_METRIC_GAUGE = 1, NGX_PROXY_WASM_METRIC_HISTOGRAM = 2, } ngx_proxy_wasm_metric_type_e; -#endif typedef struct ngx_proxy_wasm_ctx_s ngx_proxy_wasm_ctx_t; diff --git a/src/common/proxy_wasm/ngx_proxy_wasm_host.c b/src/common/proxy_wasm/ngx_proxy_wasm_host.c index b933fae29..a2c2ad23d 100644 --- a/src/common/proxy_wasm/ngx_proxy_wasm_host.c +++ b/src/common/proxy_wasm/ngx_proxy_wasm_host.c @@ -10,6 +10,7 @@ #include #include #include +#include #ifdef NGX_WASM_HTTP #include #endif @@ -1575,7 +1576,134 @@ ngx_proxy_wasm_hfuncs_dequeue_shared_queue(ngx_wavm_instance_t *instance, /* stats/metrics */ -/* NYI */ + + +static ngx_int_t +ngx_proxy_wasm_hfuncs_define_metric(ngx_wavm_instance_t *instance, + wasm_val_t args[], wasm_val_t rets[]) +{ + size_t max_len; + uint32_t *id; + ngx_str_t name, prefixed_name, *filter_name; + ngx_wa_metrics_t *metrics = ngx_wasmx_metrics(); + ngx_wa_metric_type_e type; + ngx_proxy_wasm_exec_t *pwexec; + ngx_proxy_wasm_metric_type_e pw_type; + u_char buf[metrics->config.max_metric_name_length]; + + pwexec = ngx_proxy_wasm_instance2pwexec(instance); + + pw_type = args[0].of.i32; + name.len = args[2].of.i32; + name.data = NGX_WAVM_HOST_LIFT_SLICE(instance, args[1].of.i32, name.len); + id = NGX_WAVM_HOST_LIFT_SLICE(instance, args[3].of.i32, sizeof(uint32_t)); + max_len = metrics->config.max_metric_name_length; + + switch(pw_type) { + case NGX_PROXY_WASM_METRIC_COUNTER: + type = NGX_WA_METRIC_COUNTER; + break; + + case NGX_PROXY_WASM_METRIC_GAUGE: + type = NGX_WA_METRIC_GAUGE; + break; + + default: + return ngx_proxy_wasm_result_trap(pwexec, "unknown metric type", + rets, + NGX_WAVM_ERROR); + } + + filter_name = pwexec->filter->name; + + if (filter_name->len + 4 + name.len > max_len) { + return ngx_proxy_wasm_result_trap(pwexec, "metric name too long", rets, + NGX_WAVM_ERROR); + } + + prefixed_name.data = buf; + prefixed_name.len = ngx_sprintf(buf, "pw.%V.%V", filter_name, &name) - buf; + + if (ngx_wa_metrics_add(metrics, &prefixed_name, type, id) != NGX_OK) { + return ngx_proxy_wasm_result_trap(pwexec, "could not define metric", + rets, + NGX_WAVM_ERROR); + } + + return ngx_proxy_wasm_result_ok(rets); +} + + +static ngx_int_t +ngx_proxy_wasm_hfuncs_increment_metric(ngx_wavm_instance_t *instance, + wasm_val_t args[], wasm_val_t rets[]) +{ + uint32_t metric_id; + ngx_int_t rc, offset; + ngx_wa_metrics_t *metrics = ngx_wasmx_metrics(); + ngx_proxy_wasm_exec_t *pwexec = ngx_proxy_wasm_instance2pwexec(instance); + + metric_id = args[0].of.i32; + offset = args[1].of.i64; + + rc = ngx_wa_metrics_increment(metrics, metric_id, offset); + + if (rc != NGX_OK) { + return ngx_proxy_wasm_result_trap(pwexec, "could not increment metric", + rets, NGX_WAVM_ERROR); + } + + return ngx_proxy_wasm_result_ok(rets); +} + + +static ngx_int_t +ngx_proxy_wasm_hfuncs_record_metric(ngx_wavm_instance_t *instance, + wasm_val_t args[], wasm_val_t rets[]) +{ + uint32_t metric_id; + ngx_int_t rc, offset; + ngx_wa_metrics_t *metrics = ngx_wasmx_metrics(); + ngx_proxy_wasm_exec_t *pwexec = ngx_proxy_wasm_instance2pwexec(instance); + + metric_id = args[0].of.i32; + offset = args[1].of.i64; + + rc = ngx_wa_metrics_record(metrics, metric_id, offset); + + if (rc != NGX_OK) { + return ngx_proxy_wasm_result_trap(pwexec, + "could not record metric value", rets, + NGX_WAVM_ERROR); + } + + return ngx_proxy_wasm_result_ok(rets); +} + + +static ngx_int_t +ngx_proxy_wasm_hfuncs_get_metric(ngx_wavm_instance_t *instance, + wasm_val_t args[], wasm_val_t rets[]) +{ + uint32_t metric_id; + ngx_int_t rc; + ngx_uint_t *ret_value; + ngx_wa_metrics_t *metrics = ngx_wasmx_metrics(); + ngx_proxy_wasm_exec_t *pwexec = ngx_proxy_wasm_instance2pwexec(instance); + + metric_id = args[0].of.i32; + ret_value = NGX_WAVM_HOST_LIFT(instance, args[1].of.i32, ngx_uint_t); + + rc = ngx_wa_metrics_get(metrics, metric_id, ret_value); + + if (rc == NGX_ERROR) { + /* TODO: format with key */ + return ngx_proxy_wasm_result_trap(pwexec, "could not retrieve metric", + rets, NGX_WAVM_ERROR); + } + + return ngx_proxy_wasm_result_ok(rets); +} /* custom extension points */ @@ -1957,7 +2085,7 @@ static ngx_wavm_host_func_def_t ngx_proxy_wasm_hfuncs[] = { ngx_wavm_arity_i32x4, ngx_wavm_arity_i32 }, { ngx_string("proxy_define_metric"), /* 0.2.0 && 0.2.1 */ - &ngx_proxy_wasm_hfuncs_nop, /* NYI */ + &ngx_proxy_wasm_hfuncs_define_metric, ngx_wavm_arity_i32x4, ngx_wavm_arity_i32 }, { ngx_string("proxy_get_metric_value"), /* vNEXT */ @@ -1965,7 +2093,7 @@ static ngx_wavm_host_func_def_t ngx_proxy_wasm_hfuncs[] = { ngx_wavm_arity_i32x2, ngx_wavm_arity_i32 }, { ngx_string("proxy_get_metric"), /* 0.2.0 && 0.2.1 */ - &ngx_proxy_wasm_hfuncs_nop, /* NYI */ + &ngx_proxy_wasm_hfuncs_get_metric, ngx_wavm_arity_i32x2, ngx_wavm_arity_i32 }, { ngx_string("proxy_set_metric_value"), /* vNEXT */ @@ -1973,7 +2101,7 @@ static ngx_wavm_host_func_def_t ngx_proxy_wasm_hfuncs[] = { ngx_wavm_arity_i32_i64, ngx_wavm_arity_i32 }, { ngx_string("proxy_record_metric"), /* 0.2.0 && 0.2.1 */ - &ngx_proxy_wasm_hfuncs_nop, /* NYI */ + &ngx_proxy_wasm_hfuncs_record_metric, ngx_wavm_arity_i32_i64, ngx_wavm_arity_i32 }, { ngx_string("proxy_increment_metric_value"), /* vNEXT */ @@ -1981,7 +2109,7 @@ static ngx_wavm_host_func_def_t ngx_proxy_wasm_hfuncs[] = { ngx_wavm_arity_i32_i64, ngx_wavm_arity_i32 }, { ngx_string("proxy_increment_metric"), /* 0.2.0 && 0.2.1 */ - &ngx_proxy_wasm_hfuncs_nop, /* NYI */ + &ngx_proxy_wasm_hfuncs_increment_metric, ngx_wavm_arity_i32_i64, ngx_wavm_arity_i32 }, { ngx_string("proxy_delete_metric"), /* vNEXT */ diff --git a/src/common/shm/ngx_wasm_shm.h b/src/common/shm/ngx_wasm_shm.h index 1ae08b43e..f3a885138 100644 --- a/src/common/shm/ngx_wasm_shm.h +++ b/src/common/shm/ngx_wasm_shm.h @@ -11,6 +11,7 @@ typedef enum { NGX_WASM_SHM_TYPE_KV, NGX_WASM_SHM_TYPE_QUEUE, + NGX_WASM_SHM_TYPE_METRICS, } ngx_wasm_shm_type_e; diff --git a/src/common/shm/ngx_wasm_shm_kv.c b/src/common/shm/ngx_wasm_shm_kv.c index 265023d83..94e77cb95 100644 --- a/src/common/shm/ngx_wasm_shm_kv.c +++ b/src/common/shm/ngx_wasm_shm_kv.c @@ -15,28 +15,11 @@ #define NGX_WASM_SLRU_NQUEUES(pool) (NGX_WASM_SLAB_SLOTS(pool) + 1) -typedef struct { - ngx_rbtree_t rbtree; - ngx_rbtree_node_t sentinel; - union { - ngx_queue_t lru_queue; - ngx_queue_t slru_queues[0]; - } eviction; -} ngx_wasm_shm_kv_t; - - -typedef struct { - ngx_str_node_t key; - ngx_str_t value; - uint32_t cas; - ngx_queue_t queue; -} ngx_wasm_shm_kv_node_t; - - -static ngx_inline ngx_wasm_shm_kv_t * +ngx_wasm_shm_kv_t * ngx_wasm_shm_get_kv(ngx_wasm_shm_t *shm) { - ngx_wa_assert(shm->type == NGX_WASM_SHM_TYPE_KV); + ngx_wa_assert(shm->type == NGX_WASM_SHM_TYPE_KV || \ + shm->type == NGX_WASM_SHM_TYPE_METRICS); return shm->data; } diff --git a/src/common/shm/ngx_wasm_shm_kv.h b/src/common/shm/ngx_wasm_shm_kv.h index fe280034f..d711b6b57 100644 --- a/src/common/shm/ngx_wasm_shm_kv.h +++ b/src/common/shm/ngx_wasm_shm_kv.h @@ -5,6 +5,16 @@ #include +typedef struct { + ngx_rbtree_t rbtree; + ngx_rbtree_node_t sentinel; + union { + ngx_queue_t lru_queue; + ngx_queue_t slru_queues[0]; + } eviction; +} ngx_wasm_shm_kv_t; + + typedef struct { ngx_str_t namespace; ngx_str_t key; @@ -13,6 +23,16 @@ typedef struct { } ngx_wasm_shm_kv_key_t; +typedef struct { + ngx_str_node_t key; + ngx_str_t value; + uint32_t cas; + ngx_queue_t queue; +} ngx_wasm_shm_kv_node_t; + + +ngx_wasm_shm_kv_t * ngx_wasm_shm_get_kv(ngx_wasm_shm_t *shm); + ngx_int_t ngx_wasm_shm_kv_init(ngx_wasm_shm_t *shm); ngx_int_t ngx_wasm_shm_kv_get_locked(ngx_wasm_shm_t *shm, ngx_str_t *key, uint32_t *key_hash, ngx_str_t **value_out, uint32_t *cas); diff --git a/src/ngx_wasmx.c b/src/ngx_wasmx.c index 40787eabc..df48ae6c1 100644 --- a/src/ngx_wasmx.c +++ b/src/ngx_wasmx.c @@ -23,6 +23,9 @@ ngx_uint_t ngx_wasm_max_module = 0; ngx_uint_t ngx_ipc_max_module = 0; +static ngx_wa_metrics_t *ngx_wa_metrics = NULL; + + static ngx_command_t ngx_wasmx_cmds[] = { { ngx_string("wasm"), @@ -68,6 +71,13 @@ ngx_module_t ngx_wasmx_module = { }; +ngx_wa_metrics_t * +ngx_wasmx_metrics() +{ + return ngx_wa_metrics; +} + + static char * ngx_wasmx_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf, ngx_uint_t type, ngx_uint_t conf_type) @@ -116,6 +126,11 @@ ngx_wasmx_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf, } #endif + ngx_wa_metrics = ngx_wa_metrics_create(cf->cycle); + if (ngx_wa_metrics == NULL) { + return NGX_CONF_ERROR; + } + *(ngx_wa_conf_t **) conf = wacf; } @@ -217,6 +232,10 @@ ngx_wasmx_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf, } } + if (ngx_wa_metrics_init_conf(ngx_wa_metrics, cf) != NGX_OK) { + return NGX_CONF_ERROR; + } + return NGX_CONF_OK; } @@ -249,6 +268,11 @@ ngx_wasmx_init(ngx_cycle_t *cycle) return NGX_OK; } + rc = ngx_wa_metrics_init(ngx_wa_metrics, cycle); + if (rc != NGX_OK) { + return rc; + } + /* NGX_WASM_MODULES + NGX_IPC_MODULES init */ for (i = 0; cycle->modules[i]; i++) { diff --git a/src/wasm/ngx_wasm.h b/src/wasm/ngx_wasm.h index c570d10b0..a257be2a0 100644 --- a/src/wasm/ngx_wasm.h +++ b/src/wasm/ngx_wasm.h @@ -5,6 +5,7 @@ #include #include #include +#include #if (NGX_SSL) #include #endif @@ -18,6 +19,7 @@ #define NGX_WASMTIME_CONF 0x20000000 #define NGX_WASMER_CONF 0x40000000 #define NGX_V8_CONF 0x80000000 +#define NGX_METRICS_CONF 0x16000000 #define NGX_WASM_DONE_PHASE 15 #define NGX_WASM_BACKGROUND_PHASE 16 @@ -125,12 +127,16 @@ void ngx_wasm_log_error(ngx_uint_t level, ngx_log_t *log, ngx_err_t err, void swap_modules_if_needed(ngx_conf_t *cf, const char *m1, const char *m2); #endif -/* directives */ +/* blocks */ char *ngx_wasm_core_wasmtime_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); char *ngx_wasm_core_wasmer_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); char *ngx_wasm_core_v8_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +char *ngx_wasm_core_metrics_block(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); + +/* directives */ char *ngx_wasm_core_flag_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); char *ngx_wasm_core_module_directive(ngx_conf_t *cf, ngx_command_t *cmd, @@ -139,6 +145,10 @@ char *ngx_wasm_core_shm_kv_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); char *ngx_wasm_core_shm_queue_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +char *ngx_wasm_core_metrics_slab_size_directive(ngx_conf_t *cf, + ngx_command_t *cmd, void *conf); +char *ngx_wasm_core_metrics_max_metric_name_length_directive(ngx_conf_t *cf, + ngx_command_t *cmd, void *conf); char *ngx_wasm_core_resolver_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); char *ngx_wasm_core_pwm_lua_resolver_directive(ngx_conf_t *cf, diff --git a/src/wasm/ngx_wasm_core_module.c b/src/wasm/ngx_wasm_core_module.c index cd7f8d80b..f367a44ad 100644 --- a/src/wasm/ngx_wasm_core_module.c +++ b/src/wasm/ngx_wasm_core_module.c @@ -5,6 +5,7 @@ #include #include +#include static void *ngx_wasm_core_create_conf(ngx_conf_t *cf); @@ -43,6 +44,13 @@ static ngx_command_t ngx_wasm_core_commands[] = { 0, NULL }, + { ngx_string("metrics"), + NGX_WASM_CONF|NGX_CONF_BLOCK|NGX_CONF_NOARGS, + ngx_wasm_core_metrics_block, + NGX_WA_WASM_CONF_OFFSET, + 0, + NULL }, + /* directives */ { ngx_string("flag"), @@ -90,6 +98,20 @@ static ngx_command_t ngx_wasm_core_commands[] = { 0, NULL }, + { ngx_string("slab_size"), + NGX_METRICS_CONF|NGX_CONF_TAKE1, + ngx_wasm_core_metrics_slab_size_directive, + NGX_WA_WASM_CONF_OFFSET, + 0, + NULL }, + + { ngx_string("max_metric_name_length"), + NGX_METRICS_CONF|NGX_CONF_TAKE1, + ngx_wasm_core_metrics_max_metric_name_length_directive, + NGX_WA_WASM_CONF_OFFSET, + 0, + NULL }, + { ngx_string("shm_queue"), NGX_WASM_CONF|NGX_CONF_TAKE23|NGX_CONF_TAKE4, ngx_wasm_core_shm_queue_directive, diff --git a/src/wasm/ngx_wasm_directives.c b/src/wasm/ngx_wasm_directives.c index 5437eae08..6a915a38d 100644 --- a/src/wasm/ngx_wasm_directives.c +++ b/src/wasm/ngx_wasm_directives.c @@ -7,6 +7,10 @@ #include +static ngx_int_t ngx_wasm_core_shm_validate_size(ngx_conf_t *cf, ssize_t size, + ngx_str_t *value); + + static char * ngx_wasm_core_runtime_block(ngx_conf_t *cf, ngx_uint_t cmd_type) { @@ -45,6 +49,23 @@ ngx_wasm_core_v8_block(ngx_conf_t *cf, ngx_command_t *cmd, void *dummy) } +char * +ngx_wasm_core_metrics_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + char *rv; + ngx_conf_t save = *cf; + + cf->cmd_type = NGX_METRICS_CONF; + cf->module_type = NGX_WASM_MODULE; + + rv = ngx_conf_parse(cf, NULL); + + *cf = save; + + return rv; +} + + static ngx_int_t ngx_wasm_core_current_runtime_flag(ngx_conf_t *cf) { @@ -109,6 +130,35 @@ ngx_wasm_core_flag_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } +static ngx_int_t +ngx_wasm_core_shm_validate_size(ngx_conf_t *cf, ssize_t size, ngx_str_t *value) +{ + const ssize_t min_size = 3 * ngx_pagesize; + + if (size == NGX_ERROR) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "[wasm] invalid shm size \"%V\"", value); + return NGX_ERROR; + } + + if (size < min_size) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "[wasm] shm size of %d bytes is too small, " + "minimum required is %d bytes", size, min_size); + return NGX_ERROR; + } + + if ((size & (ngx_pagesize - 1)) != 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "[wasm] shm size of %d bytes is not page-aligned, " + "must be a multiple of %d", size, ngx_pagesize); + return NGX_ERROR; + } + + return NGX_OK; +} + + static char * ngx_wasm_core_shm_generic_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf, ngx_wasm_shm_type_e type) @@ -120,7 +170,6 @@ ngx_wasm_core_shm_generic_directive(ngx_conf_t *cf, ngx_command_t *cmd, ngx_wasm_shm_mapping_t *mapping; ngx_wasm_shm_t *shm; ngx_wasm_shm_eviction_e eviction; - const ssize_t min_size = 3 * ngx_pagesize; value = cf->args->elts; name = &value[1]; @@ -134,23 +183,7 @@ ngx_wasm_core_shm_generic_directive(ngx_conf_t *cf, ngx_command_t *cmd, return NGX_CONF_ERROR; } - if (size == NGX_ERROR) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "[wasm] invalid shm size \"%V\"", &value[2]); - return NGX_CONF_ERROR; - } - - if (size < min_size) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "[wasm] shm size of %d bytes is too small, " - "minimum required is %d bytes", size, min_size); - return NGX_CONF_ERROR; - } - - if ((size & (ngx_pagesize - 1)) != 0) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "[wasm] shm size of %d bytes is not page-aligned, " - "must be a multiple of %d", size, ngx_pagesize); + if (ngx_wasm_core_shm_validate_size(cf, size, &value[2]) != NGX_OK) { return NGX_CONF_ERROR; } @@ -295,6 +328,57 @@ ngx_wasm_core_shm_queue_directive(ngx_conf_t *cf, ngx_command_t *cmd, } +char * +ngx_wasm_core_metrics_slab_size_directive(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf) +{ + ssize_t size; + ngx_str_t *value; + ngx_wa_metrics_t *metrics = ngx_wasmx_metrics(); + + if (cf->cmd_type != NGX_METRICS_CONF) { + return NGX_CONF_ERROR; + } + + if (metrics->config.slab_size != NGX_CONF_UNSET_SIZE) { + return "is duplicate"; + } + + value = cf->args->elts; + size = ngx_parse_size(&value[1]); + + if (ngx_wasm_core_shm_validate_size(cf, size, &value[1]) != NGX_OK) { + return NGX_CONF_ERROR; + } + + metrics->config.slab_size = size; + + return NGX_CONF_OK; +} + + +char * +ngx_wasm_core_metrics_max_metric_name_length_directive(ngx_conf_t *cf, + ngx_command_t *cmd, void *conf) +{ + ngx_str_t *value; + ngx_wa_metrics_t *metrics = ngx_wasmx_metrics(); + + if (cf->cmd_type != NGX_METRICS_CONF) { + return NGX_CONF_ERROR; + } + + if (metrics->config.max_metric_name_length != NGX_CONF_UNSET_SIZE) { + return "is duplicate"; + } + + value = cf->args->elts; + metrics->config.max_metric_name_length = ngx_parse_size(&value[1]); + + return NGX_CONF_OK; +} + + char * ngx_wasm_core_resolver_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { diff --git a/t/01-wasm/directives/011-metrics_directives.t b/t/01-wasm/directives/011-metrics_directives.t new file mode 100644 index 000000000..466fe0ef2 --- /dev/null +++ b/t/01-wasm/directives/011-metrics_directives.t @@ -0,0 +1,120 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +plan_tests(5); +run_tests(); + +__DATA__ + +=== TEST 1: metrics{} - empty block +--- valgrind +--- main_config + wasm { + metrics {} + } +--- no_error_log +[error] +[crit] +[emerg] +[stub] + + + +=== TEST 2: slab_size directive - sanity +--- main_config + wasm { + metrics { + slab_size 12k; + } + } +--- no_error_log +[error] +[crit] +[emerg] +[stub] + + + +=== TEST 3: slab_size directive - too small +--- main_config + wasm { + metrics { + slab_size 1k; + } + } +--- error_log eval +qr/\[emerg\] .*? \[wasm\] shm size of \d+ bytes is too small, minimum required is 12288 bytes/ +--- no_error_log +[error] +[crit] +[stub] +--- must_die + + + +=== TEST 4: slab_size directive - invalid size +--- main_config + wasm { + metrics { + slab_size 1x; + } + } +--- error_log eval +qr/\[emerg\] .*? \[wasm\] invalid shm size "1x"/ +--- no_error_log +[error] +[crit] +[stub] +--- must_die + + + +=== TEST 5: slab_size directive - duplicate +--- main_config + wasm { + metrics { + slab_size 12k; + slab_size 12k; + } + } +--- error_log: is duplicate +--- no_error_log +[error] +[crit] +[stub] +--- must_die + + + +=== TEST 6: max_metric_name_length directive - sanity +--- main_config + wasm { + metrics { + max_metric_name_length 64; + } + } +--- no_error_log +[error] +[crit] +[emerg] +[stub] + + + +=== TEST 7: max_metric_name_length directive - duplicate +--- main_config + wasm { + metrics { + max_metric_name_length 64; + max_metric_name_length 64; + } + } +--- error_log: is duplicate +--- no_error_log +[error] +[crit] +[stub] +--- must_die diff --git a/t/03-proxy_wasm/hfuncs/metrics/100-define_metric.t b/t/03-proxy_wasm/hfuncs/metrics/100-define_metric.t new file mode 100644 index 000000000..5e8aaafb0 --- /dev/null +++ b/t/03-proxy_wasm/hfuncs/metrics/100-define_metric.t @@ -0,0 +1,150 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +our $workers = 2; + +workers($workers); +if ($workers > 1) { + master_on(); +} + +plan_tests(6); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm metrics - define_metric() - on_configure +Hostcalls filter prefixes the name of a metric with the phase in which it's +defined. A metric c1 defined within on_configure ends up named c1_Configure. + +--- valgrind +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + metrics=c1,g1'; + echo ok; + } +} +--- grep_error_log eval: qr/defined metric \w+ as \d+ at \w+/ +--- grep_error_log_out eval +qr/defined metric c1_Configure as \d+ at Configure +defined metric g1_Configure as \d+ at Configure\n/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 2: proxy_wasm metrics - define_metric() - on_tick +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_tick=define_metrics \ + tick_period=500 \ + n_sync_calls=1 \ + metrics=c1,g1'; + echo ok; + } +} +--- grep_error_log eval: qr/defined metric \w+ as \d+ at \w+/ +--- grep_error_log_out eval +qr/defined metric c1_Tick as \d+ at Tick +defined metric g1_Tick as \d+ at Tick\n/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 3: proxy_wasm metrics - define_metric() - on: request_headers, request_body, response_headers, response_body +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +my $phases = CORE::join(',', qw( + request_headers + request_body + response_headers + response_body +)); + +qq{ + location /t { + proxy_wasm hostcalls 'on=$phases \ + test=/t/metrics/define \ + metrics=c1,g1'; + echo ok; + } +} +--- request +POST /t +hello +--- grep_error_log eval: qr/defined metric \w+ as \d+ at \w+/ +--- grep_error_log_out eval +my $checks; +my @phases = qw( + RequestHeaders + RequestBody + ResponseHeaders + ResponseBody + ResponseBody +); + +foreach my $p (@phases) { + my $suffixed_c1 = "c1_" . $p; + my $suffixed_g1 = "g1_" . $p; + $checks .= " +?defined metric $suffixed_c1 as [0-9]+ at $p +defined metric $suffixed_g1 as [0-9]+ at $p\n"; +} + +qr/$checks/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 4: proxy_wasm metrics - define_metric() - on_http_call_response +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- http_config eval +--- config eval +qq{ + listen unix:$ENV{TEST_NGINX_UNIX_SOCKET}; + + location /dispatched { + return 200 "Hello back"; + } + + location /t { + proxy_wasm hostcalls 'test=/t/dispatch_http_call \ + host=unix:$ENV{TEST_NGINX_UNIX_SOCKET} \ + path=/dispatched \ + on_http_call_response=define_metrics \ + metrics=c1,g1'; + echo ok; + } +} +--- grep_error_log eval: qr/defined metric \w+ as \d+ at \w+/ +--- grep_error_log_out eval +qr/defined metric c1_HTTPCallResponse as \d+ at HTTPCallResponse +defined metric g1_HTTPCallResponse as \d+ at HTTPCallResponse\n/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] diff --git a/t/03-proxy_wasm/hfuncs/metrics/101-define_metric_edge_cases.t b/t/03-proxy_wasm/hfuncs/metrics/101-define_metric_edge_cases.t new file mode 100644 index 000000000..e1ab6b300 --- /dev/null +++ b/t/03-proxy_wasm/hfuncs/metrics/101-define_metric_edge_cases.t @@ -0,0 +1,86 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +skip_hup(); +no_shuffle(); + +plan_tests(6); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm metrics - define_metric() metric name too long +In SIGHUP mode, this test fails if executed after a test that defined metrics, +as any existing metric whose name exceeds `max_metric_name_length` won't be +successfully reallocated causing the reconfiguration to fail. + +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- main_config eval +qq{ + wasm { + module hostcalls $ENV{TEST_NGINX_CRATES_DIR}/hostcalls.wasm; + + metrics { + max_metric_name_length 4; + } + } +} +--- config + location /t { + proxy_wasm hostcalls 'on=request_headers \ + test=/t/metrics/define \ + metrics=c1'; + echo ok; + } +--- error_code: 500 +--- error_log eval +[ + qr/.+on_request_headers.+/, + qr/host trap \(internal error\): metric name too long.*/, +] +--- no_error_log +[emerg] +[alert] +[stub] + + + +=== TEST 2: proxy_wasm metrics - define_metric() no memory +In SIGHUP mode, this test fails if executed after a test that defined more +metrics than it's possible to fit in `5m`. + +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- main_config eval +qq{ + wasm { + module hostcalls $ENV{TEST_NGINX_CRATES_DIR}/hostcalls.wasm; + + metrics { + slab_size 5m; + max_metric_name_length 128; + } + } +} +--- config + location /t { + proxy_wasm hostcalls 'on=request_headers \ + test=/t/metrics/define \ + metrics=c20337 \ + metrics_name_len=100'; + echo ok; + } +--- error_code: 500 +--- error_log eval +[ + qr/\[crit\] .+ \[wasm\] "metrics" shm store: no memory; cannot allocate pair with key size \d+ and value size \d+/, + qr/.+on_request_headers.+/, + qr/host trap \(internal error\): could not define metric.*/, +] +--- no_error_log +[emerg] +[alert] diff --git a/t/03-proxy_wasm/hfuncs/metrics/200-increment_metric.t b/t/03-proxy_wasm/hfuncs/metrics/200-increment_metric.t new file mode 100644 index 000000000..b0f7b4c08 --- /dev/null +++ b/t/03-proxy_wasm/hfuncs/metrics/200-increment_metric.t @@ -0,0 +1,166 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +skip_hup(); + +our $workers = 2; + +workers($workers); +if ($workers > 1) { + master_on(); +} + +plan_tests(6); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm metrics shm - increment_metric - on_configure +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'metrics=c2 \ + on_configure=define_and_increment_counters'; + echo ok; + } +} +--- grep_error_log eval: qr/(incremented \w+|\w+: \d+) at \w+/ +--- grep_error_log_out eval +my $check; +$check .= "incremented c1_Configure at Configure(\n|\n.+\n)"; +$check .= "incremented c2_Configure at Configure(\n|\n.+\n)"; +$check .= "c1_Configure: $::workers at Configure(\n|\n.+\n)"; +$check .= "c2_Configure: $::workers at Configure(\n|\n.+\n)"; + +qr/$check/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 2: proxy_wasm metrics - increment_metric() - on_tick +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on_tick=increment_counters \ + tick_period=500 \ + n_sync_calls=1 \ + metrics=c2'; + echo ok; + } +} +--- grep_error_log eval: qr/(incremented \w+|\w+: \d+) at \w+/ +--- grep_error_log_out eval +my $check; +$check .= "incremented c1_Configure at Tick(\n|\n.+\n)"; +$check .= "incremented c2_Configure at Tick(\n|\n.+\n)"; +$check .= "c1_Configure: $::workers at Tick(\n|\n.+\n)"; +$check .= "c2_Configure: $::workers at Tick(\n|\n.+\n)"; + +qr/$check/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 3: proxy_wasm metrics - increment_metric() - on: request_headers, request_body, response_headers, response_body +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +my $phases = CORE::join(',', qw( + request_headers + request_body + response_headers + response_body +)); + +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on=$phases \ + test=/t/metrics/increment_counters \ + metrics=c2'; + echo ok; + } +} +--- request +POST /t +hello +--- grep_error_log eval: qr/(incremented \w+|\w+: \d+) at \w+/ +--- grep_error_log_out eval +my $checks; +my $i = 0; +my @phases = qw( + RequestHeaders + RequestBody + ResponseHeaders + ResponseBody + ResponseBody +); + +foreach my $p (@phases) { + $i++; + $checks .= " +?incremented c1_Configure at $p +incremented c2_Configure at $p +c1_Configure: $i at $p +c2_Configure: $i at $p\n"; +} + +qr/$checks/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 4: proxy_wasm metrics - increment_metric() - on_http_call_response +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- http_config eval +--- config eval +qq{ + listen unix:$ENV{TEST_NGINX_UNIX_SOCKET}; + + location /dispatched { + return 200 "Hello back"; + } + + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + test=/t/dispatch_http_call \ + host=unix:$ENV{TEST_NGINX_UNIX_SOCKET} \ + path=/dispatched \ + on_http_call_response=increment_counters \ + metrics=c2'; + echo ok; + } +} +--- grep_error_log eval: qr/(incremented \w+|\w+: \d+) at \w+/ +--- grep_error_log_out eval +qr/incremented c1_Configure at HTTPCallResponse +incremented c2_Configure at HTTPCallResponse +c1_Configure: 1 at HTTPCallResponse +c2_Configure: 1 at HTTPCallResponse\n/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] diff --git a/t/03-proxy_wasm/hfuncs/metrics/201-increment_metric_misuse.t b/t/03-proxy_wasm/hfuncs/metrics/201-increment_metric_misuse.t new file mode 100644 index 000000000..68874ccfd --- /dev/null +++ b/t/03-proxy_wasm/hfuncs/metrics/201-increment_metric_misuse.t @@ -0,0 +1,60 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +plan_tests(7); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm metrics - increment_metric() gauge +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on=request_headers \ + test=/t/metrics/increment_counters \ + metrics=g1,g2'; + echo ok; + } +} +--- error_code: 500 +--- error_log eval +[ + qr/\[error\] .+ \[wasm\] attempt to call increment_metric on a gauge; operation not supported/, + qr/.+on_request_headers.+/, + qr/host trap \(internal error\): could not increment metric.*/, +] +--- no_error_log +[crit] +[emerg] +[alert] + + + +=== TEST 2: proxy_wasm metrics - increment_metric() invalid metric id +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config + location /t { + proxy_wasm hostcalls 'on=request_headers \ + test=/t/metrics/increment_invalid_counter'; + echo ok; + } +--- error_code: 500 +--- error_log eval +[ + qr/.+on_request_headers.+/, + qr/metric \"\d+\" not found.*/, + qr/host trap \(internal error\): could not increment metric.*/, +] +--- no_error_log +[crit] +[emerg] +[alert] diff --git a/t/03-proxy_wasm/hfuncs/metrics/300-record_metric.t b/t/03-proxy_wasm/hfuncs/metrics/300-record_metric.t new file mode 100644 index 000000000..00ad66089 --- /dev/null +++ b/t/03-proxy_wasm/hfuncs/metrics/300-record_metric.t @@ -0,0 +1,179 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +skip_hup(); + +our $workers = 2; + +workers($workers); +if ($workers > 1) { + master_on(); +} + +plan_tests(6); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm metrics shm - record_metric, gauges - on_configure +--- workers: 1 +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config + location /t { + proxy_wasm hostcalls 'metrics=g2 \ + on_configure=define_and_toggle_gauges'; + echo ok; + } +--- grep_error_log eval: qr/(toggled \w+|\w+: \d+) at \w+/ +--- grep_error_log_out eval +my $check; +$check .= "toggled g1_Configure at Configure(\n|\n.+\n)"; +$check .= "toggled g2_Configure at Configure(\n|\n.+\n)"; +$check .= "g1_Configure: 1 at Configure(\n|\n.+\n)"; +$check .= "g2_Configure: 1 at Configure(\n|\n.+\n)"; +qr/$check/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 2: proxy_wasm metrics - record_metric(), gauges - on_tick +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- config eval +my $filters; + +foreach my $wid (0 .. $::workers - 1) { + my $wait = 100 + ($wid * 500); + $filters .= " + proxy_wasm hostcalls 'on_configure=define_metrics \ + on_tick=set_gauges \ + tick_period=$wait \ + n_sync_calls=1 \ + on_worker=$wid \ + value=$wid \ + metrics=g2';"; +} +qq{ + location /t { + $filters + + echo ok; + } +} +--- wait: 1 +--- grep_error_log eval: qr/(set \w+ to \d+|\w+: \d+) at \w+/ +--- grep_error_log_out eval +my $checks; + +foreach my $worker_id (0 .. $::workers - 1) { + $checks .= "set g1_Configure to $worker_id at Tick +set g2_Configure to $worker_id at Tick +g1_Configure: $worker_id at Tick +g2_Configure: $worker_id at Tick +"; +} + +qr/$checks/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 3: proxy_wasm metrics - record_metric(), gauges - on: request_headers, request_body, response_headers, response_body +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +my $phases = CORE::join(',', qw( + request_headers + request_body + response_headers + response_body +)); + +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on=$phases \ + test=/t/metrics/toggle_gauges \ + metrics=g2'; + echo ok; + } +} +--- request +POST /t +hello +--- grep_error_log eval: qr/(toggled \w+|\w+: \d+) at \w+/ +--- grep_error_log_out eval +my $i = 0; +my $checks; +my @phases = qw( + RequestHeaders + RequestBody + ResponseHeaders + ResponseBody + ResponseBody +); + +foreach my $phase (@phases) { + $i = $i ? 0 : 1; + $checks .= " +?toggled g1_Configure at $phase +toggled g2_Configure at $phase +g1_Configure: $i at $phase +g2_Configure: $i at $phase\n"; +} + +qr/$checks/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 4: proxy_wasm metrics - record_metric(), gauges - on_http_call_response +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- http_config eval +--- config eval +qq{ + listen unix:$ENV{TEST_NGINX_UNIX_SOCKET}; + + location /dispatched { + return 200 "Hello back"; + } + + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + test=/t/dispatch_http_call \ + host=unix:$ENV{TEST_NGINX_UNIX_SOCKET} \ + path=/dispatched \ + on_http_call_response=toggle_gauges \ + metrics=g2'; + echo ok; + } +} +--- grep_error_log eval: qr/(toggled \w+|\w+: \d+) at \w+/ +--- grep_error_log_out eval +qr/toggled g1_Configure at HTTPCallResponse +toggled g2_Configure at HTTPCallResponse +g1_Configure: 1 at HTTPCallResponse +g2_Configure: 1 at HTTPCallResponse\n/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] diff --git a/t/03-proxy_wasm/hfuncs/metrics/301-record_metric_misuse.t b/t/03-proxy_wasm/hfuncs/metrics/301-record_metric_misuse.t new file mode 100644 index 000000000..954b9ebdd --- /dev/null +++ b/t/03-proxy_wasm/hfuncs/metrics/301-record_metric_misuse.t @@ -0,0 +1,58 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +plan_tests(7); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm metrics - record_metric() counter +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on=request_headers \ + test=/t/metrics/toggle_gauges \ + metrics=c1,c2'; + echo ok; + } +--- error_code: 500 +--- error_log eval +[ + qr/\[error\] .+ \[wasm\] attempt to call record_metric on a counter; operation not supported/, + qr/.+on_request_headers.+/, + qr/host trap \(internal error\): could not record metric.*/, +] +--- no_error_log +[crit] +[emerg] +[alert] + + + +=== TEST 2: proxy_wasm metrics - record_metric() invalid metric id +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config + location /t { + proxy_wasm hostcalls 'on=request_headers \ + test=/t/metrics/set_invalid_gauge'; + echo ok; + } +--- error_code: 500 +--- error_log eval +[ + qr/.+on_request_headers.+/, + qr/metric \"0\" not found.*/, +] +--- no_error_log +[crit] +[emerg] +[alert] +[stub] diff --git a/t/07-metrics/001-metrics_sighup.t b/t/07-metrics/001-metrics_sighup.t new file mode 100644 index 000000000..2bad9eefb --- /dev/null +++ b/t/07-metrics/001-metrics_sighup.t @@ -0,0 +1,211 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: +use strict; +use lib '.'; +use t::TestWasmX; + +skip_no_hup(); + +our $workers = 2; + +workers($workers); +if ($workers > 1) { + master_on(); +} + +no_shuffle(); +plan_tests(8); +run_tests(); + +__DATA__ + +=== TEST 1: SIGHUP metrics - define and increment counters +Evaluating counters values in error_log rather than in response headers as some +worker(s) might not have done their increment by the time response_headers phase +is invoked. + +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_increment_counters \ + metrics=c2'; + echo ok; + } +} +--- grep_error_log eval: qr/c\d_Configure: \d+/ +--- grep_error_log_out eval +qr/c1_Configure: $::workers.* +c2_Configure: $::workers.*/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] +[stub] +[stub] + + + +=== TEST 2: SIGHUP metrics - shm preserved, no realloc +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_increment_counters \ + on=response_headers \ + test=/t/metrics/get \ + metrics=c2'; + echo ok; + } +} +--- response_headers +c1-Configure: 4 +c2-Configure: 4 +--- no_error_log +reallocating metric +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 3: SIGHUP metrics - increased worker_processes - shm preserved, realloc +--- workers: 4 +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_increment_counters \ + on=response_headers \ + test=/t/metrics/get \ + metrics=c2'; + echo ok; + } +} +--- response_headers +c1-Configure: 8 +c2-Configure: 8 +--- grep_error_log eval: qr/reallocating metric .+/ +--- grep_error_log_out eval +qr/reallocating metric .*c1.* +reallocating metric .*c2.*/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 4: SIGHUP metrics - decreased worker_processes - shm preserved, realloc +--- workers: 2 +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_increment_counters \ + on=response_headers \ + test=/t/metrics/get \ + metrics=c2'; + echo ok; + } +} +--- response_headers +c1-Configure: 10 +c2-Configure: 10 +--- grep_error_log eval: qr/reallocating metric .+/ +--- grep_error_log_out eval +qr/reallocating metric .*c1.* +reallocating metric .*c2.*/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 5: SIGHUP metrics - decreased slab_size - shm preserved, realloc +--- workers: 2 +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- main_config eval +qq{ + wasm { + module hostcalls $ENV{TEST_NGINX_CRATES_DIR}/hostcalls.wasm; + + metrics { + slab_size 12k; + } + } +} +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_increment_counters \ + on=response_headers \ + test=/t/metrics/get \ + metrics=c2'; + echo ok; + } +} +--- response_headers +c1-Configure: 12 +c2-Configure: 12 +--- grep_error_log eval: qr/reallocating metric .+/ +--- grep_error_log_out eval +qr/reallocating metric .*c1.* +reallocating metric .*c2.*/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 6: SIGHUP metrics - increased slab_size - shm preserved, realloc +--- workers: 2 +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- main_config eval +qq{ + wasm { + module hostcalls $ENV{TEST_NGINX_CRATES_DIR}/hostcalls.wasm; + + metrics { + slab_size 16k; + } + } +} +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_increment_counters \ + on=response_headers \ + test=/t/metrics/get \ + metrics=c2'; + echo ok; + } +} +--- response_headers +c1-Configure: 14 +c2-Configure: 14 +--- grep_error_log eval: qr/reallocating metric .+/ +--- grep_error_log_out eval +qr/reallocating metric .*c1.* +reallocating metric .*c2.*/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] diff --git a/t/TestWasmX.pm b/t/TestWasmX.pm index c2c3df560..a8d207146 100644 --- a/t/TestWasmX.pm +++ b/t/TestWasmX.pm @@ -29,6 +29,7 @@ our @EXPORT = qw( load_nginx_modules plan_tests skip_hup + skip_no_hup skip_no_ssl skip_no_ipc skip_no_debug @@ -59,6 +60,12 @@ sub skip_hup { } } +sub skip_no_hup { + if ($ENV{TEST_NGINX_USE_HUP} == 0) { + plan(skip_all => "skip in default mode"); + } +} + sub skip_no_ssl { if ($nginxV !~ m/built with \S+SSL/) { plan(skip_all => "SSL support required (NGX_BUILD_SSL=1)"); @@ -201,6 +208,17 @@ add_block_preprocessor(sub { $wasm_config = $wasm_config . (join "\n", @arr); } + # --- metrics + + my $metrics = $block->metrics; + + if (defined $metrics) { + $wasm_config = $wasm_config . + " metrics {\n" . + " slab_size $metrics" . ";\n" . + " }\n"; + } + # --- shm_queue my $shm_queue = $block->shm_queue; diff --git a/t/lib/proxy-wasm-tests/hostcalls/src/filter.rs b/t/lib/proxy-wasm-tests/hostcalls/src/filter.rs index bea8a9391..380b3876e 100644 --- a/t/lib/proxy-wasm-tests/hostcalls/src/filter.rs +++ b/t/lib/proxy-wasm-tests/hostcalls/src/filter.rs @@ -1,4 +1,4 @@ -use crate::{test_http::*, types::*}; +use crate::{test_http::*, tests::*, types::*}; use http::StatusCode; use log::*; use proxy_wasm::{traits::*, types::*}; @@ -102,6 +102,9 @@ impl Context for TestHttp { Some(format!("called {} times", self.n_sync_calls + 1).as_str()), ); } + "define_metrics" => test_define_metrics(self, TestPhase::HTTPCallResponse), + "increment_counters" => test_increment_counters(self, TestPhase::HTTPCallResponse), + "toggle_gauges" => test_toggle_gauges(self, TestPhase::HTTPCallResponse), _ => {} } diff --git a/t/lib/proxy-wasm-tests/hostcalls/src/lib.rs b/t/lib/proxy-wasm-tests/hostcalls/src/lib.rs index 783432b53..147b0ab22 100644 --- a/t/lib/proxy-wasm-tests/hostcalls/src/lib.rs +++ b/t/lib/proxy-wasm-tests/hostcalls/src/lib.rs @@ -7,13 +7,14 @@ mod types; use crate::{tests::*, types::test_http::*, types::test_root::*, types::*}; use log::*; use proxy_wasm::{traits::*, types::*}; -use std::{collections::HashMap, time::Duration}; +use std::{collections::BTreeMap, collections::HashMap, time::Duration}; proxy_wasm::main! {{ proxy_wasm::set_log_level(LogLevel::Trace); proxy_wasm::set_root_context(|_| -> Box { Box::new(TestRoot { config: HashMap::new(), + metrics: BTreeMap::new(), n_sync_calls: 0, }) }); @@ -63,12 +64,19 @@ impl RootContext for TestRoot { )); } - if let Some(on_configure) = self.get_config("on_configure") { - match on_configure { - "do_trap" => panic!("trap on_configure"), - "do_return_false" => return false, - _ => (), + match self.get_config("on_configure").unwrap_or("") { + "do_trap" => panic!("trap on_configure"), + "do_return_false" => return false, + "define_metrics" => test_define_metrics(self, TestPhase::Configure), + "define_and_increment_counters" => { + test_define_metrics(self, TestPhase::Configure); + test_increment_counters(self, TestPhase::Configure); } + "define_and_toggle_gauges" => { + test_define_metrics(self, TestPhase::Configure); + test_toggle_gauges(self, TestPhase::Configure); + } + _ => (), } true @@ -84,15 +92,30 @@ impl RootContext for TestRoot { self.get_config("tick_period").unwrap() ); + let n_sync_calls = self + .config + .get("n_sync_calls") + .map_or(1, |v| v.parse().expect("bad n_sync_calls value")); + + if self.n_sync_calls >= n_sync_calls { + return; + } + match self.get_config("on_tick").unwrap_or("") { "log_property" => test_log_property(self), + "define_metrics" => { + test_define_metrics(self, TestPhase::Tick); + self.n_sync_calls += 1; + } + "increment_counters" => test_increment_counters(self, TestPhase::Tick), + "toggle_gauges" => test_toggle_gauges(self, TestPhase::Tick), + "set_gauges" => { + test_set_gauges(self, TestPhase::Tick); + self.n_sync_calls += 1; + } + "log_metrics" => test_log_metrics(self, TestPhase::Tick), "set_property" => test_set_property(self), "dispatch" => { - let n_sync_calls = self - .config - .get("n_sync_calls") - .map_or(1, |v| v.parse().expect("bad n_sync_calls value")); - if self.n_sync_calls >= n_sync_calls { return; } @@ -172,6 +195,7 @@ impl RootContext for TestRoot { Some(Box::new(TestHttp { config: self.config.clone(), on_phases: phases, + metrics: self.metrics.clone(), n_sync_calls: 0, ncalls: 0, })) diff --git a/t/lib/proxy-wasm-tests/hostcalls/src/tests/mod.rs b/t/lib/proxy-wasm-tests/hostcalls/src/tests/mod.rs index 1b98e15bd..a2151d571 100644 --- a/t/lib/proxy-wasm-tests/hostcalls/src/tests/mod.rs +++ b/t/lib/proxy-wasm-tests/hostcalls/src/tests/mod.rs @@ -123,6 +123,13 @@ pub(crate) fn test_log_properties(ctx: &(dyn TestContext + 'static), phase: Test } } +pub(crate) fn test_log_metrics(ctx: &(dyn TestContext + 'static), phase: TestPhase) { + for (n, id) in ctx.get_metrics_mapping() { + let value = get_metric(*id).unwrap(); + info!("{}: {} at {:?}", n, value, phase) + } +} + fn show_property(ctx: &(dyn TestContext + 'static), path: &[&str]) -> String { if let Some(p) = ctx.get_property(path.to_vec()) { if let Ok(value) = std::str::from_utf8(&p) { @@ -238,7 +245,9 @@ trait HeaderStr { impl HeaderStr for str { fn split_header<'a>(&'a self) -> Option<(&'a str, &'a str)> { if &self[0..1] == ":" { - self[1..].find(':').map(|n| (&self[0..n + 1], &self[n + 2..])) + self[1..] + .find(':') + .map(|n| (&self[0..n + 1], &self[n + 2..])) } else { self.split_once(':') } @@ -414,6 +423,126 @@ pub(crate) fn test_set_shared_data_by_len(ctx: &mut TestHttp) { test_set_shared_data(ctx); } +fn should_run_on_current_worker(ctx: &(dyn TestContext + 'static)) -> bool { + match ctx.get_config("on_worker") { + Some(on_worker) => { + let w_id_bytes = ctx + .get_property(vec![("worker_id")]) + .expect("could not get worker_id"); + let w_id = std::str::from_utf8(&w_id_bytes).expect("bad worker_id value"); + + on_worker == w_id + } + None => true, + } +} + +pub(crate) fn test_define_metrics(ctx: &mut (dyn TestContext + 'static), phase: TestPhase) { + if !should_run_on_current_worker(ctx) { + return; + } + + let name_len = ctx.get_config("metrics_name_len").map_or(0, |x| { + x.parse::().expect("bad metrics_name_len value") + }); + + let metrics_config = ctx + .get_config("metrics") + .map(|x| x.to_string()) + .expect("missing metrics parameter"); + + for metric in metrics_config.split(",") { + let metric_char = metric.chars().nth(0).unwrap(); + let metric_type = match metric_char { + 'c' => MetricType::Counter, + 'g' => MetricType::Gauge, + _ => panic!("unexpected metric type"), + }; + let n = metric[1..].parse::().expect("bad metrics value"); + + for i in 1..(n + 1) { + let m_id; + let name = format!("{}{}_{:?}", metric_char, i, phase); + + if name_len > 0 { + let suffixed_name = + format!("{}{}", name, "x".repeat(name_len - name.chars().count())); + m_id = + define_metric(metric_type, &suffixed_name).expect("cannot define new metric"); + } else { + m_id = define_metric(metric_type, &name).expect("cannot define new metric"); + } + + info!("defined metric {} as {:?} at {:?}", &name, m_id, phase); + + ctx.save_metric_mapping(name.as_str(), m_id); + } + } +} + +pub(crate) fn test_increment_counters(ctx: &(dyn TestContext + 'static), phase: TestPhase) { + if !should_run_on_current_worker(ctx) { + return; + } + + let n_increments = ctx + .get_config("n_increments") + .map_or(1, |x| x.parse::().expect("bad n_increments value")); + + for (n, id) in ctx.get_metrics_mapping() { + for _ in 0..n_increments { + increment_metric(*id, 1).unwrap(); + } + + info!("incremented {} at {:?}", n, phase); + } + + test_log_metrics(ctx, phase); +} + +pub(crate) fn test_toggle_gauges(ctx: &(dyn TestContext + 'static), phase: TestPhase) { + if !should_run_on_current_worker(ctx) { + return; + } + + for (n, id) in ctx.get_metrics_mapping() { + let new_value = match get_metric(*id).unwrap() { + 0 => 1, + _ => 0, + }; + + record_metric(*id, new_value).unwrap(); + info!("toggled {} at {:?}", n, phase); + } + + test_log_metrics(ctx, phase); +} + +pub(crate) fn test_set_gauges(ctx: &(dyn TestContext + 'static), phase: TestPhase) { + if !should_run_on_current_worker(ctx) { + return; + } + + let value = ctx + .get_config("value") + .map_or(1, |x| x.parse::().expect("bad value")); + + for (n, id) in ctx.get_metrics_mapping() { + record_metric(*id, value).unwrap(); + info!("set {} to {} at {:?}", n, value, phase); + } + + test_log_metrics(ctx, phase); +} + +pub(crate) fn test_get_metrics(ctx: &TestHttp) { + for (n, id) in ctx.get_metrics_mapping() { + let name = n.replace("_", "-"); + let value = get_metric(*id).unwrap().to_string(); + ctx.add_http_response_header(name.as_str(), value.as_str()); + } +} + pub(crate) fn test_shared_queue_enqueue(ctx: &TestHttp) { let queue_id: u32 = ctx .config diff --git a/t/lib/proxy-wasm-tests/hostcalls/src/types/mod.rs b/t/lib/proxy-wasm-tests/hostcalls/src/types/mod.rs index c4f07751e..fc042ecc1 100644 --- a/t/lib/proxy-wasm-tests/hostcalls/src/types/mod.rs +++ b/t/lib/proxy-wasm-tests/hostcalls/src/types/mod.rs @@ -6,16 +6,21 @@ use crate::*; #[derive(Debug, Eq, PartialEq, enum_utils::FromStr)] #[enumeration(rename_all = "snake_case")] pub enum TestPhase { + Configure, + Tick, RequestHeaders, RequestBody, ResponseHeaders, ResponseBody, ResponseTrailers, + HTTPCallResponse, Log, } pub trait TestContext { fn get_config(&self, name: &str) -> Option<&str>; + fn get_metrics_mapping(&self) -> &BTreeMap; + fn save_metric_mapping(&mut self, name: &str, metric_id: u32) -> Option; } impl Context for dyn TestContext {} @@ -24,10 +29,26 @@ impl TestContext for TestRoot { fn get_config(&self, name: &str) -> Option<&str> { self.config.get(name).map(|s| s.as_str()) } + + fn get_metrics_mapping(&self) -> &BTreeMap { + &self.metrics + } + + fn save_metric_mapping(&mut self, name: &str, metric_id: u32) -> Option { + self.metrics.insert(name.to_string(), metric_id) + } } impl TestContext for TestHttp { fn get_config(&self, name: &str) -> Option<&str> { self.config.get(name).map(|s| s.as_str()) } + + fn get_metrics_mapping(&self) -> &BTreeMap { + &self.metrics + } + + fn save_metric_mapping(&mut self, name: &str, metric_id: u32) -> Option { + self.metrics.insert(name.to_string(), metric_id) + } } diff --git a/t/lib/proxy-wasm-tests/hostcalls/src/types/test_http.rs b/t/lib/proxy-wasm-tests/hostcalls/src/types/test_http.rs index 0fda3ce29..00bf96f4c 100644 --- a/t/lib/proxy-wasm-tests/hostcalls/src/types/test_http.rs +++ b/t/lib/proxy-wasm-tests/hostcalls/src/types/test_http.rs @@ -1,11 +1,12 @@ use crate::{tests::echo::*, tests::host::*, tests::*, *}; use http::StatusCode; use log::*; -use proxy_wasm::{traits::*, types::*}; +use proxy_wasm::{hostcalls::*, traits::*, types::*}; pub struct TestHttp { pub on_phases: Vec, pub config: HashMap, + pub metrics: BTreeMap, pub n_sync_calls: usize, pub ncalls: usize, } @@ -65,6 +66,7 @@ impl TestHttp { "/t/log/response_body" => test_log_response_body(self), "/t/log/property" => test_log_property(self), "/t/log/properties" => test_log_properties(self, cur_phase), + "/t/log/metrics" => test_log_metrics(self, cur_phase), /* send_local_response */ "/t/send_local_response/status/204" => test_send_status(self, 204), @@ -115,6 +117,14 @@ impl TestHttp { "/t/shm/enqueue" => test_shared_queue_enqueue(self), "/t/shm/dequeue" => test_shared_queue_dequeue(self), + /* metrics */ + "/t/metrics/define" => test_define_metrics(self, cur_phase), + "/t/metrics/increment_counters" => test_increment_counters(self, cur_phase), + "/t/metrics/toggle_gauges" => test_toggle_gauges(self, cur_phase), + "/t/metrics/get" => test_get_metrics(self), + "/t/metrics/increment_invalid_counter" => increment_metric(0, 1).unwrap(), + "/t/metrics/set_invalid_gauge" => record_metric(0, 1).unwrap(), + /* errors */ "/t/trap" => panic!("custom trap"), "/t/error/get_response_body" => { diff --git a/t/lib/proxy-wasm-tests/hostcalls/src/types/test_root.rs b/t/lib/proxy-wasm-tests/hostcalls/src/types/test_root.rs index 95ad06857..6300c47a2 100644 --- a/t/lib/proxy-wasm-tests/hostcalls/src/types/test_root.rs +++ b/t/lib/proxy-wasm-tests/hostcalls/src/types/test_root.rs @@ -2,5 +2,6 @@ use crate::*; pub struct TestRoot { pub config: HashMap, + pub metrics: BTreeMap, pub n_sync_calls: usize, } diff --git a/util/setup_dev.sh b/util/setup_dev.sh index 02f19fbc2..4b3944401 100755 --- a/util/setup_dev.sh +++ b/util/setup_dev.sh @@ -136,6 +136,30 @@ EOF use_http2 use_http3 env_to_nginx +EOF + patch --forward --ignore-whitespace lib/perl5/Test/Nginx/Util.pm <<'EOF' + @@ -977,6 +977,11 @@ + my $post_main_config = $block->post_main_config; + my $err_log_file = $block->error_log_file; + my $server_name = $block->server_name; + + my $workers = $block->workers; + + + + if (!$workers) { + + $workers = $Workers; + + } + + if ($UseHup) { + master_on(); # config reload is buggy when master is off + @@ -1054,7 +1059,7 @@ + bail_out "Can't open $ConfFile for writing: $!\n"; + print $out "daemon $DaemonEnabled;" if ($DaemonEnabled eq 'off'); + print $out <<_EOC_; + -worker_processes $Workers; + +worker_processes $workers; + master_process $MasterProcessEnabled; + error_log $err_log_file $LogLevel; + pid $PidFile; + EOF set -e popd