diff --git a/config b/config index dd1c748ee..44f7c317c 100644 --- a/config +++ b/config @@ -129,6 +129,7 @@ NGX_WASMX_INCS="\ $ngx_addon_dir/src/common \ $ngx_addon_dir/src/common/proxy_wasm \ $ngx_addon_dir/src/common/shm \ + $ngx_addon_dir/src/common/metrics \ $ngx_addon_dir/src/common/lua" NGX_WASMX_DEPS="\ @@ -141,7 +142,9 @@ NGX_WASMX_DEPS="\ $ngx_addon_dir/src/common/proxy_wasm/ngx_proxy_wasm_properties.h \ $ngx_addon_dir/src/common/shm/ngx_wasm_shm.h \ $ngx_addon_dir/src/common/shm/ngx_wasm_shm_kv.h \ - $ngx_addon_dir/src/common/shm/ngx_wasm_shm_queue.h" + $ngx_addon_dir/src/common/shm/ngx_wasm_shm_queue.h \ + $ngx_addon_dir/src/common/metrics/ngx_wa_metrics.h \ + $ngx_addon_dir/src/common/metrics/ngx_wa_histogram.h" NGX_WASMX_SRCS="\ $ngx_addon_dir/src/ngx_wasmx.c \ @@ -155,7 +158,9 @@ NGX_WASMX_SRCS="\ $ngx_addon_dir/src/common/proxy_wasm/ngx_proxy_wasm_util.c \ $ngx_addon_dir/src/common/shm/ngx_wasm_shm.c \ $ngx_addon_dir/src/common/shm/ngx_wasm_shm_kv.c \ - $ngx_addon_dir/src/common/shm/ngx_wasm_shm_queue.c" + $ngx_addon_dir/src/common/shm/ngx_wasm_shm_queue.c \ + $ngx_addon_dir/src/common/metrics/ngx_wa_metrics.c \ + $ngx_addon_dir/src/common/metrics/ngx_wa_histogram.c" # wasm diff --git a/docs/DIRECTIVES.md b/docs/DIRECTIVES.md index 950d17f56..9f351c4c1 100644 --- a/docs/DIRECTIVES.md +++ b/docs/DIRECTIVES.md @@ -6,6 +6,7 @@ By alphabetical order: - [cache_config](#cache-config) - [compiler](#compiler) - [flag](#flag) +- [max_metric_name_length](#max_metric_name_length) - [module](#module) - [proxy_wasm](#proxy_wasm) - [proxy_wasm_isolation](#proxy_wasm_isolation) @@ -16,6 +17,7 @@ By alphabetical order: - [resolver_timeout](#resolver_timeout) - [shm_kv](#shm_kv) - [shm_queue](#shm_queue) +- [slab_size](#slab_size) - [socket_buffer_size](#socket_buffer_size) - [socket_buffer_reuse](#socket_buffer_reuse) - [socket_connect_timeout](#socket_connect_timeout) @@ -57,6 +59,9 @@ By context: - [tls_trusted_certificate](#tls_trusted_certificate) - [tls_verify_cert](#tls_verify_cert) - [tls_verify_host](#tls_verify_host) + - `metrics{}` + - [max_metric_name_length](#max_metric_name_length) + - [slab_size](#slab_size) - `wasmtime{}` - [cache_config](#cache-config) - [flag](#flag) @@ -205,6 +210,29 @@ wasm { [Back to TOC](#directives) +max_metric_name_length +---------------------- + +**usage** | `max_metric_name_length ;` +------------:|:---------------------------------------------------------------- +**contexts** | `metrics{}` +**default** | `256` +**example** | `max_metric_name_length 512;` + +Set the maximum allowed length of a metric name. + +The configured value cannot be lower than `6` due to internal metrics storage in +memory. + +> Notes + +Configuring this value allows for predictable memory usage when configuring the +metrics [slab_size](#slab_size). + +See [Metrics] for a complete description of how metrics are stored in memory. + +[Back to TOC](#directives) + module ------ @@ -525,6 +553,32 @@ policy, and writes will fail when the allocated memory slab is full. [Back to TOC](#directives) +slab_size +--------- + +**usage** | `slab_size ;` +------------:|:---------------------------------------------------------------- +**contexts** | `metrics{}` +**default** | `5m` +**example** | `slab_size 12m;` + +Set the `size` of the shared memory slab dedicated to metrics storage. The value +must be at least 3 * pagesize, e.g. `15k` on Linux. + +> Notes + +The space in memory occupied by a metric depends on its name length, type and +the number of worker processes running. As an example, if all metric names are +64 chars long and 4 workers are running, `5m` can accommodate 20k counters, 20k +gauges, or up to 16k histograms. + +See the [max_metric_name_length](#max_metric_name_length) directive to configure +the maximum allowed length of metrics names. + +See [Metrics] for a complete description of how metrics are stored in memory. + +[Back to TOC](#directives) + socket_buffer_reuse ------------------- @@ -939,7 +993,8 @@ the `http{}` contexts. [Contexts]: USER.md#contexts [Execution Chain]: USER.md#execution-chain -[SLRU eviction algorithm]: SLRU.md +[Metrics]: METRICS.md [OpenResty]: https://openresty.org/en/ [resolver]: https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver [resolver_timeout]: https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver_timeout +[SLRU eviction algorithm]: SLRU.md diff --git a/docs/METRICS.md b/docs/METRICS.md new file mode 100644 index 000000000..85d67826b --- /dev/null +++ b/docs/METRICS.md @@ -0,0 +1,139 @@ +# Metrics + +This document elaborates on the types of metrics available in ngx_wasm_module, +how they are stored in memory, and how to estimate the amount of [slab_size] +memory necessary for your use-case. + +## Table of Contents + +- [Types of Metrics](#types-of-metrics) +- [Name Prefixing](#name-prefixing) +- [Histogram Binning Strategy](#histogram-binning-strategy) +- [Histogram Update and Expansion](#histogram-update-and-expansion) +- [Memory Consumption](#memory-consumption) +- [Shared Memory Allocation](#shared-memory-allocation) +- [Nginx Reconfiguration](#nginx-reconfiguration) + +## Types of Metrics + +In accordance with Proxy-Wasm specifications, a "metric" is either a counter, a +gauge, or a histogram. + +- A counter is an unsigned 64-bit int that can only be incremented. +- A gauge is an unsigned 64-bit int that can take arbitrary values. +- A histogram represents range frequencies of a variable and can be defined as a + set of pairs of ranges and counters. + For example, the distribution of response time of HTTP requests can be + represented as a histogram with ranges `[0, 1]`, `(1, 2]`, `(2, 4]`, and `(4, + Inf]`. The 1st range counter would be the number of requests with response + time less or equal to 1ms; the 2nd range counter represents requests with + response time between 1ms and 2ms; the 3rd range counter are requests with + response time between 2ms and 4ms; and the last range counter are requests + with response time bigger than 4ms. + +[Back to TOC](#table-of-contents) + +## Name Prefixing + +To avoid naming conflicts between Proxy-Wasm filters, the name of a metric is +always prefixed with: `pw.{filter_name}.{metric_name}`. This means that a metric +named `a_counter` inserted by `a_filter` will have its name stored as: +`pw.a_filter.a_counter`. + +Thus, the maximum length of a metric name configured via +[max_metric_name_length] is enforced on the prefixed name and may need to be +increased in some cases. + +[Back to TOC](#table-of-contents) + +## Histogram Binning Strategy + +The above example demonstrates a histogram with ranges (or bins) whose +upper-bound grows in powers of 2, i.e. `2^0`, `2^1`, and `2^2`. This is usually +called "logarithmic binning" and is how histograms bins are represented in +ngx_wasm_module. + +This binning strategy implies that when a value `v` is recorded, it is matched +with the smallest power of two that is bigger than `v`. This value is the +*upper-bound* of the bin associated with `v`. If the histogram contains or can +contain such a bin, that bin's counter is incremented. If not, the bin with the +next smallest upper-bound bigger than `v` has its counter incremented instead. + +[Back to TOC](#table-of-contents) + +## Histogram Update and Expansion + +Histograms are created with 5 bins: 1 initialized and 4 uninitialized. + +The bin initialized upon histogram creation has upper-bound `2^32` and its +counter is incremented if it is the only bin whose upper-bound is bigger than +the recorded value. + +If a value `v` is recorded and its bin is not part of the initialized bins, a +new bin with the upper-bound associated with `v` is initialized, and its counter +is incremented. + +If the histogram is out of uninitialized bins, it can be expanded up to 18 +bins so as to accommodate the additional bins for other ranges of `v`. + +[Back to TOC](#table-of-contents) + +## Memory Consumption + +The space occupied by a metric in memory contains: + +1. Its name. +2. Its value. +3. And the underlying structure representing the metric in the shared key-value + store memory ([slab_size]). + +While the key-value structure has a fixed size of **96 bytes**, the sizes of +name and value vary. + +In memory, the value of a counter or gauge occupies 8 bytes + 16 bytes per +worker process. The value size grows according to the number of workers because +metric values are segmented across them: Each worker has its own segment of the +value to write updates to. When a metric is retrieved, the segments are +consolidated and returned as a single metric value. This storage strategy allows +metric updates to be performed without the aid of shared memory read/write locks +at the cost of 16 bytes per worker. + +Histogram values also have a baseline size of 8 bytes + 16 bytes per worker +process. However, histograms also need extra space per worker for bins storage. +Bins storage costs 4 bytes + 8 bytes per bin. Thus, a 5-bin histogram takes: 8 +bytes + (16 + 4 + 5*8), so 60 bytes per worker. + +As such, in a 4-workers setup, a counter or gauge whose name is 64 chars long +occupies 168 bytes, and a 5-bin histogram with the same name length occupies 408 +bytes. A 18-bin histogram with the same length name occupies 824 bytes. + +[Back to TOC](#table-of-contents) + +## Shared Memory Allocation + +Nginx employs a shared memory allocation model that enforces allocation size to +be a power of 2 greater than 8; nonconforming values are rounded up, see [Nginx +shared memory]. + +For instance, this means that an allocation of 168 bytes ends up occupying 256 +bytes of shared memory. This should be taken into consideration when estimating +the total space required for a group of metrics. + +[Back to TOC](#table-of-contents) + +## Nginx Reconfiguration + +If Nginx is reconfigured with a different number of workers or a different +[slab_size] value, existing metrics need to be reallocated into a new +shared memory zone at reconfiguration time. This is due to the metric values +being segmented across workers. + +As such, it is important to make sure that the new [slab_size] value is large +enough to accommodate existing metrics, and that the value of +[max_metric_name_length] is not less than any existing metric name. + +[Back to TOC](#table-of-contents) + +[Nginx shared memory]: https://nginx.org/en/docs/dev/development_guide.html#shared_memory +[slab_size]: DIRECTIVES.md#slab_size +[max_metric_name_length]: DIRECTIVES.md#max_metric_name_length diff --git a/docs/PROXY_WASM.md b/docs/PROXY_WASM.md index 72495ea17..bcc05eae2 100644 --- a/docs/PROXY_WASM.md +++ b/docs/PROXY_WASM.md @@ -536,10 +536,10 @@ SDK ABI `0.2.1`) and their present status in ngx_wasm_module: `proxy_enqueue_shared_queue` | :heavy_check_mark: | No automatic eviction mechanism if the queue is full. `proxy_resolve_shared_queue` | :x: | *Stats/metrics* | | -`proxy_define_metric` | :x: | -`proxy_get_metric` | :x: | -`proxy_record_metric` | :x: | -`proxy_increment_metric` | :x: | +`proxy_define_metric` | :heavy_check_mark: | +`proxy_get_metric` | :heavy_check_mark: | +`proxy_record_metric` | :heavy_check_mark: | +`proxy_increment_metric` | :heavy_check_mark: | *Custom extension points* | | `proxy_call_foreign_function` | :x: | diff --git a/docs/adr/005-metrics.md b/docs/adr/005-metrics.md new file mode 100644 index 000000000..b72a8a2d6 --- /dev/null +++ b/docs/adr/005-metrics.md @@ -0,0 +1,220 @@ +# Metrics + +* Status: proposed +* Deciders: WasmX +* Date: 2024-05-03 + +## Table of Contents + +- [Problem Statement](#problem-statement) +- [Technical Context](#technical-context) +- [Decision Drivers](#decision-drivers) +- [Proposal](#proposal) + - [Histograms](#histograms) + - [Binning Strategy](#binning-strategy) + - [Allocation and Updates](#allocation-and-updates) + - [Expansion](#expansion) + +## Problem Statement + +Support the definition, update, and retrieval of metrics from Proxy-Wasm +filters, ngx_wasm_module itself, and the Lua land. How exactly are metrics +stored and how to access them so as to ensure two Nginx workers never write to +the same memory space. + +[Back to TOC](#table-of-contents) + +## Technical Context + +A metric can either be a counter, a gauge, or a histogram: + +- A counter is an integer that can only be incremented. +- A gauge is an integer that can take arbitrary positive values. +- A histogram represents range frequencies of a variable and can be defined as a + set of pairs of ranges and counters. + For example, the distribution of response time of HTTP requests can be + represented as a histogram with ranges `[0, 1]`, `(1, 2]`, `(2, 4]`, and `(4, + Inf]`. The 1st range counter would be the number of requests with response + time less or equal to 1ms; the 2nd range counter represents requests with + response time between 1ms and 2ms; the 3rd range counter are requests with + response time between 2ms and 4ms; and the last range counter are requests + with response time bigger than 4ms. + +A metric's value should reflect updates from all worker processes. As a counter +begins at `0` and is incremented by workers 0 and 1, it should be `2` when +another worker retrieves its value. A gauge, however, is whatever value was last +set by any of the workers. Histograms, like counters, account for values +recorded by all workers. + +[Back to TOC](#table-of-contents) + +## Decision Drivers + +* Full Proxy-Wasm ABI compatibility. +* Build atop ngx_wasm_shm. +* Minimize memory usage. +* Minimize metrics access cost. +* Generalize metrics ABI beyond Proxy-Wasm facilities (WasmX-wide). + +[Back to TOC](#table-of-contents) + +## Proposal + +The proposed scheme for metrics storage builds atop ngx_wasm_shm's key-value +store. Metric name is stored as a key in a red-black tree node along with the +metric value. Metric values are represented by `ngx_wa_metric_t`, see below. The +member `type` is the metric type while the flexible array member `slots` stores +actual metric data. + +The length of `slots` equals the number of worker processes running when the +metric is defined. This ensures each worker has its own dedicated slot to write +metric updates. + +For counters, each entry in the `slots` array is simply an unsigned integer that +its corresponding worker increments. When a counter is retrieved, all values in +the `slots` array are summed and that total sum is returned. + +For gauges, each of the `slots` is a pair of unsigned integer and timestamp. +When a worker sets a gauge, the value is stored along with the current time in +its corresponding worker slot. When a gauge is retrieved, the values in the +`slots` are iterated and the most recent value is returned. + +For histograms, each of the `slots` points to a `ngx_wa_metrics_histogram_t` +instance. Each worker updates the histogram pointed to by its slot. When a +histogram is retrieved, the `slots` array is iterated and each worker's +histogram is merged into a temporary histogram, which can then be serialized. + +```c +typedef enum { + NGX_WA_METRIC_COUNTER, + NGX_WA_METRIC_GAUGE, + NGX_WA_METRIC_HISTOGRAM, +} ngx_wa_metric_type_e; + +typedef struct { + ngx_uint_t value; + ngx_msec_t last_update; +} ngx_wa_metrics_gauge_t; + +typedef struct { + uint32_t upper_bound; + uint32_t count; +} ngx_wa_metrics_bin_t; + +typedef struct { + uint8_t n_bins; + ngx_wa_metrics_bin_t bins[]; +} ngx_wa_metrics_histogram_t; + +typedef union { + ngx_uint_t counter; + ngx_wa_metrics_gauge_t gauge; + ngx_wa_metrics_histogram_t *histogram; +} ngx_wa_metric_val_t; + +typedef struct { + ngx_wa_metric_type_e type; + ngx_wa_metric_val_t slots[]; +} ngx_wa_metric_t; +``` + +This storage strategy ensures that two workers **never** write to the same +memory address when updating a metric as long as no memory allocation is +performed. This is indeed the case for counters and gauges and it is also the +case for most histogram updates. + +This is an important feature of this design as it allows the more frequent +update operations to be performed without the aid of shared memory locks. The +cost of a lock-less metric update then becomes merely the cost of searching the +red-black tree of the underlying key-value store: `O(log n)`. + +The capacity of updating a metric without having to acquire a lock is +particularly attractive when a set of worker processes is under heavy load. In +such conditions, lock contention is likely to impact proxy throughput as workers +are more likely to wait for a lock to be released before proceeding with the +metric update. + +Metric definition and removal still require locks to be safely performed as two +workers might end up attempting to write to the same memory location. This is +also true for histogram updates which cause them to grow their number of `bins`. + +The ABI proposed to accomplish the described system closely resembles the one +from Proxy-Wasm specification itself: + +```c +ngx_int_t ngx_wa_metrics_define(ngx_wa_metrics_t *metrics, ngx_str_t *name, + ngx_wa_metric_type_e type, uint32_t *out); +ngx_int_t ngx_wa_metrics_increment(ngx_wa_metrics_t *metrics, + uint32_t metric_id, ngx_int_t val); +ngx_int_t ngx_wa_metrics_record(ngx_wa_metrics_t *metrics, uint32_t metric_id, + ngx_int_t val); +ngx_int_t ngx_wa_metrics_get(ngx_wa_metrics_t *metrics, uint32_t metric_id, + ngx_uint_t *out); +``` + +[Back to TOC](#table-of-contents) + +### Histograms + +This proposal includes a scheme composed of `ngx_wa_metrics_bin_t` (a pair of +upper-bound and counter) and `ngx_wa_metrics_histogram_t` (a list of +`ngx_wa_metrics_bin_t` ordered by upper-bound) to represent histogram data in +memory. A bin's counter is the number of recorded values less than or equal to +its upper-bound and bigger than the previous bin's upper-bound. + +This storage layout can represent both histograms with user-defined bins and +histograms following an automatic binning strategy, like logarithmic binning. +This document will focus, however, on logarithmic binning only; user-defined +bins are left for a future iteration. + +[Back to TOC](#table-of-contents) + +#### Binning Strategy + +The proposed binning strategy assumes the domain of the variables being measured +is the set of non-negative integers and divides this domain into bins whose +upper-bound grows in powers of 2, i.e., 1, 2, 4, 8, 16, etc. The mapping of a +value `v` to its bin is given by the function `pow(2, ceil(log2(v)))` which +calculates the bin's upper-bound. The value 10, for example, is mapped to the +bin whose upper-bound is `pow(2, ceil(log2(10)))`, or 16. The bin with +upper-bound `16` represents recorded values between `8` and `16`. + +This logarithmic scaling provides good enough resolution for small values in +return for low resolution for large values while keeping the memory footprint +reasonably low: values up to 65,536 can be represented with only 16 bins. These +characteristics fit the typical use case of measuring HTTP response time in +milliseconds. + +[Back to TOC](#table-of-contents) + +#### Allocation and Updates + +Histograms are created with enough space for 5 bins, one of which is initialized +with `NGX_MAX_UINT32_VALUE` as upper-bound, leaving 4 uninitialized bins. + +If a value `v` is recorded into a histogram and its respective bin is part of +the histogram's bins, its counter is simply incremented. If not and there is at +least one uninitialized bin, then one bin is initialized with the `v` +upper-bound, and the bins are rearranged to ensure ascending order with respect +to upper-bound. The new bin's counter is then incremented. + +[Back to TOC](#table-of-contents) + +#### Expansion + +If a value `v` is recorded but its bin is not part of the histogram's bins and +there are not any uninitialized bins left, the histogram needs to grow to +accommodate the new value's bin. + +Expanding a histogram means allocating memory for a new histogram instance with +enough space for the additional bin, then copying memory from the old instance +to the new one, and finally releasing the old histogram from memory. The new +uninitialized bin is then initialized with the `v` upper-bound and its counter +is incremented. + +Histograms, however, can only grow up to a maximum number of bins. When a value +`v` is recorded into a histogram but its bin is not part of the bins and the +histogram has reached the total bin limit, the bin with the smallest upper-bound +bigger than `v` is incremented instead. + +[Back to TOC](#table-of-contents) diff --git a/src/common/debug/ngx_wasm_debug_module.c b/src/common/debug/ngx_wasm_debug_module.c index ba78c4aa7..175020cd4 100644 --- a/src/common/debug/ngx_wasm_debug_module.c +++ b/src/common/debug/ngx_wasm_debug_module.c @@ -9,6 +9,8 @@ #include #endif +#include + #if (!NGX_DEBUG) # error ngx_wasm_debug_module included in a non-debug build #endif @@ -20,7 +22,12 @@ static ngx_int_t ngx_wasm_debug_init(ngx_cycle_t *cycle) { - static ngx_wasm_phase_t ngx_wasm_debug_phases[] = { + static size_t long_metric_name_len = NGX_MAX_ERROR_STR; + uint32_t mid; + ngx_str_t metric_name; + u_char buf[long_metric_name_len]; + + static ngx_wasm_phase_t ngx_wasm_debug_phases[] = { { ngx_string("a_phase"), 0, 0, 0 }, { ngx_null_string, 0, 0, 0 } }; @@ -41,6 +48,31 @@ ngx_wasm_debug_init(ngx_cycle_t *cycle) ngx_wasm_phase_lookup(&ngx_wasm_debug_subsystem, 3) == NULL ); + metric_name.len = long_metric_name_len; + metric_name.data = buf; + + /* invalid metric name length */ + ngx_wa_assert( + ngx_wa_metrics_define(ngx_wasmx_metrics(cycle), + &metric_name, + NGX_WA_METRIC_COUNTER, + &mid) == NGX_ABORT + ); + + /* invalid metric type */ + ngx_wa_assert( + ngx_wa_metrics_define(ngx_wasmx_metrics(cycle), + &metric_name, + 100, + &mid) == NGX_ABORT + ); + + /* unknown metric type name */ + ngx_wa_assert( + ngx_strncmp(((ngx_str_t *) ngx_wa_metric_type_name(12))->data, + "unknown", 8) == 0 + ); + return NGX_OK; } diff --git a/src/common/metrics/ngx_wa_histogram.c b/src/common/metrics/ngx_wa_histogram.c new file mode 100644 index 000000000..b21dd385e --- /dev/null +++ b/src/common/metrics/ngx_wa_histogram.c @@ -0,0 +1,239 @@ +#ifndef DDEBUG +#define DDEBUG 0 +#endif +#include "ddebug.h" + +#include +#include + + +#define NGX_WA_BINS_INIT 5 +#define NGX_WA_BINS_MAX 18 +#define NGX_WA_BINS_INCREMENT 4 + + +static uint32_t +bin_log2_upper_bound(ngx_uint_t n) +{ + uint32_t upper_bound = 2; + + if (n <= 1) { + return 1; + } + + if (n > NGX_MAX_UINT32_VALUE / 2) { + return NGX_MAX_UINT32_VALUE; + } + + for (n = n - 1; n >>= 1; upper_bound <<= 1) { /* void */ } + + return upper_bound; +} + + +static ngx_int_t +histogram_grow(ngx_wa_metrics_t *metrics, ngx_wa_metrics_histogram_t *h, + ngx_wa_metrics_histogram_t **out) +{ + size_t old_size, size; + ngx_int_t rc = NGX_OK; + ngx_uint_t n; + ngx_wa_metrics_histogram_t *new_h = NULL; + + if (h->n_bins == NGX_WA_BINS_MAX) { + return NGX_ERROR; + } + + ngx_log_debug0(NGX_LOG_DEBUG_WASM, metrics->shm->log, 0, + "growing histogram"); + + n = ngx_min(NGX_WA_BINS_INCREMENT, NGX_WA_BINS_MAX - h->n_bins); + old_size = sizeof(ngx_wa_metrics_histogram_t) + + sizeof(ngx_wa_metrics_bin_t) * h->n_bins; + size = old_size + sizeof(ngx_wa_metrics_bin_t) * n; + + if (metrics->shm->eviction == NGX_WASM_SHM_EVICTION_NONE) { + ngx_wasm_shm_lock(metrics->shm); + } + + new_h = ngx_slab_calloc_locked(metrics->shm->shpool, size); + if (new_h == NULL) { + ngx_log_debug0(NGX_LOG_DEBUG_WASM, metrics->shm->log, 0, + "cannot expand histogram"); + rc = NGX_ERROR; + goto error; + } + + ngx_memcpy(new_h, h, old_size); + ngx_slab_free_locked(metrics->shm->shpool, h); + + new_h->n_bins += n; + *out = new_h; + +error: + + if (metrics->shm->eviction == NGX_WASM_SHM_EVICTION_NONE) { + ngx_wasm_shm_unlock(metrics->shm); + } + + return rc; +} + + +static ngx_wa_metrics_bin_t * +histogram_bin(ngx_wa_metrics_t *metrics, ngx_wa_metrics_histogram_t *h, + ngx_uint_t n, ngx_wa_metrics_histogram_t **out) +{ + size_t i, j = 0; + uint32_t ub = bin_log2_upper_bound(n); + ngx_wa_metrics_bin_t *b; + + for (i = 0; i < h->n_bins; i++) { + b = &h->bins[i]; + j = (j == 0 && ub < b->upper_bound) ? i : j; + + if (b->upper_bound == ub) { + return b; + } + + if (b->upper_bound == 0) { + break; + } + } + + if (i == h->n_bins) { + if (out && histogram_grow(metrics, h, out) == NGX_OK) { + h = *out; + + } else { + ngx_wasm_log_error(NGX_LOG_INFO, metrics->shm->log, 0, + "cannot add a new histogram bin for value " + "\"%uD\", returning next closest bin", n); + return &h->bins[j]; + } + } + + /* shift bins to create space for the new one */ + ngx_memcpy(&h->bins[j + 1], &h->bins[j], + sizeof(ngx_wa_metrics_bin_t) * (i - j)); + + h->bins[j].upper_bound = ub; + h->bins[j].count = 0; + + return &h->bins[j]; +} + + +#if (NGX_DEBUG) +static void +histogram_log(ngx_wa_metrics_t *metrics, ngx_wa_metric_t *m, uint32_t mid) +{ + size_t i, size = sizeof(ngx_wa_metrics_histogram_t) + + sizeof(ngx_wa_metrics_bin_t) + * NGX_WA_BINS_MAX; + ngx_wa_metrics_bin_t *b; + ngx_wa_metrics_histogram_t *h; + u_char *p, buf[size], s_buf[NGX_MAX_ERROR_STR]; + + ngx_memzero(buf, size); + + p = s_buf; + h = (ngx_wa_metrics_histogram_t *) buf; + h->n_bins = NGX_WA_BINS_MAX; + h->bins[0].upper_bound = NGX_MAX_UINT32_VALUE; + + ngx_wa_metrics_histogram_get(metrics, m, metrics->workers, h); + + for (i = 0; i < h->n_bins; i++) { + b = &h->bins[i]; + if (b->upper_bound == 0) { + break; + } + + p = ngx_sprintf(p, " %uD: %uD;", b->upper_bound, b->count); + } + + ngx_log_debug3(NGX_LOG_DEBUG_WASM, metrics->shm->log, 0, + "histogram \"%uD\": %*s", + mid, p - s_buf - 1, s_buf + 1); +} +#endif + + +ngx_int_t +ngx_wa_metrics_histogram_add_locked(ngx_wa_metrics_t *metrics, + ngx_wa_metric_t *m) +{ + size_t i; + static uint16_t n_bins = NGX_WA_BINS_INIT; + ngx_wa_metrics_histogram_t **h; + + for (i = 0; i < metrics->workers; i++) { + h = &m->slots[i].histogram; + *h = ngx_slab_calloc_locked(metrics->shm->shpool, + sizeof(ngx_wa_metrics_histogram_t) + + sizeof(ngx_wa_metrics_bin_t) * n_bins); + if (*h == NULL) { + goto error; + } + + (*h)->n_bins = n_bins; + (*h)->bins[0].upper_bound = NGX_MAX_UINT32_VALUE; + } + + return NGX_OK; + +error: + + ngx_wasm_log_error(NGX_LOG_ERR, metrics->shm->log, 0, + "cannot allocate histogram"); + + for (/* void */ ; i > 0; i--) { + ngx_slab_free_locked(metrics->shm->shpool, m->slots[i - 1].histogram); + } + + return NGX_ERROR; +} + + +ngx_int_t +ngx_wa_metrics_histogram_record(ngx_wa_metrics_t *metrics, ngx_wa_metric_t *m, + ngx_uint_t slot, uint32_t mid, ngx_uint_t n) +{ + ngx_wa_metrics_bin_t *b; + ngx_wa_metrics_histogram_t *h; + + h = m->slots[slot].histogram; + b = histogram_bin(metrics, h, n, &m->slots[slot].histogram); + b->count += 1; + +#if (NGX_DEBUG) + histogram_log(metrics, m, mid); +#endif + + return NGX_OK; +} + + +void +ngx_wa_metrics_histogram_get(ngx_wa_metrics_t *metrics, ngx_wa_metric_t *m, + ngx_uint_t slots, ngx_wa_metrics_histogram_t *out) +{ + size_t i, j = 0; + ngx_wa_metrics_bin_t *b, *out_b; + ngx_wa_metrics_histogram_t *h; + + for (i = 0; i < slots; i++) { + h = m->slots[i].histogram; + + for (j = 0; j < h->n_bins; j++) { + b = &h->bins[j]; + if (b->upper_bound == 0) { + break; + } + + out_b = histogram_bin(metrics, out, b->upper_bound, NULL); + out_b->count += b->count; + } + } +} diff --git a/src/common/metrics/ngx_wa_histogram.h b/src/common/metrics/ngx_wa_histogram.h new file mode 100644 index 000000000..854b1ad57 --- /dev/null +++ b/src/common/metrics/ngx_wa_histogram.h @@ -0,0 +1,16 @@ +#ifndef _NGX_WA_HISTOGRAM_H_INCLUDED_ +#define _NGX_WA_HISTOGRAM_H_INCLUDED_ + + +#include + + +ngx_int_t ngx_wa_metrics_histogram_add_locked(ngx_wa_metrics_t *metrics, + ngx_wa_metric_t *m); +ngx_int_t ngx_wa_metrics_histogram_record(ngx_wa_metrics_t *metrics, + ngx_wa_metric_t *m, ngx_uint_t slot, uint32_t mid, ngx_uint_t n); +void ngx_wa_metrics_histogram_get(ngx_wa_metrics_t *metrics, ngx_wa_metric_t *m, + ngx_uint_t slots, ngx_wa_metrics_histogram_t *out); + + +#endif /* _NGX_WA_HISTOGRAM_H_INCLUDED_ */ diff --git a/src/common/metrics/ngx_wa_metrics.c b/src/common/metrics/ngx_wa_metrics.c new file mode 100644 index 000000000..5c90e9b3e --- /dev/null +++ b/src/common/metrics/ngx_wa_metrics.c @@ -0,0 +1,460 @@ +#ifndef DDEBUG +#define DDEBUG 0 +#endif +#include "ddebug.h" + +#include +#include +#include + + +ngx_str_t * +ngx_wa_metric_type_name(ngx_wa_metric_type_e type) +{ + static ngx_str_t counter = ngx_string("counter"); + static ngx_str_t gauge = ngx_string("gauge"); + static ngx_str_t histogram = ngx_string("histogram"); + static ngx_str_t unknown = ngx_string("unknown"); + + switch (type) { + case NGX_WA_METRIC_COUNTER: + return &counter; + + case NGX_WA_METRIC_GAUGE: + return &gauge; + + case NGX_WA_METRIC_HISTOGRAM: + return &histogram; + + default: + return &unknown; + } +} + + +static ngx_uint_t +get_counter(ngx_wa_metric_t *m, ngx_uint_t slots) +{ + ngx_uint_t i, val = 0; + + for (i = 0; i < slots; i++) { + val += m->slots[i].counter; + } + + return val; +} + + +static ngx_uint_t +get_gauge(ngx_wa_metric_t *m, ngx_uint_t slots) +{ + ngx_msec_t l; + ngx_uint_t i, val = 0; + + val = m->slots[0].gauge.value; + l = m->slots[0].gauge.last_update; + + for (i = 1; i < slots; i++) { + if (m->slots[i].gauge.last_update > l) { + val = m->slots[i].gauge.value; + l = m->slots[i].gauge.last_update; + } + } + + return val; +} + + +static ngx_int_t +realloc_histogram(ngx_wa_metrics_t *metrics, ngx_wa_metric_t *old_m, + uint32_t mid) +{ + uint32_t cas, slots = metrics->old_metrics->workers; + ngx_int_t rc; + ngx_str_t *val; + ngx_wa_metric_t *m; + + rc = ngx_wasm_shm_kv_get_locked(metrics->shm, NULL, &mid, &val, &cas); + if (rc != NGX_OK) { + return rc; + } + + m = (ngx_wa_metric_t *) val->data; + + ngx_wa_metrics_histogram_get(metrics, old_m, slots, m->slots[0].histogram); + + return NGX_OK; +} + + +static ngx_int_t +realloc_metrics(ngx_wa_metrics_t *metrics, ngx_rbtree_node_t *node, + ngx_rbtree_node_t *sentinel) +{ + uint32_t mid; + ngx_int_t rc; + ngx_uint_t val; + ngx_wasm_shm_kv_node_t *n = (ngx_wasm_shm_kv_node_t *) node; + ngx_wa_metric_t *m = (ngx_wa_metric_t *) n->value.data; + + if (node == sentinel) { + return NGX_OK; + } + + ngx_log_debug1(NGX_LOG_DEBUG_WASM, metrics->shm->log, 0, + "reallocating metric \"%V\"", &n->key.str); + + if (ngx_wa_metrics_define(metrics, &n->key.str, m->type, &mid) != NGX_OK) { + ngx_wasm_log_error(NGX_LOG_ERR, metrics->shm->log, 0, + "failed redefining metric \"%V\"", &n->key.str); + + return NGX_ERROR; + } + + switch (m->type) { + case NGX_WA_METRIC_COUNTER: + val = get_counter(m, metrics->old_metrics->workers); + rc = ngx_wa_metrics_increment(metrics, mid, val); + break; + + case NGX_WA_METRIC_GAUGE: + val = get_gauge(m, metrics->old_metrics->workers); + rc = ngx_wa_metrics_record(metrics, mid, val); + break; + + case NGX_WA_METRIC_HISTOGRAM: + rc = realloc_histogram(metrics, m, mid); + break; + + default: + ngx_wa_assert(0); + return NGX_ERROR; + } + + if (rc != NGX_OK) { + ngx_wasm_log_error(NGX_LOG_ERR, metrics->shm->log, 0, + "failed updating metric \"%V\"", &n->key.str); + return NGX_ERROR; + } + + if (node->left + && realloc_metrics(metrics, node->left, sentinel) != NGX_OK) + { + return NGX_ERROR; + } + + if (node->right + && realloc_metrics(metrics, node->right, sentinel) != NGX_OK) + { + return NGX_ERROR; + } + + return NGX_OK; +} + + +ngx_wa_metrics_t * +ngx_wa_metrics_alloc(ngx_cycle_t *cycle) +{ + static ngx_str_t shm_name = ngx_string("metrics"); + ngx_wa_metrics_t *metrics; + + metrics = ngx_pcalloc(cycle->pool, sizeof(ngx_wa_metrics_t)); + if (metrics == NULL) { + return NULL; + } + + metrics->old_metrics = ngx_wasmx_metrics(cycle->old_cycle); + metrics->config.slab_size = NGX_CONF_UNSET_SIZE; + metrics->config.max_metric_name_length = NGX_CONF_UNSET_SIZE; + + metrics->shm = ngx_pcalloc(cycle->pool, sizeof(ngx_wasm_shm_t)); + if (metrics->shm == NULL) { + ngx_pfree(cycle->pool, metrics); + return NULL; + } + + metrics->shm->log = &cycle->new_log; + metrics->shm->name = shm_name; + metrics->shm->type = NGX_WASM_SHM_TYPE_METRICS; + metrics->shm->eviction = NGX_WASM_SHM_EVICTION_NONE; + + return metrics; +} + + +char * +ngx_wa_metrics_init_conf(ngx_wa_metrics_t *metrics, ngx_conf_t *cf) +{ + ngx_cycle_t *cycle = cf->cycle; + ngx_wa_metrics_t *old_metrics = metrics->old_metrics; + ngx_core_conf_t *ccf; + + ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module); + + if (metrics->config.slab_size == NGX_CONF_UNSET_SIZE) { + metrics->config.slab_size = NGX_WA_METRICS_DEFAULT_SLAB_SIZE; + } + + if (metrics->config.max_metric_name_length == NGX_CONF_UNSET_SIZE) { + metrics->config.max_metric_name_length = + NGX_WA_METRICS_DEFAULT_MAX_NAME_LEN; + } + + /* TODO: if eviction is enabled, metrics->workers must be set to 1 */ + metrics->workers = ccf->worker_processes; + metrics->shm_zone = ngx_shared_memory_add(cf, &metrics->shm->name, + metrics->config.slab_size, + &ngx_wasmx_module); + if (metrics->shm_zone == NULL) { + return NGX_CONF_ERROR; + } + + metrics->shm_zone->init = ngx_wasm_shm_init_zone; + metrics->shm_zone->data = metrics->shm; + metrics->shm_zone->noreuse = 0; + + if (old_metrics + && (metrics->workers != old_metrics->workers + || metrics->config.slab_size != old_metrics->config.slab_size)) + { + metrics->shm_zone->noreuse = 1; + } + + return NGX_CONF_OK; +} + + +ngx_int_t +ngx_wa_metrics_init(ngx_wa_metrics_t *metrics, ngx_cycle_t *cycle) +{ + ngx_wasm_shm_kv_t *old_shm_kv; + + if (metrics->old_metrics && !metrics->shm_zone->noreuse) { + /* reuse old kv store */ + metrics->shm->data = metrics->old_metrics->shm->data; + return NGX_OK; + } + + if (ngx_wasm_shm_kv_init(metrics->shm) != NGX_OK) { + return NGX_ERROR; + } + + if (metrics->old_metrics && metrics->shm_zone->noreuse) { + old_shm_kv = ngx_wasm_shm_get_kv(metrics->old_metrics->shm); + + return realloc_metrics(metrics, old_shm_kv->rbtree.root, + old_shm_kv->rbtree.sentinel); + } + + return NGX_OK; +} + + +ngx_int_t +ngx_wa_metrics_define(ngx_wa_metrics_t *metrics, ngx_str_t *name, + ngx_wa_metric_type_e type, uint32_t *out) +{ + ssize_t size = sizeof(ngx_wa_metric_t) + + sizeof(ngx_wa_metric_val_t) * metrics->workers; + uint32_t cas, mid; + ngx_int_t rc, written; + ngx_str_t *p, val; + ngx_wa_metric_t *m; + u_char buf[size]; + + if (type != NGX_WA_METRIC_COUNTER + && type != NGX_WA_METRIC_GAUGE + && type != NGX_WA_METRIC_HISTOGRAM) + { + return NGX_ABORT; + } + + if (name->len > metrics->config.max_metric_name_length) { + return NGX_ABORT; + } + + mid = ngx_crc32_long(name->data, name->len); + + ngx_wasm_shm_lock(metrics->shm); + + rc = ngx_wasm_shm_kv_get_locked(metrics->shm, NULL, &mid, &p, &cas); + if (rc == NGX_OK) { + ngx_log_debug1(NGX_LOG_DEBUG_WASM, metrics->shm->log, 0, + "wasm returning existing metric id \"%uD\"", mid); + goto done; + } + + ngx_memzero(buf, size); + m = (ngx_wa_metric_t *) buf; + m->type = type; + + if (type == NGX_WA_METRIC_HISTOGRAM) { + rc = ngx_wa_metrics_histogram_add_locked(metrics, m); + if (rc != NGX_OK) { + goto error; + } + } + + val.len = size; + val.data = buf; + + rc = ngx_wasm_shm_kv_set_locked(metrics->shm, name, &val, 0, &written); + if (rc != NGX_OK) { + goto error; + } + +done: + + *out = mid; + +error: + + ngx_wasm_shm_unlock(metrics->shm); + + if (rc == NGX_OK) { + ngx_wasm_log_error(NGX_LOG_INFO, metrics->shm->log, 0, + "defined %V \"%V\" with id %uD", + ngx_wa_metric_type_name(type), name, mid); + } + + return rc; +} + + +ngx_int_t +ngx_wa_metrics_increment(ngx_wa_metrics_t *metrics, uint32_t mid, ngx_int_t n) +{ + uint32_t cas; + ngx_uint_t slot; + ngx_int_t rc; + ngx_str_t *val; + ngx_wa_metric_t *m; + + slot = (ngx_process == NGX_PROCESS_WORKER) ? ngx_worker : 0; + +#if 0 + if (metrics->shm->eviction != NGX_WASM_EVICTION_NONE) { + slot = 0; + ngx_wasm_shm_lock(metrics->shm); + } +#endif + + rc = ngx_wasm_shm_kv_get_locked(metrics->shm, NULL, &mid, &val, &cas); + if (rc != NGX_OK) { + goto error; + } + + m = (ngx_wa_metric_t *) val->data; + + switch (m->type) { + case NGX_WA_METRIC_COUNTER: + break; + + default: + rc = NGX_ABORT; + goto error; + } + + ngx_log_debug2(NGX_LOG_DEBUG_WASM, metrics->shm->log, 0, + "wasm updating metric \"%uD\" with %d", mid, n); + + m->slots[slot].counter += n; + +error: + +#if 0 + if (metrics->shm->eviction != NGX_WASM_EVICTION_NONE) { + ngx_wasm_shm_unlock(metrics->shm); + } +#endif + + return rc; +} + + +ngx_int_t +ngx_wa_metrics_record(ngx_wa_metrics_t *metrics, uint32_t mid, ngx_int_t n) +{ + uint32_t cas; + ngx_int_t rc; + ngx_str_t *val; + ngx_uint_t slot; + ngx_wa_metric_t *m; + + slot = (ngx_process == NGX_PROCESS_WORKER) ? ngx_worker : 0; + +#if 0 + if (metrics->shm->eviction != NGX_WASM_EVICTION_NONE) { + slot = 0; + ngx_wasm_shm_lock(metrics->shm); + } +#endif + + rc = ngx_wasm_shm_kv_get_locked(metrics->shm, NULL, &mid, &val, &cas); + if (rc != NGX_OK) { + goto error; + } + + ngx_log_debug2(NGX_LOG_DEBUG_WASM, metrics->shm->log, 0, + "wasm updating metric \"%uD\" with %d", mid, n); + + m = (ngx_wa_metric_t *) val->data; + + switch (m->type) { + case NGX_WA_METRIC_GAUGE: + m->slots[slot].gauge.value = n; + m->slots[slot].gauge.last_update = ngx_current_msec; + break; + + case NGX_WA_METRIC_HISTOGRAM: + ngx_wa_metrics_histogram_record(metrics, m, slot, mid, n); + break; + + default: + rc = NGX_ABORT; + goto error; + } + +error: + +#if 0 + if (metrics->shm->eviction != NGX_WASM_EVICTION_NONE) { + ngx_wasm_shm_unlock(metrics->shm); + } +#endif + + return rc; +} + + +ngx_int_t +ngx_wa_metrics_get(ngx_wa_metrics_t *metrics, uint32_t mid, ngx_uint_t *out) +{ + uint32_t cas; + ngx_int_t rc; + ngx_str_t *n; + ngx_wa_metric_t *m; + + rc = ngx_wasm_shm_kv_get_locked(metrics->shm, NULL, &mid, &n, &cas); + if (rc != NGX_OK) { + return NGX_DECLINED; + } + + m = (ngx_wa_metric_t *) n->data; + + switch (m->type) { + case NGX_WA_METRIC_COUNTER: + *out = get_counter(m, metrics->workers); + break; + + case NGX_WA_METRIC_GAUGE: + *out = get_gauge(m, metrics->workers); + break; + + default: + return NGX_ABORT; + } + + return NGX_OK; +} diff --git a/src/common/metrics/ngx_wa_metrics.h b/src/common/metrics/ngx_wa_metrics.h new file mode 100644 index 000000000..d9929ba50 --- /dev/null +++ b/src/common/metrics/ngx_wa_metrics.h @@ -0,0 +1,81 @@ +#ifndef _NGX_WA_METRICS_H_INCLUDED_ +#define _NGX_WA_METRICS_H_INCLUDED_ + + +#include + + +typedef struct ngx_wa_metrics_s ngx_wa_metrics_t; + +typedef enum { + NGX_WA_METRIC_COUNTER, + NGX_WA_METRIC_GAUGE, + NGX_WA_METRIC_HISTOGRAM, +} ngx_wa_metric_type_e; + + +typedef struct { + ngx_uint_t value; + ngx_msec_t last_update; +} ngx_wa_metrics_gauge_t; + + +typedef struct { + uint32_t upper_bound; + uint32_t count; +} ngx_wa_metrics_bin_t; + + +typedef struct { + uint8_t n_bins; + ngx_wa_metrics_bin_t bins[]; +} ngx_wa_metrics_histogram_t; + + +typedef union { + ngx_uint_t counter; + ngx_wa_metrics_gauge_t gauge; + ngx_wa_metrics_histogram_t *histogram; +} ngx_wa_metric_val_t; + + +typedef struct { + ngx_wa_metric_type_e type; + ngx_wa_metric_val_t slots[]; +} ngx_wa_metric_t; + + +typedef struct { + size_t slab_size; + size_t max_metric_name_length; + unsigned initialized:1; +} ngx_wa_metrics_conf_t; + + +struct ngx_wa_metrics_s { + ngx_uint_t workers; + ngx_shm_zone_t *shm_zone; + ngx_wasm_shm_t *shm; + ngx_wa_metrics_t *old_metrics; + ngx_wa_metrics_conf_t config; +}; + + +ngx_wa_metrics_t *ngx_wasmx_metrics(ngx_cycle_t *cycle); +ngx_str_t *ngx_wa_metric_type_name(ngx_wa_metric_type_e type); + +ngx_wa_metrics_t *ngx_wa_metrics_alloc(ngx_cycle_t *cycle); +char *ngx_wa_metrics_init_conf(ngx_wa_metrics_t *metrics, ngx_conf_t *cf); +ngx_int_t ngx_wa_metrics_init(ngx_wa_metrics_t *metrics, ngx_cycle_t *cycle); + +ngx_int_t ngx_wa_metrics_define(ngx_wa_metrics_t *metrics, ngx_str_t *name, + ngx_wa_metric_type_e type, uint32_t *out); +ngx_int_t ngx_wa_metrics_increment(ngx_wa_metrics_t *metrics, + uint32_t metric_id, ngx_int_t val); +ngx_int_t ngx_wa_metrics_record(ngx_wa_metrics_t *metrics, uint32_t metric_id, + ngx_int_t val); +ngx_int_t ngx_wa_metrics_get(ngx_wa_metrics_t *metrics, uint32_t metric_id, + ngx_uint_t *out); + + +#endif /* _NGX_WA_METRICS_H_INCLUDED_ */ diff --git a/src/common/proxy_wasm/ngx_proxy_wasm.h b/src/common/proxy_wasm/ngx_proxy_wasm.h index 40a98472f..1d9c73981 100644 --- a/src/common/proxy_wasm/ngx_proxy_wasm.h +++ b/src/common/proxy_wasm/ngx_proxy_wasm.h @@ -140,13 +140,11 @@ typedef enum { } ngx_proxy_wasm_map_type_e; -#if 0 typedef enum { NGX_PROXY_WASM_METRIC_COUNTER = 0, NGX_PROXY_WASM_METRIC_GAUGE = 1, NGX_PROXY_WASM_METRIC_HISTOGRAM = 2, } ngx_proxy_wasm_metric_type_e; -#endif typedef struct ngx_proxy_wasm_ctx_s ngx_proxy_wasm_ctx_t; @@ -242,6 +240,9 @@ struct ngx_proxy_wasm_ctx_s { ngx_str_t root_id; /* pwexec->root_id */ ngx_str_t call_status; /* dispatch response status */ ngx_str_t response_status; /* response status */ +#if (NGX_DEBUG) + ngx_str_t worker_id; /* ngx_worker */ +#endif ngx_uint_t call_code; ngx_uint_t response_code; diff --git a/src/common/proxy_wasm/ngx_proxy_wasm_host.c b/src/common/proxy_wasm/ngx_proxy_wasm_host.c index b98f273e1..d564aba15 100644 --- a/src/common/proxy_wasm/ngx_proxy_wasm_host.c +++ b/src/common/proxy_wasm/ngx_proxy_wasm_host.c @@ -10,6 +10,7 @@ #include #include #include +#include #ifdef NGX_WASM_HTTP #include #endif @@ -1570,8 +1571,210 @@ ngx_proxy_wasm_hfuncs_dequeue_shared_queue(ngx_wavm_instance_t *instance, } -/* stats/metrics */ -/* NYI */ +/* metrics */ + + +static ngx_int_t +ngx_proxy_wasm_hfuncs_define_metric(ngx_wavm_instance_t *instance, + wasm_val_t args[], wasm_val_t rets[]) +{ + size_t max_len; + uint32_t *id; + ngx_str_t name, prefixed_name, *filter_name; + ngx_cycle_t *cycle = (ngx_cycle_t *) ngx_cycle; + ngx_wa_metrics_t *metrics = ngx_wasmx_metrics(cycle); + ngx_wa_metric_type_e type; + ngx_proxy_wasm_exec_t *pwexec; + ngx_proxy_wasm_metric_type_e pw_type; + u_char buf[metrics->config.max_metric_name_length]; + u_char trapmsg[NGX_MAX_ERROR_STR]; + + pwexec = ngx_proxy_wasm_instance2pwexec(instance); + + pw_type = args[0].of.i32; + name.len = args[2].of.i32; + name.data = NGX_WAVM_HOST_LIFT_SLICE(instance, args[1].of.i32, name.len); + id = NGX_WAVM_HOST_LIFT_SLICE(instance, args[3].of.i32, sizeof(uint32_t)); + max_len = metrics->config.max_metric_name_length; + + ngx_memzero(trapmsg, NGX_MAX_ERROR_STR); + + switch(pw_type) { + case NGX_PROXY_WASM_METRIC_COUNTER: + type = NGX_WA_METRIC_COUNTER; + break; + + case NGX_PROXY_WASM_METRIC_GAUGE: + type = NGX_WA_METRIC_GAUGE; + break; + + case NGX_PROXY_WASM_METRIC_HISTOGRAM: + type = NGX_WA_METRIC_HISTOGRAM; + break; + + default: + ngx_sprintf(trapmsg, "could not define metric \"%*s\": " + "unknown type \"%ui\"", + name.len, name.data, pw_type); + + return ngx_proxy_wasm_result_trap(pwexec, (char *) trapmsg, + rets, NGX_WAVM_ERROR); + } + + filter_name = pwexec->filter->name; + + if (4 + filter_name->len + name.len > max_len) { + ngx_sprintf(trapmsg, "could not define metric: name \"%*s\" too long", + name.len, name.data); + + return ngx_proxy_wasm_result_trap(pwexec, (char *) trapmsg, + rets, NGX_WAVM_ERROR); + } + + prefixed_name.data = buf; + prefixed_name.len = ngx_sprintf(buf, "pw.%V.%V", filter_name, &name) + - buf; + + if (ngx_wa_metrics_define(metrics, &prefixed_name, type, id) != NGX_OK) { + ngx_sprintf(trapmsg, "could not define metric \"%*s\"", + name.len, name.data); + + return ngx_proxy_wasm_result_trap(pwexec, (char *) trapmsg, + rets, NGX_WAVM_ERROR); + } + + return ngx_proxy_wasm_result_ok(rets); +} + + +static ngx_int_t +ngx_proxy_wasm_hfuncs_get_metric(ngx_wavm_instance_t *instance, + wasm_val_t args[], wasm_val_t rets[]) +{ + u_char *p; + uint32_t metric_id; + ngx_int_t rc; + ngx_uint_t *ret_value; + ngx_cycle_t *cycle = (ngx_cycle_t *) ngx_cycle; + ngx_wa_metrics_t *metrics = ngx_wasmx_metrics(cycle); + ngx_proxy_wasm_exec_t *pwexec = ngx_proxy_wasm_instance2pwexec(instance); + u_char trapmsg[NGX_MAX_ERROR_STR]; + + metric_id = args[0].of.i32; + ret_value = NGX_WAVM_HOST_LIFT(instance, args[1].of.i32, ngx_uint_t); + + rc = ngx_wa_metrics_get(metrics, metric_id, ret_value); + if (rc != NGX_OK) { + ngx_memzero(trapmsg, NGX_MAX_ERROR_STR); + p = ngx_sprintf(trapmsg, "could not retrieve metric id \"%ui\": ", + metric_id); + + switch (rc) { + case NGX_DECLINED: + ngx_sprintf(p, "metric not found"); + break; + + case NGX_ABORT: + ngx_sprintf(p, "metric is a histogram"); + break; + + default: + break; + } + + return ngx_proxy_wasm_result_trap(pwexec, (char *) trapmsg, + rets, NGX_WAVM_ERROR); + } + + return ngx_proxy_wasm_result_ok(rets); +} + + +static ngx_int_t +ngx_proxy_wasm_hfuncs_record_metric(ngx_wavm_instance_t *instance, + wasm_val_t args[], wasm_val_t rets[]) +{ + u_char *p; + uint32_t metric_id; + ngx_int_t rc, value; + ngx_cycle_t *cycle = (ngx_cycle_t *) ngx_cycle; + ngx_wa_metrics_t *metrics = ngx_wasmx_metrics(cycle); + ngx_proxy_wasm_exec_t *pwexec = ngx_proxy_wasm_instance2pwexec(instance); + u_char trapmsg[NGX_MAX_ERROR_STR]; + + metric_id = args[0].of.i32; + value = args[1].of.i64; + + rc = ngx_wa_metrics_record(metrics, metric_id, value); + if (rc != NGX_OK) { + ngx_memzero(trapmsg, NGX_MAX_ERROR_STR); + p = ngx_sprintf(trapmsg, + "could not record value in metric id \"%ui\": ", + metric_id); + + switch (rc) { + case NGX_DECLINED: + ngx_sprintf(p, "metric not found"); + break; + + case NGX_ABORT: + ngx_sprintf(p, "metric is a counter"); + break; + + default: + break; + } + + return ngx_proxy_wasm_result_trap(pwexec, (char *) trapmsg, + rets, NGX_WAVM_ERROR); + } + + return ngx_proxy_wasm_result_ok(rets); +} + + +static ngx_int_t +ngx_proxy_wasm_hfuncs_increment_metric(ngx_wavm_instance_t *instance, + wasm_val_t args[], wasm_val_t rets[]) +{ + u_char *p; + uint32_t metric_id; + ngx_int_t rc, offset; + ngx_cycle_t *cycle = (ngx_cycle_t *) ngx_cycle; + ngx_wa_metrics_t *metrics = ngx_wasmx_metrics(cycle); + ngx_proxy_wasm_exec_t *pwexec = ngx_proxy_wasm_instance2pwexec(instance); + u_char trapmsg[NGX_MAX_ERROR_STR]; + + metric_id = args[0].of.i32; + offset = args[1].of.i64; + + rc = ngx_wa_metrics_increment(metrics, metric_id, offset); + if (rc != NGX_OK) { + ngx_memzero(trapmsg, NGX_MAX_ERROR_STR); + p = ngx_sprintf(trapmsg, + "could not increment metric id \"%ui\": ", + metric_id); + + switch (rc) { + case NGX_DECLINED: + ngx_sprintf(p, "metric not found"); + break; + + case NGX_ABORT: + ngx_sprintf(p, "metric not a counter"); + break; + + default: + break; + } + + return ngx_proxy_wasm_result_trap(pwexec, (char *) trapmsg, rets, + NGX_WAVM_ERROR); + } + + + return ngx_proxy_wasm_result_ok(rets); +} /* custom extension points */ @@ -1953,7 +2156,7 @@ static ngx_wavm_host_func_def_t ngx_proxy_wasm_hfuncs[] = { ngx_wavm_arity_i32x4, ngx_wavm_arity_i32 }, { ngx_string("proxy_define_metric"), /* 0.2.0 && 0.2.1 */ - &ngx_proxy_wasm_hfuncs_nop, /* NYI */ + &ngx_proxy_wasm_hfuncs_define_metric, ngx_wavm_arity_i32x4, ngx_wavm_arity_i32 }, { ngx_string("proxy_get_metric_value"), /* vNEXT */ @@ -1961,7 +2164,7 @@ static ngx_wavm_host_func_def_t ngx_proxy_wasm_hfuncs[] = { ngx_wavm_arity_i32x2, ngx_wavm_arity_i32 }, { ngx_string("proxy_get_metric"), /* 0.2.0 && 0.2.1 */ - &ngx_proxy_wasm_hfuncs_nop, /* NYI */ + &ngx_proxy_wasm_hfuncs_get_metric, ngx_wavm_arity_i32x2, ngx_wavm_arity_i32 }, { ngx_string("proxy_set_metric_value"), /* vNEXT */ @@ -1969,7 +2172,7 @@ static ngx_wavm_host_func_def_t ngx_proxy_wasm_hfuncs[] = { ngx_wavm_arity_i32_i64, ngx_wavm_arity_i32 }, { ngx_string("proxy_record_metric"), /* 0.2.0 && 0.2.1 */ - &ngx_proxy_wasm_hfuncs_nop, /* NYI */ + &ngx_proxy_wasm_hfuncs_record_metric, ngx_wavm_arity_i32_i64, ngx_wavm_arity_i32 }, { ngx_string("proxy_increment_metric_value"), /* vNEXT */ @@ -1977,7 +2180,7 @@ static ngx_wavm_host_func_def_t ngx_proxy_wasm_hfuncs[] = { ngx_wavm_arity_i32_i64, ngx_wavm_arity_i32 }, { ngx_string("proxy_increment_metric"), /* 0.2.0 && 0.2.1 */ - &ngx_proxy_wasm_hfuncs_nop, /* NYI */ + &ngx_proxy_wasm_hfuncs_increment_metric, ngx_wavm_arity_i32_i64, ngx_wavm_arity_i32 }, { ngx_string("proxy_delete_metric"), /* vNEXT */ diff --git a/src/common/proxy_wasm/ngx_proxy_wasm_properties.c b/src/common/proxy_wasm/ngx_proxy_wasm_properties.c index e26b92411..ebe81881d 100644 --- a/src/common/proxy_wasm/ngx_proxy_wasm_properties.c +++ b/src/common/proxy_wasm/ngx_proxy_wasm_properties.c @@ -394,6 +394,34 @@ get_filter_root_id(ngx_proxy_wasm_ctx_t *pwctx, ngx_str_t *path, } +#if (NGX_DEBUG) +static ngx_int_t +get_worker_id(ngx_proxy_wasm_ctx_t *pwctx, ngx_str_t *path, + ngx_str_t *value) +{ + size_t len; + u_char buf[NGX_OFF_T_LEN]; + + if (!pwctx->worker_id.len) { + len = ngx_sprintf(buf, "%i", ngx_worker) - buf; + + pwctx->worker_id.data = ngx_pnalloc(pwctx->pool, len); + if (pwctx->worker_id.data == NULL) { + return NGX_ERROR; + } + + ngx_memcpy(pwctx->worker_id.data, buf, len); + pwctx->worker_id.len = len; + } + + value->len = pwctx->worker_id.len; + value->data = pwctx->worker_id.data; + + return NGX_OK; +} +#endif + + static pwm2ngx_mapping_t pw2ngx[] = { #ifdef NGX_WASM_HTTP @@ -599,6 +627,14 @@ static pwm2ngx_mapping_t pw2ngx[] = { ngx_null_string, ¬_supported, NULL }, + /* debugging properties */ + +#if (NGX_DEBUG) + { ngx_string("worker_id"), + ngx_null_string, + &get_worker_id, NULL }, +#endif + { ngx_null_string, ngx_null_string, NULL, NULL } }; @@ -654,7 +690,7 @@ ngx_proxy_wasm_properties_init(ngx_conf_t *cf) pwm2ngx_init.hash = &pwm2ngx_hash.hash; pwm2ngx_init.key = ngx_hash_key; - pwm2ngx_init.max_size = 256; + pwm2ngx_init.max_size = 512; pwm2ngx_init.bucket_size = ngx_align(64, ngx_cacheline_size); pwm2ngx_init.name = "pwm2ngx_properties"; pwm2ngx_init.pool = cf->pool; diff --git a/src/common/shm/ngx_wasm_shm.h b/src/common/shm/ngx_wasm_shm.h index 1ae08b43e..f3a885138 100644 --- a/src/common/shm/ngx_wasm_shm.h +++ b/src/common/shm/ngx_wasm_shm.h @@ -11,6 +11,7 @@ typedef enum { NGX_WASM_SHM_TYPE_KV, NGX_WASM_SHM_TYPE_QUEUE, + NGX_WASM_SHM_TYPE_METRICS, } ngx_wasm_shm_type_e; diff --git a/src/common/shm/ngx_wasm_shm_kv.c b/src/common/shm/ngx_wasm_shm_kv.c index 265023d83..834b46212 100644 --- a/src/common/shm/ngx_wasm_shm_kv.c +++ b/src/common/shm/ngx_wasm_shm_kv.c @@ -15,28 +15,11 @@ #define NGX_WASM_SLRU_NQUEUES(pool) (NGX_WASM_SLAB_SLOTS(pool) + 1) -typedef struct { - ngx_rbtree_t rbtree; - ngx_rbtree_node_t sentinel; - union { - ngx_queue_t lru_queue; - ngx_queue_t slru_queues[0]; - } eviction; -} ngx_wasm_shm_kv_t; - - -typedef struct { - ngx_str_node_t key; - ngx_str_t value; - uint32_t cas; - ngx_queue_t queue; -} ngx_wasm_shm_kv_node_t; - - -static ngx_inline ngx_wasm_shm_kv_t * +ngx_wasm_shm_kv_t * ngx_wasm_shm_get_kv(ngx_wasm_shm_t *shm) { - ngx_wa_assert(shm->type == NGX_WASM_SHM_TYPE_KV); + ngx_wa_assert(shm->type == NGX_WASM_SHM_TYPE_KV || \ + shm->type == NGX_WASM_SHM_TYPE_METRICS); return shm->data; } @@ -287,8 +270,7 @@ ngx_wasm_shm_kv_set_locked(ngx_wasm_shm_t *shm, ngx_str_t *key, ngx_wasm_shm_kv_node_t *n, *old; old = NULL; - n = (ngx_wasm_shm_kv_node_t *) ngx_wasm_shm_rbtree_lookup(&kv->rbtree, - key_hash); + n = ngx_wasm_shm_rbtree_lookup(&kv->rbtree, key_hash); if (cas != (n == NULL ? 0 : n->cas)) { *written = 0; diff --git a/src/common/shm/ngx_wasm_shm_kv.h b/src/common/shm/ngx_wasm_shm_kv.h index fe280034f..d711b6b57 100644 --- a/src/common/shm/ngx_wasm_shm_kv.h +++ b/src/common/shm/ngx_wasm_shm_kv.h @@ -5,6 +5,16 @@ #include +typedef struct { + ngx_rbtree_t rbtree; + ngx_rbtree_node_t sentinel; + union { + ngx_queue_t lru_queue; + ngx_queue_t slru_queues[0]; + } eviction; +} ngx_wasm_shm_kv_t; + + typedef struct { ngx_str_t namespace; ngx_str_t key; @@ -13,6 +23,16 @@ typedef struct { } ngx_wasm_shm_kv_key_t; +typedef struct { + ngx_str_node_t key; + ngx_str_t value; + uint32_t cas; + ngx_queue_t queue; +} ngx_wasm_shm_kv_node_t; + + +ngx_wasm_shm_kv_t * ngx_wasm_shm_get_kv(ngx_wasm_shm_t *shm); + ngx_int_t ngx_wasm_shm_kv_init(ngx_wasm_shm_t *shm); ngx_int_t ngx_wasm_shm_kv_get_locked(ngx_wasm_shm_t *shm, ngx_str_t *key, uint32_t *key_hash, ngx_str_t **value_out, uint32_t *cas); diff --git a/src/ngx_wasmx.c b/src/ngx_wasmx.c index 40787eabc..ba5acc860 100644 --- a/src/ngx_wasmx.c +++ b/src/ngx_wasmx.c @@ -9,9 +9,6 @@ #endif -#define NGX_WA_CONF_ERR_DUPLICATE "is duplicate" - - static char *ngx_wasm_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); #ifdef NGX_WA_IPC static char *ngx_ipc_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); @@ -116,6 +113,11 @@ ngx_wasmx_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf, } #endif + wacf->metrics = ngx_wa_metrics_alloc(cf->cycle); + if (wacf->metrics == NULL) { + return NGX_CONF_ERROR; + } + *(ngx_wa_conf_t **) conf = wacf; } @@ -217,6 +219,13 @@ ngx_wasmx_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf, } } + /* metrics init_conf */ + + rv = ngx_wa_metrics_init_conf(wacf->metrics, cf); + if (rv != NGX_CONF_OK) { + return rv; + } + return NGX_CONF_OK; } @@ -249,6 +258,11 @@ ngx_wasmx_init(ngx_cycle_t *cycle) return NGX_OK; } + rc = ngx_wa_metrics_init(wacf->metrics, cycle); + if (rc != NGX_OK) { + return rc; + } + /* NGX_WASM_MODULES + NGX_IPC_MODULES init */ for (i = 0; cycle->modules[i]; i++) { @@ -286,3 +300,21 @@ ngx_wasmx_init(ngx_cycle_t *cycle) return NGX_OK; } + + +ngx_inline ngx_wa_metrics_t * +ngx_wasmx_metrics(ngx_cycle_t *cycle) +{ + ngx_wa_conf_t *wacf; + + if (cycle->conf_ctx == NULL) { + return NULL; + } + + wacf = ngx_wa_cycle_get_conf(cycle); + if (wacf == NULL) { + return NULL; + } + + return wacf->metrics; +} diff --git a/src/ngx_wasmx.h b/src/ngx_wasmx.h index e61001fd2..1938eb7ab 100644 --- a/src/ngx_wasmx.h +++ b/src/ngx_wasmx.h @@ -3,19 +3,25 @@ #include +#include #if (NGX_DEBUG) #include -# define ngx_wa_assert(a) assert(a) +# define ngx_wa_assert(a) assert(a) #else # define ngx_wa_assert(a) #endif -#define NGX_WA_BAD_FD (ngx_socket_t) -1 +#define NGX_WA_BAD_FD (ngx_socket_t) -1 -#define NGX_WA_WASM_CONF_OFFSET offsetof(ngx_wa_conf_t, wasm_confs) -#define NGX_WA_IPC_CONF_OFFSET offsetof(ngx_wa_conf_t, ipc_confs) +#define NGX_WA_WASM_CONF_OFFSET offsetof(ngx_wa_conf_t, wasm_confs) +#define NGX_WA_IPC_CONF_OFFSET offsetof(ngx_wa_conf_t, ipc_confs) + +#define NGX_WA_CONF_ERR_DUPLICATE "is duplicate" + +#define NGX_WA_METRICS_DEFAULT_MAX_NAME_LEN 256 +#define NGX_WA_METRICS_DEFAULT_SLAB_SIZE 1024 * 1024 * 5 /* 5 MiB */ #define ngx_wa_cycle_get_conf(cycle) \ (ngx_wa_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_wasmx_module) @@ -27,11 +33,12 @@ typedef ngx_int_t (*ngx_wa_init_pt)(ngx_cycle_t *cycle); typedef struct { - ngx_uint_t initialized_types; - void **wasm_confs; + ngx_uint_t initialized_types; + void **wasm_confs; #ifdef NGX_WA_IPC - void **ipc_confs; + void **ipc_confs; #endif + ngx_wa_metrics_t *metrics; } ngx_wa_conf_t; diff --git a/src/wasm/ngx_wasm.h b/src/wasm/ngx_wasm.h index 523caf801..8fcadeba3 100644 --- a/src/wasm/ngx_wasm.h +++ b/src/wasm/ngx_wasm.h @@ -18,6 +18,7 @@ #define NGX_WASMTIME_CONF 0x20000000 #define NGX_WASMER_CONF 0x40000000 #define NGX_V8_CONF 0x80000000 +#define NGX_METRICS_CONF 0x16000000 #define NGX_WASM_DONE_PHASE 15 #define NGX_WASM_BACKGROUND_PHASE 16 @@ -125,12 +126,16 @@ void ngx_wasm_log_error(ngx_uint_t level, ngx_log_t *log, ngx_err_t err, void swap_modules_if_needed(ngx_conf_t *cf, const char *m1, const char *m2); #endif -/* directives */ +/* blocks */ char *ngx_wasm_core_wasmtime_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); char *ngx_wasm_core_wasmer_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); char *ngx_wasm_core_v8_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +char *ngx_wasm_core_metrics_block(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); + +/* directives */ char *ngx_wasm_core_flag_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); char *ngx_wasm_core_module_directive(ngx_conf_t *cf, ngx_command_t *cmd, @@ -139,6 +144,10 @@ char *ngx_wasm_core_shm_kv_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); char *ngx_wasm_core_shm_queue_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +char *ngx_wasm_core_metrics_slab_size_directive(ngx_conf_t *cf, + ngx_command_t *cmd, void *conf); +char *ngx_wasm_core_metrics_max_metric_name_length_directive(ngx_conf_t *cf, + ngx_command_t *cmd, void *conf); char *ngx_wasm_core_resolver_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); char *ngx_wasm_core_pwm_lua_resolver_directive(ngx_conf_t *cf, diff --git a/src/wasm/ngx_wasm_core_module.c b/src/wasm/ngx_wasm_core_module.c index cd7f8d80b..f367a44ad 100644 --- a/src/wasm/ngx_wasm_core_module.c +++ b/src/wasm/ngx_wasm_core_module.c @@ -5,6 +5,7 @@ #include #include +#include static void *ngx_wasm_core_create_conf(ngx_conf_t *cf); @@ -43,6 +44,13 @@ static ngx_command_t ngx_wasm_core_commands[] = { 0, NULL }, + { ngx_string("metrics"), + NGX_WASM_CONF|NGX_CONF_BLOCK|NGX_CONF_NOARGS, + ngx_wasm_core_metrics_block, + NGX_WA_WASM_CONF_OFFSET, + 0, + NULL }, + /* directives */ { ngx_string("flag"), @@ -90,6 +98,20 @@ static ngx_command_t ngx_wasm_core_commands[] = { 0, NULL }, + { ngx_string("slab_size"), + NGX_METRICS_CONF|NGX_CONF_TAKE1, + ngx_wasm_core_metrics_slab_size_directive, + NGX_WA_WASM_CONF_OFFSET, + 0, + NULL }, + + { ngx_string("max_metric_name_length"), + NGX_METRICS_CONF|NGX_CONF_TAKE1, + ngx_wasm_core_metrics_max_metric_name_length_directive, + NGX_WA_WASM_CONF_OFFSET, + 0, + NULL }, + { ngx_string("shm_queue"), NGX_WASM_CONF|NGX_CONF_TAKE23|NGX_CONF_TAKE4, ngx_wasm_core_shm_queue_directive, diff --git a/src/wasm/ngx_wasm_directives.c b/src/wasm/ngx_wasm_directives.c index 5437eae08..bd289e770 100644 --- a/src/wasm/ngx_wasm_directives.c +++ b/src/wasm/ngx_wasm_directives.c @@ -45,6 +45,30 @@ ngx_wasm_core_v8_block(ngx_conf_t *cf, ngx_command_t *cmd, void *dummy) } +char * +ngx_wasm_core_metrics_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + char *rv; + ngx_conf_t save = *cf; + ngx_wa_conf_t *wacf = (ngx_wa_conf_t *) cf->ctx; + + if (wacf->metrics->config.initialized) { + return NGX_WA_CONF_ERR_DUPLICATE; + } + + wacf->metrics->config.initialized = 1; + + cf->cmd_type = NGX_METRICS_CONF; + cf->module_type = NGX_WASM_MODULE; + + rv = ngx_conf_parse(cf, NULL); + + *cf = save; + + return rv; +} + + static ngx_int_t ngx_wasm_core_current_runtime_flag(ngx_conf_t *cf) { @@ -109,6 +133,35 @@ ngx_wasm_core_flag_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } +static ngx_int_t +validate_shm_size(ngx_conf_t *cf, ssize_t size, ngx_str_t *value) +{ + const ssize_t min_size = 3 * ngx_pagesize; + + if (size == NGX_ERROR) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "[wasm] invalid shm size \"%V\"", value); + return NGX_ERROR; + } + + if (size < min_size) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "[wasm] shm size of %d bytes is too small, " + "minimum required is %d bytes", size, min_size); + return NGX_ERROR; + } + + if ((size & (ngx_pagesize - 1)) != 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "[wasm] shm size of %d bytes is not page-aligned, " + "must be a multiple of %d", size, ngx_pagesize); + return NGX_ERROR; + } + + return NGX_OK; +} + + static char * ngx_wasm_core_shm_generic_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf, ngx_wasm_shm_type_e type) @@ -120,7 +173,6 @@ ngx_wasm_core_shm_generic_directive(ngx_conf_t *cf, ngx_command_t *cmd, ngx_wasm_shm_mapping_t *mapping; ngx_wasm_shm_t *shm; ngx_wasm_shm_eviction_e eviction; - const ssize_t min_size = 3 * ngx_pagesize; value = cf->args->elts; name = &value[1]; @@ -134,23 +186,7 @@ ngx_wasm_core_shm_generic_directive(ngx_conf_t *cf, ngx_command_t *cmd, return NGX_CONF_ERROR; } - if (size == NGX_ERROR) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "[wasm] invalid shm size \"%V\"", &value[2]); - return NGX_CONF_ERROR; - } - - if (size < min_size) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "[wasm] shm size of %d bytes is too small, " - "minimum required is %d bytes", size, min_size); - return NGX_CONF_ERROR; - } - - if ((size & (ngx_pagesize - 1)) != 0) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "[wasm] shm size of %d bytes is not page-aligned, " - "must be a multiple of %d", size, ngx_pagesize); + if (validate_shm_size(cf, size, &value[2]) != NGX_OK) { return NGX_CONF_ERROR; } @@ -295,6 +331,61 @@ ngx_wasm_core_shm_queue_directive(ngx_conf_t *cf, ngx_command_t *cmd, } +char * +ngx_wasm_core_metrics_slab_size_directive(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf) +{ + ssize_t size; + ngx_str_t *value; + ngx_wa_metrics_t *metrics = ngx_wasmx_metrics(cf->cycle); + + if (metrics->config.slab_size != NGX_CONF_UNSET_SIZE) { + return NGX_WA_CONF_ERR_DUPLICATE; + } + + value = cf->args->elts; + size = ngx_parse_size(&value[1]); + + if (validate_shm_size(cf, size, &value[1]) != NGX_OK) { + return NGX_CONF_ERROR; + } + + metrics->config.slab_size = size; + + return NGX_CONF_OK; +} + + +char * +ngx_wasm_core_metrics_max_metric_name_length_directive(ngx_conf_t *cf, + ngx_command_t *cmd, void *conf) +{ + ngx_int_t n; + ngx_str_t *value; + ngx_wa_metrics_t *metrics = ngx_wasmx_metrics(cf->cycle); + + if (metrics->config.max_metric_name_length != NGX_CONF_UNSET_SIZE) { + return NGX_WA_CONF_ERR_DUPLICATE; + } + + value = cf->args->elts; + n = ngx_atoi(value[1].data, value[1].len); + if (n == NGX_ERROR) { + /* ngx_conf_set_num_slot */ + return "invalid value"; + } + + if (n < 6) { + /* "pw.%V.%V" */ + return "value too small (min: 6)"; + } + + metrics->config.max_metric_name_length = n; + + return NGX_CONF_OK; +} + + char * ngx_wasm_core_resolver_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { @@ -302,7 +393,7 @@ ngx_wasm_core_resolver_directive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) ngx_str_t *value; if (wcf->user_resolver) { - return "is duplicate"; + return NGX_WA_CONF_ERR_DUPLICATE; } value = cf->args->elts; diff --git a/t/01-wasm/005-metrics_block.t b/t/01-wasm/005-metrics_block.t new file mode 100644 index 000000000..c2aa414ba --- /dev/null +++ b/t/01-wasm/005-metrics_block.t @@ -0,0 +1,35 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +plan_tests(4); +run_tests(); + +__DATA__ + +=== TEST 1: metrics{} - empty block +--- valgrind +--- main_config + wasm { + metrics {} + } +--- no_error_log +[error] +[crit] +[emerg] + + + +=== TEST 2: metrics{} - duplicated block +--- main_config + wasm { + metrics {} + metrics {} + } +--- error_log: is duplicate +--- no_error_log +[error] +[crit] +--- must_die diff --git a/t/01-wasm/directives/011-metrics_directives.t b/t/01-wasm/directives/011-metrics_directives.t new file mode 100644 index 000000000..08b8243b9 --- /dev/null +++ b/t/01-wasm/directives/011-metrics_directives.t @@ -0,0 +1,130 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +plan_tests(4); +run_tests(); + +__DATA__ + +=== TEST 1: slab_size directive - sanity +--- main_config + wasm { + metrics { + slab_size 12k; + } + } +--- no_error_log +[error] +[crit] +[emerg] + + + +=== TEST 2: slab_size directive - too small +--- main_config + wasm { + metrics { + slab_size 1k; + } + } +--- error_log eval +qr/\[emerg\] .*? \[wasm\] shm size of \d+ bytes is too small, minimum required is 12288 bytes/ +--- no_error_log +[error] +[crit] +--- must_die + + + +=== TEST 3: slab_size directive - invalid size +--- main_config + wasm { + metrics { + slab_size 1x; + } + } +--- error_log eval +qr/\[emerg\] .*? \[wasm\] invalid shm size "1x"/ +--- no_error_log +[error] +[crit] +--- must_die + + + +=== TEST 4: slab_size directive - duplicate +--- main_config + wasm { + metrics { + slab_size 12k; + slab_size 12k; + } + } +--- error_log: is duplicate +--- no_error_log +[error] +[crit] +--- must_die + + + +=== TEST 5: max_metric_name_length directive - sanity +--- main_config + wasm { + metrics { + max_metric_name_length 64; + } + } +--- no_error_log +[error] +[crit] +[emerg] + + + +=== TEST 6: max_metric_name_length directive - invalid value +--- main_config + wasm { + metrics { + max_metric_name_length -1; + } + } +--- error_log: invalid value +--- no_error_log +[error] +[crit] +--- must_die + + + +=== TEST 7: max_metric_name_length directive - invalid value (5) +--- main_config + wasm { + metrics { + max_metric_name_length 5; + } + } +--- error_log: value too small (min: 6) +--- no_error_log +[error] +[crit] +--- must_die + + + +=== TEST 8: max_metric_name_length directive - duplicate +--- main_config + wasm { + metrics { + max_metric_name_length 64; + max_metric_name_length 64; + } + } +--- error_log: is duplicate +--- no_error_log +[error] +[crit] +--- must_die diff --git a/t/03-proxy_wasm/hfuncs/contexts/metrics/001-proxy_define_metric.t b/t/03-proxy_wasm/hfuncs/contexts/metrics/001-proxy_define_metric.t new file mode 100644 index 000000000..fe69d804c --- /dev/null +++ b/t/03-proxy_wasm/hfuncs/contexts/metrics/001-proxy_define_metric.t @@ -0,0 +1,187 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +skip_hup(); + +plan_tests(4); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm contexts - define_metric on_vm_start +--- main_config + env WASMTIME_BACKTRACE_DETAILS=1; + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm 'define_metric'; + } +--- config + location /t { + proxy_wasm context_checks; + return 200; + } +--- ignore_response_body +--- error_log +define_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 2: proxy_wasm contexts - define_metric - on_configure +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_configure=define_metric'; + return 200; + } +--- ignore_response_body +--- error_log +define_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 3: proxy_wasm contexts - define_metric - on_tick +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_tick=define_metric'; + return 200; + } +--- ignore_response_body +--- error_log +define_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 4: proxy_wasm contexts - define_metric on_http_dispatch_response +--- wasm_modules: context_checks +--- config + location /t { + proxy_wasm context_checks 'on_http_dispatch_response=define_metric \ + host=127.0.0.1:$TEST_NGINX_SERVER_PORT'; + return 200; + } + + location /dispatch { + return 200; + } +--- ignore_response_body +--- error_log +define_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 5: proxy_wasm contexts - define_metric on_request_headers +--- wasm_modules: context_checks +--- metrics: 16k +--- config + location /t { + proxy_wasm context_checks 'on_request_headers=define_metric'; + return 200; + } +--- ignore_response_body +--- error_log +define_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 6: proxy_wasm contexts - define_metric on_request_body +--- load_nginx_modules: ngx_http_echo_module +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_request_body=define_metric'; + echo ok; + } +--- request +POST /t +payload +--- ignore_response_body +--- error_log +define_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 7: proxy_wasm contexts - define_metric on_response_headers +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_response_headers=define_metric'; + return 200; + } +--- ignore_response_body +--- error_log +define_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 8: proxy_wasm contexts - define_metric on_response_body +--- load_nginx_modules: ngx_http_echo_module +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_response_body=define_metric'; + echo ok; + } +--- ignore_response_body +--- error_log +define_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 9: proxy_wasm contexts - define_metric on_log +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_log=define_metric'; + return 200; + } +--- ignore_response_body +--- error_log +define_metric status: 0 +--- no_error_log +[error] +[crit] diff --git a/t/03-proxy_wasm/hfuncs/contexts/metrics/010-proxy_increment_metric.t b/t/03-proxy_wasm/hfuncs/contexts/metrics/010-proxy_increment_metric.t new file mode 100644 index 000000000..ecd2b3b5b --- /dev/null +++ b/t/03-proxy_wasm/hfuncs/contexts/metrics/010-proxy_increment_metric.t @@ -0,0 +1,185 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +skip_hup(); + +plan_tests(4); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm contexts - increment_metric on_vm_start +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm 'increment_metric'; + } +--- config + location /t { + proxy_wasm context_checks; + return 200; + } +--- ignore_response_body +--- error_log +increment_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 2: proxy_wasm contexts - increment_metric - on_configure +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_configure=increment_metric'; + return 200; + } +--- ignore_response_body +--- error_log +increment_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 3: proxy_wasm contexts - increment_metric - on_tick +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_tick=increment_metric'; + return 200; + } +--- ignore_response_body +--- error_log +increment_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 4: proxy_wasm contexts - increment_metric on_http_dispatch_response +--- wasm_modules: context_checks +--- config + location /t { + proxy_wasm context_checks 'on_http_dispatch_response=increment_metric \ + host=127.0.0.1:$TEST_NGINX_SERVER_PORT'; + return 200; + } + + location /dispatch { + return 200; + } +--- ignore_response_body +--- error_log +increment_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 5: proxy_wasm contexts - increment_metric on_request_headers +--- wasm_modules: context_checks +--- config + location /t { + proxy_wasm context_checks 'on_request_headers=increment_metric'; + return 200; + } +--- ignore_response_body +--- error_log +increment_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 6: proxy_wasm contexts - increment_metric on_request_body +--- load_nginx_modules: ngx_http_echo_module +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_request_body=increment_metric'; + echo ok; + } +--- request +POST /t +payload +--- ignore_response_body +--- error_log +increment_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 7: proxy_wasm contexts - increment_metric on_response_headers +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_response_headers=increment_metric'; + return 200; + } +--- ignore_response_body +--- error_log +increment_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 8: proxy_wasm contexts - increment_metric on_response_body +--- load_nginx_modules: ngx_http_echo_module +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_response_body=increment_metric'; + echo ok; + } +--- ignore_response_body +--- error_log +increment_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 9: proxy_wasm contexts - increment_metric on_log +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_log=increment_metric'; + return 200; + } +--- ignore_response_body +--- error_log +increment_metric status: 0 +--- no_error_log +[error] +[crit] diff --git a/t/03-proxy_wasm/hfuncs/contexts/metrics/020-proxy_record_metric.t b/t/03-proxy_wasm/hfuncs/contexts/metrics/020-proxy_record_metric.t new file mode 100644 index 000000000..3ce178c37 --- /dev/null +++ b/t/03-proxy_wasm/hfuncs/contexts/metrics/020-proxy_record_metric.t @@ -0,0 +1,185 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +skip_hup(); + +plan_tests(4); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm contexts - record_metric on_vm_start +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm 'record_metric'; + } +--- config + location /t { + proxy_wasm context_checks; + return 200; + } +--- ignore_response_body +--- error_log +record_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 2: proxy_wasm contexts - record_metric - on_configure +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_configure=record_metric'; + return 200; + } +--- ignore_response_body +--- error_log +record_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 3: proxy_wasm contexts - record_metric - on_tick +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_tick=record_metric'; + return 200; + } +--- ignore_response_body +--- error_log +record_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 4: proxy_wasm contexts - record_metric on_http_dispatch_response +--- wasm_modules: context_checks +--- config + location /t { + proxy_wasm context_checks 'on_http_dispatch_response=record_metric \ + host=127.0.0.1:$TEST_NGINX_SERVER_PORT'; + return 200; + } + + location /dispatch { + return 200; + } +--- ignore_response_body +--- error_log +record_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 5: proxy_wasm contexts - record_metric on_request_headers +--- wasm_modules: context_checks +--- config + location /t { + proxy_wasm context_checks 'on_request_headers=record_metric'; + return 200; + } +--- ignore_response_body +--- error_log +record_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 6: proxy_wasm contexts - record_metric on_request_body +--- load_nginx_modules: ngx_http_echo_module +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_request_body=record_metric'; + echo ok; + } +--- request +POST /t +payload +--- ignore_response_body +--- error_log +record_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 7: proxy_wasm contexts - record_metric on_response_headers +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_response_headers=record_metric'; + return 200; + } +--- ignore_response_body +--- error_log +record_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 8: proxy_wasm contexts - record_metric on_response_body +--- load_nginx_modules: ngx_http_echo_module +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_response_body=record_metric'; + echo ok; + } +--- ignore_response_body +--- error_log +record_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 9: proxy_wasm contexts - record_metric on_log +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_log=record_metric'; + return 200; + } +--- ignore_response_body +--- error_log +record_metric status: 0 +--- no_error_log +[error] +[crit] diff --git a/t/03-proxy_wasm/hfuncs/contexts/metrics/030-proxy_get_metric.t b/t/03-proxy_wasm/hfuncs/contexts/metrics/030-proxy_get_metric.t new file mode 100644 index 000000000..61e80e4f8 --- /dev/null +++ b/t/03-proxy_wasm/hfuncs/contexts/metrics/030-proxy_get_metric.t @@ -0,0 +1,185 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +skip_hup(); + +plan_tests(4); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm contexts - get_metric on_vm_start +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm 'get_metric'; + } +--- config + location /t { + proxy_wasm context_checks; + return 200; + } +--- ignore_response_body +--- error_log +get_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 2: proxy_wasm contexts - get_metric - on_configure +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_configure=get_metric'; + return 200; + } +--- ignore_response_body +--- error_log +get_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 3: proxy_wasm contexts - get_metric - on_tick +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_tick=get_metric'; + return 200; + } +--- ignore_response_body +--- error_log +get_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 4: proxy_wasm contexts - get_metric on_http_dispatch_response +--- wasm_modules: context_checks +--- config + location /t { + proxy_wasm context_checks 'on_http_dispatch_response=get_metric \ + host=127.0.0.1:$TEST_NGINX_SERVER_PORT'; + return 200; + } + + location /dispatch { + return 200; + } +--- ignore_response_body +--- error_log +get_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 5: proxy_wasm contexts - get_metric on_request_headers +--- wasm_modules: context_checks +--- config + location /t { + proxy_wasm context_checks 'on_request_headers=get_metric'; + return 200; + } +--- ignore_response_body +--- error_log +get_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 6: proxy_wasm contexts - get_metric on_request_body +--- load_nginx_modules: ngx_http_echo_module +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_request_body=get_metric'; + echo ok; + } +--- request +POST /t +payload +--- ignore_response_body +--- error_log +get_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 7: proxy_wasm contexts - get_metric on_response_headers +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_response_headers=get_metric'; + return 200; + } +--- ignore_response_body +--- error_log +get_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 8: proxy_wasm contexts - get_metric on_response_body +--- load_nginx_modules: ngx_http_echo_module +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_response_body=get_metric'; + echo ok; + } +--- ignore_response_body +--- error_log +get_metric status: 0 +--- no_error_log +[error] +[crit] + + + +=== TEST 9: proxy_wasm contexts - get_metric on_log +--- main_config + wasm { + module context_checks $TEST_NGINX_CRATES_DIR/context_checks.wasm; + } +--- config + location /t { + proxy_wasm context_checks 'on_log=get_metric'; + return 200; + } +--- ignore_response_body +--- error_log +get_metric status: 0 +--- no_error_log +[error] +[crit] diff --git a/t/03-proxy_wasm/hfuncs/metrics/001-proxy_define_metric.t b/t/03-proxy_wasm/hfuncs/metrics/001-proxy_define_metric.t new file mode 100644 index 000000000..ccb5c13c3 --- /dev/null +++ b/t/03-proxy_wasm/hfuncs/metrics/001-proxy_define_metric.t @@ -0,0 +1,94 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +our $workers = 2; + +workers($workers); +if ($workers > 1) { + master_on(); +} + +plan_tests(4); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm - define_metric() counter +--- valgrind +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- config + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + metrics=c1'; + echo ok; + } +--- grep_error_log eval: qr/\["hostcalls" \#\d\] defined metric \w+ as \d+/ +--- grep_error_log_out eval +qr/.*? defined metric c1 as \d+/ +--- no_error_log +[error] +[crit] + + + +=== TEST 2: proxy_wasm - define_metric() gauge +--- valgrind +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- config + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + metrics=g1'; + echo ok; + } +--- grep_error_log eval: qr/\["hostcalls" \#\d\] defined metric \w+ as \d+/ +--- grep_error_log_out eval +qr/.*? defined metric g1 as \d+/ +--- no_error_log +[error] +[crit] + + + +=== TEST 3: proxy_wasm - define_metric() histogram +--- valgrind +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- config + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + metrics=h1'; + echo ok; + } +--- grep_error_log eval: qr/\["hostcalls" \#\d\] defined metric \w+ as \d+/ +--- grep_error_log_out eval +qr/.*? defined metric h1 as \d+/ +--- no_error_log +[error] +[crit] + + + +=== TEST 4: proxy_wasm - define_metric() redefinition +Definition happens only once, a second call defining an existing metric simply returns its id. +--- skip_no_debug +--- valgrind +--- wasm_modules: hostcalls +--- load_nginx_modules: ngx_http_echo_module +--- config + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + metrics=c1,g1,h1'; + proxy_wasm hostcalls 'on_configure=define_metrics \ + metrics=c1,g1,h1'; + echo ok; + } +--- error_log eval +qr/wasm returning existing metric id "\d+"/ +--- no_error_log +[error] +[crit] diff --git a/t/03-proxy_wasm/hfuncs/metrics/002-proxy_define_metric_edge_cases.t b/t/03-proxy_wasm/hfuncs/metrics/002-proxy_define_metric_edge_cases.t new file mode 100644 index 000000000..bca15b650 --- /dev/null +++ b/t/03-proxy_wasm/hfuncs/metrics/002-proxy_define_metric_edge_cases.t @@ -0,0 +1,119 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +use strict; +use lib '.'; +use t::TestWasmX; + +skip_hup(); +no_shuffle(); + +plan_tests(5); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm - define_metric() metric name too long +In SIGHUP mode, this test fails if executed after a test that +defined metrics since any existing metric whose name exceeds +max_metric_name_length won't be successfully reallocated, causing +the reconfiguration to fail. + +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- main_config eval +qq{ + wasm { + module hostcalls $ENV{TEST_NGINX_CRATES_DIR}/hostcalls.wasm; + + metrics { + max_metric_name_length 6; + } + } +} +--- config + location /t { + proxy_wasm hostcalls 'on=request_headers \ + test=/t/metrics/define \ + metrics=c1'; + echo ok; + } +--- error_code: 500 +--- error_log eval +qr/host trap \(internal error\): could not define metric: name "\w+" too long/ +--- no_error_log +[emerg] +[alert] +[stub] + + + +=== TEST 2: proxy_wasm - define_metric() no memory (counter) +In SIGHUP mode, this test fails if executed after a test that +defined more metrics than is possible to fit in 5m. + +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- main_config eval +qq{ + wasm { + module hostcalls $ENV{TEST_NGINX_CRATES_DIR}/hostcalls.wasm; + + metrics { + slab_size 5m; + max_metric_name_length 128; + } + } +} +--- config + location /t { + proxy_wasm hostcalls 'on=request_headers \ + test=/t/metrics/define \ + metrics=c20337 \ + metrics_name_len=100'; + echo ok; + } +--- error_code: 500 +--- error_log eval +[ + qr/\[crit\] .+ \[wasm\] "metrics" shm store: no memory; cannot allocate pair with key size \d+ and value size \d+/, + qr/host trap \(internal error\): could not define metric "\S+" 1) { + master_on(); +} + +no_shuffle(); +plan_tests(6); +run_tests(); + +__DATA__ + +=== TEST 1: SIGHUP metrics - define metrics and increment counters +Evaluating counters values in error_log rather than in response headers as some +workers might not have done their increment by the time response_headers step +is invoked. + +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_increment_counters \ + metrics=$::metrics'; + echo ok; + } +} +--- error_log eval +qr/c2: $::workers.*/ +--- no_error_log +[error] +[crit] +[emerg] +[alert] + + + +=== TEST 2: SIGHUP metrics - shm preserved, no realloc +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_increment_counters \ + on=response_headers \ + test=/t/metrics/get \ + metrics=$::metrics'; + echo ok; + } +} +--- response_headers +c1: 4 +c2: 4 +--- no_error_log +reallocating metric +[error] +[crit] + + + +=== TEST 3: SIGHUP metrics - increased worker_processes - shm preserved, realloc +--- workers: 4 +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_increment_counters \ + on=response_headers \ + test=/t/metrics/get \ + metrics=$::metrics'; + echo ok; + } +} +--- response_headers +c1: 8 +c2: 8 +--- error_log: reallocating metric +--- no_error_log +[error] +[crit] + + + +=== TEST 4: SIGHUP metrics - decreased worker_processes - shm preserved, realloc +--- workers: 2 +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_increment_counters \ + on=response_headers \ + test=/t/metrics/get \ + metrics=$::metrics'; + echo ok; + } +} +--- response_headers +c1: 10 +c2: 10 +--- error_log: reallocating metric +--- no_error_log +[error] +[crit] + + + +=== TEST 5: SIGHUP metrics - decreased slab_size - shm preserved, realloc +--- workers: 2 +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- main_config eval +qq{ + wasm { + module hostcalls $ENV{TEST_NGINX_CRATES_DIR}/hostcalls.wasm; + + metrics { + slab_size 4m; + } + } +} +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_increment_counters \ + on=response_headers \ + test=/t/metrics/get \ + metrics=$::metrics'; + echo ok; + } +} +--- response_headers +c1: 12 +c2: 12 +--- error_log: reallocating metric +--- no_error_log +[error] +[crit] + + + +=== TEST 6: SIGHUP metrics - increased slab_size - shm preserved, realloc +--- workers: 2 +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- main_config eval +qq{ + wasm { + module hostcalls $ENV{TEST_NGINX_CRATES_DIR}/hostcalls.wasm; + + metrics { + slab_size 5m; + } + } +} +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_increment_counters \ + on=response_headers \ + test=/t/metrics/get \ + metrics=$::metrics'; + echo ok; + } +} +--- response_headers +c1: 14 +c2: 14 +--- error_log: reallocating metric +--- no_error_log +[error] +[crit] diff --git a/t/07-metrics/002-histograms_sighup.t b/t/07-metrics/002-histograms_sighup.t new file mode 100644 index 000000000..b4623dddd --- /dev/null +++ b/t/07-metrics/002-histograms_sighup.t @@ -0,0 +1,174 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: +use strict; +use lib '.'; +use t::TestWasmX; + +skip_no_hup(); + +our $workers = 2; +our $total = 0; + +workers($workers); +if ($workers > 1) { + master_on(); +} + +no_shuffle(); +plan_tests(4); +run_tests(); + +__DATA__ + +=== TEST 1: SIGHUP metrics - define metrics and record histograms +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_record_histograms \ + metrics=c2,g2,h2'; + echo ok; + } +} +--- grep_error_log eval: qr/histogram "\d+":( \d+: \d+;)+/ +--- grep_error_log_out eval +$::total += $::workers; +qr/histogram "\d+": 1: $::total; 4294967295: 0;/ +--- no_error_log +[error] +[crit] + + + +=== TEST 2: SIGHUP metrics - shm preserved, no realloc +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_record_histograms \ + metrics=h2'; + echo ok; + } +} +--- grep_error_log eval: qr/histogram "\d+":( \d+: \d+;)+/ +--- grep_error_log_out eval +$::total += $::workers; +qr/histogram "\d+": 1: $::total; 4294967295: 0;/ +--- no_error_log +[error] +[crit] + + + +=== TEST 3: SIGHUP metrics - increased worker_processes - shm preserved, realloc +--- workers: 4 +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_record_histograms \ + metrics=h2'; + echo ok; + } +} +--- grep_error_log eval: qr/histogram "\d+":( \d+: \d+;)+/ +--- grep_error_log_out eval +$::total += 4; +qr/histogram "\d+": 1: $::total; 4294967295: 0;/ +--- no_error_log +[error] +[crit] + + + +=== TEST 4: SIGHUP metrics - decreased worker_processes - shm preserved, realloc +--- workers: 2 +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: hostcalls +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_and_record_histograms \ + metrics=h2'; + echo ok; + } +} +--- grep_error_log eval: qr/histogram "\d+":( \d+: \d+;)+/ +--- grep_error_log_out eval +$::total += 2; +qr/histogram "\d+": 1: $::total; 4294967295: 0;/ +--- no_error_log +[error] +[crit] + + + +=== TEST 5: SIGHUP metrics - decreased slab_size - shm preserved, realloc +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- main_config eval +qq{ + wasm { + module hostcalls $ENV{TEST_NGINX_CRATES_DIR}/hostcalls.wasm; + + metrics { + slab_size 120k; + } + } +} +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on=request_headers \ + test=/t/metrics/record_histograms \ + metrics=h2'; + echo ok; + } +} +--- grep_error_log eval: qr/histogram "\d+":( \d+: \d+;)+/ +--- grep_error_log_out eval +$::total += 1; +qr/histogram "\d+": 1: $::total; 4294967295: 0;/ +--- no_error_log +[error] +[crit] + + + +=== TEST 6: SIGHUP metrics - increased slab_size - shm preserved, realloc +--- valgrind +--- load_nginx_modules: ngx_http_echo_module +--- main_config eval +qq{ + wasm { + module hostcalls $ENV{TEST_NGINX_CRATES_DIR}/hostcalls.wasm; + + metrics { + slab_size 16k; + } + } +} +--- config eval +qq{ + location /t { + proxy_wasm hostcalls 'on_configure=define_metrics \ + on=response_headers \ + test=/t/metrics/record_histograms \ + metrics=h2'; + echo ok; + } +} +--- grep_error_log eval: qr/histogram "\d+":( \d+: \d+;)+/ +--- grep_error_log_out eval +$::total += 1; +qr/histogram "\d+": 1: $::total; 4294967295: 0;/ +--- no_error_log +[error] +[crit] diff --git a/t/TestWasmX.pm b/t/TestWasmX.pm index e8dc2b646..ce871c6cc 100644 --- a/t/TestWasmX.pm +++ b/t/TestWasmX.pm @@ -29,6 +29,7 @@ our @EXPORT = qw( load_nginx_modules plan_tests skip_hup + skip_no_hup skip_no_ssl skip_no_ipc skip_no_debug @@ -59,6 +60,12 @@ sub skip_hup { } } +sub skip_no_hup { + if ($ENV{TEST_NGINX_USE_HUP} == 0) { + plan(skip_all => "skip without HUP mode"); + } +} + sub skip_no_ssl { if ($nginxV !~ m/built with \S+SSL/) { plan(skip_all => "SSL support required (NGX_BUILD_SSL=1)"); @@ -201,6 +208,17 @@ add_block_preprocessor(sub { $wasm_config = $wasm_config . (join "\n", @arr); } + # --- metrics + + my $metrics = $block->metrics; + + if (defined $metrics) { + $wasm_config = $wasm_config . + " metrics {\n" . + " slab_size $metrics" . ";\n" . + " }\n"; + } + # --- shm_queue my $shm_queue = $block->shm_queue; diff --git a/t/lib/proxy-wasm-tests/context-checks/src/hostcalls.rs b/t/lib/proxy-wasm-tests/context-checks/src/hostcalls.rs index b83e7cd8e..66a939603 100644 --- a/t/lib/proxy-wasm-tests/context-checks/src/hostcalls.rs +++ b/t/lib/proxy-wasm-tests/context-checks/src/hostcalls.rs @@ -381,3 +381,77 @@ pub fn dispatch_http_call(_ctx: &TestContext) { info!("dispatch_http_call status: {}", status as u32); } } + +#[allow(improper_ctypes)] +extern "C" { + fn proxy_define_metric( + metric_type: MetricType, + name_data: *const u8, + name_size: usize, + return_id: *mut u32, + ) -> Status; +} + +pub fn define_metric(_ctx: &TestContext) { + let name = "a_counter"; + let metric_type = MetricType::Counter; + let mut return_id: u32 = 0; + + unsafe { + let status = proxy_define_metric(metric_type, name.as_ptr(), name.len(), &mut return_id); + + info!("define_metric status: {}", status as u32); + } +} + +#[allow(improper_ctypes)] +extern "C" { + fn proxy_get_metric(metric_id: u32, return_value: *mut u64) -> Status; +} + +pub fn get_metric(_ctx: &TestContext) { + let name = "a_counter"; + let metric_type = MetricType::Counter; + let mut metric_id: u32 = 0; + let mut value: u64 = 0; + + unsafe { + proxy_define_metric(metric_type, name.as_ptr(), name.len(), &mut metric_id); + let status = proxy_get_metric(metric_id, &mut value); + info!("get_metric status: {}", status as u32); + } +} + +#[allow(improper_ctypes)] +extern "C" { + fn proxy_record_metric(metric_id: u32, value: u64) -> Status; +} + +pub fn record_metric(_ctx: &TestContext) { + let name = "a_gauge"; + let metric_type = MetricType::Gauge; + let mut metric_id: u32 = 0; + + unsafe { + proxy_define_metric(metric_type, name.as_ptr(), name.len(), &mut metric_id); + let status = proxy_record_metric(metric_id, 1); + info!("record_metric status: {}", status as u32); + } +} + +#[allow(improper_ctypes)] +extern "C" { + fn proxy_increment_metric(metric_id: u32, offset: i64) -> Status; +} + +pub fn increment_metric(_ctx: &TestContext) { + let name = "a_counter"; + let metric_type = MetricType::Counter; + let mut metric_id: u32 = 0; + + unsafe { + proxy_define_metric(metric_type, name.as_ptr(), name.len(), &mut metric_id); + let status = proxy_increment_metric(metric_id, 1); + info!("increment_metric status: {}", status as u32); + } +} diff --git a/t/lib/proxy-wasm-tests/context-checks/src/lib.rs b/t/lib/proxy-wasm-tests/context-checks/src/lib.rs index 084b19ad9..eea6eae50 100644 --- a/t/lib/proxy-wasm-tests/context-checks/src/lib.rs +++ b/t/lib/proxy-wasm-tests/context-checks/src/lib.rs @@ -19,6 +19,10 @@ impl TestContext { let (name, arg) = name.split_once('|').unwrap_or_else(|| (name, "")); match name { "proxy_log" => log_something(self), + "define_metric" => define_metric(self), + "increment_metric" => increment_metric(self), + "record_metric" => record_metric(self), + "get_metric" => get_metric(self), "set_tick_period" => set_tick_period(self), "add_request_header" => add_request_header(self, arg), "add_response_header" => add_response_header(self, arg), diff --git a/t/lib/proxy-wasm-tests/hostcalls/src/lib.rs b/t/lib/proxy-wasm-tests/hostcalls/src/lib.rs index 7e992e757..aaf25142d 100644 --- a/t/lib/proxy-wasm-tests/hostcalls/src/lib.rs +++ b/t/lib/proxy-wasm-tests/hostcalls/src/lib.rs @@ -7,13 +7,14 @@ mod types; use crate::{tests::*, types::test_http::*, types::test_root::*, types::*}; use log::*; use proxy_wasm::{traits::*, types::*}; -use std::{collections::HashMap, time::Duration}; +use std::{collections::BTreeMap, collections::HashMap, time::Duration}; proxy_wasm::main! {{ proxy_wasm::set_log_level(LogLevel::Trace); proxy_wasm::set_root_context(|_| -> Box { Box::new(TestRoot { config: HashMap::new(), + metrics: BTreeMap::new(), n_sync_calls: 0, }) }); @@ -27,11 +28,13 @@ impl RootContext for TestRoot { if let Ok(text) = std::str::from_utf8(&config) { info!("vm config: {}", text); - if text == "do_trap" { - panic!("trap on_vm_start"); - } else if text == "do_false" { - info!("on_vm_start returning false"); - return false; + match text { + "do_trap" => panic!("trap on_vm_start"), + "do_false" => { + info!("on_vm_start returning false"); + return false; + } + _ => (), } } else { info!("cannot parse vm config"); @@ -63,12 +66,23 @@ impl RootContext for TestRoot { )); } - if let Some(on_configure) = self.get_config("on_configure") { - match on_configure { - "do_trap" => panic!("trap on_configure"), - "do_return_false" => return false, - _ => (), + match self.get_config("on_configure").unwrap_or("") { + "do_trap" => panic!("trap on_configure"), + "do_return_false" => return false, + "define_metrics" => test_define_metrics(self), + "define_and_increment_counters" => { + test_define_metrics(self); + test_increment_counters(self, TestPhase::Configure, None); + } + "define_and_toggle_gauges" => { + test_define_metrics(self); + test_toggle_gauges(self, TestPhase::Configure, None); } + "define_and_record_histograms" => { + test_define_metrics(self); + test_record_metric(self, TestPhase::Configure); + } + _ => (), } true @@ -84,19 +98,23 @@ impl RootContext for TestRoot { self.get_config("tick_period").unwrap() ); + let n_sync_calls = self + .config + .get("n_sync_calls") + .map_or(1, |v| v.parse().expect("bad n_sync_calls value")); + + if self.n_sync_calls >= n_sync_calls { + return; + } + match self.get_config("on_tick").unwrap_or("") { "log_property" => test_log_property(self), + "set_gauges" => { + test_record_metric(self, TestPhase::Tick); + self.n_sync_calls += 1; + } "set_property" => test_set_property(self), "dispatch" => { - let n_sync_calls = self - .config - .get("n_sync_calls") - .map_or(1, |v| v.parse().expect("bad n_sync_calls value")); - - if self.n_sync_calls >= n_sync_calls { - return; - } - let mut timeout = Duration::from_secs(0); let mut headers = Vec::new(); let path = self @@ -172,6 +190,7 @@ impl RootContext for TestRoot { Some(Box::new(TestHttp { config: self.config.clone(), on_phases: phases, + metrics: self.metrics.clone(), n_sync_calls: 0, })) } diff --git a/t/lib/proxy-wasm-tests/hostcalls/src/tests/mod.rs b/t/lib/proxy-wasm-tests/hostcalls/src/tests/mod.rs index 1b98e15bd..17acfb3c3 100644 --- a/t/lib/proxy-wasm-tests/hostcalls/src/tests/mod.rs +++ b/t/lib/proxy-wasm-tests/hostcalls/src/tests/mod.rs @@ -123,6 +123,14 @@ pub(crate) fn test_log_properties(ctx: &(dyn TestContext + 'static), phase: Test } } +pub(crate) fn test_log_metrics(ctx: &(dyn TestContext + 'static), phase: TestPhase) { + for (n, id) in ctx.get_metrics_mapping() { + if n.starts_with('h') { continue } + let value = get_metric(*id).unwrap(); + info!("{}: {} at {:?}", n, value, phase) + } +} + fn show_property(ctx: &(dyn TestContext + 'static), path: &[&str]) -> String { if let Some(p) = ctx.get_property(path.to_vec()) { if let Ok(value) = std::str::from_utf8(&p) { @@ -238,7 +246,9 @@ trait HeaderStr { impl HeaderStr for str { fn split_header<'a>(&'a self) -> Option<(&'a str, &'a str)> { if &self[0..1] == ":" { - self[1..].find(':').map(|n| (&self[0..n + 1], &self[n + 2..])) + self[1..] + .find(':') + .map(|n| (&self[0..n + 1], &self[n + 2..])) } else { self.split_once(':') } @@ -414,6 +424,130 @@ pub(crate) fn test_set_shared_data_by_len(ctx: &mut TestHttp) { test_set_shared_data(ctx); } +fn should_run_on_current_worker(ctx: &(dyn TestContext + 'static)) -> bool { + match ctx.get_config("on_worker") { + Some(on_worker) => { + let w_id_bytes = ctx + .get_property(vec![("worker_id")]) + .expect("could not get worker_id"); + let w_id = std::str::from_utf8(&w_id_bytes).expect("bad worker_id value"); + + on_worker == w_id + } + None => true, + } +} + +pub(crate) fn test_define_metrics(ctx: &mut (dyn TestContext + 'static)) { + if !should_run_on_current_worker(ctx) { + return; + } + + let name_len = ctx.get_config("metrics_name_len").map_or(0, |x| { + x.parse::().expect("bad metrics_name_len value") + }); + + let metrics_config = ctx + .get_config("metrics") + .map(|x| x.to_string()) + .expect("missing metrics parameter"); + + for metric in metrics_config.split(",") { + let metric_char = metric.chars().nth(0).unwrap(); + let metric_type = match metric_char { + 'c' => MetricType::Counter, + 'g' => MetricType::Gauge, + 'h' => MetricType::Histogram, + _ => panic!("unexpected metric type"), + }; + let n = metric[1..].parse::().expect("bad metrics value"); + + for i in 1..(n + 1) { + let mut name = format!("{}{}", metric_char, i); + + if name_len > 0 { + name = format!("{}{}", name, "x".repeat(name_len - name.chars().count())); + } + + let m_id = define_metric(metric_type, &name).expect("cannot define new metric"); + + info!("defined metric {} as {:?}", &name, m_id); + + ctx.save_metric_mapping(name.as_str(), m_id); + } + } +} + +pub(crate) fn test_increment_counters(ctx: &(dyn TestContext + 'static), phase: TestPhase, skip_others: Option) { + if !should_run_on_current_worker(ctx) { + return; + } + + let n_increments = ctx + .get_config("n_increments") + .map_or(1, |x| x.parse::().expect("bad n_increments value")); + + for (n, id) in ctx.get_metrics_mapping() { + if skip_others.unwrap_or(true) && !n.starts_with('c') { continue } + + for _ in 0..n_increments { + increment_metric(*id, 1).unwrap(); + } + + info!("incremented {} at {:?}", n, phase); + } + + test_log_metrics(ctx, phase); +} + +pub(crate) fn test_toggle_gauges(ctx: &(dyn TestContext + 'static), phase: TestPhase, skip_others: Option) { + if !should_run_on_current_worker(ctx) { + return; + } + + for (n, id) in ctx.get_metrics_mapping() { + if skip_others.unwrap_or(true) && !n.starts_with('g') { continue } + + let new_value = match get_metric(*id).unwrap() { + 0 => 1, + _ => 0, + }; + + record_metric(*id, new_value).unwrap(); + info!("toggled {} at {:?}", n, phase); + } + + test_log_metrics(ctx, phase); +} + +pub(crate) fn test_record_metric(ctx: &(dyn TestContext + 'static), phase: TestPhase) { + if !should_run_on_current_worker(ctx) { + return; + } + + let value = ctx + .get_config("value") + .map_or(1, |x| x.parse::().expect("bad value")); + + for (n, id) in ctx.get_metrics_mapping() { + if n.starts_with('c') { continue } + record_metric(*id, value).unwrap(); + info!("record {} on {} at {:?}", value, n, phase); + } + + test_log_metrics(ctx, phase); +} + +pub(crate) fn test_get_metrics(ctx: &TestHttp) { + for (n, id) in ctx.get_metrics_mapping() { + if n.starts_with('h') { continue } + + let name = n.replace("_", "-"); + let value = get_metric(*id).unwrap().to_string(); + ctx.add_http_response_header(name.as_str(), value.as_str()); + } +} + pub(crate) fn test_shared_queue_enqueue(ctx: &TestHttp) { let queue_id: u32 = ctx .config diff --git a/t/lib/proxy-wasm-tests/hostcalls/src/types/mod.rs b/t/lib/proxy-wasm-tests/hostcalls/src/types/mod.rs index c4f07751e..315efbf9b 100644 --- a/t/lib/proxy-wasm-tests/hostcalls/src/types/mod.rs +++ b/t/lib/proxy-wasm-tests/hostcalls/src/types/mod.rs @@ -6,6 +6,8 @@ use crate::*; #[derive(Debug, Eq, PartialEq, enum_utils::FromStr)] #[enumeration(rename_all = "snake_case")] pub enum TestPhase { + Configure, + Tick, RequestHeaders, RequestBody, ResponseHeaders, @@ -16,6 +18,8 @@ pub enum TestPhase { pub trait TestContext { fn get_config(&self, name: &str) -> Option<&str>; + fn get_metrics_mapping(&self) -> &BTreeMap; + fn save_metric_mapping(&mut self, name: &str, metric_id: u32) -> Option; } impl Context for dyn TestContext {} @@ -24,10 +28,26 @@ impl TestContext for TestRoot { fn get_config(&self, name: &str) -> Option<&str> { self.config.get(name).map(|s| s.as_str()) } + + fn get_metrics_mapping(&self) -> &BTreeMap { + &self.metrics + } + + fn save_metric_mapping(&mut self, name: &str, metric_id: u32) -> Option { + self.metrics.insert(name.to_string(), metric_id) + } } impl TestContext for TestHttp { fn get_config(&self, name: &str) -> Option<&str> { self.config.get(name).map(|s| s.as_str()) } + + fn get_metrics_mapping(&self) -> &BTreeMap { + &self.metrics + } + + fn save_metric_mapping(&mut self, name: &str, metric_id: u32) -> Option { + self.metrics.insert(name.to_string(), metric_id) + } } diff --git a/t/lib/proxy-wasm-tests/hostcalls/src/types/test_http.rs b/t/lib/proxy-wasm-tests/hostcalls/src/types/test_http.rs index 993c4a917..a097f0415 100644 --- a/t/lib/proxy-wasm-tests/hostcalls/src/types/test_http.rs +++ b/t/lib/proxy-wasm-tests/hostcalls/src/types/test_http.rs @@ -1,11 +1,12 @@ use crate::{tests::echo::*, tests::host::*, tests::*, *}; use http::StatusCode; use log::*; -use proxy_wasm::{traits::*, types::*}; +use proxy_wasm::{hostcalls::*, traits::*, types::*}; pub struct TestHttp { pub on_phases: Vec, pub config: HashMap, + pub metrics: BTreeMap, pub n_sync_calls: usize, } @@ -64,6 +65,7 @@ impl TestHttp { "/t/log/response_body" => test_log_response_body(self), "/t/log/property" => test_log_property(self), "/t/log/properties" => test_log_properties(self, cur_phase), + "/t/log/metrics" => test_log_metrics(self, cur_phase), /* send_local_response */ "/t/send_local_response/status/204" => test_send_status(self, 204), @@ -121,6 +123,31 @@ impl TestHttp { "/t/shm/enqueue" => test_shared_queue_enqueue(self), "/t/shm/dequeue" => test_shared_queue_dequeue(self), + /* metrics */ + "/t/metrics/define" => test_define_metrics(self), + "/t/metrics/increment_counters" => test_increment_counters(self, cur_phase, None), + "/t/metrics/increment_gauges" => { + let skip_non_counters = false; + test_increment_counters(self, cur_phase, Some(skip_non_counters)); + } + "/t/metrics/toggle_gauges" => test_toggle_gauges(self, cur_phase, None), + "/t/metrics/toggle_counters" => { + let skip_non_gauges = false; + test_toggle_gauges(self, cur_phase, Some(skip_non_gauges)); + } + "/t/metrics/record_histograms" => test_record_metric(self, cur_phase), + "/t/metrics/get" => test_get_metrics(self), + "/t/metrics/increment_invalid_counter" => increment_metric(0, 1).unwrap(), + "/t/metrics/set_invalid_gauge" => record_metric(0, 1).unwrap(), + "/t/metrics/get_invalid_metric" => { + get_metric(0).unwrap(); + } + "/t/metrics/get_histogram" => { + info!("retrieving histogram in \"{:?}\"", cur_phase); + let h_id = define_metric(MetricType::Histogram, "h1").expect("cannot define new metric"); + get_metric(h_id).unwrap(); + } + /* errors */ "/t/trap" => panic!("custom trap"), "/t/error/get_response_body" => { diff --git a/t/lib/proxy-wasm-tests/hostcalls/src/types/test_root.rs b/t/lib/proxy-wasm-tests/hostcalls/src/types/test_root.rs index 95ad06857..6300c47a2 100644 --- a/t/lib/proxy-wasm-tests/hostcalls/src/types/test_root.rs +++ b/t/lib/proxy-wasm-tests/hostcalls/src/types/test_root.rs @@ -2,5 +2,6 @@ use crate::*; pub struct TestRoot { pub config: HashMap, + pub metrics: BTreeMap, pub n_sync_calls: usize, } diff --git a/util/setup_dev.sh b/util/setup_dev.sh index e5eed89e1..00d10b997 100755 --- a/util/setup_dev.sh +++ b/util/setup_dev.sh @@ -113,6 +113,31 @@ EOF worker_processes $Workers; -daemon $DaemonEnabled; master_process $MasterProcessEnabled; +EOF + # --- workers block + patch --forward --ignore-whitespace lib/perl5/Test/Nginx/Util.pm <<'EOF' + @@ -977,6 +977,11 @@ + my $post_main_config = $block->post_main_config; + my $err_log_file = $block->error_log_file; + my $server_name = $block->server_name; + + my $workers = $block->workers; + + + + if (!$workers) { + + $workers = $Workers; + + } + + if ($UseHup) { + master_on(); # config reload is buggy when master is off + @@ -1054,7 +1059,7 @@ + bail_out "Can't open $ConfFile for writing: $!\n"; + print $out "daemon $DaemonEnabled;" if ($DaemonEnabled eq 'off'); + print $out <<_EOC_; + -worker_processes $Workers; + +worker_processes $workers; + master_process $MasterProcessEnabled; + error_log $err_log_file $LogLevel; + pid $PidFile; + EOF patch --forward --ignore-whitespace lib/perl5/Test/Nginx/Util.pm <<'EOF' @@ -484,6 +484,7 @@ sub master_process_enabled (@) {