From c815acbc2e166ebd68d35295aa793f05d6d2f5f4 Mon Sep 17 00:00:00 2001 From: Thibault Charbonnier Date: Sat, 2 Dec 2023 09:34:29 -0800 Subject: [PATCH 1/3] chore(util) always invoke the Proxy-Wasm SDK scripts This is a fix for CI, in which the SDK directories may exist but the examples are not built and/or copied to `t/`. In any case, better making the scripts idempotent than not invoking them at all and risking an inconsistent state. --- util/sdks/assemblyscript.sh | 3 ++- util/sdks/go.sh | 3 ++- util/setup_dev.sh | 44 +++++++++++++++++++++++++------------ 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/util/sdks/assemblyscript.sh b/util/sdks/assemblyscript.sh index 2aa931c01..f97128fc1 100755 --- a/util/sdks/assemblyscript.sh +++ b/util/sdks/assemblyscript.sh @@ -70,8 +70,9 @@ build_assemblyscript_sdk() { if [[ -f "$DIR_PROXY_WASM_ASSEMBLYSCRIPT_SDK/.hash" \ && $(cat "$DIR_PROXY_WASM_ASSEMBLYSCRIPT_SDK/.hash") == $(echo $hash_src) - && -z $clean ]]; + && -z "$clean" ]]; then + notice "AssemblyScript examples already built" exit fi diff --git a/util/sdks/go.sh b/util/sdks/go.sh index 0c5c82c85..b7bed5964 100755 --- a/util/sdks/go.sh +++ b/util/sdks/go.sh @@ -66,8 +66,9 @@ build_go_sdk() { if [[ -d "$DIR_PATCHED_PROXY_WASM_GO_SDK" \ && -f "$DIR_PATCHED_PROXY_WASM_GO_SDK/.hash" \ && $(cat "$DIR_PATCHED_PROXY_WASM_GO_SDK/.hash") == $(echo $hash_src) - && -z $clean ]]; + && -z "$clean" ]]; then + notice "Go examples already built" exit fi diff --git a/util/setup_dev.sh b/util/setup_dev.sh index 4fc576cd4..fe0f0742f 100755 --- a/util/setup_dev.sh +++ b/util/setup_dev.sh @@ -17,8 +17,12 @@ source $NGX_WASM_DIR/util/_lib.sh mkdir -p $DIR_CPANM $DIR_BIN $DIR_TESTS_LIB_WASM +# OpenSSL + install_openssl +# Test::Nginx + pushd $DIR_CPANM if [[ ! -x "cpanm" ]]; then notice "downloading cpanm..." @@ -136,6 +140,8 @@ EOF set -e popd +# reindex + pushd $DIR_BIN notice "downloading the reindex script..." download reindex https://raw.githubusercontent.com/openresty/openresty-devel-utils/master/reindex @@ -146,6 +152,8 @@ pushd $DIR_BIN chmod +x style popd +# echo-module + if [[ -d "$DIR_NGX_ECHO_MODULE" ]]; then notice "updating the echo-nginx-module repository..." pushd $DIR_NGX_ECHO_MODULE @@ -158,6 +166,8 @@ else git clone https://github.com/openresty/echo-nginx-module.git $DIR_NGX_ECHO_MODULE fi +# headers-more-module + if [[ -d "$DIR_NGX_HEADERS_MORE_MODULE" ]]; then notice "updating the headers-more-nginx-module repository..." pushd $DIR_NGX_HEADERS_MORE_MODULE @@ -170,6 +180,8 @@ else git clone https://github.com/openresty/headers-more-nginx-module.git $DIR_NGX_HEADERS_MORE_MODULE fi +# mockeagain + if [[ -d "$DIR_MOCKEAGAIN" ]]; then notice "updating the mockeagain repository..." pushd $DIR_MOCKEAGAIN @@ -186,31 +198,35 @@ pushd $DIR_MOCKEAGAIN make popd +# no-pool-patch + get_no_pool_nginx 1 +# runtime + if [[ -n "$NGX_WASM_RUNTIME" ]] && ! [[ -n "$NGX_WASM_RUNTIME_LIB" ]]; then notice "fetching the \"$NGX_WASM_RUNTIME\" runtime..." $NGX_WASM_DIR/util/runtime.sh -R "$NGX_WASM_RUNTIME" fi -if [[ ! -d "$DIR_PROXY_WASM_GO_SDK" ]]; then - if [[ ! -x "$(command -v tinygo)" ]]; then - notice "missing 'tinygo', skipping proxy-wasm-go-sdk" +# proxy-wasm-go-sdk - else - notice "downloading/building proxy-wasm-go-sdk..." - $NGX_WASM_DIR/util/sdk.sh -S go --build --clean - fi +if [[ ! -x "$(command -v tinygo)" ]]; then + notice "missing 'tinygo', skipping proxy-wasm-go-sdk" + +else + notice "building proxy-wasm-go-sdk..." + $NGX_WASM_DIR/util/sdk.sh -S go --build fi +# +# proxy-wasm-assemblyscript-sdk -if [[ ! -d "$DIR_PROXY_WASM_ASSEMBLYSCRIPT_SDK" ]]; then - if [[ ! -x "$(command -v npm)" ]]; then - notice "missing 'npm', skipping proxy-wasm-assemblyscript-sdk" +if [[ ! -x "$(command -v npm)" ]]; then + notice "missing 'npm', skipping proxy-wasm-assemblyscript-sdk" - else - notice "downloading/building proxy-wasm-assemblyscript-sdk..." - $NGX_WASM_DIR/util/sdk.sh -S assemblyscript --build --clean - fi +else + notice "building proxy-wasm-assemblyscript-sdk..." + $NGX_WASM_DIR/util/sdk.sh -S assemblyscript --build fi notice "done" From 7199954d4af1467e72a2eb4aa945ff42116928ad Mon Sep 17 00:00:00 2001 From: Thibault Charbonnier Date: Thu, 10 Aug 2023 13:45:12 -0700 Subject: [PATCH 2/3] feat(proxy-wasm) implement response body buffering First, this commit fixes `on_response_body` execution of requests when they issue subrequests that produce a body (see 004-on_http_phases.t). This unlocks testing of response body buffering via subrequests producing chunked responses (`ngx_chain_t` buffers). Response body buffering itself is implemented as part of ngx_http_wasm_filter_module, so as to be usable from other components than just proxy-wasm. Response body buffering is enabled via `rctx->resp_buffering` which is enabled when `ngx_wasm_ops_resume` returns `NGX_AGAIN`, which for filters translates to returning `PAUSE` from `on_http_response_body`. If and when the response body buffers are full, then the next `on_http_response_body` call will have `eof=false`, and more invocations can be expected, as the body *must* be proxied in chunks. If the response body fit in the buffers, then the next `on_http_response_body` call will have `eof=true` as one would expected. We catch "response body already requested" in ngx_proxy_wasm to produce a proxy-wasm error message, as this feature is currently only exposed through it. --- src/common/proxy_wasm/ngx_proxy_wasm.c | 10 +- src/http/ngx_http_wasm.h | 10 +- src/http/ngx_http_wasm_filter_module.c | 240 +++++++- src/http/ngx_http_wasm_module.c | 12 + src/http/proxy_wasm/ngx_http_proxy_wasm.c | 12 + src/wasm/ngx_wasm.h | 3 + src/wasm/ngx_wasm_ops.c | 1 + src/wasm/ngx_wasm_util.c | 20 +- t/03-proxy_wasm/004-on_http_phases.t | 49 +- t/03-proxy_wasm/006-on_http_next_action.t | 29 +- .../008-on_http_response_body_buffering.t | 520 ++++++++++++++++++ ...-proxy_wasm_oob.t => 009-proxy_wasm_oob.t} | 0 .../proxy-wasm-tests/on-phases/src/filter.rs | 20 +- 13 files changed, 867 insertions(+), 59 deletions(-) create mode 100644 t/03-proxy_wasm/008-on_http_response_body_buffering.t rename t/03-proxy_wasm/{008-proxy_wasm_oob.t => 009-proxy_wasm_oob.t} (100%) diff --git a/src/common/proxy_wasm/ngx_proxy_wasm.c b/src/common/proxy_wasm/ngx_proxy_wasm.c index 287091f98..55f3ac4a1 100644 --- a/src/common/proxy_wasm/ngx_proxy_wasm.c +++ b/src/common/proxy_wasm/ngx_proxy_wasm.c @@ -498,6 +498,13 @@ action2rc(ngx_proxy_wasm_ctx_t *pwctx, case NGX_PROXY_WASM_ACTION_PAUSE: switch (pwctx->phase->index) { #ifdef NGX_WASM_HTTP + case NGX_HTTP_WASM_BODY_FILTER_PHASE: + ngx_log_debug3(NGX_LOG_DEBUG_WASM, pwctx->log, 0, + "proxy_wasm buffering response after " + "\"ResponseBody\" step " + "(filter: %l/%l, pwctx: %p)", + pwexec->index + 1, pwctx->nfilters, pwctx); + goto yield; case NGX_HTTP_REWRITE_PHASE: case NGX_HTTP_ACCESS_PHASE: case NGX_HTTP_CONTENT_PHASE: @@ -640,7 +647,8 @@ ngx_proxy_wasm_resume(ngx_proxy_wasm_ctx_t *pwctx, rc = action2rc(pwctx, pwexec); if (rc != NGX_OK) { if (rc == NGX_AGAIN - && pwctx->exec_index + 1 <= pwctx->nfilters) + && pwctx->exec_index + 1 <= pwctx->nfilters + && step != NGX_PROXY_WASM_STEP_RESP_BODY) { dd("yield: resume on next filter " "(idx: %ld -> %ld, nelts: %ld)", diff --git a/src/http/ngx_http_wasm.h b/src/http/ngx_http_wasm.h index 94043832b..3a2ea697b 100644 --- a/src/http/ngx_http_wasm.h +++ b/src/http/ngx_http_wasm.h @@ -47,10 +47,12 @@ struct ngx_http_wasm_req_ctx_s { ngx_http_handler_pt r_content_handler; ngx_array_t resp_shim_headers; + ngx_uint_t resp_bufs_count; /* response buffers count */ + ngx_chain_t *resp_bufs; /* response buffers */ + ngx_chain_t *resp_buf_last; /* last response buffers */ ngx_chain_t *resp_chunk; - off_t resp_chunk_len; unsigned resp_chunk_eof; /* seen last buf flag */ - + off_t resp_chunk_len; off_t req_content_length_n; off_t resp_content_length_n; @@ -73,6 +75,7 @@ struct ngx_http_wasm_req_ctx_s { unsigned in_wev:1; /* in wev_handler */ unsigned resp_content_chosen:1; /* content handler has an output to produce */ + unsigned resp_buffering:1; /* enable response buffering */ unsigned resp_content_sent:1; /* has started sending output (may have yielded) */ unsigned resp_finalized:1; /* finalized connection (ourselves) */ unsigned fake_request:1; @@ -91,8 +94,9 @@ typedef struct { ngx_msec_t recv_timeout; size_t socket_buffer_size; /* wasm_socket_buffer_size */ - ngx_bufs_t socket_large_buffers; /* wasm_socket_large_buffer_size */ ngx_flag_t socket_buffer_reuse; /* wasm_socket_buffer_reuse */ + ngx_bufs_t socket_large_buffers; /* wasm_socket_large_buffer_size */ + ngx_bufs_t resp_body_buffers; /* wasm_response_body_buffers */ ngx_flag_t pwm_req_headers_in_access; ngx_flag_t pwm_lua_resolver; diff --git a/src/http/ngx_http_wasm_filter_module.c b/src/http/ngx_http_wasm_filter_module.c index ca3d537dd..1237144a6 100644 --- a/src/http/ngx_http_wasm_filter_module.c +++ b/src/http/ngx_http_wasm_filter_module.c @@ -44,6 +44,12 @@ static ngx_http_output_header_filter_pt ngx_http_next_header_filter; static ngx_http_output_body_filter_pt ngx_http_next_body_filter; +static void ngx_http_wasm_body_filter_resume(ngx_http_wasm_req_ctx_t *rctx, + ngx_chain_t *in); +static ngx_int_t ngx_http_wasm_body_filter_buffer( + ngx_http_wasm_req_ctx_t *rctx, ngx_chain_t *in); + + static ngx_int_t ngx_http_wasm_filter_init(ngx_conf_t *cf) { @@ -112,10 +118,8 @@ ngx_http_wasm_header_filter_handler(ngx_http_request_t *r) done: -#if (DDEBUG) ngx_log_debug1(NGX_LOG_DEBUG_WASM, r->connection->log, 0, "wasm \"header_filter\" phase rc: %d", rc); -#endif return rc; } @@ -125,7 +129,7 @@ static ngx_int_t ngx_http_wasm_body_filter_handler(ngx_http_request_t *r, ngx_chain_t *in) { ngx_int_t rc; - ngx_http_wasm_req_ctx_t *rctx; + ngx_http_wasm_req_ctx_t *rctx = NULL, *mrctx = NULL; dd("enter"); @@ -134,22 +138,63 @@ ngx_http_wasm_body_filter_handler(ngx_http_request_t *r, ngx_chain_t *in) goto done; } - if (rc != NGX_DECLINED && !rctx->entered_header_filter) { - rc = NGX_DECLINED; - } + if (rc == NGX_DECLINED || !rctx->entered_header_filter) { + if (r != r->main) { + /* get main rctx */ + rc = ngx_http_wasm_rctx(r->main, &mrctx); + if (rc == NGX_ERROR) { + goto done; + } + } - if (rc == NGX_DECLINED) { - rc = ngx_http_next_body_filter(r, in); - goto done; + if (rc == NGX_DECLINED && rctx == NULL) { + /* r == r->main or r->main has no rctx */ + rc = ngx_http_next_body_filter(r, in); + goto done; + } + + if (r != r->main && rctx == NULL) { + /* subrequest with no rctx; merge main rctx for buffering */ + ngx_wasm_assert(mrctx); + rctx = mrctx; + } } - ngx_wasm_assert(rc == NGX_OK); + ngx_http_wasm_body_filter_resume(rctx, in); - rctx->resp_chunk = in; - rctx->resp_chunk_len = ngx_wasm_chain_len(in, &rctx->resp_chunk_eof); + if (rctx->resp_buffering) { + rc = ngx_http_wasm_body_filter_buffer(rctx, in); + dd("ngx_http_wasm_body_filter_buffer rc: %ld", rc); + switch (rc) { + case NGX_ERROR: + goto done; + case NGX_OK: + /* chunk in buffers */ + ngx_wasm_assert(rctx->resp_bufs); + + if (!rctx->resp_chunk_eof) { + /* more to come; go again */ + rc = NGX_AGAIN; + goto done; + } + + dd("eof, resume"); + rctx->resp_buffering = 0; + ngx_http_wasm_body_filter_resume(rctx, rctx->resp_bufs); + break; + case NGX_DONE: + dd("buffers full, resume"); + rctx->resp_buffering = 0; + ngx_http_wasm_body_filter_resume(rctx, rctx->resp_bufs); + break; + default: + ngx_wasm_assert(0); + break; + } + } - (void) ngx_wasm_ops_resume(&rctx->opctx, - NGX_HTTP_WASM_BODY_FILTER_PHASE); + ngx_wasm_chain_log_debug(r->connection->log, rctx->resp_chunk, + "rctx->resp_chunk"); rc = ngx_http_next_body_filter(r, rctx->resp_chunk); dd("ngx_http_next_body_filter rc: %ld", rc); @@ -159,10 +204,6 @@ ngx_http_wasm_body_filter_handler(ngx_http_request_t *r, ngx_chain_t *in) rctx->resp_chunk = NULL; - ngx_chain_update_chains(r->connection->pool, - &rctx->free_bufs, &rctx->busy_bufs, - &rctx->resp_chunk, buf_tag); - #ifdef NGX_WASM_RESPONSE_TRAILERS if (rctx->resp_chunk_eof && r->parent == NULL) { (void) ngx_wasm_ops_resume(&rctx->opctx, @@ -172,10 +213,169 @@ ngx_http_wasm_body_filter_handler(ngx_http_request_t *r, ngx_chain_t *in) done: -#if (DDEBUG) + ngx_wasm_assert(rc == NGX_OK || rc == NGX_AGAIN || rc == NGX_ERROR); + ngx_log_debug1(NGX_LOG_DEBUG_WASM, r->connection->log, 0, "wasm \"body_filter\" phase rc: %d", rc); -#endif + + if (rc == NGX_OK && rctx && rctx->resp_chunk_eof) { + ngx_chain_update_chains(rctx->pool, + &rctx->free_bufs, &rctx->busy_bufs, + &rctx->resp_bufs, rctx->env.buf_tag); + } return rc; } + + +static void +ngx_http_wasm_body_filter_resume(ngx_http_wasm_req_ctx_t *rctx, ngx_chain_t *in) +{ + ngx_int_t rc; + ngx_chain_t *cl; + + rctx->resp_chunk = in; + rctx->resp_chunk_len = 0; + + for (cl = rctx->resp_chunk; cl; cl = cl->next) { + rctx->resp_chunk_len += ngx_buf_size(cl->buf); + + if (cl->buf->last_buf) { + rctx->resp_chunk_eof = 1; + break; + + } else if (cl->buf->last_in_chain) { + if (rctx->r != rctx->r->main) { + rctx->resp_chunk_eof = 1; + } + + break; + } + } + + if (!rctx->resp_buffering) { + rc = ngx_wasm_ops_resume(&rctx->opctx, + NGX_HTTP_WASM_BODY_FILTER_PHASE); + if (rc == NGX_AGAIN) { + ngx_wasm_assert(rctx->resp_bufs == NULL); + rctx->resp_buffering = 1; + } + } +} + + +static ngx_int_t +ngx_http_wasm_body_filter_buffer(ngx_http_wasm_req_ctx_t *rctx, ngx_chain_t *in) +{ + off_t n, avail, copy; + ngx_chain_t *cl, *ll, *rl; + ngx_http_request_t *r = rctx->r; + ngx_http_wasm_loc_conf_t *loc; + + dd("enter"); + + ngx_wasm_assert(rctx->resp_buffering); + + cl = rctx->resp_buf_last; + loc = ngx_http_get_module_loc_conf(r, ngx_http_wasm_module); + + for (ll = in; ll; ll = ll->next) { + + n = ngx_buf_size(ll->buf); + dd("n: %ld", n); + + if (n == 0) { + if (ll->buf->last_in_chain || ll->buf->last_buf) { + if (rctx->resp_bufs == NULL) { + rctx->resp_bufs = ll; + + } else { + rctx->resp_buf_last->next = ll; + rctx->resp_buf_last = ll; + } + } + } + + while (n) { + if (cl == NULL) { + if (rctx->resp_bufs_count + >= (ngx_uint_t) loc->resp_body_buffers.num) + { + if (rctx->resp_buf_last) { + for (rl = ll; rl; rl = rl->next) { + if (ngx_buf_size(rl->buf)) { + rctx->resp_buf_last->next = rl; + rctx->resp_buf_last = rl; + } + } + } + + ngx_wasm_chain_log_debug(r->connection->log, + rctx->resp_bufs, + "response buffers: "); + + return NGX_DONE; + } + + cl = ngx_wasm_chain_get_free_buf(rctx->pool, + &rctx->free_bufs, + loc->resp_body_buffers.size, + rctx->env.buf_tag, 1); + if (cl == NULL) { + return NGX_ERROR; + } + + cl->buf->pos = cl->buf->start; + cl->buf->last = cl->buf->pos; + cl->buf->tag = rctx->env.buf_tag; + cl->buf->memory = 1; + cl->next = NULL; + + if (rctx->resp_bufs == NULL) { + rctx->resp_bufs = cl; + + } else if (rctx->resp_buf_last) { + rctx->resp_buf_last->next = cl; + } + + rctx->resp_buf_last = cl; + rctx->resp_bufs_count++; + } + + avail = cl->buf->end - cl->buf->last; + copy = ngx_min(n, avail); + dd("avail: %ld, copy: %ld", avail, copy); + if (copy == 0) { + cl = NULL; + continue; + } + + ngx_memcpy(cl->buf->last, ll->buf->pos, copy); + + ll->buf->pos += copy; + cl->buf->last += copy; + + cl->buf->flush = ll->buf->flush; + cl->buf->last_buf = ll->buf->last_buf; + cl->buf->last_in_chain = ll->buf->last_in_chain; + + dd("f: %d, l: %d, lic: %d", ll->buf->flush, + ll->buf->last_buf, ll->buf->last_in_chain); + + if (copy < n) { + /* more to copy, next buffer */ + ngx_wasm_assert(cl->buf->last == cl->buf->end); + rctx->resp_buf_last = cl; + cl = NULL; + } + + n -= copy; + dd("next n: %ld", n); + } + } + + ngx_wasm_chain_log_debug(r->connection->log, rctx->resp_bufs, + "response buffers: "); + + return NGX_OK; +} diff --git a/src/http/ngx_http_wasm_module.c b/src/http/ngx_http_wasm_module.c index 483c6528e..2fdbbf2ca 100644 --- a/src/http/ngx_http_wasm_module.c +++ b/src/http/ngx_http_wasm_module.c @@ -163,6 +163,13 @@ static ngx_command_t ngx_http_wasm_module_cmds[] = { offsetof(ngx_http_wasm_loc_conf_t, socket_large_buffers), NULL }, + { ngx_string("wasm_response_body_buffers"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE2, + ngx_conf_set_bufs_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_wasm_loc_conf_t, resp_body_buffers), + NULL }, + { ngx_string("proxy_wasm"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE12, ngx_http_wasm_proxy_wasm_directive, @@ -355,6 +362,11 @@ ngx_http_wasm_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) conf->socket_buffer_reuse = 1; #endif + ngx_conf_merge_bufs_value(conf->resp_body_buffers, + prev->resp_body_buffers, + NGX_WASM_DEFAULT_RESP_BODY_BUF_NUM, + NGX_WASM_DEFAULT_RESP_BODY_BUF_SIZE); + ngx_conf_merge_value(conf->pwm_req_headers_in_access, prev->pwm_req_headers_in_access, 0); diff --git a/src/http/proxy_wasm/ngx_http_proxy_wasm.c b/src/http/proxy_wasm/ngx_http_proxy_wasm.c index 92cea0bc4..743407fae 100644 --- a/src/http/proxy_wasm/ngx_http_proxy_wasm.c +++ b/src/http/proxy_wasm/ngx_http_proxy_wasm.c @@ -241,6 +241,18 @@ ngx_http_proxy_wasm_on_response_body(ngx_proxy_wasm_exec_t *pwexec, *out = rets->data[0].of.i32; + if (*out == NGX_PROXY_WASM_ACTION_PAUSE && rctx->resp_bufs) { + /** + * If 'on_response_body' has been called again, buffering is over. + * The buffers could be full, or eof has been reached. + */ + ngx_proxy_wasm_log_error(NGX_LOG_ERR, pwexec->log, 0, + "invalid \"on_response_body\" return " + "action (PAUSE): response body buffering " + "already requested"); + *out = NGX_PROXY_WASM_ACTION_CONTINUE; + } + return rc; } diff --git a/src/wasm/ngx_wasm.h b/src/wasm/ngx_wasm.h index e857d8e9c..1dbad995d 100644 --- a/src/wasm/ngx_wasm.h +++ b/src/wasm/ngx_wasm.h @@ -42,6 +42,8 @@ #define NGX_WASM_DEFAULT_SOCK_BUF_SIZE 1024 #define NGX_WASM_DEFAULT_SOCK_LARGE_BUF_NUM 4 #define NGX_WASM_DEFAULT_SOCK_LARGE_BUF_SIZE 8192 +#define NGX_WASM_DEFAULT_RESP_BODY_BUF_NUM 4 +#define NGX_WASM_DEFAULT_RESP_BODY_BUF_SIZE 4096 #define ngx_wasm_core_cycle_get_conf(cycle) \ (ngx_get_conf(cycle->conf_ctx, ngx_wasm_module)) \ @@ -109,6 +111,7 @@ typedef struct { ngx_wavm_t *ngx_wasm_main_vm(ngx_cycle_t *cycle); +void ngx_wasm_chain_log_debug(ngx_log_t *log, ngx_chain_t *in, char *label); size_t ngx_wasm_chain_len(ngx_chain_t *in, unsigned *eof); ngx_uint_t ngx_wasm_chain_clear(ngx_chain_t *in, size_t offset, unsigned *eof, unsigned *flush); diff --git a/src/wasm/ngx_wasm_ops.c b/src/wasm/ngx_wasm_ops.c index 4c4a03e7b..73f44f6bd 100644 --- a/src/wasm/ngx_wasm_ops.c +++ b/src/wasm/ngx_wasm_ops.c @@ -408,6 +408,7 @@ ngx_wasm_op_proxy_wasm_handler(ngx_wasm_op_ctx_t *opctx, } pwctx->phase = phase; + pwctx->action = NGX_PROXY_WASM_ACTION_CONTINUE; switch (phase->index) { diff --git a/src/wasm/ngx_wasm_util.c b/src/wasm/ngx_wasm_util.c index e0800165e..dcc7b74f4 100644 --- a/src/wasm/ngx_wasm_util.c +++ b/src/wasm/ngx_wasm_util.c @@ -6,15 +6,14 @@ #include -#if 0 -static void -ngx_wasm_chain_log_debug(ngx_log_t *log, ngx_chain_t *in, char *fmt) +void +ngx_wasm_chain_log_debug(ngx_log_t *log, ngx_chain_t *in, char *label) { -#if (NGX_DEBUG) +#if (DDEBUG) size_t len; - ngx_chain_t *cl; - ngx_buf_t *buf; ngx_str_t s; + ngx_buf_t *buf; + ngx_chain_t *cl; cl = in; @@ -25,17 +24,16 @@ ngx_wasm_chain_log_debug(ngx_log_t *log, ngx_chain_t *in, char *fmt) s.len = len; s.data = buf->pos; - ngx_log_debug7(NGX_LOG_DEBUG_WASM, log, 0, + ngx_log_debug8(NGX_LOG_DEBUG_WASM, log, 0, "%s: \"%V\" (buf: %p, len: %d, last_buf: %d," - " last_in_chain: %d, flush: %d)", - fmt, &s, buf, len, buf->last_buf, - buf->last_in_chain, buf->flush); + " last_in_chain: %d, flush: %d, cl: %p)", + label, &s, buf, len, buf->last_buf, + buf->last_in_chain, buf->flush, cl); cl = cl->next; } #endif } -#endif size_t diff --git a/t/03-proxy_wasm/004-on_http_phases.t b/t/03-proxy_wasm/004-on_http_phases.t index 03e155d9a..1fff94531 100644 --- a/t/03-proxy_wasm/004-on_http_phases.t +++ b/t/03-proxy_wasm/004-on_http_phases.t @@ -599,7 +599,42 @@ qr#filter 1/2 resuming "on_log" step in "log" phase -=== TEST 24: proxy_wasm - same module in multiple location{} blocks +=== TEST 24: proxy_wasm - as a parent of several subrequests +--- load_nginx_modules: ngx_http_echo_module +--- wasm_modules: on_phases +--- config + location /a { + internal; + echo a; + } + + location /b { + internal; + echo bb; + } + + location /t { + echo_subrequest GET '/a'; + echo_subrequest GET '/b'; + proxy_wasm on_phases; + } +--- response_body +a +bb +--- grep_error_log eval: qr/#\d+ on_(request|response|log).*/ +--- grep_error_log_out eval +qr/#\d+ on_request_headers, 2 headers.* +#\d+ on_response_headers, 5 headers.* +#\d+ on_response_body, 2 bytes, eof: false.* +#\d+ on_response_body, 3 bytes, eof: false.* +#\d+ on_response_body, 0 bytes, eof: true/ +--- no_error_log +[error] +[crit] + + + +=== TEST 25: proxy_wasm - same module in multiple location{} blocks --- load_nginx_modules: ngx_http_echo_module --- wasm_modules: on_phases --- config @@ -637,7 +672,7 @@ on_log -=== TEST 25: proxy_wasm - chained filters in same location{} block +=== TEST 26: proxy_wasm - chained filters in same location{} block should run each filter after the other within each phase --- skip_no_debug --- load_nginx_modules: ngx_http_echo_module @@ -673,7 +708,7 @@ qr/#\d+ on_request_headers, 3 headers.* -=== TEST 26: proxy_wasm - chained filters in server{} block +=== TEST 27: proxy_wasm - chained filters in server{} block should run each filter after the other within each phase --- load_nginx_modules: ngx_http_echo_module --- wasm_modules: on_phases @@ -704,7 +739,7 @@ qr/#\d+ on_request_headers, 2 headers.* -=== TEST 27: proxy_wasm - chained filters in http{} block +=== TEST 28: proxy_wasm - chained filters in http{} block should run each filter after the other within each phase --- load_nginx_modules: ngx_http_echo_module --- wasm_modules: on_phases @@ -735,7 +770,7 @@ qr/#\d+ on_request_headers, 2 headers.* -=== TEST 28: proxy_wasm - mixed filters in server{} and http{} blocks +=== TEST 29: proxy_wasm - mixed filters in server{} and http{} blocks should run root context of both filters should not chain in request; instead, server{} overrides http{} --- load_nginx_modules: ngx_http_echo_module @@ -765,7 +800,7 @@ qr/log_msg: server .*? request: "GET \/t\s+/ -=== TEST 29: proxy_wasm - mixed filters in server{} and location{} blocks (return in rewrite) +=== TEST 30: proxy_wasm - mixed filters in server{} and location{} blocks (return in rewrite) should not chain; instead, location{} overrides server{} --- wasm_modules: on_phases --- config @@ -790,7 +825,7 @@ qr/log_msg: location .*? request: "GET \/t\s+/ -=== TEST 30: proxy_wasm - mixed filters in http{}, server{}, and location{} blocks +=== TEST 31: proxy_wasm - mixed filters in http{}, server{}, and location{} blocks should not chain; instead, location{} overrides server{}, server{} overrides http{} --- wasm_modules: on_phases --- http_config diff --git a/t/03-proxy_wasm/006-on_http_next_action.t b/t/03-proxy_wasm/006-on_http_next_action.t index d2286b191..b45ac74f3 100644 --- a/t/03-proxy_wasm/006-on_http_next_action.t +++ b/t/03-proxy_wasm/006-on_http_next_action.t @@ -72,7 +72,8 @@ NYI === TEST 4: proxy_wasm - on_response_body -> Pause -NYI +Triggers response buffering. +--- skip_no_debug: 5 --- wasm_modules: on_phases --- config location /t { @@ -80,12 +81,11 @@ NYI return 200; } --- response_body ---- error_log eval -[ - qr/pausing after "ResponseBody"/, - qr#\[error\] .*? bad "on_response_body" return action: "PAUSE"#, - qr#\[info\] .*? filter chain failed resuming: previous error \(invalid return action\)# -] +--- error_log +buffering response after "ResponseBody" step +--- no_error_log +[error] +[crit] @@ -193,8 +193,7 @@ NYI === TEST 8: proxy_wasm - subrequest on_response_body -> Pause NYI ---- timeout_expected: 1 ---- abort +--- skip_no_debug: 5 --- load_nginx_modules: ngx_http_echo_module --- wasm_modules: on_phases --- config @@ -214,13 +213,11 @@ NYI echo_subrequest GET /pause; echo_subrequest GET /nop; } ---- error_code: 200 --- response_body ok ok ---- error_log eval -[ - qr/pausing after "ResponseBody"/, - qr#\[error\] .*? bad "on_response_body" return action: "PAUSE"#, - qr#\[info\] .*? filter chain failed resuming: previous error \(invalid return action\)# -] +--- error_log +buffering response after "ResponseBody" step +--- no_error_log +[error] +[crit] diff --git a/t/03-proxy_wasm/008-on_http_response_body_buffering.t b/t/03-proxy_wasm/008-on_http_response_body_buffering.t new file mode 100644 index 000000000..e26767ee8 --- /dev/null +++ b/t/03-proxy_wasm/008-on_http_response_body_buffering.t @@ -0,0 +1,520 @@ +# vim:set ft= ts=4 sts=4 sw=4 et fdm=marker: + +BEGIN { + $ENV{TEST_NGINX_EVENT_TYPE} = 'poll'; +} + +use strict; +use lib '.'; +use t::TestWasm; + +add_block_preprocessor(sub { + my $block = shift; + + if (!defined $block->no_error_log) { + $block->set_value("no_error_log", "[error]\n[crit]"); + } + + if (!defined $block->load_nginx_modules) { + $block->set_value("load_nginx_modules", + "ngx_http_echo_module " . + "ngx_http_headers_more_filter_module"); + } + + if (!defined $block->wasm_modules) { + $block->set_value("wasm_modules", "on_phases"); + } +}); + +plan_tests(5); +run_tests(); + +__DATA__ + +=== TEST 1: proxy_wasm - on_response_body, no buffering +--- config + location /t { + echo -n 'a\n'; + echo_flush; + echo -n 'bb\n'; + echo_flush; + echo -n 'ccc\n'; + echo_flush; + echo -n 'dddd\n'; + proxy_wasm on_phases; + } +--- response_body +a +bb +ccc +dddd +--- grep_error_log eval: qr/on_response_body, .*?eof: (true|false)/ +--- grep_error_log_out +on_response_body, 2 bytes, eof: false +on_response_body, 3 bytes, eof: false +on_response_body, 4 bytes, eof: false +on_response_body, 5 bytes, eof: false +on_response_body, 0 bytes, eof: true + + + +=== TEST 2: proxy_wasm - on_response_body buffering chunks, default buffers +--- config + location /t { + wasm_response_body_buffers 4 32; + echo -n 'a\n'; + echo_flush; + echo -n 'bb\n'; + echo_flush; + echo -n 'ccc\n'; + echo_flush; + echo -n 'dddd\n'; + proxy_wasm on_phases 'pause_on=response_body'; + } +--- response_body +a +bb +ccc +dddd +--- grep_error_log eval: qr/on_response_body, .*?eof: (true|false)/ +--- grep_error_log_out +on_response_body, 2 bytes, eof: false +on_response_body, 14 bytes, eof: true + + + +=== TEST 3: proxy_wasm - on_response_body buffering chunks, buffers too small for body +--- config + location /t { + wasm_response_body_buffers 3 2; + echo -n 'a\n'; + echo_flush; + echo -n 'bb\n'; + echo_flush; + echo -n 'ccc\n'; + echo_flush; + echo -n 'dddd\n'; + proxy_wasm on_phases 'pause_on=response_body'; + } +--- response_body +a +bb +ccc +dddd +--- grep_error_log eval: qr/on_response_body, .*?eof: (true|false)/ +--- grep_error_log_out +on_response_body, 2 bytes, eof: false +on_response_body, 9 bytes, eof: false +on_response_body, 5 bytes, eof: false +on_response_body, 0 bytes, eof: true + + + +=== TEST 4: proxy_wasm - on_response_body buffering chunks, buffers larger than body +--- config + location /t { + wasm_response_body_buffers 4 32; + echo -n 'a\n'; + echo_flush; + echo -n 'bb\n'; + echo_flush; + echo -n 'ccc\n'; + echo_flush; + echo -n 'dddd\n'; + proxy_wasm on_phases 'pause_on=response_body'; + } +--- response_body +a +bb +ccc +dddd +--- grep_error_log eval: qr/on_response_body, .*?eof: (true|false)/ +--- grep_error_log_out +on_response_body, 2 bytes, eof: false +on_response_body, 14 bytes, eof: true + + + +=== TEST 5: proxy_wasm - on_response_body buffering chunks, buffers same size as body +--- config + location /t { + wasm_response_body_buffers 1 14; + echo -n 'a\n'; + echo_flush; + echo -n 'bb\n'; + echo_flush; + echo -n 'ccc\n'; + echo_flush; + echo -n 'dddd\n'; + proxy_wasm on_phases 'pause_on=response_body'; + } +--- response_body +a +bb +ccc +dddd +--- grep_error_log eval: qr/on_response_body, .*?eof: (true|false)/ +--- grep_error_log_out +on_response_body, 2 bytes, eof: false +on_response_body, 14 bytes, eof: true + + + +=== TEST 6: proxy_wasm - on_response_body buffering chunks, buffer of size 1 for larger body +--- config + location /t { + wasm_response_body_buffers 1 1; + echo -n 'a\n'; + echo_flush; + echo -n 'bb\n'; + echo_flush; + echo -n 'ccc\n'; + echo_flush; + echo -n 'dddd\n'; + proxy_wasm on_phases 'pause_on=response_body'; + } +--- response_body +a +bb +ccc +dddd +--- grep_error_log eval: qr/on_response_body, .*?eof: (true|false)/ +--- grep_error_log_out +on_response_body, 2 bytes, eof: false +on_response_body, 2 bytes, eof: false +on_response_body, 3 bytes, eof: false +on_response_body, 4 bytes, eof: false +on_response_body, 5 bytes, eof: false +on_response_body, 0 bytes, eof: true + + + +=== TEST 7: proxy_wasm - on_response_body buffering chunks, buffers same size as body +--- config + location /t { + wasm_response_body_buffers 1 14; + # unlike 'echo -n', this format creates 2 chained buffers for each 'echo', + # the 2nd one being the newline buffer. + echo 'a'; + echo_flush; + echo 'bb'; + echo_flush; + echo 'ccc'; + echo_flush; + echo 'dddd'; + proxy_wasm on_phases 'pause_on=response_body'; + } +--- response_body +a +bb +ccc +dddd +--- grep_error_log eval: qr/on_response_body, .*?eof: (true|false)/ +--- grep_error_log_out +on_response_body, 2 bytes, eof: false +on_response_body, 14 bytes, eof: true + + + +=== TEST 8: proxy_wasm - on_response_body buffering chunks, buffer of size 1 for larger body +--- config + location /t { + wasm_response_body_buffers 1 1; + # unlike 'echo -n', this format creates 2 chained buffers for each 'echo', + # the 2nd one being the newline buffer. + echo 'a'; + echo_flush; + echo 'bb'; + echo_flush; + echo 'ccc'; + echo_flush; + echo 'dddd'; + proxy_wasm on_phases 'pause_on=response_body'; + } +--- response_body +a +bb +ccc +dddd +--- grep_error_log eval: qr/on_response_body, .*?eof: (true|false)/ +--- grep_error_log_out +on_response_body, 2 bytes, eof: false +on_response_body, 2 bytes, eof: false +on_response_body, 3 bytes, eof: false +on_response_body, 4 bytes, eof: false +on_response_body, 5 bytes, eof: false +on_response_body, 0 bytes, eof: true + + + +=== TEST 9: proxy_wasm - on_response_body buffering chunks, buffers smaller than subrequests bodies +--- config + location /a { + internal; + echo a; + } + + location /b { + internal; + echo bb; + } + + location /c { + internal; + echo ccc; + } + + location /d { + internal; + echo dddd; + } + + location /t { + wasm_response_body_buffers 1 1; + echo_subrequest GET '/a'; + echo_subrequest GET '/b'; + echo_subrequest GET '/c'; + echo_subrequest GET '/d'; + proxy_wasm on_phases 'pause_on=response_body'; + } +--- response_body +a +bb +ccc +dddd +--- grep_error_log eval: qr/on_response_body, .*?eof: (true|false)/ +--- grep_error_log_out +on_response_body, 2 bytes, eof: false +on_response_body, 2 bytes, eof: false +on_response_body, 3 bytes, eof: false +on_response_body, 4 bytes, eof: false +on_response_body, 5 bytes, eof: false +on_response_body, 0 bytes, eof: true + + + +=== TEST 10: proxy_wasm - on_response_body buffering with proxy_pass (buffers too small for body) +Clear Server header or else different build modes produce different +bufferring results (no-pool/openresty) +--- http_config eval +qq{ + upstream test_upstream { + server unix:$ENV{TEST_NGINX_UNIX_SOCKET}; + } + + server { + listen unix:$ENV{TEST_NGINX_UNIX_SOCKET}; + + location / { + more_clear_headers 'Server'; + echo_duplicate 128 'a'; + echo_flush; + echo_duplicate 128 'b'; + } + } +} +--- config + location /t { + wasm_response_body_buffers 1 64; + + proxy_buffer_size 256; + proxy_buffers 3 128; + proxy_busy_buffers_size 256; + + proxy_pass http://test_upstream/; + proxy_wasm on_phases 'pause_on=response_body'; + } +--- response_body_like: a{128}b{128} +--- grep_error_log eval: qr/on_response_body, .*?eof: (true|false)/ +--- grep_error_log_out +on_response_body, 155 bytes, eof: false +on_response_body, 155 bytes, eof: false +on_response_body, 101 bytes, eof: false +on_response_body, 0 bytes, eof: true + + + +=== TEST 11: proxy_wasm - on_response_body buffering with proxy_pass (buffers larger than body) +Clear Server header or else different build modes produce different +bufferring results (no-pool/openresty) +--- http_config eval +qq{ + upstream test_upstream { + server unix:$ENV{TEST_NGINX_UNIX_SOCKET}; + } + + server { + listen unix:$ENV{TEST_NGINX_UNIX_SOCKET}; + + location / { + more_clear_headers 'Server'; + echo_duplicate 128 "a"; + echo_flush; + echo_duplicate 128 "b"; + echo_flush; + echo_duplicate 128 "c"; + echo_flush; + echo_duplicate 128 "d"; + } + } +} +--- config + location /t { + wasm_response_body_buffers 6 256; + + proxy_buffer_size 256; + proxy_buffers 3 128; + proxy_busy_buffers_size 256; + + proxy_pass http://test_upstream/; + proxy_wasm on_phases 'pause_on=response_body'; + } +--- response_body_like: a{128}b{128}c{128}d{128} +--- grep_error_log eval: qr/on_response_body, .*?eof: (true|false)/ +--- grep_error_log_out +on_response_body, 155 bytes, eof: false +on_response_body, 512 bytes, eof: true + + + +=== TEST 12: proxy_wasm - on_response_body buffering with proxy_pass (buffers same size as body) +Clear Server header or else different build modes produce different +bufferring results (no-pool/openresty) +--- http_config eval +qq{ + upstream test_upstream { + server unix:$ENV{TEST_NGINX_UNIX_SOCKET}; + } + + server { + listen unix:$ENV{TEST_NGINX_UNIX_SOCKET}; + + location / { + more_clear_headers 'Server'; + echo_duplicate 128 "a"; + echo_flush; + echo_duplicate 128 "b"; + echo_flush; + echo_duplicate 128 "c"; + echo_flush; + echo_duplicate 128 "d"; + } + } +} +--- config + location /t { + wasm_response_body_buffers 4 128; + + proxy_buffer_size 256; + proxy_buffers 3 128; + proxy_busy_buffers_size 256; + + proxy_pass http://test_upstream/; + proxy_wasm on_phases 'pause_on=response_body'; + } +--- response_body_like: a{128}b{128}c{128}d{128} +--- grep_error_log eval: qr/on_response_body, .*?eof: (true|false)/ +--- grep_error_log_out +on_response_body, 155 bytes, eof: false +on_response_body, 512 bytes, eof: true + + + +=== TEST 13: proxy_wasm - on_response_body buffering, get_http_response_body() when buffers larger than body +--- valgrind +--- config + location /t { + wasm_response_body_buffers 4 32; + echo -n 'a\n'; + echo_flush; + echo -n 'bb\n'; + echo_flush; + echo -n 'ccc\n'; + echo_flush; + echo -n 'dddd\n'; + proxy_wasm on_phases 'pause_on=response_body'; + proxy_wasm on_phases 'log_response_body=true'; + } +--- response_body +a +bb +ccc +dddd +--- grep_error_log eval: qr/on_response_body, .*?eof: (true|false)/ +--- grep_error_log_out +on_response_body, 2 bytes, eof: false +on_response_body, 14 bytes, eof: true +on_response_body, 14 bytes, eof: true +--- error_log +response body chunk: "a\nbb\nccc\ndddd\n" +--- no_error_log +[error] + + + +=== TEST 14: proxy_wasm - on_response_body buffering, get_http_response_body() when buffers too small for body +--- valgrind +--- config + location /t { + wasm_response_body_buffers 3 2; + echo -n 'a\n'; + echo_flush; + echo -n 'bb\n'; + echo_flush; + echo -n 'ccc\n'; + echo_flush; + echo -n 'dddd\n'; + proxy_wasm on_phases 'pause_on=response_body'; + proxy_wasm on_phases 'log_response_body=true'; + } +--- response_body +a +bb +ccc +dddd +--- grep_error_log eval: qr/on_response_body, .*?eof: (true|false)/ +--- grep_error_log_out +on_response_body, 2 bytes, eof: false +on_response_body, 9 bytes, eof: false +on_response_body, 9 bytes, eof: false +on_response_body, 5 bytes, eof: false +on_response_body, 5 bytes, eof: false +on_response_body, 0 bytes, eof: true +on_response_body, 0 bytes, eof: true +--- error_log +response body chunk: "a\nbb\nccc\n" +response body chunk: "dddd\n" +--- no_error_log + + + +=== TEST 15: proxy_wasm - on_response_body buffering, then ask for buffering again +--- config + location /t { + wasm_response_body_buffers 3 32; + echo -n 'a\n'; + echo_flush; + echo -n 'bb\n'; + echo_flush; + echo -n 'ccc\n'; + echo_flush; + echo -n 'dddd\n'; + proxy_wasm on_phases 'pause_on=response_body pause_times=2'; + proxy_wasm on_phases 'log_response_body=true'; + } +--- response_body +a +bb +ccc +dddd +--- grep_error_log eval: qr/on_response_body, .*?eof: (true|false)/ +--- grep_error_log_out +on_response_body, 2 bytes, eof: false +on_response_body, 14 bytes, eof: true +on_response_body, 14 bytes, eof: true +--- error_log eval +[ + "response body chunk: \"a\\nbb\\nccc\\ndddd\\n\"", + qr/\[error\] .*? invalid "on_response_body" return action \(PAUSE\): response body buffering already requested/ +] +--- no_error_log diff --git a/t/03-proxy_wasm/008-proxy_wasm_oob.t b/t/03-proxy_wasm/009-proxy_wasm_oob.t similarity index 100% rename from t/03-proxy_wasm/008-proxy_wasm_oob.t rename to t/03-proxy_wasm/009-proxy_wasm_oob.t diff --git a/t/lib/proxy-wasm-tests/on-phases/src/filter.rs b/t/lib/proxy-wasm-tests/on-phases/src/filter.rs index 496a962db..10151abe6 100644 --- a/t/lib/proxy-wasm-tests/on-phases/src/filter.rs +++ b/t/lib/proxy-wasm-tests/on-phases/src/filter.rs @@ -88,6 +88,11 @@ impl RootContext for HttpHeadersRoot { .config .get("pause_on") .map(|s| s.parse().expect("bad pause_on")), + pause_times: self + .config + .get("pause_times") + .map_or(1, |v| v.parse().expect("bad pause_on value")), + pause_n: 0, })) } } @@ -96,21 +101,34 @@ struct OnPhases { context_id: u32, config: HashMap, pause_on: Option, + pause_times: usize, + pause_n: usize, } impl OnPhases { fn next_action(&mut self, phase: Phase) -> Action { if let Some(pause_on) = &self.pause_on { - if pause_on == &phase { + if pause_on == &phase && self.pause_n < self.pause_times { info!( "#{} pausing after \"{:?}\" phase", self.context_id, pause_on ); + self.pause_n += 1; + return Action::Pause; } } + if phase == Phase::ResponseBody && self.config.get("log_response_body").is_some() { + if let Some(bytes) = self.get_http_response_body(0, usize::MAX) { + match String::from_utf8(bytes) { + Ok(s) => info!("response body chunk: {:?}", s), + Err(e) => panic!("Invalid UTF-8 sequence: {}", e), + } + } + } + Action::Continue } } From 2f0486cf5f4133ec4c4301061bbf081da3f996f0 Mon Sep 17 00:00:00 2001 From: Thibault Charbonnier Date: Thu, 10 Aug 2023 17:01:12 -0700 Subject: [PATCH 3/3] docs(proxy-wasm) document response body buffering --- docs/DIRECTIVES.md | 16 +++++++++++++ docs/PROXY_WASM.md | 59 ++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/docs/DIRECTIVES.md b/docs/DIRECTIVES.md index 6dd2c7cbf..d8bd99ad2 100644 --- a/docs/DIRECTIVES.md +++ b/docs/DIRECTIVES.md @@ -26,6 +26,7 @@ By alphabetical order: - [tls_verify_cert](#tls_verify_cert) - [tls_verify_host](#tls_verify_host) - [wasm_call](#wasm_call) +- [wasm_response_body_buffers](#wasm_response_body_buffers) - [wasm_socket_buffer_reuse](#wasm_socket_buffer_reuse) - [wasm_socket_buffer_size](#wasm_socket_buffer_size) - [wasm_socket_connect_timeout](#wasm_socket_connect_timeout) @@ -66,6 +67,7 @@ By context: - [proxy_wasm_request_headers_in_access](#proxy_wasm_request_headers_in_access) - [resolver_add](#resolver_add) - [wasm_call](#wasm_call) + - [wasm_response_body_buffers](#wasm_response_body_buffers) - [wasm_socket_buffer_reuse](#wasm_socket_buffer_reuse) - [wasm_socket_buffer_size](#wasm_socket_buffer_size) - [wasm_socket_connect_timeout](#wasm_socket_connect_timeout) @@ -758,6 +760,20 @@ return `HTTP 500`. [Back to TOC](#directives) +wasm_response_body_buffers +-------------------------- + +**usage** | `wasm_response_body_buffers ;` +------------:|:---------------------------------------------------------------- +**contexts** | `http{}`, `server{}`, `location{}` +**default** | `4 4096` +**example** | `wasm_response_body_buffers 2 16k;` + +Set the maximum `number` and `size` of buffers used for [response body +buffering](PROXY_WASM.md#response-body-buffering). + +[Back to TOC](#directives) + wasm_socket_buffer_reuse ------------------------ diff --git a/docs/PROXY_WASM.md b/docs/PROXY_WASM.md index 8a1a68b5a..bdf808b18 100644 --- a/docs/PROXY_WASM.md +++ b/docs/PROXY_WASM.md @@ -15,6 +15,7 @@ - [Supported Entrypoints](#supported-entrypoints) - [Supported Host ABI](#supported-host-abi) - [Supported Properties](#supported-properties) + - [Response Body Buffering](#response-body-buffering) - [Examples] - [Current Limitations] @@ -383,6 +384,8 @@ specifications and different SDK libraries: - [Tested SDKs](#tested-sdks) - [Supported Entrypoints](#supported-entrypoints) - [Supported Host ABI](#supported-host-abi) +- [Supported Properties](#supported-properties) +- [Response Body Buffering](#response-body-buffering) [Back to TOC](#table-of-contents) @@ -578,7 +581,7 @@ implementation state in ngx_wasm_module: -------------------------------------------:|:------------------:|:-------------------:|:---------------- *Request properties* | | `request.path` | :heavy_check_mark: | :x: | Maps to [ngx.request_uri](https://nginx.org/en/docs/http/ngx_http_core_module.html#var_request_uri). -`request.url_path` | :heavy_check_mark: | :x: | Maps to [ngx.uri](https://nginx.org/en/docs/http/ngx_http_core_module.html#uri). +`request.url_path` | :heavy_check_mark: | :x: | Maps to [ngx.uri](https://nginx.org/en/docs/http/ngx_http_core_module.html#uri). `request.host` | :heavy_check_mark: | :x: | Maps to [ngx.hostname](https://nginx.org/en/docs/http/ngx_http_core_module.html#hostname). `request.scheme` | :heavy_check_mark: | :x: | Maps to [ngx.scheme](https://nginx.org/en/docs/http/ngx_http_core_module.html#scheme). `request.method` | :heavy_check_mark: | :x: | Maps to [ngx.request_method](https://nginx.org/en/docs/http/ngx_http_core_module.html#request_method). @@ -651,6 +654,55 @@ ngx_wasm_module, most likely due to a Host incompatibility. [Back to TOC](#table-of-contents) +### Response Body Buffering + +Buffering of response body chunks is supported within ngx_wasm_module so filters +don't have to implement buffering themselves. This allows the `on_response_body` +step to be invoked with the full response body available for read via +`get_http_response_body`. + +When response buffering is enabled, response chunks will be copied to buffers +defined by the [wasm_response_body_buffers] directive while execution of the +Proxy-Wasm filter chain is temporarily suspended until buffering is complete, at +which point `on_response_body` will be invoked again. + +To enable this behavior from a filter based on Proxy-Wasm ABI v0.2.1, the filter +must return `Action::Pause` from `on_response_body`. Once enabled, +ngx_wasm_module will accumulate subsequent body chunks until either `eof` is +reached, or the buffers are full. When either of these conditions are met, +`on_response_body` will be invoked again and the body buffer will contain the +buffered chunks. + +In other words, once body buffering is enabled, the next `on_response_body` +invocation will contain the buffered body **and may be invoked again** +if `eof` was not reached but the buffers are full. + +A typical response buffering flow could be: + +1. 1st `on_response_body` call: *ignore 1st chunk, requesting buffering.* + 1. Check for `eof=false`. + 2. Ensure buffering was not already requested. + 3. Return `Action::Pause`, requesting buffering. +2. 2nd `on_response_body` call: *buffering ended, but how?* + 1. If `eof=true`, the full response body is in the buffers. + 2. If `eof=false`, the buffers are full, but more chunks are expected + (users should treat the buffers as if it were a single, non-buffered + chunk). +3. nth `on_response_body` call: *next chunks, if any.* + +Returning `Action::Pause` when buffering has already taken place will be ignored +(i.e. treated as `Action::Continue`) and an error log will be printed. + +> Notes + +Keep in mind there are fundamental issues with buffering bodies at scale due to +the nature of the workload, hard buffer limits defined by +[wasm_response_body_buffers], and Wasm memory limits themselves (loading and +manipulating the body in filters). This feature should be used with extreme +caution in production environments. + +[Back to TOC](#table-of-contents) + ## Examples - Functional filters written by the WasmX team: @@ -672,12 +724,13 @@ factors are at play when porting the SDK to a new Host proxy runtime. Proxy-Wasm's design was primarily influenced by Envoy concepts and features, but because Envoy and Nginx differ in underlying implementations there remains a few -limitations on some supported features: +limitations on some supported features (non-exhaustive list): 1. Pausing a filter (i.e. `Action::Pause`) can only be done in the following steps: - `on_http_request_headers` - `on_http_request_body` + - `on_http_response_body` (to enable body buffering) - `on_http_call_response` 2. The "queue" shared memory implementation does not implement an automatic @@ -698,6 +751,8 @@ limitations and increasing overall surface support for the Proxy-Wasm SDK. [Examples]: #examples [Current Limitations]: #current-limitations +[wasm_response_body_buffers]: DIRECTIVES.md#wasm_response_body_buffers + [WebAssembly]: https://webassembly.org/ [Nginx Variables]: https://nginx.org/en/docs/varindex.html [Envoy Attributes]: https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/advanced/attributes.html?highlight=properties#request-attributes