Skip to content

Commit

Permalink
feat(proxy-wasm) invoke 'on_http_call_response' on dispatch failures
Browse files Browse the repository at this point in the history 
Invoke `on_http_call_response` on dispatch connection failures. This
allows catching failures such as:

- timeout
- broken connection
- resolver failures
- TLS handshake failures
- Any other failures *after the connection has attempted to be
  established*

Dispatch failures occurring *before* a connection has attempted to be
established will not trigger `on_http_call_response` as they are already
supposed to trigger a Wasm exception (i.e. trap).

When a dispatch connection failure occurs, `on_http_call_response` is
invoked with: `on_http_call_response(call_id, 0, 0, 0)` (i.e. no header,
no body, no trailers). Calls to retrieve the response `:status` will
return `None`. A user may triage the connection failure by querying the
`:dispatch_status` pseudo-header:

```rust
fn on_http_call_response(
    &mut self,
    token_id: u32,
    nheaders: usize,
    body_size: usize,
    ntrailers: usize,
) {
    let dispatch_status = self.get_http_call_response_header(":dispatch_status");

    match dispatch_status.as_deref() {
        Some("timeout") => {},
        Some("broken connection") => {},
        Some("tls handshake failure") => {},
        Some("resolver failure") => {},
        Some("reader failure") => {},
        Some(s) => info!("dispatch_status: {}", s),
        None => {}
    }

    self.resume_http_request()
}
```

The socket status `"bad argument"` also exists but since the connection
has not been attempted yet, it won't show up during
`on_http_call_response` and is for internal use only (i.e. could be
added to the socket error log for example).

Fix #622.
  • Loading branch information
@thibaultcha
thibaultcha committed Nov 19, 2024
1 parent 9238d4a commit d7f6736
Show file tree
Hide file tree
Showing 12 changed files with 412 additions and 125 deletions.
98 changes: 74 additions & 24 deletions src/common/ngx_wasm_socket_tcp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@


static void ngx_wasm_socket_tcp_err(ngx_wasm_socket_tcp_t *sock,
const char *fmt, ...);
ngx_wasm_socket_status_e status, const char *fmt, ...);
static void ngx_wasm_socket_resolve_handler(ngx_resolver_ctx_t *ctx);
static ngx_int_t ngx_wasm_socket_tcp_connect_peer(ngx_wasm_socket_tcp_t *sock);
static ngx_int_t ngx_wasm_socket_tcp_get_peer(ngx_peer_connection_t *pc,
Expand All @@ -36,14 +36,43 @@ static ngx_int_t ngx_wasm_socket_tcp_ssl_set_server_name(ngx_connection_t *c,
#endif


static ngx_str_t ngx_wasm_socket_errlist[] = {
ngx_string("ok"),
ngx_string("bad argument"),
ngx_string("timeout"),
ngx_string("broken connection"),
#if (NGX_SSL)
ngx_string("tls handshake failure"),
#endif
ngx_string("resolver failure"),
ngx_string("reader failure"),
ngx_string("internal failure")
};


ngx_str_t *
ngx_wasm_socket_tcp_status_strerror(ngx_wasm_socket_status_e status)
{
ngx_str_t *msg;

msg = ((ngx_uint_t) status < NGX_WASM_SOCKET_STATUS_INTERNAL_FAILURE)
? &ngx_wasm_socket_errlist[status]
: &ngx_wasm_socket_errlist[NGX_WASM_SOCKET_STATUS_INTERNAL_FAILURE];

return msg;
}


static void
ngx_wasm_socket_tcp_err(ngx_wasm_socket_tcp_t *sock,
ngx_wasm_socket_status_e status,
const char *fmt, ...)
{
va_list args;
u_char *p, *last;

if (sock->err == NULL) {
sock->status = status;
sock->err = ngx_pnalloc(sock->pool, NGX_MAX_ERROR_STR);
if (sock->err == NULL) {
return;
Expand Down Expand Up @@ -193,7 +222,8 @@ ngx_wasm_socket_tcp_init(ngx_wasm_socket_tcp_t *sock,
sock->url.no_resolve = 1;

if (ngx_parse_url(sock->pool, &sock->url) != NGX_OK) {
ngx_wasm_socket_tcp_err(sock, "%s", sock->url.err);
ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_BAD_ARGUMENT,
"%s", sock->url.err);
return NGX_ERROR;
}

Expand Down Expand Up @@ -385,7 +415,8 @@ ngx_wasm_socket_tcp_connect(ngx_wasm_socket_tcp_t *sock)
}

if (rslv_ctx == NULL) {
ngx_wasm_socket_tcp_err(sock, "failed starting resolver");
ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_INTERNAL_FAILURE,
"failed starting resolver");
return NGX_ERROR;
}

Expand Down Expand Up @@ -429,7 +460,9 @@ ngx_wasm_socket_resolve_handler(ngx_resolver_ctx_t *ctx)
if (ctx->state || !ctx->naddrs) {
#if (NGX_WASM_LUA)
if (ctx->state == NGX_WASM_LUA_RESOLVE_ERR) {
ngx_wasm_socket_tcp_err(sock, "lua resolver failed");
ngx_wasm_socket_tcp_err(sock,
NGX_WASM_SOCKET_STATUS_RESOLVER_FAILURE,
"lua resolver failed");
goto error;
}
#endif
Expand All @@ -440,7 +473,8 @@ ngx_wasm_socket_resolve_handler(ngx_resolver_ctx_t *ctx)
}
#endif

ngx_wasm_socket_tcp_err(sock, "resolver error: %s",
ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_RESOLVER_FAILURE,
"resolver error: %s",
ngx_resolver_strerror(ctx->state));
goto error;
}
Expand Down Expand Up @@ -571,12 +605,14 @@ ngx_wasm_socket_tcp_connect_peer(ngx_wasm_socket_tcp_t *sock)
return NGX_ERROR;

} else if (rc == NGX_BUSY) {
ngx_wasm_socket_tcp_err(sock, "no live connection");
ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_INTERNAL_FAILURE,
"no live connection");
return NGX_BUSY;

} else if (rc == NGX_DECLINED) {
sock->socket_errno = ngx_socket_errno;
ngx_wasm_socket_tcp_err(sock, NULL);
ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_BROKEN_CONNECTION,
NULL);
return NGX_ERROR;
}

Expand Down Expand Up @@ -686,11 +722,13 @@ ngx_wasm_socket_tcp_ssl_handshake_handler(ngx_connection_t *c)
}

if (c->write->timedout) {
ngx_wasm_socket_tcp_err(sock, "tls handshake timed out");
ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_TLS_FAILURE,
"tls handshake timed out");
goto resume;
}

ngx_wasm_socket_tcp_err(sock, "tls handshake failed");
ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_TLS_FAILURE,
"tls handshake failed");

resume:

Expand Down Expand Up @@ -722,6 +760,7 @@ ngx_wasm_socket_tcp_ssl_handshake_done(ngx_connection_t *c)
rc = SSL_get_verify_result(c->ssl->connection);
if (rc != X509_V_OK) {
ngx_wasm_socket_tcp_err(sock,
NGX_WASM_SOCKET_STATUS_TLS_FAILURE,
"tls certificate verify error: (%l:%s)",
rc, X509_verify_cert_error_string(rc));
return NGX_ERROR;
Expand All @@ -740,6 +779,7 @@ ngx_wasm_socket_tcp_ssl_handshake_done(ngx_connection_t *c)

if (ngx_ssl_check_host(c, &sock->ssl_server_name) != NGX_OK) {
ngx_wasm_socket_tcp_err(sock,
NGX_WASM_SOCKET_STATUS_TLS_FAILURE,
"tls certificate CN "
"does not match \"%V\" sni",
&sock->ssl_server_name);
Expand Down Expand Up @@ -809,6 +849,7 @@ ngx_wasm_socket_tcp_ssl_set_server_name(ngx_connection_t *c,
|| ngx_inet_addr(name->data, len) != INADDR_NONE)
{
ngx_wasm_socket_tcp_err(sock,
NGX_WASM_SOCKET_STATUS_TLS_FAILURE,
"could not derive tls sni from host (\"%V\")",
&sock->host);
goto error;
Expand Down Expand Up @@ -877,7 +918,8 @@ ngx_wasm_socket_tcp_send(ngx_wasm_socket_tcp_t *sock, ngx_chain_t *cl)
ngx_connection_t *c;

if (!sock->connected) {
ngx_wasm_socket_tcp_err(sock, "not connected");
ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_INTERNAL_FAILURE,
"not connected");
return NGX_ERROR;
}

Expand Down Expand Up @@ -930,7 +972,8 @@ ngx_wasm_socket_tcp_send(ngx_wasm_socket_tcp_t *sock, ngx_chain_t *cl)
if (n == NGX_ERROR) {
c->error = 1;
sock->socket_errno = ngx_socket_errno;
ngx_wasm_socket_tcp_err(sock, NULL);
ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_BROKEN_CONNECTION,
NULL);
return NGX_ERROR;
}

Expand Down Expand Up @@ -1054,7 +1097,8 @@ ngx_wasm_socket_tcp_read(ngx_wasm_socket_tcp_t *sock,
ngx_connection_t *c;

if (!sock->connected) {
ngx_wasm_socket_tcp_err(sock, "not connected");
ngx_wasm_socket_tcp_err(sock, NGX_WASM_SOCKET_STATUS_INTERNAL_FAILURE,
"not connected");
return NGX_ERROR;
}

Expand Down Expand Up @@ -1091,7 +1135,9 @@ ngx_wasm_socket_tcp_read(ngx_wasm_socket_tcp_t *sock,

rc = reader(sock, size, reader_ctx);
if (rc == NGX_ERROR) {
ngx_wasm_socket_tcp_err(sock, "parser error");
ngx_wasm_socket_tcp_err(sock,
NGX_WASM_SOCKET_STATUS_READER_FAILURE,
"parser error");
return NGX_ERROR;
}

Expand Down Expand Up @@ -1159,7 +1205,9 @@ ngx_wasm_socket_tcp_read(ngx_wasm_socket_tcp_t *sock,

if (n == NGX_ERROR) {
sock->socket_errno = ngx_socket_errno;
ngx_wasm_socket_tcp_err(sock, NULL);
ngx_wasm_socket_tcp_err(sock,
NGX_WASM_SOCKET_STATUS_BROKEN_CONNECTION,
NULL);
return NGX_ERROR;
}

Expand Down Expand Up @@ -1505,11 +1553,11 @@ ngx_wasm_socket_tcp_connect_handler(ngx_wasm_socket_tcp_t *sock)
"wasm tcp socket timed out connecting to \"%V:%ud\"",
&c->addr_text, ngx_inet_get_port(sock->peer.sockaddr));

ngx_wasm_socket_tcp_err(sock, "timed out connecting to \"%V:%ud\"",
ngx_wasm_socket_tcp_err(sock,
NGX_WASM_SOCKET_STATUS_TIMEOUT,
"timed out connecting to \"%V:%ud\"",
&c->addr_text,
ngx_inet_get_port(sock->peer.sockaddr));

sock->timedout = 1;
return;
}

Expand All @@ -1520,7 +1568,9 @@ ngx_wasm_socket_tcp_connect_handler(ngx_wasm_socket_tcp_t *sock)
if (rc != NGX_OK) {
if (rc > 0) {
sock->socket_errno = (ngx_err_t) rc;
ngx_wasm_socket_tcp_err(sock, NULL);
ngx_wasm_socket_tcp_err(sock,
NGX_WASM_SOCKET_STATUS_BROKEN_CONNECTION,
NULL);
}

return;
Expand Down Expand Up @@ -1560,11 +1610,11 @@ ngx_wasm_socket_tcp_send_handler(ngx_wasm_socket_tcp_t *sock)
"wasm tcp socket timed out writing to \"%V:%ud\"",
&c->addr_text, ngx_inet_get_port(sock->peer.sockaddr));

ngx_wasm_socket_tcp_err(sock, "timed out writing to \"%V:%ud\"",
ngx_wasm_socket_tcp_err(sock,
NGX_WASM_SOCKET_STATUS_TIMEOUT,
"timed out writing to \"%V:%ud\"",
&c->addr_text,
ngx_inet_get_port(sock->peer.sockaddr));

sock->timedout = 1;
return;
}
}
Expand All @@ -1584,11 +1634,11 @@ ngx_wasm_socket_tcp_receive_handler(ngx_wasm_socket_tcp_t *sock)
"wasm tcp socket timed out reading from \"%V:%ud\"",
&c->addr_text, ngx_inet_get_port(sock->peer.sockaddr));

ngx_wasm_socket_tcp_err(sock, "timed out reading from \"%V:%ud\"",
ngx_wasm_socket_tcp_err(sock,
NGX_WASM_SOCKET_STATUS_TIMEOUT,
"timed out reading from \"%V:%ud\"",
&c->addr_text,
ngx_inet_get_port(sock->peer.sockaddr));

sock->timedout = 1;
return;
}

Expand Down
19 changes: 17 additions & 2 deletions src/common/ngx_wasm_socket_tcp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,20 @@ typedef ngx_int_t (*ngx_wasm_socket_tcp_dns_resolver_pt)(
ngx_resolver_ctx_t *rslv_ctx);


typedef enum {
NGX_WASM_SOCKET_STATUS_OK = 0,
NGX_WASM_SOCKET_STATUS_BAD_ARGUMENT,
NGX_WASM_SOCKET_STATUS_TIMEOUT,
NGX_WASM_SOCKET_STATUS_BROKEN_CONNECTION,
#if (NGX_SSL)
NGX_WASM_SOCKET_STATUS_TLS_FAILURE,
#endif
NGX_WASM_SOCKET_STATUS_RESOLVER_FAILURE,
NGX_WASM_SOCKET_STATUS_READER_FAILURE,
NGX_WASM_SOCKET_STATUS_INTERNAL_FAILURE,
} ngx_wasm_socket_status_e;


typedef struct {
ngx_str_t host;
in_port_t port;
Expand All @@ -39,6 +53,7 @@ struct ngx_wasm_socket_tcp_s {
ngx_wasm_subsys_env_t *env;

ngx_wasm_socket_tcp_resume_handler_pt resume_handler;
ngx_wasm_socket_status_e status;
void *data;
#if (NGX_WASM_LUA)
ngx_wasm_lua_ctx_t *lctx;
Expand Down Expand Up @@ -85,19 +100,19 @@ struct ngx_wasm_socket_tcp_s {

/* flags */

unsigned timedout:1;
unsigned connected:1;
unsigned eof:1;
unsigned closed:1;
unsigned read_closed:1;
unsigned write_closed:1;

#if (NGX_SSL)
unsigned ssl_ready:1;
#endif
};


ngx_str_t *ngx_wasm_socket_tcp_status_strerror(ngx_wasm_socket_status_e status);

ngx_int_t ngx_wasm_socket_tcp_init(ngx_wasm_socket_tcp_t *sock,
ngx_str_t *host, unsigned tls, ngx_str_t *sni, ngx_wasm_subsys_env_t *env);
ngx_int_t ngx_wasm_socket_tcp_connect(ngx_wasm_socket_tcp_t *sock);
Expand Down
38 changes: 34 additions & 4 deletions src/common/proxy_wasm/ngx_proxy_wasm_maps.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ static ngx_str_t *ngx_proxy_wasm_maps_get_response_status(
static ngx_int_t ngx_proxy_wasm_maps_set_response_status(
ngx_wavm_instance_t *instance, ngx_str_t *value,
ngx_proxy_wasm_map_type_e map_type);
static ngx_str_t *ngx_proxy_wasm_maps_get_dispatch_response_status(
ngx_wavm_instance_t *instance, ngx_proxy_wasm_map_type_e map_type);
static ngx_str_t *ngx_proxy_wasm_maps_get_dispatch_status(
ngx_wavm_instance_t *instance, ngx_proxy_wasm_map_type_e map_type);
#endif
Expand Down Expand Up @@ -75,6 +77,11 @@ static ngx_proxy_wasm_maps_key_t ngx_proxy_wasm_maps_special_keys[] = {
/* dispatch response */

{ ngx_string(":status"),
NGX_PROXY_WASM_MAP_HTTP_CALL_RESPONSE_HEADERS,
ngx_proxy_wasm_maps_get_dispatch_response_status,
NULL },

{ ngx_string(":dispatch_status"),
NGX_PROXY_WASM_MAP_HTTP_CALL_RESPONSE_HEADERS,
ngx_proxy_wasm_maps_get_dispatch_status,
NULL },
Expand Down Expand Up @@ -118,8 +125,14 @@ ngx_proxy_wasm_maps_get_map(ngx_wavm_instance_t *instance,
}

reader = &call->http_reader;
r = &reader->fake_r;

if (!r->signature) {
/* reader not initialized */
return NULL;
}

return &reader->fake_r.upstream->headers_in.headers;
return &r->upstream->headers_in.headers;
#endif

default:
Expand Down Expand Up @@ -753,7 +766,7 @@ ngx_proxy_wasm_maps_set_response_status(ngx_wavm_instance_t *instance,


static ngx_str_t *
ngx_proxy_wasm_maps_get_dispatch_status(ngx_wavm_instance_t *instance,
ngx_proxy_wasm_maps_get_dispatch_response_status(ngx_wavm_instance_t *instance,
ngx_proxy_wasm_map_type_e map_type)
{
ngx_uint_t status;
Expand All @@ -771,9 +784,8 @@ ngx_proxy_wasm_maps_get_dispatch_status(ngx_wavm_instance_t *instance,

/* status */

ngx_wa_assert(reader->fake_r.upstream);

if (reader->fake_r.upstream == NULL) {
/* response not received */
return NULL;
}

Expand Down Expand Up @@ -810,4 +822,22 @@ ngx_proxy_wasm_maps_get_dispatch_status(ngx_wavm_instance_t *instance,

return &pwctx->call_status;
}


static ngx_str_t *
ngx_proxy_wasm_maps_get_dispatch_status(ngx_wavm_instance_t *instance,
ngx_proxy_wasm_map_type_e map_type)
{
ngx_proxy_wasm_exec_t *pwexec;
ngx_http_proxy_wasm_dispatch_t *call;
ngx_wasm_socket_tcp_t *sock;

ngx_wa_assert(map_type == NGX_PROXY_WASM_MAP_HTTP_CALL_RESPONSE_HEADERS);

pwexec = ngx_proxy_wasm_instance2pwexec(instance);
call = pwexec->call;
sock = &call->sock;

return ngx_wasm_socket_tcp_status_strerror(sock->status);
}
#endif
Loading

0 comments on commit d7f6736

Please sign in to comment.