Skip to content

Commit

Permalink
fix(runloop): upstream ssl failure when plugins use response handler (#…
Browse files Browse the repository at this point in the history
…11502)

* fix(runloop): upstream ssl failure when plugins use response handler

If a plugin has response() handler, in `Kong.response` it will emits
a subrequest by calling `ngx.location.capture("/kong_buffered_http", options)`.
`ngx.location.capture` will create a new nginx request, so the overwritten
ssl info (client key & cert etc.) get lost in the new nginx request.

To fix this, those ssl info need to be re-set in the new request
context. We choose to do this in the early rewrite phase of the new
request before `Kong.balancer()` getting executed.

[FTI-5347](https://konghq.atlassian.net/browse/FTI-5347)

(cherry picked from commit 54468c4)
  • Loading branch information
catbro666 committed Sep 21, 2023
1 parent dff6860 commit 4672a21
Show file tree
Hide file tree
Showing 7 changed files with 447 additions and 87 deletions.
7 changes: 7 additions & 0 deletions CHANGELOG/unreleased/kong/11502.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
message: Fix upstream ssl failure when plugins use response handler
type: bugfix
scope: Core
prs:
- 11502
jiras:
- "FTI-5347"
1 change: 1 addition & 0 deletions kong-3.4.1-0.rockspec
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,7 @@ build = {
["kong.runloop.log_level"] = "kong/runloop/log_level.lua",
["kong.runloop.certificate"] = "kong/runloop/certificate.lua",
["kong.runloop.plugins_iterator"] = "kong/runloop/plugins_iterator.lua",
["kong.runloop.upstream_ssl"] = "kong/runloop/upstream_ssl.lua",
["kong.runloop.balancer"] = "kong/runloop/balancer/init.lua",
["kong.runloop.balancer.balancers"] = "kong/runloop/balancer/balancers.lua",
["kong.runloop.balancer.consistent_hashing"] = "kong/runloop/balancer/consistent_hashing.lua",
Expand Down
28 changes: 5 additions & 23 deletions kong/runloop/balancer/init.lua
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
local pl_tablex = require "pl.tablex"
local utils = require "kong.tools.utils"
local hooks = require "kong.hooks"
local get_certificate = require("kong.runloop.certificate").get_certificate
local recreate_request = require("ngx.balancer").recreate_request

local healthcheckers = require "kong.runloop.balancer.healthcheckers"
local balancers = require "kong.runloop.balancer.balancers"
local upstream_ssl = require "kong.runloop.upstream_ssl"
local upstreams = require "kong.runloop.balancer.upstreams"
local targets = require "kong.runloop.balancer.targets"

Expand Down Expand Up @@ -38,7 +38,7 @@ local EMPTY_T = pl_tablex.readonly {}

local set_authority

local set_upstream_cert_and_key = require("resty.kong.tls").set_upstream_cert_and_key
local fallback_upstream_client_cert = upstream_ssl.fallback_upstream_client_cert

if ngx.config.subsystem ~= "stream" then
set_authority = require("resty.kong.grpc").set_authority
Expand Down Expand Up @@ -321,6 +321,8 @@ local function execute(balancer_data, ctx)

-- store for retries
balancer_data.balancer = balancer
-- store for use in subrequest `ngx.location.capture("kong_buffered_http")`
balancer_data.upstream = upstream

-- calculate hash-value
-- only add it if it doesn't exist, in case a plugin inserted one
Expand All @@ -330,27 +332,7 @@ local function execute(balancer_data, ctx)
balancer_data.hash_value = hash_value
end

if ctx and ctx.service and not ctx.service.client_certificate then
-- service level client_certificate is not set
local cert, res, err
local client_certificate = upstream.client_certificate

-- does the upstream object contains a client certificate?
if client_certificate then
cert, err = get_certificate(client_certificate)
if not cert then
log(ERR, "unable to fetch upstream client TLS certificate ",
client_certificate.id, ": ", err)
return
end

res, err = set_upstream_cert_and_key(cert.cert, cert.key)
if not res then
log(ERR, "unable to apply upstream client TLS certificate ",
client_certificate.id, ": ", err)
end
end
end
fallback_upstream_client_cert(ctx, upstream)
end
end

Expand Down
66 changes: 3 additions & 63 deletions kong/runloop/handler.lua
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ local Router = require "kong.router"
local balancer = require "kong.runloop.balancer"
local events = require "kong.runloop.events"
local wasm = require "kong.runloop.wasm"
local upstream_ssl = require "kong.runloop.upstream_ssl"
local reports = require "kong.reports"
local constants = require "kong.constants"
local certificate = require "kong.runloop.certificate"
local concurrency = require "kong.concurrency"
local lrucache = require "resty.lrucache"
local ktls = require "resty.kong.tls"
Expand Down Expand Up @@ -72,7 +72,6 @@ local NOOP = function() end


local ERR = ngx.ERR
local CRIT = ngx.CRIT
local NOTICE = ngx.NOTICE
local WARN = ngx.WARN
local INFO = ngx.INFO
Expand Down Expand Up @@ -110,10 +109,7 @@ local STREAM_TLS_PASSTHROUGH_SOCK


local set_authority
local set_upstream_cert_and_key = ktls.set_upstream_cert_and_key
local set_upstream_ssl_verify = ktls.set_upstream_ssl_verify
local set_upstream_ssl_verify_depth = ktls.set_upstream_ssl_verify_depth
local set_upstream_ssl_trusted_store = ktls.set_upstream_ssl_trusted_store
local set_service_ssl = upstream_ssl.set_service_ssl

if is_http_module then
set_authority = require("resty.kong.grpc").set_authority
Expand Down Expand Up @@ -763,9 +759,6 @@ end

local balancer_prepare
do
local get_certificate = certificate.get_certificate
local get_ca_certificate_store = certificate.get_ca_certificate_store

local function sleep_once_for_balancer_init()
ngx.sleep(0)
sleep_once_for_balancer_init = NOOP
Expand Down Expand Up @@ -814,60 +807,7 @@ do
ctx.route = route
ctx.balancer_data = balancer_data

if service then
local res, err
local client_certificate = service.client_certificate

if client_certificate then
local cert, err = get_certificate(client_certificate)
if not cert then
log(ERR, "unable to fetch upstream client TLS certificate ",
client_certificate.id, ": ", err)
return
end

res, err = set_upstream_cert_and_key(cert.cert, cert.key)
if not res then
log(ERR, "unable to apply upstream client TLS certificate ",
client_certificate.id, ": ", err)
end
end

local tls_verify = service.tls_verify
if tls_verify then
res, err = set_upstream_ssl_verify(tls_verify)
if not res then
log(CRIT, "unable to set upstream TLS verification to: ",
tls_verify, ", err: ", err)
end
end

local tls_verify_depth = service.tls_verify_depth
if tls_verify_depth then
res, err = set_upstream_ssl_verify_depth(tls_verify_depth)
if not res then
log(CRIT, "unable to set upstream TLS verification to: ",
tls_verify, ", err: ", err)
-- in case verify can not be enabled, request can no longer be
-- processed without potentially compromising security
return kong.response.exit(500)
end
end

local ca_certificates = service.ca_certificates
if ca_certificates then
res, err = get_ca_certificate_store(ca_certificates)
if not res then
log(CRIT, "unable to get upstream TLS CA store, err: ", err)

else
res, err = set_upstream_ssl_trusted_store(res)
if not res then
log(CRIT, "unable to set upstream TLS CA store, err: ", err)
end
end
end
end
set_service_ssl(ctx)

if is_stream_module and scheme == "tcp" then
local res, err = disable_proxy_ssl()
Expand Down
115 changes: 115 additions & 0 deletions kong/runloop/upstream_ssl.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
local certificate = require "kong.runloop.certificate"
local ktls = require "resty.kong.tls"


local kong = kong
local ngx = ngx
local log = ngx.log
local ERR = ngx.ERR
local CRIT = ngx.CRIT

local get_certificate = certificate.get_certificate
local get_ca_certificate_store = certificate.get_ca_certificate_store
local set_upstream_cert_and_key = ktls.set_upstream_cert_and_key
local set_upstream_ssl_verify = ktls.set_upstream_ssl_verify
local set_upstream_ssl_verify_depth = ktls.set_upstream_ssl_verify_depth
local set_upstream_ssl_trusted_store = ktls.set_upstream_ssl_trusted_store


local function set_service_ssl(ctx)
local service = ctx and ctx.service

if service then
local res, err
local client_certificate = service.client_certificate

if client_certificate then
local cert, err = get_certificate(client_certificate)
if not cert then
log(ERR, "unable to fetch upstream client TLS certificate ",
client_certificate.id, ": ", err)
return
end

res, err = set_upstream_cert_and_key(cert.cert, cert.key)
if not res then
log(ERR, "unable to apply upstream client TLS certificate ",
client_certificate.id, ": ", err)
end
end

local tls_verify = service.tls_verify
if tls_verify then
res, err = set_upstream_ssl_verify(tls_verify)
if not res then
log(CRIT, "unable to set upstream TLS verification to: ",
tls_verify, ", err: ", err)
end
end

local tls_verify_depth = service.tls_verify_depth
if tls_verify_depth then
res, err = set_upstream_ssl_verify_depth(tls_verify_depth)
if not res then
log(CRIT, "unable to set upstream TLS verification to: ",
tls_verify, ", err: ", err)
-- in case verify can not be enabled, request can no longer be
-- processed without potentially compromising security
return kong.response.exit(500)
end
end

local ca_certificates = service.ca_certificates
if ca_certificates then
res, err = get_ca_certificate_store(ca_certificates)
if not res then
log(CRIT, "unable to get upstream TLS CA store, err: ", err)

else
res, err = set_upstream_ssl_trusted_store(res)
if not res then
log(CRIT, "unable to set upstream TLS CA store, err: ", err)
end
end
end
end
end

local function fallback_upstream_client_cert(ctx, upstream)
if not ctx then
return
end

upstream = upstream or (ctx.balancer_data and ctx.balancer_data.upstream)

if not upstream then
return
end

if ctx.service and not ctx.service.client_certificate then
-- service level client_certificate is not set
local cert, res, err
local client_certificate = upstream.client_certificate

-- does the upstream object contains a client certificate?
if client_certificate then
cert, err = get_certificate(client_certificate)
if not cert then
log(ERR, "unable to fetch upstream client TLS certificate ",
client_certificate.id, ": ", err)
return
end

res, err = set_upstream_cert_and_key(cert.cert, cert.key)
if not res then
log(ERR, "unable to apply upstream client TLS certificate ",
client_certificate.id, ": ", err)
end
end
end
end

return {
set_service_ssl = set_service_ssl,
fallback_upstream_client_cert = fallback_upstream_client_cert,
}
11 changes: 10 additions & 1 deletion kong/templates/nginx_kong.lua
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,16 @@ server {
default_type '';
set $kong_proxy_mode 'http';
rewrite_by_lua_block {;}
rewrite_by_lua_block {
-- ngx.localtion.capture will create a new nginx request,
-- so the upstream ssl-related info attached to the `r` gets lost.
-- we need to re-set them here to the new nginx request.
local ctx = ngx.ctx
local upstream_ssl = require("kong.runloop.upstream_ssl")
upstream_ssl.set_service_ssl(ctx)
upstream_ssl.fallback_upstream_client_cert(ctx)
}
access_by_lua_block {;}
header_filter_by_lua_block {;}
body_filter_by_lua_block {;}
Expand Down
Loading

0 comments on commit 4672a21

Please sign in to comment.