fix(runloop): upstream ssl failure when plugins use response handler

CHANGELOG/unreleased/kong/11502.yaml

@@ -0,0 +1,7 @@
+message: Fix upstream ssl failure when plugins use response handler
+type: bugfix
+scope: Core
+prs:
+  - 11502
+jiras:
+  - "FTI-5347"

kong-3.5.0-0.rockspec

@@ -168,6 +168,7 @@ build = {
     ["kong.runloop.log_level"] = "kong/runloop/log_level.lua",
     ["kong.runloop.certificate"] = "kong/runloop/certificate.lua",
     ["kong.runloop.plugins_iterator"] = "kong/runloop/plugins_iterator.lua",
+    ["kong.runloop.upstream_ssl"] = "kong/runloop/upstream_ssl.lua",
     ["kong.runloop.balancer"] = "kong/runloop/balancer/init.lua",
     ["kong.runloop.balancer.balancers"] = "kong/runloop/balancer/balancers.lua",
     ["kong.runloop.balancer.consistent_hashing"] = "kong/runloop/balancer/consistent_hashing.lua",

kong/runloop/balancer/init.lua

@@ -1,11 +1,11 @@
 local pl_tablex = require "pl.tablex"
 local utils = require "kong.tools.utils"
 local hooks = require "kong.hooks"
-local get_certificate = require("kong.runloop.certificate").get_certificate
 local recreate_request = require("ngx.balancer").recreate_request
 
 local healthcheckers = require "kong.runloop.balancer.healthcheckers"
 local balancers = require "kong.runloop.balancer.balancers"
+local upstream_ssl = require "kong.runloop.upstream_ssl"
 local upstreams = require "kong.runloop.balancer.upstreams"
 local targets = require "kong.runloop.balancer.targets"
 
@@ -38,7 +38,7 @@ local EMPTY_T = pl_tablex.readonly {}
 
 local set_authority
 
-local set_upstream_cert_and_key = require("resty.kong.tls").set_upstream_cert_and_key
+local fallback_upstream_client_cert = upstream_ssl.fallback_upstream_client_cert
 
 if ngx.config.subsystem ~= "stream" then
   set_authority = require("resty.kong.grpc").set_authority
@@ -321,6 +321,8 @@ local function execute(balancer_data, ctx)
 
       -- store for retries
       balancer_data.balancer = balancer
+      -- store for use in subrequest `ngx.location.capture("kong_buffered_http")`
+      balancer_data.upstream = upstream
 
       -- calculate hash-value
       -- only add it if it doesn't exist, in case a plugin inserted one
@@ -330,27 +332,7 @@ local function execute(balancer_data, ctx)
         balancer_data.hash_value = hash_value
       end
 
-      if ctx and ctx.service and not ctx.service.client_certificate then
-        -- service level client_certificate is not set
-        local cert, res, err
-        local client_certificate = upstream.client_certificate
-
-        -- does the upstream object contains a client certificate?
-        if client_certificate then
-          cert, err = get_certificate(client_certificate)
-          if not cert then
-            log(ERR, "unable to fetch upstream client TLS certificate ",
-                     client_certificate.id, ": ", err)
-            return
-          end
-
-          res, err = set_upstream_cert_and_key(cert.cert, cert.key)
-          if not res then
-            log(ERR, "unable to apply upstream client TLS certificate ",
-                     client_certificate.id, ": ", err)
-          end
-        end
-      end
+      fallback_upstream_client_cert(ctx, upstream)
     end
   end
 

kong/runloop/handler.lua

@@ -6,9 +6,9 @@ local Router       = require "kong.router"
 local balancer     = require "kong.runloop.balancer"
 local events       = require "kong.runloop.events"
 local wasm         = require "kong.runloop.wasm"
+local upstream_ssl = require "kong.runloop.upstream_ssl"
 local reports      = require "kong.reports"
 local constants    = require "kong.constants"
-local certificate  = require "kong.runloop.certificate"
 local concurrency  = require "kong.concurrency"
 local lrucache     = require "resty.lrucache"
 local ktls         = require "resty.kong.tls"
@@ -72,7 +72,6 @@ local NOOP = function() end
 
 
 local ERR   = ngx.ERR
-local CRIT  = ngx.CRIT
 local NOTICE = ngx.NOTICE
 local WARN  = ngx.WARN
 local INFO  = ngx.INFO
@@ -110,10 +109,7 @@ local STREAM_TLS_PASSTHROUGH_SOCK
 
 
 local set_authority
-local set_upstream_cert_and_key = ktls.set_upstream_cert_and_key
-local set_upstream_ssl_verify = ktls.set_upstream_ssl_verify
-local set_upstream_ssl_verify_depth = ktls.set_upstream_ssl_verify_depth
-local set_upstream_ssl_trusted_store = ktls.set_upstream_ssl_trusted_store
+local set_service_ssl = upstream_ssl.set_service_ssl
 
 if is_http_module then
   set_authority = require("resty.kong.grpc").set_authority
@@ -763,9 +759,6 @@ end
 
 local balancer_prepare
 do
-  local get_certificate = certificate.get_certificate
-  local get_ca_certificate_store = certificate.get_ca_certificate_store
-
   local function sleep_once_for_balancer_init()
     ngx.sleep(0)
     sleep_once_for_balancer_init = NOOP
@@ -814,60 +807,7 @@ do
     ctx.route            = route
     ctx.balancer_data    = balancer_data
 
-    if service then
-      local res, err
-      local client_certificate = service.client_certificate
-
-      if client_certificate then
-        local cert, err = get_certificate(client_certificate)
-        if not cert then
-          log(ERR, "unable to fetch upstream client TLS certificate ",
-                   client_certificate.id, ": ", err)
-          return
-        end
-
-        res, err = set_upstream_cert_and_key(cert.cert, cert.key)
-        if not res then
-          log(ERR, "unable to apply upstream client TLS certificate ",
-                   client_certificate.id, ": ", err)
-        end
-      end
-
-      local tls_verify = service.tls_verify
-      if tls_verify then
-        res, err = set_upstream_ssl_verify(tls_verify)
-        if not res then
-          log(CRIT, "unable to set upstream TLS verification to: ",
-                   tls_verify, ", err: ", err)
-        end
-      end
-
-      local tls_verify_depth = service.tls_verify_depth
-      if tls_verify_depth then
-        res, err = set_upstream_ssl_verify_depth(tls_verify_depth)
-        if not res then
-          log(CRIT, "unable to set upstream TLS verification to: ",
-                   tls_verify, ", err: ", err)
-          -- in case verify can not be enabled, request can no longer be
-          -- processed without potentially compromising security
-          return kong.response.exit(500)
-        end
-      end
-
-      local ca_certificates = service.ca_certificates
-      if ca_certificates then
-        res, err = get_ca_certificate_store(ca_certificates)
-        if not res then
-          log(CRIT, "unable to get upstream TLS CA store, err: ", err)
-
-        else
-          res, err = set_upstream_ssl_trusted_store(res)
-          if not res then
-            log(CRIT, "unable to set upstream TLS CA store, err: ", err)
-          end
-        end
-      end
-    end
+    set_service_ssl(ctx)
 
     if is_stream_module and scheme == "tcp" then
       local res, err = disable_proxy_ssl()

kong/runloop/upstream_ssl.lua

@@ -0,0 +1,115 @@
+local certificate  = require "kong.runloop.certificate"
+local ktls         = require "resty.kong.tls"
+
+
+local kong         = kong
+local ngx          = ngx
+local log          = ngx.log
+local ERR          = ngx.ERR
+local CRIT         = ngx.CRIT
+
+local get_certificate                = certificate.get_certificate
+local get_ca_certificate_store       = certificate.get_ca_certificate_store
+local set_upstream_cert_and_key      = ktls.set_upstream_cert_and_key
+local set_upstream_ssl_verify        = ktls.set_upstream_ssl_verify
+local set_upstream_ssl_verify_depth  = ktls.set_upstream_ssl_verify_depth
+local set_upstream_ssl_trusted_store = ktls.set_upstream_ssl_trusted_store
+
+
+local function set_service_ssl(ctx)
+  local service = ctx and ctx.service 
+
+  if service then
+    local res, err
+    local client_certificate = service.client_certificate
+
+    if client_certificate then
+      local cert, err = get_certificate(client_certificate)
+      if not cert then
+        log(ERR, "unable to fetch upstream client TLS certificate ",
+                 client_certificate.id, ": ", err)
+        return
+      end
+
+      res, err = set_upstream_cert_and_key(cert.cert, cert.key)
+      if not res then
+        log(ERR, "unable to apply upstream client TLS certificate ",
+                 client_certificate.id, ": ", err)
+      end
+    end
+
+    local tls_verify = service.tls_verify
+    if tls_verify then
+      res, err = set_upstream_ssl_verify(tls_verify)
+      if not res then
+        log(CRIT, "unable to set upstream TLS verification to: ",
+                 tls_verify, ", err: ", err)
+      end
+    end
+
+    local tls_verify_depth = service.tls_verify_depth
+    if tls_verify_depth then
+      res, err = set_upstream_ssl_verify_depth(tls_verify_depth)
+      if not res then
+        log(CRIT, "unable to set upstream TLS verification to: ",
+                 tls_verify, ", err: ", err)
+        -- in case verify can not be enabled, request can no longer be
+        -- processed without potentially compromising security
+        return kong.response.exit(500)
+      end
+    end
+
+    local ca_certificates = service.ca_certificates
+    if ca_certificates then
+      res, err = get_ca_certificate_store(ca_certificates)
+      if not res then
+        log(CRIT, "unable to get upstream TLS CA store, err: ", err)
+
+      else
+        res, err = set_upstream_ssl_trusted_store(res)
+        if not res then
+          log(CRIT, "unable to set upstream TLS CA store, err: ", err)
+        end
+      end
+    end
+  end
+end
+
+local function fallback_upstream_client_cert(ctx, upstream)
+  if not ctx then
+    return
+  end
+
+  upstream = upstream or (ctx.balancer_data and ctx.balancer_data.upstream)
+
+  if not upstream then
+    return
+  end
+
+  if ctx.service and not ctx.service.client_certificate then
+    -- service level client_certificate is not set
+    local cert, res, err
+    local client_certificate = upstream.client_certificate
+
+    -- does the upstream object contains a client certificate?
+    if client_certificate then
+      cert, err = get_certificate(client_certificate)
+      if not cert then
+        log(ERR, "unable to fetch upstream client TLS certificate ",
+                 client_certificate.id, ": ", err)
+        return
+      end
+
+      res, err = set_upstream_cert_and_key(cert.cert, cert.key)
+      if not res then
+        log(ERR, "unable to apply upstream client TLS certificate ",
+                 client_certificate.id, ": ", err)
+      end
+    end
+  end
+end
+
+return {
+  set_service_ssl = set_service_ssl,
+  fallback_upstream_client_cert = fallback_upstream_client_cert,
+}

kong/templates/nginx_kong.lua

@@ -284,7 +284,16 @@ server {
         default_type         '';
         set $kong_proxy_mode 'http';
 
-        rewrite_by_lua_block       {;}
+        rewrite_by_lua_block       {
+          -- ngx.localtion.capture will create a new nginx request,
+          -- so the upstream ssl-related info attached to the `r` gets lost.
+          -- we need to re-set them here to the new nginx request.
+          local ctx = ngx.ctx
+          local upstream_ssl = require("kong.runloop.upstream_ssl")
+
+          upstream_ssl.set_service_ssl(ctx)
+          upstream_ssl.fallback_upstream_client_cert(ctx)
+        }
         access_by_lua_block        {;}
         header_filter_by_lua_block {;}
         body_filter_by_lua_block   {;}