fix(vault): support array type config reference and multiple nginx su…

…bsystem (#13855)

* fix(vault): allow arrays in conf loader to be referenced

### Summary

The commit includes the following fix for vault:

Fix the issue that array-type configuration's element cannot be referenced. conf_loader now supports iterate through array-like configuration field and deref the secrets one by one.

Enable the vault to dereference secrets using partial SSL configurations, for example, lua_ssl_trusted_certificate=system, {vault://abc/def}. This is implemented especially for resty environments that execute the actual kong command, with the opts pre_cmd = true. Vault related functionalities can work normally by using valid partial configuration generated in the pre_cmd environment. This change does not have a separate changelog entry because it is part of the previous "array-type config vault ref" fix and is more inline with user's intuition.

Fix the issue that vault reference cannot work well when multiple subsystems(http/stream) are enabled. The kong_processed_secrets environment variable/file are now generated by subsystems, so enabling multiple subsystem will generates multiple env var/secret file for storing vault deref result.

* fix(vault): allow arrays in conf loader to be referenced (#12672)

* fix(*): special treatment for pre_cmd

* fix(cmd): fix vault refs when both http and stream are enabled

* tests(*): fix prepare cmd test

* tests(*): add vault related cmd tests

* docs(changelog): add changelog

* test(*): try to fix test

* fix(*): unsetenv unconditionally clean http/stream env

---------

Signed-off-by: Aapo Talvensaari <[email protected]>
Co-authored-by: Aapo Talvensaari <[email protected]>
(cherry picked from commit bc0acea)

changelog/unreleased/kong/fix-vault-array-config.yml

@@ -0,0 +1,4 @@
+message: |
+   **Vault**: Fixed an issue where array-like configuration fields cannot contain vault reference.
+type: bugfix
+scope: Core

changelog/unreleased/kong/fix-vault-stream-subsystem.yml

@@ -0,0 +1,4 @@
+message: |
+   **Vault**: Fixed an issue where vault reference in kong configuration cannot be dereferenced when both http and stream subsystems are enabled.
+type: bugfix
+scope: Core

kong/cmd/stop.lua

@@ -22,7 +22,7 @@ local function execute(args, opts)
   -- load <PREFIX>/kong.conf containing running node's config
   local conf = assert(conf_loader(default_conf.kong_env, {
     prefix = args.prefix
-  }))
+  }, { stopping = true }))
   assert(nginx_signals.stop(conf))
 
   if opts.quiet then

kong/cmd/utils/inject_confs.lua

@@ -20,7 +20,9 @@ local function load_conf(args)
 
   if pl_path.exists(conf.kong_env) then
     -- load <PREFIX>/kong.conf containing running node's config
-    conf = assert(conf_loader(conf.kong_env))
+    conf = assert(conf_loader(conf.kong_env, {
+      prefix = conf.prefix
+    }))
   end
 
   -- make sure necessary files like `.ca_combined` exist

kong/cmd/utils/nginx_signals.lua

@@ -84,13 +84,25 @@ local function set_process_secrets_env(kong_conf)
     return nil, err
   end
 
-  return C.setenv("KONG_PROCESS_SECRETS", secrets, 1) == 0
+  local ok_sub_http
+  if kong_conf.role == "control_plane" or #kong_conf.proxy_listeners > 0
+    or #kong_conf.admin_listeners > 0 or #kong_conf.status_listeners > 0 then
+      ok_sub_http = C.setenv("KONG_PROCESS_SECRETS_HTTP", secrets, 1) == 0
+  end
+
+  local ok_sub_stream
+  if #kong_conf.stream_listeners > 0 then
+    ok_sub_stream = C.setenv("KONG_PROCESS_SECRETS_STREAM", secrets, 1) == 0
+  end
+
+  return ok_sub_http or ok_sub_stream
 end
 
 
 local function unset_process_secrets_env(has_process_secrets)
   if has_process_secrets then
-    C.unsetenv("KONG_PROCESS_SECRETS")
+    C.unsetenv("KONG_PROCESS_SECRETS_HTTP")
+    C.unsetenv("KONG_PROCESS_SECRETS_STREAM")
   end
 end
 

kong/cmd/utils/prefix_handler.lua

@@ -28,6 +28,7 @@ local nginx_signals = require "kong.cmd.utils.nginx_signals"
 
 local strip = require("kong.tools.string").strip
 local split = require("kong.tools.string").split
+local shallow_copy = require("kong.tools.table").shallow_copy
 
 
 local getmetatable = getmetatable
@@ -373,40 +374,62 @@ local function write_env_file(path, data)
   return true
 end
 
-local function write_process_secrets_file(path, data)
-  os.remove(path)
+local function write_process_secrets_file(kong_conf, data)
+  local path = kong_conf.kong_process_secrets
 
-  local flags = bit.bor(system_constants.O_RDONLY(),
-                        system_constants.O_CREAT())
+  local function write_single_secret_file(path, data)
+    os.remove(path)
 
-  local mode = ffi.new("int", bit.bor(system_constants.S_IRUSR(),
-                                      system_constants.S_IWUSR()))
+    local flags = bit.bor(system_constants.O_RDONLY(),
+                          system_constants.O_CREAT())
 
-  local fd = ffi.C.open(path, flags, mode)
-  if fd < 0 then
-    local errno = ffi.errno()
-    return nil, "unable to open process secrets path " .. path .. " (" ..
-                ffi.string(ffi.C.strerror(errno)) .. ")"
-  end
+    local mode = ffi.new("int", bit.bor(system_constants.S_IRUSR(),
+                                        system_constants.S_IWUSR()))
 
-  local ok = ffi.C.close(fd)
-  if ok ~= 0 then
-    local errno = ffi.errno()
-    return nil, "failed to close fd (" ..
-                ffi.string(ffi.C.strerror(errno)) .. ")"
-  end
+    local fd = ffi.C.open(path, flags, mode)
+    if fd < 0 then
+      local errno = ffi.errno()
+      return nil, "unable to open process secrets path " .. path .. " (" ..
+                  ffi.string(ffi.C.strerror(errno)) .. ")"
+    end
 
-  local file, err = io.open(path, "w+b")
-  if not file then
-    return nil, "unable to open process secrets path " .. path .. " (" .. err .. ")"
-  end
+    local ok = ffi.C.close(fd)
+    if ok ~= 0 then
+      local errno = ffi.errno()
+      return nil, "failed to close fd (" ..
+                  ffi.string(ffi.C.strerror(errno)) .. ")"
+    end
 
-  local ok, err = file:write(data)
+    local file, err = io.open(path, "w+b")
+    if not file then
+      return nil, "unable to open process secrets path " .. path .. " (" .. err .. ")"
+    end
 
-  file:close()
+    local ok, err = file:write(data)
 
-  if not ok then
-    return nil, "unable to write process secrets path " .. path .. " (" .. err .. ")"
+    file:close()
+
+    if not ok then
+      return nil, "unable to write process secrets path " .. path .. " (" .. err .. ")"
+    end
+
+    return true
+
+  end
+
+  if kong_conf.role == "control_plane" or #kong_conf.proxy_listeners > 0
+    or #kong_conf.admin_listeners > 0 or #kong_conf.status_listeners > 0 then
+    local ok, err = write_single_secret_file(path .. "_http", data)
+    if not ok then
+      return nil, err
+    end
+  end
+
+  if #kong_conf.stream_listeners > 0 then
+    local ok, err = write_single_secret_file(path .. "_stream", data)
+    if not ok then
+      return nil, err
+    end
   end
 
   return true
@@ -829,21 +852,32 @@ local function prepare_prefix(kong_config, nginx_custom_template_path, skip_writ
     "",
   }
 
-  local refs = kong_config["$refs"]
-  local has_refs = refs and type(refs) == "table"
-
-  local secrets
-  if write_process_secrets and has_refs then
-    secrets = process_secrets.extract(kong_config)
-  end
-
   local function quote_hash(s)
     return s:gsub("#", "\\#")
   end
 
+  local refs = kong_config["$refs"]
+  local has_refs = refs and type(refs) == "table"
+  local secrets = process_secrets.extract(kong_config)
+
   for k, v in pairs(kong_config) do
+    -- do not output secrets in .kong_env
     if has_refs and refs[k] then
-      v = refs[k]
+      local ref = refs[k]
+      if type(ref) == "table" then
+        if type(v) ~= "table" then
+          v = { v }
+        else
+          v = shallow_copy(v)
+        end
+
+        for i, r in pairs(ref) do
+          v[i] = r
+        end
+
+      elseif ref then
+        v = ref
+      end
     end
 
     if type(v) == "table" then
@@ -869,13 +903,13 @@ local function prepare_prefix(kong_config, nginx_custom_template_path, skip_writ
     prepare_prefixed_interface_dir("/usr/local/kong", "gui", kong_config)
   end
 
-  if secrets then
+  if secrets and write_process_secrets then
     secrets, err = process_secrets.serialize(secrets, kong_config.kong_env)
     if not secrets then
       return nil, err
     end
 
-    ok, err = write_process_secrets_file(kong_config.kong_process_secrets, secrets)
+    ok, err = write_process_secrets_file(kong_config, secrets)
     if not ok then
       return nil, err
     end

kong/cmd/utils/process_secrets.lua

@@ -11,6 +11,7 @@ local fmt = string.format
 local sub = string.sub
 local type = type
 local pairs = pairs
+local shallow_copy = require("kong.tools.table").shallow_copy
 
 
 local CIPHER_ALG = "aes-256-gcm"
@@ -64,7 +65,7 @@ local function extract(conf)
 
   local secrets = {}
   for k in pairs(refs) do
-    secrets[k] = conf[k]
+    secrets[k] = shallow_copy(conf[k])
   end
 
   return secrets

kong/conf_loader/constants.lua

@@ -591,6 +591,7 @@ local CONF_SENSITIVE = {
   admin_gui_ssl_cert_key = true,
   status_ssl_cert_key = true,
   debug_ssl_cert_key = true,
+  ["$refs"] = true, -- for internal use only, no need to log
 }