From a064063a223eb00239ca1645ad0b7e91b33d5963 Mon Sep 17 00:00:00 2001 From: Aapo Talvensaari Date: Fri, 1 Mar 2024 01:14:11 +0200 Subject: [PATCH] fix(vault): allow arrays in conf loader to be referenced ### Summary Some properties, like `KONG_SSL_CERT` and `KONG_SSL_CERT_KEY` are arrays and users can specify many. Vaults didn't work in this scenario: For example below didn't work before: ``` CERT_1=$( --- kong/cmd/utils/prefix_handler.lua | 14 +--- kong/cmd/utils/process_secrets.lua | 17 ++++- kong/conf_loader/constants.lua | 1 + kong/conf_loader/init.lua | 76 +++++++++++++------ kong/conf_loader/parse.lua | 24 +++--- kong/pdk/vault.lua | 1 + .../02-integration/13-vaults/03-mock_spec.lua | 2 +- 7 files changed, 86 insertions(+), 49 deletions(-) diff --git a/kong/cmd/utils/prefix_handler.lua b/kong/cmd/utils/prefix_handler.lua index 189c3a03981c..4309e9bf2e96 100644 --- a/kong/cmd/utils/prefix_handler.lua +++ b/kong/cmd/utils/prefix_handler.lua @@ -791,23 +791,13 @@ local function prepare_prefix(kong_config, nginx_custom_template_path, skip_writ "", } - local refs = kong_config["$refs"] - local has_refs = refs and type(refs) == "table" - - local secrets - if write_process_secrets and has_refs then - secrets = process_secrets.extract(kong_config) - end - local function quote_hash(s) return s:gsub("#", "\\#") end - for k, v in pairs(kong_config) do - if has_refs and refs[k] then - v = refs[k] - end + local secrets = write_process_secrets and process_secrets.extract(kong_config, true) + for k, v in pairs(kong_config) do if type(v) == "table" then if (getmetatable(v) or {}).__tostring then -- the 'tostring' meta-method knows how to serialize diff --git a/kong/cmd/utils/process_secrets.lua b/kong/cmd/utils/process_secrets.lua index 85e54d15c93b..fa5596129adf 100644 --- a/kong/cmd/utils/process_secrets.lua +++ b/kong/cmd/utils/process_secrets.lua @@ -11,6 +11,7 @@ local fmt = string.format local sub = string.sub local type = type local pairs = pairs +local shallow_copy = require("kong.tools.table").shallow_copy local CIPHER_ALG = "aes-256-gcm" @@ -56,15 +57,25 @@ local function hash_key_data(key_data) end -local function extract(conf) +local function extract(conf, manipulate_conf) local refs = conf["$refs"] if not refs or type(refs) ~= "table" then return end local secrets = {} - for k in pairs(refs) do - secrets[k] = conf[k] + for k, v in pairs(refs) do + secrets[k] = shallow_copy(conf[k]) + if manipulate_conf then + if type(v) == "table" then + for i, r in pairs(v) do + conf[k][i] = r + end + + else + conf[k] = v + end + end end return secrets diff --git a/kong/conf_loader/constants.lua b/kong/conf_loader/constants.lua index cda8a9a9ccdb..8469e779794b 100644 --- a/kong/conf_loader/constants.lua +++ b/kong/conf_loader/constants.lua @@ -574,6 +574,7 @@ local CONF_SENSITIVE = { admin_gui_ssl_cert_key = true, status_ssl_cert_key = true, debug_ssl_cert_key = true, + ["$refs"] = true, -- for internal use only, no need to log } diff --git a/kong/conf_loader/init.lua b/kong/conf_loader/init.lua index 2088aaa4a292..e1a53e098a6d 100644 --- a/kong/conf_loader/init.lua +++ b/kong/conf_loader/init.lua @@ -399,6 +399,39 @@ local function load(path, custom_conf, opts) loaded_vaults = setmetatable(vaults, conf_constants._NOP_TOSTRING_MT) + -- collect references + local is_reference = require "kong.pdk.vault".is_reference + for k, v in pairs(conf) do + local typ = (conf_constants.CONF_PARSERS[k] or {}).typ or "string" + v = parse_value(v, typ) + if typ == "array" then + local found + + for i, r in ipairs(v) do + if is_reference(r) then + found = true + if not refs then + refs = setmetatable({}, conf_constants._NOP_TOSTRING_MT) + end + if not refs[k] then + refs[k] = {} + end + refs[k][i] = r + end + end + + if found then + conf[k] = v + end + + elseif is_reference(v) then + if not refs then + refs = setmetatable({}, conf_constants._NOP_TOSTRING_MT) + end + refs[k] = v + end + end + if get_phase() == "init" then local secrets = getenv("KONG_PROCESS_SECRETS") if secrets then @@ -422,36 +455,34 @@ local function load(path, custom_conf, opts) end for k, deref in pairs(secrets) do - local v = parse_value(conf[k], "string") - if refs then - refs[k] = v - else - refs = setmetatable({ [k] = v }, conf_constants._NOP_TOSTRING_MT) - end - conf[k] = deref end end - else + elseif refs then local vault_conf = { loaded_vaults = loaded_vaults } for k, v in pairs(conf) do if sub(k, 1, 6) == "vault_" then vault_conf[k] = parse_value(v, "string") end end - local vault = require("kong.pdk.vault").new({ configuration = vault_conf }) - for k, v in pairs(conf) do - v = parse_value(v, "string") - if vault.is_reference(v) then - if refs then - refs[k] = v - else - refs = setmetatable({ [k] = v }, conf_constants._NOP_TOSTRING_MT) + for k, v in pairs(refs) do + if type(v) == "table" then + for i, r in pairs(v) do + local deref, deref_err = vault.get(r) + if deref == nil or deref_err then + if opts.starting then + return nil, fmt("failed to dereference '%s': %s for config option '%s[%d]'", r, deref_err, k, i) + end + + else + conf[k][i] = deref + end end + else local deref, deref_err = vault.get(v) if deref == nil or deref_err then if opts.starting then @@ -468,7 +499,6 @@ local function load(path, custom_conf, opts) -- validation local ok, err, errors = check_and_parse(conf, opts) - if not opts.starting then log.enable() end @@ -703,12 +733,14 @@ local function load(path, custom_conf, opts) local conf_arr = {} for k, v in pairs(conf) do - local to_print = v - if conf_constants.CONF_SENSITIVE[k] then - to_print = conf_constants.CONF_SENSITIVE_PLACEHOLDER - end + if k ~= "$refs" then + local to_print = v + if conf_constants.CONF_SENSITIVE[k] then + to_print = conf_constants.CONF_SENSITIVE_PLACEHOLDER + end - conf_arr[#conf_arr+1] = k .. " = " .. pl_pretty.write(to_print, "") + conf_arr[#conf_arr+1] = k .. " = " .. pl_pretty.write(to_print, "") + end end sort(conf_arr) diff --git a/kong/conf_loader/parse.lua b/kong/conf_loader/parse.lua index eeeac125c25d..f99dd1ba1478 100644 --- a/kong/conf_loader/parse.lua +++ b/kong/conf_loader/parse.lua @@ -221,22 +221,24 @@ local function check_and_parse(conf, opts) local errors = {} for k, value in pairs(conf) do - local v_schema = conf_constants.CONF_PARSERS[k] or {} + if k ~= "$refs" then + local v_schema = conf_constants.CONF_PARSERS[k] or {} - value = parse_value(value, v_schema.typ) + value = parse_value(value, v_schema.typ) - local typ = v_schema.typ or "string" - if value and not conf_constants.TYP_CHECKS[typ](value) then - errors[#errors + 1] = fmt("%s is not a %s: '%s'", k, typ, - tostring(value)) + local typ = v_schema.typ or "string" + if value and not conf_constants.TYP_CHECKS[typ](value) then + errors[#errors + 1] = fmt("%s is not a %s: '%s'", k, typ, + tostring(value)) - elseif v_schema.enum and not tablex.find(v_schema.enum, value) then - errors[#errors + 1] = fmt("%s has an invalid value: '%s' (%s)", k, - tostring(value), concat(v_schema.enum, ", ")) + elseif v_schema.enum and not tablex.find(v_schema.enum, value) then + errors[#errors + 1] = fmt("%s has an invalid value: '%s' (%s)", k, + tostring(value), concat(v_schema.enum, ", ")) - end + end - conf[k] = value + conf[k] = value + end end --------------------- diff --git a/kong/pdk/vault.lua b/kong/pdk/vault.lua index 4a29f405aff8..f5294c364ea6 100644 --- a/kong/pdk/vault.lua +++ b/kong/pdk/vault.lua @@ -1673,6 +1673,7 @@ local function new(self) init_worker() end + --- -- Warmups vault caches from config. -- diff --git a/spec/02-integration/13-vaults/03-mock_spec.lua b/spec/02-integration/13-vaults/03-mock_spec.lua index 77508422afea..f05da23e1731 100644 --- a/spec/02-integration/13-vaults/03-mock_spec.lua +++ b/spec/02-integration/13-vaults/03-mock_spec.lua @@ -110,7 +110,7 @@ for _, strategy in helpers.each_strategy() do local body = assert.res_status(200, res) local json = cjson.decode(body) assert.equal(meta._VERSION, json.version) - assert.equal("{vault://mock/admin-listen}", json.configuration.admin_listen) + assert.same({ "127.0.0.1:9001" }, json.configuration.admin_listen) assert.falsy(exists(join(helpers.test_conf.prefix, ".kong_process_secrets"))) end) end)