From 0f73339b558497d4ecd4d0fb64ad866270ff164b Mon Sep 17 00:00:00 2001 From: Xiaochen Wang Date: Thu, 8 Aug 2024 17:23:19 +0800 Subject: [PATCH] feat(dns): use new `resolver_*` directives for new DNS client (#13427) This avoids confusion with old DNS client config as not all of them works in the new client. KAG-4994 --- .../unreleased/kong/refactor_dns_client.yml | 4 +- kong.conf.default | 101 ++++++++++++++++++ kong/conf_loader/constants.lua | 10 ++ kong/conf_loader/parse.lua | 31 ++++++ kong/dns/client.lua | 7 -- kong/templates/kong_defaults.lua | 10 ++ kong/templates/nginx_kong.lua | 2 +- kong/tools/dns.lua | 24 +++++ spec/01-unit/03-conf_loader_spec.lua | 52 +++++++++ spec/02-integration/02-cmd/11-config_spec.lua | 1 + .../04-admin_api/11-reports_spec.lua | 1 + .../05-proxy/22-reports_spec.lua | 1 + .../10-go_plugins/01-reports_spec.lua | 1 + .../17-admin_gui/03-reports_spec.lua | 3 + .../20-wasm/04-proxy-wasm_spec.lua | 1 + .../20-wasm/07-reports_spec.lua | 1 + .../22-ai_plugins/01-reports_spec.lua | 1 + spec/fixtures/default_status_listen.conf | 1 + spec/kong_tests.conf | 1 + 19 files changed, 242 insertions(+), 11 deletions(-) diff --git a/changelog/unreleased/kong/refactor_dns_client.yml b/changelog/unreleased/kong/refactor_dns_client.yml index da5cd40f65ca..8de130ca54b3 100644 --- a/changelog/unreleased/kong/refactor_dns_client.yml +++ b/changelog/unreleased/kong/refactor_dns_client.yml @@ -2,8 +2,6 @@ message: > Starting from this version, a new DNS client library has been implemented and added into Kong. The new DNS client library has the following changes - Introduced global caching for DNS records across workers, significantly reducing the query load on DNS servers. - Introduced observable statistics for the new DNS client, and a new Admin API `/status/dns` to retrieve them. - - Deprecated the `dns_no_sync` option. Multiple DNS queries for the same name will always be synchronized (even across workers). This remains functional with the legacy DNS client library. - - Deprecated the `dns_not_found_ttl` option. It uses the `dns_error_ttl` option for all error responses. This option remains functional with the legacy DNS client library. - - Deprecated the `dns_order` option. By default, SRV, A, and AAAA are supported. Only names in the SRV format (`_service._proto.name`) enable resolving of DNS SRV records. + - Simplified the logic and make it more standardized type: feature scope: Core diff --git a/kong.conf.default b/kong.conf.default index 87c9b12cf96d..b9f80f810c6a 100644 --- a/kong.conf.default +++ b/kong.conf.default @@ -1532,6 +1532,107 @@ # same name/type will be synchronized to a # single query. +#------------------------------------------------------------------------------ +# New DNS RESOLVER +#------------------------------------------------------------------------------ + +# This DNS resolver introduces global caching for DNS records across workers, +# significantly reducing the query load on DNS servers. +# +# It provides observable statistics, you can retrieve them through the Admin API +# `/status/dns`. + +#legacy_dns_client = off # Disable the new DNS resolver, using the + # original DNS resolver. See above `dns_xxx` + # options for the original DNS resolver. + +#resolver_address = + # Comma-separated list of nameservers, each + # entry in `ip[:port]` format to be used by + # Kong. If not specified, the nameservers in + # the local `resolv.conf` file will be used. + # Port defaults to 53 if omitted. Accepts + # both IPv4 and IPv6 addresses. + # + # Examples: + # + # ``` + # resolver_address = 8.8.8.8 + # resolver_address = 8.8.8.8, [::1] + # resolver_address = 8.8.8.8:53, [::1]:53 + # ``` + +#resolver_hosts_file = /etc/hosts + # The hosts file to use. This file is read + # once and its content is static in memory. + # To read the file again after modifying it, + # Kong must be reloaded. + +#resolver_family = A,SRV # The supported query types. + # + # For a domain name, Kong will only query + # either IP addresses (A or AAAA) or SRV + # records, but not both. + # + # It will query SRV records only when the + # domain matches the + # "_._." format, for + # example, "_ldap._tcp.example.com". + # + # For IP addresses (A or AAAA) resolution, it + # first attempts IPv4 (A) and then queries + # IPv6 (AAAA). + +#resolver_valid_ttl = + # By default, DNS records are cached using + # the TTL value of a response. This optional + # parameter (in seconds) allows overriding it. + +#resolver_error_ttl = 1 # TTL in seconds for error responses and empty + # responses. + +#resolver_stale_ttl = 3600 # Defines, in seconds, how long a record will + # remain in cache past its TTL. This value + # will be used while the new DNS record is + # fetched in the background. + # + # Stale data will be used from expiry of a + # record until either the refresh query + # completes, or the `resolver_stale_ttl` number + # of seconds have passed. + # + # This configuration enables Kong to be more + # resilient during the DNS server downtime. + +#resolver_lru_cache_size = 10000 # The DNS client uses a two-layer cache system: + # L1 - worker-level LRU Lua VM cache + # L2 - across-workers shared memory cache + # + # This value specifies the maximum allowed + # number of DNS responses stored in the L1 LRU + # lua VM cache. + # + # A single name query can easily take up 1~10 + # slots, depending on attempted query types and + # extended domains from /etc/resolv.conf + # options `domain` or `search`. + +#resolver_mem_cache_size = 5m # This value specifies the size of the L2 + # shared memory cache for DNS responses, + # `kong_dns_cache`. + # + # Accepted units are `k` and `m`, with a + # minimum recommended value of a few MBs. + # + # 5MB shared memory size could store + # ~20000 DNS responeses with single A record or + # ~10000 DNS responeses with 2~3 A records. + # + # 10MB shared memory size could store + # ~40000 DNS responeses with single A record or + # ~20000 DNS responeses with 2~3 A records. + + #------------------------------------------------------------------------------ # VAULTS #------------------------------------------------------------------------------ diff --git a/kong/conf_loader/constants.lua b/kong/conf_loader/constants.lua index 59bd482cce66..116ef68c1d04 100644 --- a/kong/conf_loader/constants.lua +++ b/kong/conf_loader/constants.lua @@ -370,7 +370,17 @@ local CONF_PARSERS = { dns_not_found_ttl = { typ = "number" }, dns_error_ttl = { typ = "number" }, dns_no_sync = { typ = "boolean" }, + legacy_dns_client = { typ = "boolean" }, + + resolver_address = { typ = "array" }, + resolver_hosts_file = { typ = "string" }, + resolver_family = { typ = "array" }, + resolver_valid_ttl = { typ = "number" }, + resolver_stale_ttl = { typ = "number" }, + resolver_error_ttl = { typ = "number" }, + resolver_lru_cache_size = { typ = "number" }, + privileged_worker = { typ = "boolean", deprecated = { diff --git a/kong/conf_loader/parse.lua b/kong/conf_loader/parse.lua index 5a585fdde034..198913c0a22a 100644 --- a/kong/conf_loader/parse.lua +++ b/kong/conf_loader/parse.lua @@ -529,6 +529,37 @@ local function check_and_parse(conf, opts) end end + --- new dns client + + if conf.resolver_address then + for _, server in ipairs(conf.resolver_address) do + local dns = normalize_ip(server) + + if not dns or dns.type == "name" then + errors[#errors + 1] = "resolver_address must be a comma separated list " .. + "in the form of IPv4/6 or IPv4/6:port, got '" .. + server .. "'" + end + end + end + + if conf.resolver_hosts_file then + if not pl_path.isfile(conf.resolver_hosts_file) then + errors[#errors + 1] = "resolver_hosts_file: file does not exist" + end + end + + if conf.resolver_family then + local allowed = { A = true, AAAA = true, SRV = true } + + for _, name in ipairs(conf.resolver_family) do + if not allowed[upper(name)] then + errors[#errors + 1] = fmt("resolver_family: invalid entry '%s'", + tostring(name)) + end + end + end + if not conf.lua_package_cpath then conf.lua_package_cpath = "" end diff --git a/kong/dns/client.lua b/kong/dns/client.lua index f4a2072397d1..64eb3b14a934 100644 --- a/kong/dns/client.lua +++ b/kong/dns/client.lua @@ -617,13 +617,6 @@ local dns_client function _M.init(opts) log(DEBUG, PREFIX, "(re)configuring dns client") - if opts then - opts.valid_ttl = opts.valid_ttl or opts.validTtl - opts.error_ttl = opts.error_ttl or opts.badTtl - opts.stale_ttl = opts.stale_ttl or opts.staleTtl - opts.cache_size = opts.cache_size or opts.cacheSize - end - local client, err = _M.new(opts) if not client then return nil, err diff --git a/kong/templates/kong_defaults.lua b/kong/templates/kong_defaults.lua index b03980d9fb97..153a61ad076c 100644 --- a/kong/templates/kong_defaults.lua +++ b/kong/templates/kong_defaults.lua @@ -169,8 +169,18 @@ dns_cache_size = 10000 dns_not_found_ttl = 30 dns_error_ttl = 1 dns_no_sync = off + legacy_dns_client = off +resolver_address = NONE +resolver_hosts_file = /etc/hosts +resolver_family = A,SRV +resolver_valid_ttl = NONE +resolver_stale_ttl = 3600 +resolver_lru_cache_size = 10000 +resolver_mem_cache_size = 5m +resolver_error_ttl = 1 + dedicated_config_processing = on worker_consistency = eventual worker_state_update_frequency = 5 diff --git a/kong/templates/nginx_kong.lua b/kong/templates/nginx_kong.lua index c024a723a688..a02336697f76 100644 --- a/kong/templates/nginx_kong.lua +++ b/kong/templates/nginx_kong.lua @@ -24,7 +24,7 @@ lua_shared_dict kong_db_cache_miss 12m; lua_shared_dict kong_secrets 5m; > if not legacy_dns_client then -lua_shared_dict kong_dns_cache 5m; +lua_shared_dict kong_dns_cache ${{RESOLVER_MEM_CACHE_SIZE}}; > end underscores_in_headers on; diff --git a/kong/tools/dns.lua b/kong/tools/dns.lua index 60d9aca446b6..22f32a8f9c1e 100644 --- a/kong/tools/dns.lua +++ b/kong/tools/dns.lua @@ -37,6 +37,30 @@ local setup_client = function(conf) noSynchronisation = conf.dns_no_sync, } + -- new dns client + if ngx.shared.kong_dns_cache and not _G.busted_legacy_dns_client then + + servers = {} + + if conf.resolver_address then + for i, server in ipairs(conf.resolver_address) do + local s = normalize_ip(server) + servers[i] = { s.host, s.port or 53 } -- inserting port if omitted + end + end + + opts = { + nameservers = servers, + hosts = conf.resolver_hosts_file, + family = conf.resolver_family, + valid_ttl = conf.resolver_valid_ttl, + error_ttl = conf.resolver_error_ttl, + stale_ttl = conf.resolver_stale_ttl, + cache_size = conf.resolver_lru_cache_size, + enable_ipv6 = true, -- allow for IPv6 nameserver addresses + } + end + assert(dns_client.init(opts)) return dns_client diff --git a/spec/01-unit/03-conf_loader_spec.lua b/spec/01-unit/03-conf_loader_spec.lua index 604792c60476..4957aee346ed 100644 --- a/spec/01-unit/03-conf_loader_spec.lua +++ b/spec/01-unit/03-conf_loader_spec.lua @@ -642,8 +642,10 @@ describe("Configuration loader", function() local conf = assert(conf_loader()) assert.same({"bundled"}, conf.plugins) assert.same({"LAST", "SRV", "A", "CNAME"}, conf.dns_order) + assert.same({"A", "SRV"}, conf.resolver_family) assert.is_nil(getmetatable(conf.plugins)) assert.is_nil(getmetatable(conf.dns_order)) + assert.is_nil(getmetatable(conf.resolver_family)) end) it("trims array values", function() local conf = assert(conf_loader("spec/fixtures/to-strip.conf")) @@ -788,6 +790,31 @@ describe("Configuration loader", function() assert.is_nil(err) assert.is_table(conf) end) + it("errors when resolver_address is not a list in ipv4/6[:port] format (new dns)", function() + local conf, err = conf_loader(nil, { + resolver_address = "1.2.3.4:53;4.3.2.1" -- ; as separator + }) + assert.equal("resolver_address must be a comma separated list in the form of IPv4/6 or IPv4/6:port, got '1.2.3.4:53;4.3.2.1'", err) + assert.is_nil(conf) + + conf, err = conf_loader(nil, { + resolver_address = "198.51.100.0:53" + }) + assert.is_nil(err) + assert.is_table(conf) + + conf, err = conf_loader(nil, { + resolver_address = "[::1]:53" + }) + assert.is_nil(err) + assert.is_table(conf) + + conf, err = conf_loader(nil, { + resolver_address = "198.51.100.0,1.2.3.4:53,::1,[::1]:53" + }) + assert.is_nil(err) + assert.is_table(conf) + end) it("errors when node_id is not a valid uuid", function() local conf, err = conf_loader(nil, { node_id = "foobar", @@ -810,6 +837,15 @@ describe("Configuration loader", function() assert.equal([[dns_hostsfile: file does not exist]], err) assert.is_nil(conf) end) + it("errors when the hosts file does not exist (new dns)", function() + -- new dns + local tmpfile = "/a_file_that_does_not_exist" + local conf, err = conf_loader(nil, { + resolver_hosts_file = tmpfile, + }) + assert.equal([[resolver_hosts_file: file does not exist]], err) + assert.is_nil(conf) + end) it("accepts an existing hosts file", function() local tmpfile = require("pl.path").tmpname() -- this creates the file! finally(function() os.remove(tmpfile) end) @@ -819,6 +855,15 @@ describe("Configuration loader", function() assert.is_nil(err) assert.equal(tmpfile, conf.dns_hostsfile) end) + it("accepts an existing hosts file (new dns)", function() + local tmpfile = require("pl.path").tmpname() -- this creates the file! + finally(function() os.remove(tmpfile) end) + local conf, err = conf_loader(nil, { + resolver_hosts_file = tmpfile, + }) + assert.is_nil(err) + assert.equal(tmpfile, conf.resolver_hosts_file) + end) it("errors on bad entries in the order list", function() local conf, err = conf_loader(nil, { dns_order = "A,CXAME,SRV", @@ -826,6 +871,13 @@ describe("Configuration loader", function() assert.is_nil(conf) assert.equal([[dns_order: invalid entry 'CXAME']], err) end) + it("errors on bad entries in the family list", function() + local conf, err = conf_loader(nil, { + resolver_family = "A,AAAX,SRV", + }) + assert.is_nil(conf) + assert.equal([[resolver_family: invalid entry 'AAAX']], err) + end) it("errors on bad entries in headers", function() local conf, err = conf_loader(nil, { headers = "server_tokens,Foo-Bar", diff --git a/spec/02-integration/02-cmd/11-config_spec.lua b/spec/02-integration/02-cmd/11-config_spec.lua index 0a32456f26ac..e3d37f79a462 100644 --- a/spec/02-integration/02-cmd/11-config_spec.lua +++ b/spec/02-integration/02-cmd/11-config_spec.lua @@ -104,6 +104,7 @@ describe("kong config", function() assert(helpers.start_kong({ nginx_conf = "spec/fixtures/custom_nginx.template", dns_hostsfile = dns_hostsfile, + resolver_hosts_file = dns_hostsfile, anonymous_reports = "on", })) diff --git a/spec/02-integration/04-admin_api/11-reports_spec.lua b/spec/02-integration/04-admin_api/11-reports_spec.lua index 3abb81a1b87e..e07ce95d5d2a 100644 --- a/spec/02-integration/04-admin_api/11-reports_spec.lua +++ b/spec/02-integration/04-admin_api/11-reports_spec.lua @@ -61,6 +61,7 @@ for _, strategy in helpers.each_strategy() do nginx_conf = "spec/fixtures/custom_nginx.template", database = strategy, dns_hostsfile = dns_hostsfile, + resolver_hosts_file = dns_hostsfile, anonymous_reports = "on", declarative_config = yaml_file, }, {"routes", "services"})) diff --git a/spec/02-integration/05-proxy/22-reports_spec.lua b/spec/02-integration/05-proxy/22-reports_spec.lua index 7b43172860ea..5f82748efdbc 100644 --- a/spec/02-integration/05-proxy/22-reports_spec.lua +++ b/spec/02-integration/05-proxy/22-reports_spec.lua @@ -177,6 +177,7 @@ for _, strategy in helpers.each_strategy() do nginx_conf = "spec/fixtures/custom_nginx.template", database = strategy, dns_hostsfile = dns_hostsfile, + resolver_hosts_file = dns_hostsfile, anonymous_reports = true, plugins = "reports-api", stream_listen = helpers.get_proxy_ip(false) .. ":19000," .. diff --git a/spec/02-integration/10-go_plugins/01-reports_spec.lua b/spec/02-integration/10-go_plugins/01-reports_spec.lua index d3457d1683fe..6e6d1a321536 100644 --- a/spec/02-integration/10-go_plugins/01-reports_spec.lua +++ b/spec/02-integration/10-go_plugins/01-reports_spec.lua @@ -50,6 +50,7 @@ for _, strategy in helpers.each_strategy() do nginx_conf = "spec/fixtures/custom_nginx.template", database = strategy, dns_hostsfile = dns_hostsfile, + resolver_hosts_file = dns_hostsfile, plugins = "bundled,reports-api,go-hello", pluginserver_names = "test", pluginserver_test_socket = kong_prefix .. "/go-hello.socket", diff --git a/spec/02-integration/17-admin_gui/03-reports_spec.lua b/spec/02-integration/17-admin_gui/03-reports_spec.lua index d8de7e69e487..858fc840fed4 100644 --- a/spec/02-integration/17-admin_gui/03-reports_spec.lua +++ b/spec/02-integration/17-admin_gui/03-reports_spec.lua @@ -69,6 +69,7 @@ describe("anonymous reports for kong manager", function () anonymous_reports = true, plugins = "bundled,reports-api", dns_hostsfile = dns_hostsfile, + resolver_hosts_file = dns_hostsfile, })) finally(function() @@ -84,6 +85,7 @@ describe("anonymous reports for kong manager", function () anonymous_reports = true, plugins = "bundled,reports-api", dns_hostsfile = dns_hostsfile, + resolver_hosts_file = dns_hostsfile, })) finally(function() @@ -101,6 +103,7 @@ describe("anonymous reports for kong manager", function () anonymous_reports = true, plugins = "bundled,reports-api", dns_hostsfile = dns_hostsfile, + resolver_hosts_file = dns_hostsfile, })) local gui_dir_path = prepare_gui_dir() diff --git a/spec/02-integration/20-wasm/04-proxy-wasm_spec.lua b/spec/02-integration/20-wasm/04-proxy-wasm_spec.lua index f2c623128f92..4518eb657d53 100644 --- a/spec/02-integration/20-wasm/04-proxy-wasm_spec.lua +++ b/spec/02-integration/20-wasm/04-proxy-wasm_spec.lua @@ -138,6 +138,7 @@ describe("proxy-wasm filters (#wasm) (#" .. strategy .. ")", function() nginx_conf = "spec/fixtures/custom_nginx.template", wasm = true, dns_hostsfile = hosts_file, + resolver_hosts_file = hosts_file, plugins = "pre-function,post-function", })) end) diff --git a/spec/02-integration/20-wasm/07-reports_spec.lua b/spec/02-integration/20-wasm/07-reports_spec.lua index f305bdc26d2e..d62569c7fd4c 100644 --- a/spec/02-integration/20-wasm/07-reports_spec.lua +++ b/spec/02-integration/20-wasm/07-reports_spec.lua @@ -59,6 +59,7 @@ for _, strategy in helpers.each_strategy() do nginx_conf = "spec/fixtures/custom_nginx.template", database = strategy, dns_hostsfile = dns_hostsfile, + resolver_hosts_file = dns_hostsfile, plugins = "bundled,reports-api", wasm = true, anonymous_reports = true, diff --git a/spec/02-integration/22-ai_plugins/01-reports_spec.lua b/spec/02-integration/22-ai_plugins/01-reports_spec.lua index 9c4858e7127a..ab80378da634 100644 --- a/spec/02-integration/22-ai_plugins/01-reports_spec.lua +++ b/spec/02-integration/22-ai_plugins/01-reports_spec.lua @@ -157,6 +157,7 @@ for _, strategy in helpers.each_strategy() do nginx_conf = "spec/fixtures/custom_nginx.template", database = strategy, dns_hostsfile = dns_hostsfile, + resolver_hosts_file = dns_hostsfile, plugins = "bundled,reports-api", anonymous_reports = true, }, nil, nil, fixtures)) diff --git a/spec/fixtures/default_status_listen.conf b/spec/fixtures/default_status_listen.conf index 88a615dad0b8..43544b7aea12 100644 --- a/spec/fixtures/default_status_listen.conf +++ b/spec/fixtures/default_status_listen.conf @@ -16,6 +16,7 @@ pg_database = kong_tests anonymous_reports = off dns_hostsfile = spec/fixtures/hosts +resolver_hosts_file = spec/fixtures/hosts nginx_main_worker_processes = 1 nginx_main_worker_rlimit_nofile = 4096 diff --git a/spec/kong_tests.conf b/spec/kong_tests.conf index b6736f7cbf52..25d07a1f661f 100644 --- a/spec/kong_tests.conf +++ b/spec/kong_tests.conf @@ -30,6 +30,7 @@ worker_consistency = strict dedicated_config_processing = on dns_hostsfile = spec/fixtures/hosts +resolver_hosts_file = spec/fixtures/hosts nginx_main_worker_processes = 1 nginx_main_worker_rlimit_nofile = 4096