diff --git a/bin/kong b/bin/kong index d403aa2ef6c0..e03b945314ca 100755 --- a/bin/kong +++ b/bin/kong @@ -31,6 +31,7 @@ local cmds = { roar = true, hybrid = true, vault = true, + drain = true, } -- unnecessary to inject nginx directives for these simple cmds @@ -42,6 +43,7 @@ local skip_inject_cmds = { quit = true, health = true, hybrid = true, + drain = true, } for k in pairs(cmds) do diff --git a/changelog/unreleased/kong/feat-kong-drain-cmd.yml b/changelog/unreleased/kong/feat-kong-drain-cmd.yml new file mode 100644 index 000000000000..820825f95c41 --- /dev/null +++ b/changelog/unreleased/kong/feat-kong-drain-cmd.yml @@ -0,0 +1,3 @@ +message: Add the `kong drain` CLI command to make the `/status/ready` endpoint return `503 Service Unavailable` response. +type: feature +scope: CLI Command diff --git a/kong-3.9.0-0.rockspec b/kong-3.9.0-0.rockspec index 0cbe859d1e2c..3a7ddf689b58 100644 --- a/kong-3.9.0-0.rockspec +++ b/kong-3.9.0-0.rockspec @@ -145,6 +145,7 @@ build = { ["kong.cmd.vault"] = "kong/cmd/vault.lua", ["kong.cmd.version"] = "kong/cmd/version.lua", ["kong.cmd.hybrid"] = "kong/cmd/hybrid.lua", + ["kong.cmd.drain"] = "kong/cmd/drain.lua", ["kong.cmd.utils.log"] = "kong/cmd/utils/log.lua", ["kong.cmd.utils.kill"] = "kong/cmd/utils/kill.lua", ["kong.cmd.utils.env"] = "kong/cmd/utils/env.lua", diff --git a/kong/cmd/drain.lua b/kong/cmd/drain.lua new file mode 100644 index 000000000000..9cb41d337250 --- /dev/null +++ b/kong/cmd/drain.lua @@ -0,0 +1,88 @@ +local http = require "resty.luasocket.http" +local print = print +local fmt = string.format + +local conf_loader = require "kong.conf_loader" +local pl_path = require "pl.path" +local log = require "kong.cmd.utils.log" +local json_encode = require("cjson.safe").encode + +local function execute(args) + log.disable() + + -- only to retrieve the default prefix or use given one + local conf = assert(conf_loader(args.conf, { + prefix = args.prefix + })) + + if pl_path.exists(conf.kong_env) then + -- load /kong.conf containing running node's config + conf = assert(conf_loader(conf.kong_env)) + end + + log.enable() + + if #conf.status_listeners == 0 then + print("No status listeners found in configuration.") + return + end + + local status_listener = conf.status_listeners[1] + + local scheme = "http" + if status_listener.ssl then + scheme = "https" + end + + local url = scheme .. "://" .. status_listener.ip .. ":" .. status_listener.port .. "/status/ready" + + local httpc = http.new() + httpc:set_timeout(1000) + + + local res, err = httpc:request_uri(url, { + method = "POST", + headers = { + ["Content-Type"] = "application/json" + }, + body = json_encode({ + status = "draining" + }), + -- we don't need to verify the SSL certificate for this request + ssl_verify = false, + }) + + httpc:close() + + if not res then + print(fmt("Failed to send request to %s: %s", url, err)) + return + end + + if res.status ~= 204 then + print(fmt("Unexpected response status from %s: %d", url, res.status)) + return + end + + print("Kong's status successfully changed to 'draining'") +end + + +local lapp = [[ +Usage: kong drain [OPTIONS] + +Make status listeners(`/status/ready`) return 503 Service Unavailable. + +Example usage: + kong drain + +Options: + -c,--conf (optional string) configuration file + -p,--prefix (optional string) override prefix directory +]] + + +return { + lapp = lapp, + execute = execute, +} diff --git a/kong/status/ready.lua b/kong/status/ready.lua index 4058fb2af1e3..d83757fc24c7 100644 --- a/kong/status/ready.lua +++ b/kong/status/ready.lua @@ -23,6 +23,7 @@ local PLUGINS_REBUILD_COUNTER_KEY = constants.PLUGINS_REBUILD_COUNTER_KEY local ROUTERS_REBUILD_COUNTER_KEY = constants.ROUTERS_REBUILD_COUNTER_KEY local DECLARATIVE_EMPTY_CONFIG_HASH = constants.DECLARATIVE_EMPTY_CONFIG_HASH +local KONG_STATUS_READY = "kong:status:ready" local function is_dbless_ready(router_rebuilds, plugins_iterator_rebuilds) if router_rebuilds < worker_count then @@ -75,16 +76,16 @@ local function is_ready() if not ok then return false, "failed to connect to database" end - + kong.db:close() if is_control_plane then return true end - local router_rebuilds = + local router_rebuilds = tonumber(kong_shm:get(ROUTERS_REBUILD_COUNTER_KEY)) or 0 - local plugins_iterator_rebuilds = + local plugins_iterator_rebuilds = tonumber(kong_shm:get(PLUGINS_REBUILD_COUNTER_KEY)) or 0 local err @@ -102,6 +103,15 @@ end return { ["/status/ready"] = { GET = function(self, dao, helpers) + local ready = kong_shm:get(KONG_STATUS_READY) + if ready == nil then + kong_shm:set(KONG_STATUS_READY, true) + end + + if ready == false then + return kong.response.exit(503, { message = "draining" }) + end + local ok, err = is_ready() if ok then ngx_log(ngx_DEBUG, "ready for proxying") @@ -111,6 +121,15 @@ return { ngx_log(ngx_NOTICE, "not ready for proxying: ", err) return kong.response.exit(503, { message = err }) end + end, + + POST = function(self, dao, helpers) + if self.params and self.params.status == "draining" then + kong_shm:set(KONG_STATUS_READY, false) + return kong.response.exit(204) + end + + return kong.response.exit(400) end - } + }, } diff --git a/spec/02-integration/02-cmd/17-drain_spec.lua b/spec/02-integration/02-cmd/17-drain_spec.lua new file mode 100644 index 000000000000..a2f87b143a4a --- /dev/null +++ b/spec/02-integration/02-cmd/17-drain_spec.lua @@ -0,0 +1,297 @@ +local helpers = require "spec.helpers" +local http = require "resty.http" + +local cp_status_port = helpers.get_available_port() +local dp_status_port = 8100 + +local function get_status_no_ssl_verify() + local httpc = http.new() + + local ok, err = httpc:connect({ + scheme = "https", + host = "127.0.0.1", + port = dp_status_port, + ssl_verify = false, + }) + if not ok then + return nil, err + end + + local res, err = httpc:request({ + path = "/status/ready", + headers = { + ["Content-Type"] = "application/json", + } + }) + + if not res then + return nil, err + end + + return res.status +end + +local function verify_status(port, code) + local status_client = assert(helpers.http_client("127.0.0.1", port, 20000)) + + local res = status_client:send({ + method = "GET", + path = "/status/ready", + }) + + status_client:close() + local status = res and res.status + + if status == code then + return true + end + + return false +end + +for _, strategy in helpers.each_strategy() do + if strategy ~= "off" then + -- skip the "off" strategy, as dbless has its own test suite + describe("kong drain with #" .. strategy .. " backend", function() + lazy_setup(function() + helpers.get_db_utils(strategy, {}) -- runs migrations + + assert(helpers.start_kong({ + database = strategy, + nginx_conf = "spec/fixtures/custom_nginx.template", + status_listen = "127.0.0.1:8100", + nginx_main_worker_processes = 8, + log_level = "info", + })) + end) + + lazy_teardown(function() + helpers.stop_kong() + end) + + it("should set Kong to 'draining'", function() + helpers.wait_until(function() + return verify_status(dp_status_port, 200) + end, 10) + + local ok, err, msg = helpers.kong_exec("drain", { + prefix = helpers.test_conf.prefix, + }) + assert.equal("", err) + assert.equal("Kong's status successfully changed to 'draining'\n", msg) + assert.equal(true, ok) + + + helpers.wait_until(function() + return verify_status(dp_status_port, 503) + end, 10) + end) + end) + + describe("Kong without a status listener", function() + lazy_setup(function() + helpers.get_db_utils(strategy, {}) -- runs migrations + + assert(helpers.start_kong({ + database = strategy, + nginx_conf = "spec/fixtures/custom_nginx.template", + })) + end) + + lazy_teardown(function() + helpers.stop_kong() + end) + + it("should return an error when trying to set 'draining' without a status listener", function() + local ok, err, msg = helpers.kong_exec("drain", { + prefix = helpers.test_conf.prefix, + }) + assert.equal("", err) + assert.equal("No status listeners found in configuration.\n", msg) + assert.equal(true, ok) + end) + + end) + + describe("Kong with SSL-enabled status listener", function() + lazy_setup(function() + helpers.get_db_utils(strategy, {}) -- runs migrations + + assert(helpers.start_kong({ + database = strategy, + nginx_conf = "spec/fixtures/custom_nginx.template", + status_listen = "127.0.0.1:8100 ssl", + nginx_main_worker_processes = 8, + })) + end) + + lazy_teardown(function() + helpers.stop_kong() + end) + + it("should set Kong to 'draining' with SSL-enabled status listener", function() + helpers.wait_until(function() + local status = get_status_no_ssl_verify() + if status == 200 then + return true + end + end, 10) + + local ok, err, msg = helpers.kong_exec("drain", { + prefix = helpers.test_conf.prefix, + }) + assert.equal("", err) + assert.equal("Kong's status successfully changed to 'draining'\n", msg) + assert.equal(true, ok) + + helpers.wait_until(function() + local status = get_status_no_ssl_verify() + if status == 503 then + return true + end + end, 10) + end) + end) + end +end + +for _, strategy in helpers.each_strategy({"postgres"}) do + describe("kong drain in hybrid mode #" .. strategy, function() + local bp = helpers.get_db_utils(strategy, { + "services", + }) + + -- insert some data to make sure the control plane is ready and send the configuration to dp + -- so that `current_hash` of dp wouldn't be DECLARATIVE_EMPTY_CONFIG_HASH, so that dp would be ready + assert(bp.services:insert { + name = "example", + }) + + lazy_setup(function() + assert(helpers.start_kong({ + role = "data_plane", + database = "off", + cluster_cert = "spec/fixtures/kong_clustering.crt", + cluster_cert_key = "spec/fixtures/kong_clustering.key", + cluster_control_plane = "127.0.0.1:9005", + proxy_listen = "127.0.0.1:9002", + nginx_worker_processes = 8, + status_listen = "127.0.0.1:" .. dp_status_port, + prefix = "serve_dp", + log_level = "info", + })) + + assert(helpers.start_kong({ + role = "control_plane", + cluster_cert = "spec/fixtures/kong_clustering.crt", + cluster_cert_key = "spec/fixtures/kong_clustering.key", + database = strategy, + prefix = "serve_cp", + cluster_listen = "127.0.0.1:9005", + nginx_conf = "spec/fixtures/custom_nginx.template", + status_listen = "127.0.0.1:" .. cp_status_port + })) + end) + + lazy_teardown(function() + helpers.stop_kong("serve_dp") + helpers.stop_kong("serve_cp") + end) + + it("should set Kong to 'draining'", function() + helpers.wait_until(function() + return verify_status(dp_status_port, 200) + end, 10) + + -- set dp to draining + local ok, err, msg = helpers.kong_exec("drain --prefix serve_dp", { + prefix = helpers.test_conf.prefix, + database = "off", + }) + assert.equal("", err) + assert.equal("Kong's status successfully changed to 'draining'\n", msg) + assert.equal(true, ok) + + helpers.wait_until(function() + return verify_status(dp_status_port, 503) + end, 10) + + -- set cp to draining + local ok, err, msg = helpers.kong_exec("drain --prefix serve_cp", { + prefix = helpers.test_conf.prefix, + }) + assert.equal("", err) + assert.equal("Kong's status successfully changed to 'draining'\n", msg) + assert.equal(true, ok) + + helpers.wait_until(function() + return verify_status(cp_status_port, 503) + end, 10) + end) + end) +end + +describe("kong drain in DB-less mode #off", function() + local admin_client + + lazy_setup(function() + assert(helpers.start_kong ({ + status_listen = "127.0.0.1:8100", + plugins = "admin-api-method", + database = "off", + nginx_main_worker_processes = 8, + log_level = "info", + })) + end) + + before_each(function() + admin_client = helpers.admin_client() + end) + + after_each(function() + if admin_client then + admin_client:close() + end + end) + + lazy_teardown(function() + helpers.stop_kong() + end) + + it("should set Kong to 'draining'", function() + local res = assert(admin_client:send { + method = "POST", + path = "/config", + body = { + config = [[ + _format_version: "3.0" + services: + - name: example + url: http://mockbin.org + ]] + }, + headers = { + ["Content-Type"] = "multipart/form-data" + }, + }) + + assert.res_status(201, res) + + helpers.wait_until(function() + return verify_status(dp_status_port, 200) + end, 10) + + + local ok, err, msg = helpers.kong_exec("drain", { + prefix = helpers.test_conf.prefix, + database = "off", + }) + assert.equal("", err) + assert.equal("Kong's status successfully changed to 'draining'\n", msg) + assert.equal(true, ok) + + helpers.wait_until(function() + return verify_status(dp_status_port, 503) + end, 10) + end) +end)