feat(test): add reconfiguration completion detection test plugin

Unlike the previous implementation, this one does not require changes
to Kong and its proxy path.  It works based on the assumption that the
order of admin API changes is preserved.  The admin API client marks
the end of the changes that it needs to see propagated to the data
plane(s) by changing the configuration of this plugin, setting a
particular configuration version number.  On the proxy path, a header
X-Kong-Configuration-Version is sent with that version number.  The
plugin's access handler verifies that the version number configured in
the plugin (on the dataplane) matches the version number requested by
the client.  If the version numbers do not match, a 503 error is
generated, which causes the client to retry.

The plugin is available only to busted tests.  It needs to be enabled
when starting Kong.

A new busted test helper function make_synchronized_clients is
provided that automatically synchronizes a proxy client and an admin
API client.  The the test can freely mix invocations to either
endpoints.  Whenever a change is made through the admin API, the proxy
path request is delayed until the change has propagated to the data
plane.  spec/02-integration/13-vaults/06-refresh-secrets_spec.lua has
been updated to use the function as an illustration.

spec/02-integration/13-vaults/06-refresh-secrets_spec.lua

@@ -34,14 +34,13 @@ for _, strategy in helpers.each_strategy() do
         database = strategy,
         prefix = helpers.test_conf.prefix,
         nginx_conf = "spec/fixtures/custom_nginx.template",
-        plugins = "dummy",
+        plugins = "dummy,reconfiguration-completion",
         vaults = "env",
       })
     end)
 
     before_each(function()
-      admin_client = assert(helpers.admin_client())
-      proxy_client = assert(helpers.proxy_client())
+      proxy_client, admin_client = helpers.make_synchronized_clients()
     end)
 
     after_each(function()
@@ -76,15 +75,13 @@ for _, strategy in helpers.each_strategy() do
       })
       assert.res_status(200, res)
 
-      assert
-          .with_timeout(10)
-          .eventually(function()
-            local res = proxy_client:send {
-              method = "GET",
-              path = "/",
-            }
-            return res and res.status == 200 and res.headers["Dummy-Plugin"] == "MONSTER" and res.headers["X-Test-This"] == "SPIRIT"
-          end).is_truthy("Could not find header in request")
+      local res = proxy_client:send {
+        method = "GET",
+        path = "/",
+      }
+      assert.res_status(200, res)
+      assert.is_same("MONSTER", res.headers["Dummy-Plugin"])
+      assert.is_same("SPIRIT", res.headers["X-Test-This"])
     end)
   end)
 end

spec/03-plugins/39-reconfiguration-completion/01-access_spec.lua

@@ -0,0 +1,186 @@
+local helpers = require "spec.helpers"
+local cjson = require "cjson"
+local utils = require "kong.tools.utils"
+
+describe("Reconfiguration completion detection plugin", function()
+
+  local STATE_UPDATE_FREQUENCY = .2
+
+  local admin_client
+  local proxy_client
+
+  local function plugin_tests()
+
+    local configuration_version = utils.uuid()
+
+    local res = admin_client:post("/plugins", {
+      body = {
+        name = "reconfiguration-completion",
+        config = {
+          version = configuration_version,
+        }
+      },
+      headers = { ["Content-Type"] = "application/json" },
+    })
+    local body = assert.res_status(201, res)
+    local plugin = cjson.decode(body)
+    local reconfiguration_completion_plugin_id = plugin.id
+
+    res = admin_client:post("/plugins", {
+      body = {
+        name = "request-termination",
+        config = {
+          status_code = 200,
+          body = "kong terminated the request",
+        }
+      },
+      headers = { ["Content-Type"] = "application/json" },
+    })
+    assert.res_status(201, res)
+
+    res = admin_client:post("/services", {
+      body = {
+        name = "test-service",
+        url = "http://127.0.0.1",
+      },
+      headers = { ["Content-Type"] = "application/json" },
+    })
+    body = assert.res_status(201, res)
+    local service = cjson.decode(body)
+
+    -- We're running the route setup in `eventually` to cover for the unlikely case that reconfiguration completes
+    -- between adding the route, updating the plugin and requesting the path through the proxy path.
+
+    local next_path do
+      local path_suffix = 0
+      function next_path()
+        path_suffix = path_suffix + 1
+        return "/" .. tostring(path_suffix)
+      end
+    end
+
+    local service_path
+
+    assert.eventually(function()
+      service_path = next_path()
+
+      res = admin_client:post("/services/" .. service.id .. "/routes", {
+        body = {
+          paths = { service_path }
+        },
+        headers = { ["Content-Type"] = "application/json" },
+      })
+      assert.res_status(201, res)
+
+      configuration_version = utils.uuid()
+      res = admin_client:patch("/plugins/" .. reconfiguration_completion_plugin_id, {
+        body = {
+          config = {
+            version = configuration_version,
+          }
+        },
+        headers = { ["Content-Type"] = "application/json" },
+      })
+      assert.res_status(200, res)
+
+      res = proxy_client:get(service_path,
+              {
+                headers = {
+                  ["If-Kong-Configuration-Version"] = configuration_version
+                }
+              })
+      assert.res_status(503, res)
+      assert.equals("pending", res.headers['x-kong-reconfiguration-status'])
+      local retry_after = tonumber(res.headers['retry-after'])
+      ngx.sleep(retry_after)
+    end)
+            .with_timeout(10)
+            .has_no_error()
+
+    assert.eventually(function()
+      res = proxy_client:get(service_path,
+              {
+                headers = {
+                  ["If-Kong-Configuration-Version"] = configuration_version
+                }
+              })
+      body = assert.res_status(200, res)
+      assert.equals("kong terminated the request", body)
+    end)
+            .has_no_error()
+  end
+
+  describe("#traditional mode", function()
+    lazy_setup(function()
+      helpers.get_db_utils()
+      assert(helpers.start_kong({
+        plugins = "bundled,reconfiguration-completion",
+        worker_consistency = "eventual",
+        worker_state_update_frequency = STATE_UPDATE_FREQUENCY,
+      }))
+      admin_client = helpers.admin_client()
+      proxy_client = helpers.proxy_client()
+    end)
+
+    teardown(function()
+      if admin_client then
+        admin_client:close()
+      end
+      if proxy_client then
+        proxy_client:close()
+      end
+      helpers.stop_kong()
+    end)
+
+    it('', plugin_tests)
+  end)
+
+  describe("#hybrid mode", function()
+    lazy_setup(function()
+      helpers.get_db_utils()
+
+      assert(helpers.start_kong({
+        plugins = "bundled,reconfiguration-completion",
+        role = "control_plane",
+        database = "postgres",
+        prefix = "cp",
+        cluster_cert = "spec/fixtures/kong_clustering.crt",
+        cluster_cert_key = "spec/fixtures/kong_clustering.key",
+        lua_ssl_trusted_certificate = "spec/fixtures/kong_clustering.crt",
+        cluster_listen = "127.0.0.1:9005",
+        cluster_telemetry_listen = "127.0.0.1:9006",
+        nginx_conf = "spec/fixtures/custom_nginx.template",
+        db_update_frequency = STATE_UPDATE_FREQUENCY,
+     }))
+
+      assert(helpers.start_kong({
+        plugins = "bundled,reconfiguration-completion",
+        role = "data_plane",
+        database = "off",
+        prefix = "dp",
+        cluster_cert = "spec/fixtures/kong_clustering.crt",
+        cluster_cert_key = "spec/fixtures/kong_clustering.key",
+        lua_ssl_trusted_certificate = "spec/fixtures/kong_clustering.crt",
+        cluster_control_plane = "127.0.0.1:9005",
+        cluster_telemetry_endpoint = "127.0.0.1:9006",
+        proxy_listen = "0.0.0.0:9002",
+        worker_state_update_frequency = STATE_UPDATE_FREQUENCY,
+      }))
+      admin_client = helpers.admin_client()
+      proxy_client = helpers.proxy_client("127.0.0.1", 9002)
+    end)
+
+    teardown(function()
+      if admin_client then
+        admin_client:close()
+      end
+      if proxy_client then
+        proxy_client:close()
+      end
+      helpers.stop_kong("dp")
+      helpers.stop_kong("cp")
+    end)
+
+    it('', plugin_tests)
+  end)
+end)

spec/03-plugins/39-reconfiguration-completion/02-helper_spec.lua

@@ -0,0 +1,167 @@
+local helpers = require "spec.helpers"
+local cjson = require "cjson"
+
+describe("Reconfiguration completion detection helper", function()
+
+  local STATE_UPDATE_FREQUENCY = .2
+
+  local admin_client
+  local proxy_client
+
+  local function helper_tests(make_proxy_client)
+    local res = admin_client:post("/plugins", {
+      body = {
+        name = "request-termination",
+        config = {
+          status_code = 200,
+          body = "kong terminated the request",
+        }
+      },
+      headers = { ["Content-Type"] = "application/json" },
+    })
+    local body = assert.res_status(201, res)
+    local request_termination_plugin_id = cjson.decode(body).id
+
+    res = admin_client:post("/services", {
+      body = {
+        name = "test-service",
+        url = "http://127.0.0.1",
+      },
+      headers = { ["Content-Type"] = "application/json" },
+    })
+    body = assert.res_status(201, res)
+    local service = cjson.decode(body)
+
+    local path = "/foo-barak"
+
+    res = admin_client:post("/services/" .. service.id .. "/routes", {
+      body = {
+        paths = { path }
+      },
+      headers = { ["Content-Type"] = "application/json" },
+    })
+    assert.res_status(201, res)
+
+    res = proxy_client:get(path)
+    body = assert.res_status(200, res)
+    assert.equals("kong terminated the request", body)
+
+    res = admin_client:patch("/plugins/" .. request_termination_plugin_id, {
+      body = {
+        config = {
+          status_code = 404,
+          body = "kong terminated the request with 404",
+        }
+      },
+      headers = { ["Content-Type"] = "application/json" },
+    })
+    assert.res_status(200, res)
+
+    res = proxy_client:get(path)
+    body = assert.res_status(404, res)
+    assert.equals("kong terminated the request with 404", body)
+
+    local second_admin_client = helpers.admin_client()
+    admin_client:synchronize_sibling(second_admin_client)
+
+    res = second_admin_client:patch("/plugins/" .. request_termination_plugin_id, {
+      body = {
+        config = {
+          status_code = 405,
+          body = "kong terminated the request with 405",
+        }
+      },
+      headers = { ["Content-Type"] = "application/json" },
+    })
+    assert.res_status(200, res)
+
+    local second_proxy_client = make_proxy_client()
+    proxy_client:synchronize_sibling(second_proxy_client)
+
+    res = second_proxy_client:get(path)
+    body = assert.res_status(405, res)
+    assert.equals("kong terminated the request with 405", body)
+  end
+
+  describe("#traditional mode", function()
+
+    local function make_proxy_client()
+      return helpers.proxy_client()
+    end
+
+    lazy_setup(function()
+      helpers.get_db_utils()
+      assert(helpers.start_kong({
+        plugins = "bundled,reconfiguration-completion",
+        worker_consistency = "eventual",
+        worker_state_update_frequency = STATE_UPDATE_FREQUENCY,
+      }))
+      proxy_client, admin_client = helpers.make_synchronized_clients()
+    end)
+
+    teardown(function()
+      if admin_client then
+        admin_client:close()
+      end
+      if proxy_client then
+        proxy_client:close()
+      end
+      helpers.stop_kong()
+    end)
+
+    it('', function () helper_tests(make_proxy_client) end)
+  end)
+
+  describe("#hybrid mode", function()
+
+    local function make_proxy_client()
+      return helpers.proxy_client("127.0.0.1", 9002)
+    end
+
+    lazy_setup(function()
+      helpers.get_db_utils()
+
+      assert(helpers.start_kong({
+        plugins = "bundled,reconfiguration-completion",
+        role = "control_plane",
+        database = "postgres",
+        prefix = "cp",
+        cluster_cert = "spec/fixtures/kong_clustering.crt",
+        cluster_cert_key = "spec/fixtures/kong_clustering.key",
+        lua_ssl_trusted_certificate = "spec/fixtures/kong_clustering.crt",
+        cluster_listen = "127.0.0.1:9005",
+        cluster_telemetry_listen = "127.0.0.1:9006",
+        nginx_conf = "spec/fixtures/custom_nginx.template",
+        db_update_frequency = STATE_UPDATE_FREQUENCY,
+      }))
+
+      assert(helpers.start_kong({
+        plugins = "bundled,reconfiguration-completion",
+        role = "data_plane",
+        database = "off",
+        prefix = "dp",
+        cluster_cert = "spec/fixtures/kong_clustering.crt",
+        cluster_cert_key = "spec/fixtures/kong_clustering.key",
+        lua_ssl_trusted_certificate = "spec/fixtures/kong_clustering.crt",
+        cluster_control_plane = "127.0.0.1:9005",
+        cluster_telemetry_endpoint = "127.0.0.1:9006",
+        proxy_listen = "0.0.0.0:9002",
+        worker_state_update_frequency = STATE_UPDATE_FREQUENCY,
+      }))
+      proxy_client, admin_client = helpers.make_synchronized_clients({ proxy_client = make_proxy_client() })
+    end)
+
+    teardown(function()
+      if admin_client then
+        admin_client:close()
+      end
+      if proxy_client then
+        proxy_client:close()
+      end
+      helpers.stop_kong("dp")
+      helpers.stop_kong("cp")
+    end)
+
+    it('', function () helper_tests(make_proxy_client) end)
+  end)
+end)