tests(clustering): enable incremental sync in hybrid mode (#13914)

See: #13906, #13876

KAG-5918, KAG-5876

spec/02-integration/07-sdk/03-cluster_spec.lua

@@ -40,8 +40,9 @@ fixtures_cp.http_mock.my_server_block = [[
   }
 ]]
 
--- TODO: reenable the inc sync test
-for _, inc_sync in ipairs { "off"  } do
+for _, v in ipairs({ {"off", "off"}, {"on", "off"}, {"on", "on"}, }) do
+  local rpc, inc_sync = v[1], v[2]
+
 for _, strategy in helpers.each_strategy() do
   describe("PDK: kong.cluster for #" .. strategy .. " inc_sync=" .. inc_sync, function()
     local proxy_client
@@ -63,6 +64,7 @@ for _, strategy in helpers.each_strategy() do
         db_update_frequency = 0.1,
         cluster_listen = "127.0.0.1:9005",
         nginx_conf = "spec/fixtures/custom_nginx.template",
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }, nil, nil, fixtures_cp))
 
@@ -75,6 +77,7 @@ for _, strategy in helpers.each_strategy() do
         cluster_control_plane = "127.0.0.1:9005",
         proxy_listen = "0.0.0.0:9002",
         nginx_conf = "spec/fixtures/custom_nginx.template",
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }, nil, nil, fixtures_dp))
     end)

spec/02-integration/09-hybrid_mode/01-sync_spec.lua

@@ -12,8 +12,9 @@ local uuid = require("kong.tools.uuid").uuid
 local KEY_AUTH_PLUGIN
 
 
---- XXX FIXME: enable incremental sync "on"
-for _, inc_sync in ipairs { "off" } do
+for _, v in ipairs({ {"off", "off"}, {"on", "off"}, {"on", "on"}, }) do
+  local rpc, inc_sync = v[1], v[2]
+
 for _, strategy in helpers.each_strategy() do
 
 --- XXX FIXME: enable inc_sync = on
@@ -33,6 +34,7 @@ describe("CP/DP communication #" .. strategy .. " inc_sync=" .. inc_sync, functi
       db_update_frequency = 0.1,
       cluster_listen = "127.0.0.1:9005",
       nginx_conf = "spec/fixtures/custom_nginx.template",
+      cluster_rpc = rpc,
       cluster_incremental_sync = inc_sync,
     }))
 
@@ -45,6 +47,7 @@ describe("CP/DP communication #" .. strategy .. " inc_sync=" .. inc_sync, functi
       cluster_control_plane = "127.0.0.1:9005",
       proxy_listen = "0.0.0.0:9002",
       nginx_conf = "spec/fixtures/custom_nginx.template",
+      cluster_rpc = rpc,
       cluster_incremental_sync = inc_sync,
       worker_state_update_frequency = 1,
     }))
@@ -366,6 +369,7 @@ describe("CP/DP #version check #" .. strategy .. " inc_sync=" .. inc_sync, funct
         cluster_listen = "127.0.0.1:9005",
         nginx_conf = "spec/fixtures/custom_nginx.template",
         cluster_version_check = "major_minor",
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }))
 
@@ -634,6 +638,7 @@ describe("CP/DP config sync #" .. strategy .. " inc_sync=" .. inc_sync, function
       database = strategy,
       db_update_frequency = 3,
       cluster_listen = "127.0.0.1:9005",
+      cluster_rpc = rpc,
       cluster_incremental_sync = inc_sync,
     }))
 
@@ -646,6 +651,7 @@ describe("CP/DP config sync #" .. strategy .. " inc_sync=" .. inc_sync, function
       cluster_control_plane = "127.0.0.1:9005",
       proxy_listen = "0.0.0.0:9002",
       cluster_incremental_sync = inc_sync,
+      cluster_rpc = rpc,
       worker_state_update_frequency = 1,
     }))
   end)
@@ -749,6 +755,7 @@ skip_inc_sync("CP/DP labels #" .. strategy, function()
       db_update_frequency = 0.1,
       cluster_listen = "127.0.0.1:9005",
       nginx_conf = "spec/fixtures/custom_nginx.template",
+      cluster_rpc = rpc,
       cluster_incremental_sync = inc_sync,
     }))
 
@@ -762,6 +769,7 @@ skip_inc_sync("CP/DP labels #" .. strategy, function()
       proxy_listen = "0.0.0.0:9002",
       nginx_conf = "spec/fixtures/custom_nginx.template",
       cluster_dp_labels="deployment:mycloud,region:us-east-1",
+      cluster_rpc = rpc,
       cluster_incremental_sync = inc_sync,
     }))
   end)
@@ -811,6 +819,7 @@ skip_inc_sync("CP/DP cert details(cluster_mtls = shared) #" .. strategy, functio
       db_update_frequency = 0.1,
       cluster_listen = "127.0.0.1:9005",
       nginx_conf = "spec/fixtures/custom_nginx.template",
+      cluster_rpc = rpc,
       cluster_incremental_sync = inc_sync,
     }))
 
@@ -824,6 +833,7 @@ skip_inc_sync("CP/DP cert details(cluster_mtls = shared) #" .. strategy, functio
       proxy_listen = "0.0.0.0:9002",
       nginx_conf = "spec/fixtures/custom_nginx.template",
       cluster_dp_labels="deployment:mycloud,region:us-east-1",
+      cluster_rpc = rpc,
       cluster_incremental_sync = inc_sync,
     }))
   end)
@@ -871,6 +881,7 @@ skip_inc_sync("CP/DP cert details(cluster_mtls = pki) #" .. strategy, function()
       -- additional attributes for PKI:
       cluster_mtls = "pki",
       cluster_ca_cert = "spec/fixtures/kong_clustering_ca.crt",
+      cluster_rpc = rpc,
       cluster_incremental_sync = inc_sync,
     }))
 
@@ -887,6 +898,7 @@ skip_inc_sync("CP/DP cert details(cluster_mtls = pki) #" .. strategy, function()
       cluster_mtls = "pki",
       cluster_server_name = "kong_clustering",
       cluster_ca_cert = "spec/fixtures/kong_clustering.crt",
+      cluster_rpc = rpc,
       cluster_incremental_sync = inc_sync,
     }))
   end)

spec/02-integration/09-hybrid_mode/03-pki_spec.lua

@@ -2,8 +2,9 @@ local helpers = require "spec.helpers"
 local cjson = require "cjson.safe"
 
 
--- TODO: reenable the inc sync test
-for _, inc_sync in ipairs { "off"  } do
+for _, v in ipairs({ {"off", "off"}, {"on", "off"}, {"on", "on"}, }) do
+  local rpc, inc_sync = v[1], v[2]
+
 for _, strategy in helpers.each_strategy() do
 
 describe("CP/DP PKI sync #" .. strategy .. " inc_sync=" .. inc_sync, function()
@@ -27,6 +28,7 @@ describe("CP/DP PKI sync #" .. strategy .. " inc_sync=" .. inc_sync, function()
       -- additional attributes for PKI:
       cluster_mtls = "pki",
       cluster_ca_cert = "spec/fixtures/kong_clustering_ca.crt",
+      cluster_rpc = rpc,
       cluster_incremental_sync = inc_sync,
     }))
 
@@ -43,6 +45,7 @@ describe("CP/DP PKI sync #" .. strategy .. " inc_sync=" .. inc_sync, function()
       cluster_mtls = "pki",
       cluster_server_name = "kong_clustering",
       cluster_ca_cert = "spec/fixtures/kong_clustering.crt",
+      cluster_rpc = rpc,
       cluster_incremental_sync = inc_sync,
       worker_state_update_frequency = 1,
     }))

spec/02-integration/09-hybrid_mode/04-cp_cluster_sync_spec.lua

@@ -20,8 +20,9 @@ local function find_in_file(filepath, pat)
 end
 
 
--- TODO: reenable the inc sync test
-for _, inc_sync in ipairs { "off"  } do
+for _, v in ipairs({ {"off", "off"}, {"on", "off"}, {"on", "on"}, }) do
+  local rpc, inc_sync = v[1], v[2]
+
 for _, strategy in helpers.each_strategy() do
   describe("CP/CP sync works with #" .. strategy .. " inc_sync=" .. inc_sync .. " backend", function()
     lazy_setup(function()
@@ -37,6 +38,7 @@ for _, strategy in helpers.each_strategy() do
         cluster_cert = "spec/fixtures/kong_clustering.crt",
         cluster_cert_key = "spec/fixtures/kong_clustering.key",
         database = strategy,
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }))
 
@@ -50,6 +52,7 @@ for _, strategy in helpers.each_strategy() do
         cluster_cert = "spec/fixtures/kong_clustering.crt",
         cluster_cert_key = "spec/fixtures/kong_clustering.key",
         database = strategy,
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }))
 

spec/02-integration/09-hybrid_mode/05-ocsp_spec.lua

@@ -14,8 +14,9 @@ local function set_ocsp_status(status)
 end
 
 
--- TODO: reenable the inc sync test
-for _, inc_sync in ipairs { "off"  } do
+for _, v in ipairs({ {"off", "off"}, {"on", "off"}, {"on", "on"}, }) do
+  local rpc, inc_sync = v[1], v[2]
+
 for _, strategy in helpers.each_strategy() do
 
 describe("cluster_ocsp = on works #" .. strategy .. " inc_sync=" .. inc_sync, function()
@@ -42,6 +43,7 @@ describe("cluster_ocsp = on works #" .. strategy .. " inc_sync=" .. inc_sync, fu
         -- additional attributes for PKI:
         cluster_mtls = "pki",
         cluster_ca_cert = "spec/fixtures/ocsp_certs/ca.crt",
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }))
 
@@ -60,6 +62,7 @@ describe("cluster_ocsp = on works #" .. strategy .. " inc_sync=" .. inc_sync, fu
         cluster_mtls = "pki",
         cluster_server_name = "kong_clustering",
         cluster_ca_cert = "spec/fixtures/ocsp_certs/ca.crt",
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }))
     end)
@@ -114,6 +117,7 @@ describe("cluster_ocsp = on works #" .. strategy .. " inc_sync=" .. inc_sync, fu
         -- additional attributes for PKI:
         cluster_mtls = "pki",
         cluster_ca_cert = "spec/fixtures/ocsp_certs/ca.crt",
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }))
 
@@ -132,6 +136,7 @@ describe("cluster_ocsp = on works #" .. strategy .. " inc_sync=" .. inc_sync, fu
         cluster_server_name = "kong_clustering",
         cluster_ca_cert = "spec/fixtures/ocsp_certs/ca.crt",
         nginx_conf = "spec/fixtures/custom_nginx.template",
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }))
     end)
@@ -184,6 +189,7 @@ describe("cluster_ocsp = on works #" .. strategy .. " inc_sync=" .. inc_sync, fu
         -- additional attributes for PKI:
         cluster_mtls = "pki",
         cluster_ca_cert = "spec/fixtures/ocsp_certs/ca.crt",
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }))
 
@@ -202,6 +208,7 @@ describe("cluster_ocsp = on works #" .. strategy .. " inc_sync=" .. inc_sync, fu
         cluster_server_name = "kong_clustering",
         cluster_ca_cert = "spec/fixtures/ocsp_certs/ca.crt",
         nginx_conf = "spec/fixtures/custom_nginx.template",
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }))
     end)
@@ -257,6 +264,7 @@ describe("cluster_ocsp = off works with #" .. strategy .. " inc_sync=" .. inc_sy
         -- additional attributes for PKI:
         cluster_mtls = "pki",
         cluster_ca_cert = "spec/fixtures/ocsp_certs/ca.crt",
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }))
 
@@ -275,6 +283,7 @@ describe("cluster_ocsp = off works with #" .. strategy .. " inc_sync=" .. inc_sy
         cluster_mtls = "pki",
         cluster_server_name = "kong_clustering",
         cluster_ca_cert = "spec/fixtures/ocsp_certs/ca.crt",
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }))
     end)
@@ -331,6 +340,7 @@ describe("cluster_ocsp = optional works with #" .. strategy .. " inc_sync=" .. i
         -- additional attributes for PKI:
         cluster_mtls = "pki",
         cluster_ca_cert = "spec/fixtures/ocsp_certs/ca.crt",
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }))
 
@@ -349,6 +359,7 @@ describe("cluster_ocsp = optional works with #" .. strategy .. " inc_sync=" .. i
         cluster_server_name = "kong_clustering",
         cluster_ca_cert = "spec/fixtures/ocsp_certs/ca.crt",
         nginx_conf = "spec/fixtures/custom_nginx.template",
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }))
     end)
@@ -401,6 +412,7 @@ describe("cluster_ocsp = optional works with #" .. strategy .. " inc_sync=" .. i
         -- additional attributes for PKI:
         cluster_mtls = "pki",
         cluster_ca_cert = "spec/fixtures/ocsp_certs/ca.crt",
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }))
 
@@ -419,6 +431,7 @@ describe("cluster_ocsp = optional works with #" .. strategy .. " inc_sync=" .. i
         cluster_server_name = "kong_clustering",
         cluster_ca_cert = "spec/fixtures/ocsp_certs/ca.crt",
         nginx_conf = "spec/fixtures/custom_nginx.template",
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }))
     end)

spec/02-integration/09-hybrid_mode/08-lazy_export_spec.lua

@@ -2,7 +2,7 @@ local helpers = require "spec.helpers"
 
 local admin_client
 
-local function cp(strategy, inc_sync)
+local function cp(strategy, rpc, inc_sync)
   helpers.get_db_utils(strategy) -- make sure the DB is fresh n' clean
   assert(helpers.start_kong({
     role = "control_plane",
@@ -14,6 +14,7 @@ local function cp(strategy, inc_sync)
     -- additional attributes for PKI:
     cluster_mtls = "pki",
     cluster_ca_cert = "spec/fixtures/ocsp_certs/ca.crt",
+    cluster_rpc = rpc,
     cluster_incremental_sync = inc_sync,
   }))
   admin_client = assert(helpers.admin_client())
@@ -35,7 +36,7 @@ local function touch_config()
   }))
 end
 
-local function json_dp(inc_sync)
+local function json_dp(rpc, inc_sync)
   assert(helpers.start_kong({
     role = "data_plane",
     database = "off",
@@ -48,19 +49,21 @@ local function json_dp(inc_sync)
     cluster_mtls = "pki",
     cluster_server_name = "kong_clustering",
     cluster_ca_cert = "spec/fixtures/ocsp_certs/ca.crt",
+    cluster_rpc = rpc,
     cluster_incremental_sync = inc_sync,
   }))
 end
 
 
--- TODO: reenable the inc sync test
-for _, inc_sync in ipairs { "off"  } do
+for _, v in ipairs({ {"off", "off"}, {"on", "off"}, {"on", "on"}, }) do
+  local rpc, inc_sync = v[1], v[2]
+
 for _, strategy in helpers.each_strategy() do
 
 describe("lazy_export with #".. strategy .. " inc_sync=" .. inc_sync, function()
   describe("no DP", function ()
     setup(function()
-      cp(strategy, inc_sync)
+      cp(strategy, rpc, inc_sync)
     end)
     teardown(function ()
       helpers.stop_kong()
@@ -78,8 +81,8 @@ describe("lazy_export with #".. strategy .. " inc_sync=" .. inc_sync, function()
 
   describe("only json DP", function()
     setup(function()
-      cp(strategy, inc_sync)
-      json_dp(inc_sync)
+      cp(strategy, rpc, inc_sync)
+      json_dp(rpc, inc_sync)
     end)
     teardown(function ()
       helpers.stop_kong("dp1")

spec/02-integration/09-hybrid_mode/13-deprecations_spec.lua

@@ -3,8 +3,9 @@ local join = require("pl.stringx").join
 
 local ENABLED_PLUGINS = { "dummy" , "reconfiguration-completion"}
 
--- TODO: reenable the inc sync test
-for _, inc_sync in ipairs { "off"  } do
+for _, v in ipairs({ {"off", "off"}, {"on", "off"}, {"on", "on"}, }) do
+  local rpc, inc_sync = v[1], v[2]
+
 for _, strategy in helpers.each_strategy({"postgres"}) do
   describe("deprecations are not reported on DP but on CP " .. " inc_sync=" .. inc_sync, function()
     local cp_prefix = "servroot1"
@@ -43,6 +44,7 @@ for _, strategy in helpers.each_strategy({"postgres"}) do
         nginx_conf = "spec/fixtures/custom_nginx.template",
         admin_listen = "0.0.0.0:9001",
         proxy_listen = "off",
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }))
 
@@ -58,6 +60,7 @@ for _, strategy in helpers.each_strategy({"postgres"}) do
         plugins = "bundled," .. join(",", ENABLED_PLUGINS),
         admin_listen = "off",
         proxy_listen = "0.0.0.0:9002",
+        cluster_rpc = rpc,
         cluster_incremental_sync = inc_sync,
       }))
       dp_logfile = helpers.get_running_conf(dp_prefix).nginx_err_logs