Merge branch 'master' into fix/flattened-errors-json-array

.luacheckrc

@@ -30,7 +30,7 @@ exclude_files = {
     "bazel-kong",
 }
 
-files["kong/tools/kong-lua-sandbox.lua"] = {
+files["kong/tools/sandbox/kong.lua"] = {
      read_globals = {
         "_ENV",
         "table.pack",

build/build_system.bzl

@@ -171,6 +171,15 @@ def _copyright_header(ctx):
                 # while writing utf-8 content read by |ctx.read|, let's disable it
                 ctx.file(path, copyright_content_html + content, legacy_utf8 = False)
 
+_GITHUB_RELEASE_SINGLE_FILE_BUILD = """\
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "file",
+    srcs = ["{}"],
+)
+"""
+
 def _github_release_impl(ctx):
     ctx.file("WORKSPACE", "workspace(name = \"%s\")\n" % ctx.name)
 
@@ -195,20 +204,25 @@ def _github_release_impl(ctx):
         fail("Unsupported OS %s" % os_name)
 
     gh_bin = "%s" % ctx.path(Label("@gh_%s_%s//:bin/gh" % (os_name, os_arch)))
-    args = [gh_bin, "release", "download", ctx.attr.tag, "-R", ctx.attr.repo]
+    args = [gh_bin, "release", "download", ctx.attr.tag, "--repo", ctx.attr.repo]
     downloaded_file = None
     if ctx.attr.pattern:
         if "/" in ctx.attr.pattern or ".." in ctx.attr.pattern:
             fail("/ and .. are not allowed in pattern")
         downloaded_file = ctx.attr.pattern.replace("*", "_")
-        args += ["-p", ctx.attr.pattern]
+        args += ["--pattern", ctx.attr.pattern]
     elif ctx.attr.archive:
         args.append("--archive=" + ctx.attr.archive)
         downloaded_file = "gh-release." + ctx.attr.archive.split(".")[-1]
     else:
         fail("at least one of pattern or archive must be set")
 
-    args += ["-O", downloaded_file]
+    downloaded_file_path = downloaded_file
+    if not ctx.attr.extract:
+        ctx.file("file/BUILD", _GITHUB_RELEASE_SINGLE_FILE_BUILD.format(downloaded_file))
+        downloaded_file_path = "file/" + downloaded_file
+
+    args += ["--output", downloaded_file_path]
 
     ret = ctx.execute(args)
 
@@ -218,10 +232,23 @@ def _github_release_impl(ctx):
             gh_token_set = "GITHUB_TOKEN is not set, is this a private repo?"
         fail("Failed to download release (%s): %s, exit: %d" % (gh_token_set, ret.stderr, ret.return_code))
 
-    ctx.extract(downloaded_file, stripPrefix = ctx.attr.strip_prefix)
+    if ctx.attr.sha256:
+        if os_name == "macOS":
+            sha256_cmd = ["shasum", "-a", "256", downloaded_file_path]
+        else:
+            sha256_cmd = ["sha256sum", downloaded_file_path]
+        ret = ctx.execute(sha256_cmd)
+        checksum = ret.stdout.split(" ")[0]
+        if checksum != ctx.attr.sha256:
+            fail("Checksum mismatch: expected %s, got %s" % (ctx.attr.sha256, checksum))
+
+    if ctx.attr.extract:
+        ctx.extract(downloaded_file_path, stripPrefix = ctx.attr.strip_prefix)
 
     # only used in EE: always skip here in CE
     if not ctx.attr.skip_add_copyright_header and False:
+        if not ctx.attr.extract:
+            fail("Writing copyright header is only supported for extracted archives")
         _copyright_header(ctx)
 
 github_release = repository_rule(
@@ -231,11 +258,13 @@ github_release = repository_rule(
         "tag": attr.string(mandatory = True),
         "pattern": attr.string(mandatory = False),
         "archive": attr.string(mandatory = False, values = ["zip", "tar.gz"]),
+        "extract": attr.bool(default = True, doc = "Whether to extract the downloaded archive"),
         "strip_prefix": attr.string(default = "", doc = "Strip prefix from downloaded files"),
         "repo": attr.string(mandatory = True),
         "build_file": attr.label(allow_single_file = True),
         "build_file_content": attr.string(),
         "skip_add_copyright_header": attr.bool(default = False, doc = "Whether to inject COPYRIGHT-HEADER into downloaded files, only required for webuis"),
+        "sha256": attr.string(mandatory = False),
     },
 )
 

build/openresty/wasmx/filters/variables.bzl

@@ -2,17 +2,7 @@
 A list of wasm filters.
 """
 
-WASM_FILTERS = [
-    {
-        "name": "datakit-filter",
-        "repo": "Kong/datakit",
-        "tag": "0.3.1",
-        "files": {
-            "datakit.meta.json": "acd16448615ea23315e68d4516edd79135bae13469f7bf9129f7b1139cd2b873",
-            "datakit.wasm": "c086e6fb36a6ed8c9ff3284805485c7280380469b6a556ccf7e5bc06edce27e7",
-        },
-    },
-]
+WASM_FILTERS = []
 
 WASM_FILTERS_TARGETS = [
     "@%s-%s//file" % (filter["name"], file)

changelog/unreleased/kong/fix-dbless-consumer-credential-error.yml

@@ -0,0 +1,3 @@
+message: "Fixed an issue where `POST /config?flatten_errors=1` could not return a proper response if the input contained duplicate consumer credentials."
+type: bugfix
+scope: Core

changelog/unreleased/kong/remove-datakit.yml

@@ -0,0 +1,2 @@
+message: "**Wasm**: Removed the experimental datakit Wasm filter"
+type: dependency

kong-3.10.0-0.rockspec

@@ -194,7 +194,6 @@ build = {
     ["kong.tools.stream_api"] = "kong/tools/stream_api.lua",
     ["kong.tools.queue"] = "kong/tools/queue.lua",
     ["kong.tools.queue_schema"] = "kong/tools/queue_schema.lua",
-    ["kong.tools.sandbox"] = "kong/tools/sandbox.lua",
     ["kong.tools.uri"] = "kong/tools/uri.lua",
     ["kong.tools.kong-lua-sandbox"] = "kong/tools/kong-lua-sandbox.lua",
     ["kong.tools.protobuf"] = "kong/tools/protobuf.lua",
@@ -217,6 +216,17 @@ build = {
     ["kong.tools.redis.schema"] = "kong/tools/redis/schema.lua",
     ["kong.tools.aws_stream"] = "kong/tools/aws_stream.lua",
 
+    ["kong.tools.sandbox"] = "kong/tools/sandbox/init.lua",
+    ["kong.tools.sandbox.kong"] = "kong/tools/sandbox/kong.lua",
+    ["kong.tools.sandbox.environment"] = "kong/tools/sandbox/environment/init.lua",
+    ["kong.tools.sandbox.environment.handler"] = "kong/tools/sandbox/environment/handler.lua",
+    ["kong.tools.sandbox.environment.lua"] = "kong/tools/sandbox/environment/lua.lua",
+    ["kong.tools.sandbox.environment.schema"] = "kong/tools/sandbox/environment/schema.lua",
+    ["kong.tools.sandbox.require"] = "kong/tools/sandbox/require/init.lua",
+    ["kong.tools.sandbox.require.handler"] = "kong/tools/sandbox/require/handler.lua",
+    ["kong.tools.sandbox.require.lua"] = "kong/tools/sandbox/require/lua.lua",
+    ["kong.tools.sandbox.require.schema"] = "kong/tools/sandbox/require/schema.lua",
+
     ["kong.runloop.handler"] = "kong/runloop/handler.lua",
     ["kong.runloop.events"] = "kong/runloop/events.lua",
     ["kong.runloop.log_level"] = "kong/runloop/log_level.lua",

kong.conf.default

@@ -1894,9 +1894,10 @@
                                  #              them. The sandboxed function has
                                  #              restricted access to the global
                                  #              environment and only has access
-                                 #              to standard Lua functions that
-                                 #              will generally not cause harm to
-                                 #              the Kong Gateway node.
+                                 #              to Kong PDK, OpenResty, and
+                                 #              standard Lua functions that will
+                                 #              generally not cause harm to the
+                                 #              Kong Gateway node.
                                  #
                                  # * `on`: Functions have unrestricted
                                  #         access to the global environment and
@@ -1920,9 +1921,6 @@
                                  # functions are not allowed, like:
                                  # `os.execute('rm -rf /*')`.
                                  #
-                                 # For a full allowed/disallowed list, see:
-                                 # https://github.com/kikito/sandbox.lua/blob/master/sandbox.lua
-                                 #
                                  # To customize the sandbox environment, use
                                  # the `untrusted_lua_sandbox_requires` and
                                  # `untrusted_lua_sandbox_environment`

kong/clustering/rpc/manager.lua

@@ -30,6 +30,7 @@ local cjson_encode = cjson.encode
 local cjson_decode = cjson.decode
 local validate_client_cert = clustering_tls.validate_client_cert
 local CLUSTERING_PING_INTERVAL = constants.CLUSTERING_PING_INTERVAL
+local parse_proxy_url = require("kong.clustering.utils").parse_proxy_url
 
 
 local RPC_MATA_V1 = "kong.meta.v1"
@@ -474,6 +475,17 @@ function _M:connect(premature, node_id, host, path, cert, key)
 
   local c = assert(client:new(WS_OPTS))
 
+  if self.conf.cluster_use_proxy then
+    local proxy_opts = parse_proxy_url(self.conf.proxy_server)
+    opts.proxy_opts = {
+      wss_proxy = proxy_opts.proxy_url,
+      wss_proxy_authorization = proxy_opts.proxy_authorization,
+    }
+
+    ngx_log(ngx_DEBUG, "[rpc] using proxy ", proxy_opts.proxy_url,
+                       " to connect control plane")
+  end
+
   local ok, err = c:connect(uri, opts)
   if not ok then
     ngx_log(ngx_ERR, "[rpc] unable to connect to peer: ", err)

kong/clustering/services/sync/rpc.lua

@@ -422,26 +422,28 @@ function sync_once_impl(premature, retry_count)
     return
   end
 
-  sync_handler()  
-
-  local latest_notified_version = ngx.shared.kong:get(CLUSTERING_DATA_PLANES_LATEST_VERSION_KEY)
-  local current_version = tonumber(declarative.get_current_hash()) or 0
+  sync_handler()
 
+  local latest_notified_version = ngx.shared.kong:get(CLUSTERING_DATA_PLANES_LATEST_VERSION_KEY)
   if not latest_notified_version then
     ngx_log(ngx_DEBUG, "no version notified yet")
     return
   end
 
-  -- retry if the version is not updated
-  if current_version < latest_notified_version then
-    retry_count = retry_count or 0
-    if retry_count > MAX_RETRY then
-      ngx_log(ngx_ERR, "sync_once retry count exceeded. retry_count: ", retry_count)
-      return
-    end
+  local current_version = tonumber(declarative.get_current_hash()) or 0
+  if current_version >= latest_notified_version then
+    ngx_log(ngx_DEBUG, "version already updated")
+    return
+  end
 
-    return start_sync_once_timer(retry_count + 1)
+  -- retry if the version is not updated
+  retry_count = retry_count or 0
+  if retry_count > MAX_RETRY then
+    ngx_log(ngx_ERR, "sync_once retry count exceeded. retry_count: ", retry_count)
+    return
   end
+
+  return start_sync_once_timer(retry_count + 1)
 end
 
 

kong/clustering/utils.lua

@@ -33,7 +33,7 @@ local CLUSTER_PROXY_SSL_TERMINATOR_SOCK = fmt("unix:%s/%s",
 local _M = {}
 
 
-local function parse_proxy_url(proxy_server)
+function _M.parse_proxy_url(proxy_server)
   local ret = {}
 
   if proxy_server then
@@ -84,7 +84,7 @@ function _M.connect_cp(dp, endpoint, protocols)
   }
 
   if conf.cluster_use_proxy then
-    local proxy_opts = parse_proxy_url(conf.proxy_server)
+    local proxy_opts = _M.parse_proxy_url(conf.proxy_server)
     opts.proxy_opts = {
       wss_proxy = proxy_opts.proxy_url,
       wss_proxy_authorization = proxy_opts.proxy_authorization,

kong/conf_loader/init.lua

@@ -675,7 +675,7 @@ local function load(path, custom_conf, opts)
         bundled_filter_path = alt_path
 
       else
-        log.warn("Bundled proxy-wasm filters path (%s) does not exist " ..
+        log.debug("Bundled proxy-wasm filters path (%s) does not exist " ..
                  "or is not a directory. Bundled filters may not be " ..
                  "available", bundled_filter_path)
       end

kong/db/init.lua

@@ -102,7 +102,7 @@ function DB.new(kong_config, strategy)
     strategy   = strategy,
     errors     = errors,
     infos      = connector:infos(),
-    kong_config = kong_config,
+    loaded_plugins = kong_config.loaded_plugins, -- left for MigrationsState.load
   }
 
   do
@@ -444,8 +444,7 @@ do
       return nil, prefix_err(self, err)
     end
 
-    local ok, err = self.connector:schema_bootstrap(self.kong_config,
-                                                    DEFAULT_LOCKS_TTL)
+    local ok, err = self.connector:schema_bootstrap(DEFAULT_LOCKS_TTL)
 
     self.connector:close()
 

kong/db/migrations/state.lua

@@ -184,7 +184,7 @@ function State.load(db)
 
   log.debug("loading subsystems migrations...")
 
-  local subsystems, err = load_subsystems(db, db.kong_config.loaded_plugins)
+  local subsystems, err = load_subsystems(db, db.loaded_plugins)
   if not subsystems then
     return nil, prefix_err(db, err)
   end

kong/db/schema/others/declarative_config.lua

@@ -361,6 +361,22 @@ local function uniqueness_error_msg(entity, key, value)
          "with " .. key .. " set to '" .. value .. "' already declared"
 end
 
+local function add_error(errs, parent_entity, parent_idx, entity, entity_idx, err)
+  if parent_entity and parent_idx then
+    errs[parent_entity]                     = errs[parent_entity] or {}
+    errs[parent_entity][parent_idx]         = errs[parent_entity][parent_idx] or {}
+    errs[parent_entity][parent_idx][entity] = errs[parent_entity][parent_idx][entity] or {}
+
+    -- e.g. errs["upstreams"][5]["targets"][2]
+    errs[parent_entity][parent_idx][entity][entity_idx] = err
+
+  else
+    errs[entity] = errs[entity] or {}
+
+    -- e.g. errs["consumers"][3]
+    errs[entity][entity_idx] = err
+  end
+end
 
 local function populate_references(input, known_entities, by_id, by_key, expected, errs, parent_entity, parent_idx)
   for _, entity in ipairs(known_entities) do
@@ -400,31 +416,17 @@ local function populate_references(input, known_entities, by_id, by_key, expecte
       if key and key ~= ngx.null then
         local ok = add_to_by_key(by_key, entity_schema, item, entity, key)
         if not ok then
-          errs[entity] = errs[entity] or {}
-          errs[entity][i] = uniqueness_error_msg(entity, endpoint_key, key)
+          add_error(errs, parent_entity, parent_idx, entity, i,
+                    uniqueness_error_msg(entity, endpoint_key, key))
           failed = true
         end
       end
 
       if item_id then
         by_id[entity] = by_id[entity] or {}
         if (not failed) and by_id[entity][item_id] then
-          local err_t
-
-          if parent_entity and parent_idx then
-            errs[parent_entity]                     = errs[parent_entity] or {}
-            errs[parent_entity][parent_idx]         = errs[parent_entity][parent_idx] or {}
-            errs[parent_entity][parent_idx][entity] = errs[parent_entity][parent_idx][entity] or {}
-
-            -- e.g. errs["upstreams"][5]["targets"]
-            err_t = errs[parent_entity][parent_idx][entity]
-
-          else
-            errs[entity] = errs[entity] or {}
-            err_t = errs[entity]
-          end
-
-          err_t[i] = uniqueness_error_msg(entity, "primary key", item_id)
+          add_error(errs, parent_entity, parent_idx, entity, i,
+                    uniqueness_error_msg(entity, "primary key", item_id))
 
         else
           by_id[entity][item_id] = item

kong/db/strategies/postgres/connector.lua

@@ -775,7 +775,7 @@ function _mt:schema_migrations()
 end
 
 
-function _mt:schema_bootstrap(kong_config, default_locks_ttl)
+function _mt:schema_bootstrap(default_locks_ttl)
   local conn = self:get_stored_connection()
   if not conn then
     error("no connection")

kong/plugins/file-log/handler.lua

@@ -24,9 +24,6 @@ local oflags = bit.bor(O_WRONLY, O_CREAT, O_APPEND)
 local mode = ffi.new("int", bit.bor(S_IRUSR, S_IWUSR, S_IRGRP, S_IROTH))
 
 
-local sandbox_opts = { env = { kong = kong, ngx = ngx } }
-
-
 local C = ffi.C
 
 
@@ -73,7 +70,7 @@ function FileLogHandler:log(conf)
   if conf.custom_fields_by_lua then
     local set_serialize_value = kong.log.set_serialize_value
     for key, expression in pairs(conf.custom_fields_by_lua) do
-      set_serialize_value(key, sandbox(expression, sandbox_opts)())
+      set_serialize_value(key, sandbox(expression)())
     end
   end
 

kong/plugins/http-log/handler.lua

@@ -16,8 +16,6 @@ local pairs = pairs
 local max = math.max
 
 
-local sandbox_opts = { env = { kong = kong, ngx = ngx } }
-
 -- Create a function that concatenates multiple JSON objects into a JSON array.
 -- This saves us from rendering all entries into one large JSON string.
 -- Each invocation of the function returns the next bit of JSON, i.e. the opening
@@ -183,7 +181,7 @@ function HttpLogHandler:log(conf)
   if conf.custom_fields_by_lua then
     local set_serialize_value = kong.log.set_serialize_value
     for key, expression in pairs(conf.custom_fields_by_lua) do
-      set_serialize_value(key, sandbox(expression, sandbox_opts)())
+      set_serialize_value(key, sandbox(expression)())
     end
   end