backport 20231130

Dockerfile

@@ -35,5 +35,6 @@ RUN ls /usr/local/apisix/patches | sort | xargs -L1 -I __patch_file__ sh -c 'cat
 
 RUN chmod 755 /data/bkgateway/bin/* && chmod 777 /usr/local/apisix/logs
 
-
 CMD ["sh", "-c", "/usr/bin/apisix init && /usr/bin/apisix init_etcd && /usr/local/openresty/bin/openresty -p /usr/local/apisix -g 'daemon off;'"]
+
+STOPSIGNAL SIGQUIT

src/apisix/plugins/README.md

@@ -18,11 +18,12 @@
 
 上下文注入，优先级：18000 ~ 19000
 
+- bk-legacy-invalid-params                  # priority: 18880  # 用于兼容老版本 go1.16 使用 `;` 作为 query string 分隔符
 - bk-opentelemetry                          # priority: 18870  # 这个插件用于 opentelemetry, 需要尽量精准统计全局的耗时，同时需要注入 trace_id/span_id 作为后面所有插件自定义 opentelemetry 上报的 trace_id 即 parent span_id
 - bk-not-found-handler                      # priority: 18860  # 该插件仅适用于由 operator 创建的默认根路由，用以规范化 404 消息。该插件以较高优先级结束请求返回 404 错误信息
 - bk-request-id                             # priority: 18850
 - bk-stage-context                          # priority: 18840
-- bk-service-context                        # priority: 18830
+- bk-service-context                        # priority: 18830 (abandonned)
 - bk-resource-context                       # priority: 18820
 - bk-status-rewrite                         # priority: 18815
 - bk-verified-user-exempted-apps            # priority: 18810 (will be deprecated)
@@ -31,8 +32,8 @@
 
 认证：
 
-- bk-workflow-parameters                    # priority: 18750
-- bk-auth-parameters                        # priority: 18740
+- bk-workflow-parameters                    # priority: 18750 (abandonned)
+- bk-auth-parameters                        # priority: 18740 (abandonned)
 - bk-auth-verify                            # priority: 18730
 
 执行 - 响应：优先级：17500 ~ 18000

src/apisix/plugins/bk-auth-verify.lua

@@ -118,11 +118,25 @@ local function get_auth_params_from_request(ctx, authorization_keys)
     if err ~= nil then
         return nil, err
     elseif auth_params ~= nil then
+        -- 记录认证参数位置，便于统计哪些请求将认证参数放到请求参数，推动优化
+        ctx.var.auth_params_location = "header"
         return auth_params, nil
     end
 
+    if not ctx.var.bk_api_auth:allow_get_auth_params_from_parameters() then
+        -- 不允许从请求参数获取认证参数，直接返回
+        return {}, nil
+    end
+
     -- from the querystring and body
-    return get_auth_params_from_parameters(ctx, authorization_keys), nil
+    auth_params = get_auth_params_from_parameters(ctx, authorization_keys)
+
+    if not pl_types.is_empty(auth_params) then
+        -- 记录认证参数位置，便于统计哪些请求将认证参数放到请求参数，推动优化
+        ctx.var.auth_params_location = "params"
+    end
+
+    return auth_params
 end
 -- utils end
 
@@ -174,6 +188,8 @@ function _M.rewrite(conf, ctx) -- luacheck: no unused
     ctx.var.bk_user = user
     ctx.var.bk_app_code = app["app_code"]
     ctx.var.bk_username = user["username"]
+    -- 记录认证参数位置，便于统计哪些请求将认证参数放到请求参数，推动优化
+    ctx.var.auth_params_location = ctx.var.auth_params_location or ""
 end
 
 if _TEST then -- luacheck: ignore

src/apisix/plugins/bk-auth-verify/app-account-verifier.lua

@@ -20,6 +20,7 @@ local app_account_utils = require("apisix.plugins.bk-auth-verify.app-account-uti
 local bk_app_define = require("apisix.plugins.bk-define.app")
 local bk_cache = require("apisix.plugins.bk-cache.init")
 local setmetatable = setmetatable
+local string       = string
 
 local _M = {}
 
@@ -46,6 +47,14 @@ function _M.verify_app(self)
     end
 
     if not pl_types.is_empty(self.app_secret) then
+        -- check the length before call bkauth apis
+        if string.len(self.app_code) > 32 then
+            return bk_app_define.new_anonymous_app("app code cannot be longer than 32 characters")
+        end
+        if string.len(self.app_secret) > 128 then
+            return bk_app_define.new_anonymous_app("app secret cannot be longer than 128 characters")
+        end
+
         return self:verify_by_app_secret()
     end
 

src/apisix/plugins/bk-define/context-api-bkauth.lua

@@ -112,6 +112,8 @@ function ContextApiBkAuth.new(bk_api_auth)
             api_type = bk_api_auth.api_type,
             unfiltered_sensitive_keys = bk_api_auth.unfiltered_sensitive_keys or {},
             include_system_headers_mapping = include_system_headers_mapping,
+            allow_auth_from_params = bk_api_auth.allow_auth_from_params,
+            allow_delete_sensitive_params = bk_api_auth.allow_delete_sensitive_params,
             uin_conf = UinConf.new(bk_api_auth.uin_conf),
             rtx_conf = RtxConf.new(bk_api_auth.rtx_conf),
             user_conf = UserConf.new(bk_api_auth.user_conf),
@@ -123,6 +125,13 @@ function ContextApiBkAuth.get_api_type(self)
     return self.api_type
 end
 
+---Allow get auth_params from request parameters, such as querystring, body
+---@return boolean
+function ContextApiBkAuth.allow_get_auth_params_from_parameters(self)
+    -- 默认允许从参数获取认证信息
+    return self.allow_auth_from_params ~= false
+end
+
 ---Get the unfiltered sensitive keys.
 ---@return table
 function ContextApiBkAuth.get_unfiltered_sensitive_keys(self)
@@ -147,8 +156,9 @@ end
 
 ---Filter the sensitive params or not, do the filter if api_type is not ESB.
 ---@return boolean
-function ContextApiBkAuth.is_filter_sensitive_params(self)
-    return self.api_type ~= ESB
+function ContextApiBkAuth.should_delete_sensitive_params(self)
+    -- 非 esb，且允许删除敏感参数（默认允许删除）
+    return self.api_type ~= ESB and self.allow_delete_sensitive_params ~= false
 end
 
 function ContextApiBkAuth.is_user_type_uin(self)

src/apisix/plugins/bk-delete-sensitive.lua

@@ -34,6 +34,7 @@ local core = require("apisix.core")
 local bk_core = require("apisix.plugins.bk-core.init")
 local ngx = ngx -- luacheck: ignore
 local ipairs = ipairs
+local tostring = tostring
 
 local plugin_name = "bk-delete-sensitive"
 
@@ -55,7 +56,6 @@ function _M.check_schema(conf)
     return core.schema.check(schema, conf)
 end
 
-
 ---Delete the sensitive parameters in the request header, uri args and body,
 ---it will check the first, then do the modification.
 ---@param ctx apisix.Context
@@ -103,6 +103,15 @@ local function delete_sensitive_params(ctx, sensitive_keys, unfiltered_sensitive
         ::continue::
     end
 
+    if ctx.var.auth_params_location == "header" and (query_changed or form_changed or body_changed) then
+        core.log.warn(
+            "auth params are present in both header and request parameters, request_id: " ..
+                tostring(ctx.var.bk_request_id)
+        )
+        -- 记录认证参数位置，便于统计哪些请求将认证参数放到请求参数，推动优化
+        ctx.var.auth_params_location = "header_and_params"
+    end
+
     if check_query and query_changed then
         core.request.set_uri_args(ctx, uri_args)
     end
@@ -127,7 +136,7 @@ local function delete_sensitive_headers()
 end
 
 function _M.rewrite(conf, ctx) -- luacheck: no unused
-    if ctx.var.bk_api_auth and ctx.var.bk_api_auth:is_filter_sensitive_params() then
+    if ctx.var.bk_api_auth and ctx.var.bk_api_auth:should_delete_sensitive_params() then
         delete_sensitive_params(
             ctx, bk_core.config.get_sensitive_keys(), ctx.var.bk_api_auth:get_unfiltered_sensitive_keys()
         )

src/apisix/plugins/bk-legacy-invalid-params.lua

@@ -0,0 +1,66 @@
+--
+-- TencentBlueKing is pleased to support the open source community by making
+-- 蓝鲸智云 - API 网关(BlueKing - APIGateway) available.
+-- Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+-- Licensed under the MIT License (the "License"); you may not use this file except
+-- in compliance with the License. You may obtain a copy of the License at
+--
+--     http://opensource.org/licenses/MIT
+--
+-- Unless required by applicable law or agreed to in writing, software distributed under
+-- the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+-- either express or implied. See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- We undertake not to change the open source license (MIT license) applicable
+-- to the current version of the project delivered to anyone in the future.
+--
+
+-- # bk-legacy-invalid-params
+--
+-- For old gateway calling, because go 1.16 support both `&` and `;` as query string separator, but lua only support `&`
+-- and in some case, the caller html escaped the `&` to `&`(it's ok for go 1.16 gateway)
+-- so we need to adapat the old gateway calling.
+-- e.g.
+-- ?app_code=appC&app_secret=appC
+-- ?app_code=appC&app_secret=appC
+-- ?app_code=appC;app_secret=appC
+-- ?a=1;a=2
+
+local string_replace = require("pl.stringx").replace
+local string_find = string.find
+local core = require("apisix.core")
+
+local schema = {}
+
+local _M = {
+    version = 0.1,
+    priority = 18880,
+    name = "bk-legacy-invalid-params",
+    schema = schema,
+}
+
+function _M.check_schema(conf)
+    return core.schema.check(schema, conf)
+end
+
+function _M.rewrite(conf, ctx)
+    -- FIXME: 未来新的接口使用`;`也不生效, 怎么控制范围?
+
+    -- FIX 1
+    -- in golang 1.16: strings.IndexAny(key, "&;")
+    -- so here we just need to replace `;` to `&`, then reset the uri_args
+    -- args will be decoded like golang version
+
+    -- core.log.error(ctx.var.args)
+    -- only query string contains `;` should be processed
+    if ctx.var.args ~= nil and string_find(ctx.var.args, ";") then
+        local new_args = string_replace(ctx.var.args, ";", "&")
+        -- core.log.error("replace ; to &: ", new_args)
+        core.request.set_uri_args(ctx, new_args)
+    end
+    -- local args = core.request.get_uri_args()
+    -- core.log.error(core.json.delay_encode(args))
+end
+
+return _M

src/apisix/plugins/bk-rate-limit/init.lua

@@ -35,6 +35,7 @@
 --     redis_database: 0
 --     redis_timeout: 1001
 --
+local apisix_plugin = require("apisix.plugin")
 local core = require("apisix.core")
 local rate_limit_redis = require("apisix.plugins.bk-rate-limit.rate-limit-redis")
 local lrucache = core.lrucache.new(
@@ -77,6 +78,14 @@ local _M = {
     },
 }
 
+local function gen_limit_key(conf, key)
+    -- Here we use plugin-level conf version to prevent the counter from being resetting
+    -- because of the change elsewhere.
+    -- e.g. conf_version = 1969078430
+    local new_key = key .. ':' .. apisix_plugin.conf_version(conf)
+    return new_key
+end
+
 ---Create rate-limit-redis object
 ---@param plugin_name string @apisix plugin name
 ---@return table @rate-limit-redis object
@@ -108,6 +117,7 @@ function _M.rate_limit(conf, ctx, plugin_name, key, count, time_window)
         return 500
     end
 
+    key = gen_limit_key(conf, key)
     core.log.info("limit key: ", key)
 
     local delay, remaining, reset = lim:incoming(key, count, time_window)
@@ -145,4 +155,8 @@ function _M.rate_limit(conf, ctx, plugin_name, key, count, time_window)
     end
 end
 
+if _TEST then
+    _M.gen_limit_key = gen_limit_key
+end
+
 return _M

src/apisix/plugins/bk-resource-rate-limit.lua

@@ -62,6 +62,8 @@ function _M.access(conf, ctx)
     end
 
     -- TODO: make it lazy, share the key with other plugins
+    -- FIXME: should change the bk_reosurce_name to bk_resource_id if you need more accurate rate limit
+    --        while the developer may change the bk_resource_name from the frontend page.
     local key = table_concat(
         {
             bk_app_code,

src/apisix/plugins/bk-stage-context.lua

@@ -15,7 +15,6 @@
 -- We undertake not to change the open source license (MIT license) applicable
 -- to the current version of the project delivered to anyone in the future.
 --
-
 local core = require("apisix.core")
 local bk_core = require("apisix.plugins.bk-core.init")
 local context_api_bkauth = require("apisix.plugins.bk-define.context-api-bkauth")
@@ -63,6 +62,14 @@ local schema = {
                         type = "string",
                     },
                 },
+                allow_auth_from_params = {
+                    type = "boolean",
+                    default = true,
+                },
+                allow_delete_sensitive_params = {
+                    type = "boolean",
+                    default = true,
+                },
                 uin_conf = {
                     type = "object",
                     properties = {
@@ -132,20 +139,17 @@ local _M = {
     schema = schema,
 }
 
-
 ---@param jwt_private_key string: JWT private key (encoded in Base64)
----@return string|nil, string: The decoded JWT private key, or nil and error message if the decoding fails
+---@return string|nil, string|nil: The decoded JWT private key, or nil and error message if the decoding fails
 local function decode_jwt_private_key(jwt_private_key)
     local decoded_private_key = ngx_decode_base64(jwt_private_key)
     if not decoded_private_key then
         core.log.error("failed to decode jwt_private_key with base64. jwt_private_key=", jwt_private_key)
         return nil, "failed to decode jwt_private_key with base64"
     end
-    return decoded_private_key
+    return decoded_private_key, nil
 end
 
-
-
 ---@param conf table: The user-provided configuration for the plugin instance
 ---@return boolean, string|nil: true and nil if the configuration is valid, false and error message if not
 function _M.check_schema(conf)
@@ -165,7 +169,6 @@ function _M.check_schema(conf)
     return true
 end
 
-
 ---@param conf table: The user-provided configuration for the plugin instance
 ---@param ctx  api.Context: The request context
 function _M.rewrite(conf, ctx)

src/apisix/plugins/bk-stage-rate-limit.lua

@@ -85,7 +85,6 @@ function _M.access(conf, ctx)
     else
         for i, rate in ipairs(rates) do
             -- here we should add the rate index into key, otherwise the rate limit will be shared(will be wrong)
-            -- FIXME: if the rate changes, will wait for the period to effect
             local limit_key = key .. ":" .. tostring(i)
             local code = ratelimit.rate_limit(conf, ctx, plugin_name, limit_key, rate.tokens, rate.period)
             if code then

src/apisix/t/README.md

@@ -33,4 +33,4 @@
 - [apisix source code : t](https://github.com/apache/apisix/tree/master/t)
 - [test-nginx](https://github.com/openresty/test-nginx)
 - [test-nginx doc: user guide](https://openresty.gitbooks.io/programming-openresty/content/testing/index.html)
-
+- [Test::Nginx::Socket](https://metacpan.org/pod/Test::Nginx::Socket)