From 45bf89f5ae64ccf824773e41d8f94f3c1acd8e4b Mon Sep 17 00:00:00 2001 From: Xiaoyan Rao <270668624@qq.com> Date: Mon, 1 Jul 2024 11:22:47 +0800 Subject: [PATCH 1/4] fix(dao): can't be deleted or selected an entity after TTL expires. --- changelog/unreleased/kong/ttl_expires.yml | 3 + kong/db/dao/init.lua | 21 ++++-- kong/db/strategies/postgres/init.lua | 80 +++++++++++++++++++++-- spec/02-integration/03-db/14-dao_spec.lua | 32 +++++++++ 4 files changed, 126 insertions(+), 10 deletions(-) create mode 100644 changelog/unreleased/kong/ttl_expires.yml diff --git a/changelog/unreleased/kong/ttl_expires.yml b/changelog/unreleased/kong/ttl_expires.yml new file mode 100644 index 000000000000..57bec033dfbd --- /dev/null +++ b/changelog/unreleased/kong/ttl_expires.yml @@ -0,0 +1,3 @@ +message: Fixed an issue where can't be deleted or selected an entity after TTL expires. +type: bugfix +scope: Core \ No newline at end of file diff --git a/kong/db/dao/init.lua b/kong/db/dao/init.lua index f84801c85a1e..515e6c0c719c 100644 --- a/kong/db/dao/init.lua +++ b/kong/db/dao/init.lua @@ -7,6 +7,7 @@ local workspaces = require "kong.workspaces" local new_tab = require "table.new" local DAO_MAX_TTL = require("kong.constants").DATABASE.DAO_MAX_TTL local is_valid_uuid = require("kong.tools.uuid").is_valid_uuid +local deep_copy = require("kong.tools.table").deep_copy local setmetatable = setmetatable local tostring = tostring @@ -290,6 +291,12 @@ local function validate_options_value(self, options) end end + if options.skip_ttl ~= nil then + if type(options.skip_ttl) ~= "boolean" then + errors.skip_ttl = "must be a boolean" + end + end + if next(errors) then return nil, errors end @@ -896,8 +903,9 @@ local function generate_foreign_key_methods(schema) return nil, err, err_t end - local show_ws_id = { show_ws_id = true } - local entity, err, err_t = self["select_by_" .. name](self, unique_value, show_ws_id) + local select_options = deep_copy(options or {}) + select_options["show_ws_id"] = true + local entity, err, err_t = self["select_by_" .. name](self, unique_value, select_options) if err then return nil, err, err_t end @@ -906,7 +914,7 @@ local function generate_foreign_key_methods(schema) return true end - local cascade_entries = find_cascade_delete_entities(self, entity, show_ws_id) + local cascade_entries = find_cascade_delete_entities(self, entity, select_options) local ok, err_t = run_hook("dao:delete_by:pre", entity, @@ -1293,8 +1301,9 @@ function DAO:delete(pk_or_entity, options) return nil, tostring(err_t), err_t end - local show_ws_id = { show_ws_id = true } - local entity, err, err_t = self:select(primary_key, show_ws_id) + local select_options = deep_copy(options or {}) + select_options["show_ws_id"] = true + local entity, err, err_t = self:select(primary_key, select_options) if err then return nil, err, err_t end @@ -1311,7 +1320,7 @@ function DAO:delete(pk_or_entity, options) end end - local cascade_entries = find_cascade_delete_entities(self, primary_key, show_ws_id) + local cascade_entries = find_cascade_delete_entities(self, primary_key, select_options) local ws_id = entity.ws_id local _ diff --git a/kong/db/strategies/postgres/init.lua b/kong/db/strategies/postgres/init.lua index b7b0571459c8..14d4f50508a2 100644 --- a/kong/db/strategies/postgres/init.lua +++ b/kong/db/strategies/postgres/init.lua @@ -424,7 +424,7 @@ local function execute(strategy, statement_name, attributes, options) local is_update = options and options.update local has_ttl = strategy.schema.ttl - + local is_skip_ttl = options and options.skip_ttl if has_ws_id then assert(ws_id == nil or type(ws_id) == "string") argv[0] = escape_literal(connector, ws_id, "ws_id") @@ -433,7 +433,7 @@ local function execute(strategy, statement_name, attributes, options) for i = 1, argc do local name = argn[i] local value - if has_ttl and name == "ttl" then + if has_ttl and name == "ttl" and not is_skip_ttl then value = (options and options.ttl) and get_ttl_value(strategy, attributes, options) @@ -576,7 +576,12 @@ end function _mt:select(primary_key, options) - local res, err = execute(self, "select", self.collapse(primary_key), options) + local statement_name = "select" + if self.schema.ttl and options and options.skip_ttl then + statement_name = "select_skip_ttl" + end + + local res, err = execute(self, statement_name, self.collapse(primary_key), options) if res then local row = res[1] if row then @@ -592,6 +597,11 @@ end function _mt:select_by_field(field_name, unique_value, options) local statement_name = "select_by_" .. field_name + + if self.schema.ttl and options and options.skip_ttl then + statement_name = statement_name .. "_skip_ttl" + end + local filter = { [field_name] = unique_value, } @@ -695,7 +705,11 @@ end function _mt:delete(primary_key, options) - local res, err = execute(self, "delete", self.collapse(primary_key), options) + local statement_name = "delete" + if self.schema.ttl and options and options.skip_ttl then + statement_name = "delete_skip_ttl" + end + local res, err = execute(self, statement_name, self.collapse(primary_key), options) if res then if res.affected_rows == 0 then return nil, nil @@ -710,6 +724,9 @@ end function _mt:delete_by_field(field_name, unique_value, options) local statement_name = "delete_by_" .. field_name + if self.schema.ttl and options and options.skip_ttl then + statement_name = statement_name .. "_skip_ttl" + end local filter = { [field_name] = unique_value, } @@ -1189,6 +1206,19 @@ function _M.new(connector, schema, errors) } }) + add_statement("delete_skip_ttl", { + operation = "write", + argn = primary_key_names, + argv = primary_key_args, + code = { + "DELETE\n", + " FROM ", table_name_escaped, "\n", + where_clause( + " WHERE ", "(" .. pk_escaped .. ") = (" .. primary_key_placeholders .. ")", + ws_id_select_where), ";" + } + }) + add_statement("select", { operation = "read", expr = select_expressions, @@ -1205,6 +1235,21 @@ function _M.new(connector, schema, errors) } }) + add_statement("select_skip_ttl", { + operation = "read", + expr = select_expressions, + argn = primary_key_names, + argv = primary_key_args, + code = { + "SELECT ", select_expressions, "\n", + " FROM ", table_name_escaped, "\n", + where_clause( + " WHERE ", "(" .. pk_escaped .. ") = (" .. primary_key_placeholders .. ")", + ws_id_select_where), + " LIMIT 1;" + } + }) + add_statement_for_export("page_first", { operation = "read", argn = { LIMIT }, @@ -1387,6 +1432,20 @@ function _M.new(connector, schema, errors) }, }) + add_statement("select_by_" .. field_name .. "_skip_ttl", { + operation = "read", + argn = single_names, + argv = single_args, + code = { + "SELECT ", select_expressions, "\n", + " FROM ", table_name_escaped, "\n", + where_clause( + " WHERE ", unique_escaped .. " = $1", + ws_id_select_where), + " LIMIT 1;" + }, + }) + local update_by_args_names = {} for _, update_name in ipairs(update_names) do insert(update_by_args_names, update_name) @@ -1442,6 +1501,19 @@ function _M.new(connector, schema, errors) ws_id_select_where), ";" } }) + + add_statement("delete_by_" .. field_name .. "_skip_ttl", { + operation = "write", + argn = single_names, + argv = single_args, + code = { + "DELETE\n", + " FROM ", table_name_escaped, "\n", + where_clause( + " WHERE ", unique_escaped .. " = $1", + ws_id_select_where), ";" + } + }) end end diff --git a/spec/02-integration/03-db/14-dao_spec.lua b/spec/02-integration/03-db/14-dao_spec.lua index 313ebd9bd650..a8f20498d1c2 100644 --- a/spec/02-integration/03-db/14-dao_spec.lua +++ b/spec/02-integration/03-db/14-dao_spec.lua @@ -16,6 +16,7 @@ for _, strategy in helpers.all_strategies() do "services", "consumers", "acls", + "keyauth_credentials", }) _G.kong.db = db @@ -98,6 +99,7 @@ for _, strategy in helpers.all_strategies() do db.consumers:truncate() db.plugins:truncate() db.services:truncate() + db.keyauth_credentials:truncate() end) it("select_by_cache_key()", function() @@ -185,6 +187,36 @@ for _, strategy in helpers.all_strategies() do assert.same(new_plugin_config.config.redis.host, read_plugin.config.redis.host) assert.same(new_plugin_config.config.redis.host, read_plugin.config.redis_host) -- legacy field is included end) + + it("keyauth_credentials can be deleted or selected before run ttl cleanup in background timer", function() + local key = uuid() + local original_keyauth_credentials = bp.keyauth_credentials:insert({ + consumer = { id = consumer.id }, + key = key, + }, { ttl = 5 }) + + -- wait for 5 seconds. + ngx.sleep(5) + + -- select or delete keyauth_credentials after ttl expired. + local expired_keyauth_credentials + helpers.wait_until(function() + expired_keyauth_credentials = kong.db.keyauth_credentials:select_by_key(key) + return not expired_keyauth_credentials + end, 1) + assert.is_nil(expired_keyauth_credentials) + kong.db.keyauth_credentials:delete_by_key(key) + + -- select or delete keyauth_credentials with skip_ttl=true after ttl expired. + expired_keyauth_credentials = kong.db.keyauth_credentials:select_by_key(key, { skip_ttl = true }) + assert.not_nil(expired_keyauth_credentials) + assert.same(expired_keyauth_credentials.id, original_keyauth_credentials.id) + kong.db.keyauth_credentials:delete_by_key(key, { skip_ttl = true }) + + -- check again + expired_keyauth_credentials = kong.db.keyauth_credentials:select_by_key(key, { skip_ttl = true }) + assert.is_nil(expired_keyauth_credentials) + end) end) end From 6019379d1f29a0bf53d784d2b5776278043268ce Mon Sep 17 00:00:00 2001 From: Xiaoyan Rao <270668624@qq.com> Date: Tue, 2 Jul 2024 15:24:41 +0800 Subject: [PATCH 2/4] fix code style --- kong/db/strategies/postgres/init.lua | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kong/db/strategies/postgres/init.lua b/kong/db/strategies/postgres/init.lua index 14d4f50508a2..963536a3c072 100644 --- a/kong/db/strategies/postgres/init.lua +++ b/kong/db/strategies/postgres/init.lua @@ -1215,7 +1215,7 @@ function _M.new(connector, schema, errors) " FROM ", table_name_escaped, "\n", where_clause( " WHERE ", "(" .. pk_escaped .. ") = (" .. primary_key_placeholders .. ")", - ws_id_select_where), ";" + ws_id_select_where), ";" } }) @@ -1245,7 +1245,7 @@ function _M.new(connector, schema, errors) " FROM ", table_name_escaped, "\n", where_clause( " WHERE ", "(" .. pk_escaped .. ") = (" .. primary_key_placeholders .. ")", - ws_id_select_where), + ws_id_select_where), " LIMIT 1;" } }) @@ -1441,7 +1441,7 @@ function _M.new(connector, schema, errors) " FROM ", table_name_escaped, "\n", where_clause( " WHERE ", unique_escaped .. " = $1", - ws_id_select_where), + ws_id_select_where), " LIMIT 1;" }, }) @@ -1511,7 +1511,7 @@ function _M.new(connector, schema, errors) " FROM ", table_name_escaped, "\n", where_clause( " WHERE ", unique_escaped .. " = $1", - ws_id_select_where), ";" + ws_id_select_where), ";" } }) end From e1dcccd73229e67c63c326c9972725cf08fac9d5 Mon Sep 17 00:00:00 2001 From: Xiaoyan Rao <270668624@qq.com> Date: Wed, 3 Jul 2024 14:28:13 +0800 Subject: [PATCH 3/4] fix code style --- kong/db/strategies/postgres/init.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kong/db/strategies/postgres/init.lua b/kong/db/strategies/postgres/init.lua index 963536a3c072..a19f1230dc91 100644 --- a/kong/db/strategies/postgres/init.lua +++ b/kong/db/strategies/postgres/init.lua @@ -424,7 +424,7 @@ local function execute(strategy, statement_name, attributes, options) local is_update = options and options.update local has_ttl = strategy.schema.ttl - local is_skip_ttl = options and options.skip_ttl + local skip_ttl = options and options.skip_ttl if has_ws_id then assert(ws_id == nil or type(ws_id) == "string") argv[0] = escape_literal(connector, ws_id, "ws_id") @@ -433,7 +433,7 @@ local function execute(strategy, statement_name, attributes, options) for i = 1, argc do local name = argn[i] local value - if has_ttl and name == "ttl" and not is_skip_ttl then + if has_ttl and name == "ttl" and not skip_ttl then value = (options and options.ttl) and get_ttl_value(strategy, attributes, options) From 5da83edf8d43d727e227100bd80ff9f4e8b8dead Mon Sep 17 00:00:00 2001 From: Xiaoyan Rao <270668624@qq.com> Date: Wed, 3 Jul 2024 16:21:29 +0800 Subject: [PATCH 4/4] remove changelog --- changelog/unreleased/kong/ttl_expires.yml | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 changelog/unreleased/kong/ttl_expires.yml diff --git a/changelog/unreleased/kong/ttl_expires.yml b/changelog/unreleased/kong/ttl_expires.yml deleted file mode 100644 index 57bec033dfbd..000000000000 --- a/changelog/unreleased/kong/ttl_expires.yml +++ /dev/null @@ -1,3 +0,0 @@ -message: Fixed an issue where can't be deleted or selected an entity after TTL expires. -type: bugfix -scope: Core \ No newline at end of file