From 938747a141b32aa6ad580dde92ef5fa202830218 Mon Sep 17 00:00:00 2001 From: Xiaoyan Rao <270668624@qq.com> Date: Tue, 31 Dec 2024 11:14:15 +0800 Subject: [PATCH] feat(sessions): store session's meatadata (#13990) Engineering brief: https://docs.google.com/document/d/1r7nSZGOlTVhupoVGQ6Ymqsaw64eGHKSMibko8FgrwAQ/edit?tab=t.0 **### This PR won't bring a break change.** Add two new configurations`hash_subject`(by default false) and `store_metadata`(by default false). Please ref [here](https://github.com/bungle/lua-resty-session?tab=readme-ov-file#session-configuration). Add a new table `session_metadatas` to store the session's metadata. ``` CREATE TABLE IF NOT EXISTS session_metadatas( id uuid, session_id uuid REFERENCES "sessions" ("id") ON DELETE CASCADE, sid text, subject text, audience text, created_at timestamp WITH TIME ZONE, PRIMARY KEY (id) ); ``` --- .../kong/session_store_metadata.yml | 4 + kong-3.10.0-0.rockspec | 3 + kong/plugins/session/daos.lua | 49 ++++--- .../session/daos/session_metadatas.lua | 7 + .../session/migrations/003_330_to_3100.lua | 23 ++++ kong/plugins/session/migrations/init.lua | 1 + kong/plugins/session/schema.lua | 15 +++ kong/plugins/session/session.lua | 2 + kong/plugins/session/storage/kong.lua | 37 +++++- .../strategies/postgres/session_metadatas.lua | 22 ++++ .../02-kong_storage_adapter_spec.lua | 120 ++++++++++++++++++ .../migrations/003_330_to_3100_spec.lua | 15 +++ 12 files changed, 280 insertions(+), 18 deletions(-) create mode 100644 changelog/unreleased/kong/session_store_metadata.yml create mode 100644 kong/plugins/session/daos/session_metadatas.lua create mode 100644 kong/plugins/session/migrations/003_330_to_3100.lua create mode 100644 kong/plugins/session/strategies/postgres/session_metadatas.lua create mode 100644 spec/05-migration/plugins/session/migrations/003_330_to_3100_spec.lua diff --git a/changelog/unreleased/kong/session_store_metadata.yml b/changelog/unreleased/kong/session_store_metadata.yml new file mode 100644 index 000000000000..8369a55e025a --- /dev/null +++ b/changelog/unreleased/kong/session_store_metadata.yml @@ -0,0 +1,4 @@ +message: | + **session**: Added two boolean configuration fields `hash_subject` (default `false`) and `store_metadata` (default `false`) to store session's metadata in the database. +type: feature +scope: "Plugin" \ No newline at end of file diff --git a/kong-3.10.0-0.rockspec b/kong-3.10.0-0.rockspec index 9aafa5c0c0a2..1d17a970c9cd 100644 --- a/kong-3.10.0-0.rockspec +++ b/kong-3.10.0-0.rockspec @@ -567,10 +567,13 @@ build = { ["kong.plugins.session.header_filter"] = "kong/plugins/session/header_filter.lua", ["kong.plugins.session.session"] = "kong/plugins/session/session.lua", ["kong.plugins.session.daos"] = "kong/plugins/session/daos.lua", + ["kong.plugins.session.daos.session_metadatas"] = "kong/plugins/session/daos/session_metadatas.lua", + ["kong.plugins.session.strategies.postgres.session_metadatas"] = "kong/plugins/session/strategies/postgres/session_metadatas.lua", ["kong.plugins.session.storage.kong"] = "kong/plugins/session/storage/kong.lua", ["kong.plugins.session.migrations.000_base_session"] = "kong/plugins/session/migrations/000_base_session.lua", ["kong.plugins.session.migrations.001_add_ttl_index"] = "kong/plugins/session/migrations/001_add_ttl_index.lua", ["kong.plugins.session.migrations.002_320_to_330"] = "kong/plugins/session/migrations/002_320_to_330.lua", + ["kong.plugins.session.migrations.003_330_to_3100"] = "kong/plugins/session/migrations/003_330_to_3100.lua", ["kong.plugins.session.migrations"] = "kong/plugins/session/migrations/init.lua", ["kong.plugins.proxy-cache.handler"] = "kong/plugins/proxy-cache/handler.lua", diff --git a/kong/plugins/session/daos.lua b/kong/plugins/session/daos.lua index 75f85ab5c63b..d55d2d3f370a 100644 --- a/kong/plugins/session/daos.lua +++ b/kong/plugins/session/daos.lua @@ -1,20 +1,39 @@ local typedefs = require "kong.db.schema.typedefs" -return { - { - primary_key = { "id" }, - endpoint_key = "session_id", - name = "sessions", - cache_key = { "session_id" }, - ttl = true, - db_export = false, - fields = { - { id = typedefs.uuid }, - { session_id = { type = "string", unique = true, required = true } }, - { expires = { type = "integer" } }, - { data = { type = "string" } }, - { created_at = typedefs.auto_timestamp_s }, - } +local sessions = { + primary_key = { "id" }, + endpoint_key = "session_id", + name = "sessions", + cache_key = { "session_id" }, + ttl = true, + db_export = false, + fields = { + { id = typedefs.uuid }, + { session_id = { type = "string", unique = true, required = true } }, + { expires = { type = "integer" } }, + { data = { type = "string" } }, + { created_at = typedefs.auto_timestamp_s }, + } +} + +local session_metadatas = { + primary_key = { "id" }, + name = "session_metadatas", + dao = "kong.plugins.session.daos.session_metadatas", + generate_admin_api = false, + db_export = false, + fields = { + { id = typedefs.uuid }, + { session = { type = "foreign", reference = "sessions", required = true, on_delete = "cascade" } }, + { sid = { type = "string" } }, + { audience = { type = "string" } }, + { subject = { type = "string" } }, + { created_at = typedefs.auto_timestamp_s }, } } + +return { + sessions, + session_metadatas, +} diff --git a/kong/plugins/session/daos/session_metadatas.lua b/kong/plugins/session/daos/session_metadatas.lua new file mode 100644 index 000000000000..f4ca7eec46b3 --- /dev/null +++ b/kong/plugins/session/daos/session_metadatas.lua @@ -0,0 +1,7 @@ +local session_metadatas = {} + +function session_metadatas:select_by_audience_and_subject(audience, subject) + return self.strategy:select_by_audience_and_subject(audience, subject) +end + +return session_metadatas diff --git a/kong/plugins/session/migrations/003_330_to_3100.lua b/kong/plugins/session/migrations/003_330_to_3100.lua new file mode 100644 index 000000000000..2b1389a3e4da --- /dev/null +++ b/kong/plugins/session/migrations/003_330_to_3100.lua @@ -0,0 +1,23 @@ +return { + postgres = { + up = [[ + CREATE TABLE IF NOT EXISTS session_metadatas( + id uuid, + session_id uuid REFERENCES "sessions" ("id") ON DELETE CASCADE, + sid text, + subject text, + audience text, + created_at timestamp WITH TIME ZONE, + PRIMARY KEY (id) + ); + + DO $$ + BEGIN + CREATE INDEX IF NOT EXISTS "session_id_idx" ON "session_metadatas" ("session_id"); + CREATE INDEX IF NOT EXISTS "subject_audience_idx" ON "session_metadatas" ("subject", "audience"); + EXCEPTION WHEN UNDEFINED_COLUMN THEN + -- Do nothing, accept existing state + END$$; + ]], + }, +} diff --git a/kong/plugins/session/migrations/init.lua b/kong/plugins/session/migrations/init.lua index 71910b2e7551..59e50200f674 100644 --- a/kong/plugins/session/migrations/init.lua +++ b/kong/plugins/session/migrations/init.lua @@ -2,4 +2,5 @@ return { "000_base_session", "001_add_ttl_index", "002_320_to_330", + "003_330_to_3100", } diff --git a/kong/plugins/session/schema.lua b/kong/plugins/session/schema.lua index 41e77699c0ed..d1a1a472fde2 100644 --- a/kong/plugins/session/schema.lua +++ b/kong/plugins/session/schema.lua @@ -212,6 +212,21 @@ return { default = "session_logout" } }, + { + hash_subject = { + description = "Whether to hash or not the subject when store_metadata is enabled.", + type = "boolean", + default = false + } + }, + { + store_metadata = { + description = + "Whether to also store metadata of sessions, such as collecting data of sessions for a specific audience belonging to a specific subject.", + type = "boolean", + default = false + } + }, }, shorthand_fields = { -- TODO: deprecated forms, to be removed in Kong 4.0 diff --git a/kong/plugins/session/session.lua b/kong/plugins/session/session.lua index d94797708226..20566231858c 100644 --- a/kong/plugins/session/session.lua +++ b/kong/plugins/session/session.lua @@ -43,6 +43,8 @@ function _M.open_session(conf) remember_absolute_timeout = conf.remember_absolute_timeout, response_headers = conf.response_headers, request_headers = conf.request_headers, + hash_subject = conf.hash_subject, + store_metadata = conf.store_metadata, }) end diff --git a/kong/plugins/session/storage/kong.lua b/kong/plugins/session/storage/kong.lua index bc27ae8be818..b07ca80c20db 100644 --- a/kong/plugins/session/storage/kong.lua +++ b/kong/plugins/session/storage/kong.lua @@ -38,8 +38,32 @@ local function load_session_from_cache(key) return kong.cache:get(cache_key, nil, load_session_from_db, key) end +local function insert_session_metadata(metadata, session) + if not metadata then + return + end + + local audiences = metadata.audiences + local subjects = metadata.subjects + local count = #audiences + for i = 1, count do + local _, err = kong.db.session_metadatas:insert({ + sid = session.session_id, + audience = audiences[i], + subject = subjects[i], + session = session, + }) + + if err then + kong.db.sessions:delete(session.id) + return false, err + end + end + + return true +end -local function insert_session(key, value, ttl, current_time, old_key, stale_ttl, remember) +local function insert_session(key, value, ttl, current_time, old_key, stale_ttl, metadata, remember) DATA.session_id = key DATA.data = value DATA.expires = current_time + ttl @@ -47,6 +71,13 @@ local function insert_session(key, value, ttl, current_time, old_key, stale_ttl, TTL.ttl = ttl local insert_ok, insert_err = kong.db.sessions:insert(DATA, TTL) + if not insert_err then + local ok, err = insert_session_metadata(metadata, insert_ok) + if not ok and err then + return nil, err + end + end + if not old_key then return insert_ok, insert_err end @@ -103,11 +134,11 @@ end function storage:set(name, key, value, ttl, current_time, old_key, stale_ttl, metadata, remember) if get_phase() == "header_filter" then - timer_at(0, insert_session_timer, key, value, ttl, current_time, old_key, stale_ttl, remember) + timer_at(0, insert_session_timer, key, value, ttl, current_time, old_key, stale_ttl, metadata, remember) return true end - return insert_session(key, value, ttl, current_time, old_key, stale_ttl, remember) + return insert_session(key, value, ttl, current_time, old_key, stale_ttl, metadata, remember) end diff --git a/kong/plugins/session/strategies/postgres/session_metadatas.lua b/kong/plugins/session/strategies/postgres/session_metadatas.lua new file mode 100644 index 000000000000..9285c957df86 --- /dev/null +++ b/kong/plugins/session/strategies/postgres/session_metadatas.lua @@ -0,0 +1,22 @@ +local fmt = string.format + +local session_metadatas = {} + +function session_metadatas:select_by_audience_and_subject(audience, subject) + if type(audience) ~= "string" then + error("audience must be string") + end + + if type(subject) ~= "string" then + error("subject must be string") + end + + local qs = fmt( + "SELECT * FROM session_metadatas WHERE audience = %s AND subject = %s;", + kong.db.connector:escape_literal(audience), + kong.db.connector:escape_literal(subject)) + + return kong.db.connector:query(qs, "read") +end + +return session_metadatas diff --git a/spec/03-plugins/30-session/02-kong_storage_adapter_spec.lua b/spec/03-plugins/30-session/02-kong_storage_adapter_spec.lua index 509f2556cd75..060f326e711b 100644 --- a/spec/03-plugins/30-session/02-kong_storage_adapter_spec.lua +++ b/spec/03-plugins/30-session/02-kong_storage_adapter_spec.lua @@ -1,6 +1,17 @@ local helpers = require "spec.helpers" local cjson = require "cjson" +local sub = string.sub +local sha256_bin = require "kong.tools.sha256".sha256_bin +local encode_base64url = require "ngx.base64".encode_base64url +local function sha256_subject(key) + local subject, err = sha256_bin(key) + if err then + return nil, err + end + + return encode_base64url(sub(subject, 1, 16)) +end for _, strategy in helpers.each_strategy() do describe("Plugin: Session (kong storage adapter) [#" .. strategy .. "]", function() @@ -31,6 +42,16 @@ for _, strategy in helpers.each_strategy() do hosts = {"konghq.test"}, } + local route4 = bp.routes:insert { + paths = { "/metadata1" }, + hosts = { "konghq.metadata1" }, + } + + local route5 = bp.routes:insert { + paths = { "/hash_subject" }, + hosts = { "konghq.hash_subject" }, + } + assert(bp.plugins:insert { name = "session", route = { @@ -68,6 +89,33 @@ for _, strategy in helpers.each_strategy() do } }) + assert(bp.plugins:insert { + name = "session", + route = { + id = route4.id, + }, + config = { + storage = "kong", + store_metadata = true, + secret = "ultra top secret session", + response_headers = { "id", "timeout", "audience", "subject" } + } + }) + + assert(bp.plugins:insert { + name = "session", + route = { + id = route5.id, + }, + config = { + storage = "kong", + hash_subject = true, + store_metadata = true, + secret = "ultra top secret session", + response_headers = { "id", "timeout", "audience", "subject" } + } + }) + bp.plugins:insert { name = "ctx-checker", route = { id = route3.id }, @@ -117,6 +165,26 @@ for _, strategy in helpers.each_strategy() do } } + bp.plugins:insert { + name = "key-auth", + route = { + id = route4.id, + }, + config = { + anonymous = anonymous.id + } + } + + bp.plugins:insert { + name = "key-auth", + route = { + id = route5.id, + }, + config = { + anonymous = anonymous.id + } + } + bp.plugins:insert { name = "request-termination", consumer = { @@ -330,6 +398,58 @@ for _, strategy in helpers.each_strategy() do local json = cjson.decode(assert.res_status(200, res)) assert.equal('beatles, ramones', json.headers['x-authenticated-groups']) end) + + it("store metadata", function() + local request = { + method = "GET", + path = "/metadata1", + headers = { host = "konghq.metadata1", }, + } + + request.headers.apikey = "kong" + client = helpers.proxy_ssl_client() + local res = assert(client:send(request)) + assert.response(res).has.status(200) + + local sid = res.headers["Session-Id"] + local audience = res.headers["Session-audience"] + local subject = res.headers["Session-subject"] + + ngx.sleep(2) + subject = encode_base64url(subject) + audience = encode_base64url(audience) + + local session_metadatas = kong.db.session_metadatas:select_by_audience_and_subject(audience, subject) + assert.equal(1, #session_metadatas) + local metadata = session_metadatas[1] + assert.equal(sid, metadata.sid) + end) + + it("store metadata with hash_subject", function() + local request = { + method = "GET", + path = "/hash_subject", + headers = { host = "konghq.hash_subject", }, + } + + request.headers.apikey = "kong" + client = helpers.proxy_ssl_client() + local res = assert(client:send(request)) + assert.response(res).has.status(200) + + local sid = res.headers["Session-Id"] + local audience = res.headers["Session-audience"] + local subject = res.headers["Session-subject"] + ngx.sleep(2) + subject = sha256_subject(subject) + audience = encode_base64url(audience) + local session_metadatas = kong.db.session_metadatas:select_by_audience_and_subject(audience, subject) + assert.equal(1, #session_metadatas) + local metadata = session_metadatas[1] + assert.equal(subject, metadata.subject) + assert.equal(audience, metadata.audience) + assert.equal(sid, metadata.sid) + end) end) end) end diff --git a/spec/05-migration/plugins/session/migrations/003_330_to_3100_spec.lua b/spec/05-migration/plugins/session/migrations/003_330_to_3100_spec.lua new file mode 100644 index 000000000000..d7ef3c71fd01 --- /dev/null +++ b/spec/05-migration/plugins/session/migrations/003_330_to_3100_spec.lua @@ -0,0 +1,15 @@ +local uh = require "spec/upgrade_helpers" + +describe("database migration", function() + if uh.database_type() == "postgres" then + uh.all_phases("has created the \"session_metadatas\" table", function() + assert.database_has_relation("session_metadatas") + assert.table_has_column("session_metadatas", "id", "uuid") + assert.table_has_column("session_metadatas", "session_id", "uuid") + assert.table_has_column("session_metadatas", "sid", "text") + assert.table_has_column("session_metadatas", "subject", "text") + assert.table_has_column("session_metadatas", "audience", "text") + assert.table_has_column("session_metadatas", "created_at", "timestamp with time zone", "timestamp") + end) + end +end)