fix(ca_certificates): invalidate ca store caches when a ca cert is up…

…dated and prevent ca_certificates that are still being referenced by other entities from being deleted (#11789)

* fix(ca_certificates): invalidate ca store caches when a ca cert is
                      updated and prevent ca_certificates that are still
                      being referenced by other entities from being deleted.

Fix [FTI-2060](https://konghq.atlassian.net/browse/FTI-2060)

* apply comments

* change plugin tables from maps to arrays

* fix plugin_name double check

* remove `search_fields` for now as it is EE-only

* do the iteration and filtering in dao by adding `select_by_ca_certificate`

* auto-detect the entities and plugins that reference ca certificates to
make it more generic.
create a custom ca_certificates dao and put the check_ca_reference logic into
the `:delete()` method instead of a custom API route

* update the schema of ca_certificates

* fix: fields in schema is an array and cert_pk is a table

* add services:select_by_ca_certificate() tests

* fix lint

* add custom plugin "reference-ca-cert" and plugins:select_by_ca_certificate() tests

* add ca_certificates:delete() tests

* Apply suggestions from code review

Co-authored-by: Michael Martin <[email protected]>

* fix typo

* remove plugins.lua and services.lua for `off` as they're not
currently being used

---------

Co-authored-by: Michael Martin <[email protected]>

changelog/unreleased/kong/ca_certificates_reference_check.yml

@@ -0,0 +1,3 @@
+message: prevent ca to be deleted when it's still referenced by other entities and invalidate the related ca store caches when a ca cert is updated.
+type: bugfix
+scope: Core

kong-3.6.0-0.rockspec

@@ -212,6 +212,8 @@ build = {
     ["kong.db.dao.tags"] = "kong/db/dao/tags.lua",
     ["kong.db.dao.vaults"] = "kong/db/dao/vaults.lua",
     ["kong.db.dao.workspaces"] = "kong/db/dao/workspaces.lua",
+    ["kong.db.dao.services"] = "kong/db/dao/services.lua",
+    ["kong.db.dao.ca_certificates"] = "kong/db/dao/ca_certificates.lua",
     ["kong.db.declarative"] = "kong/db/declarative/init.lua",
     ["kong.db.declarative.marshaller"] = "kong/db/declarative/marshaller.lua",
     ["kong.db.declarative.export"] = "kong/db/declarative/export.lua",
@@ -251,6 +253,8 @@ build = {
     ["kong.db.strategies.postgres"] = "kong/db/strategies/postgres/init.lua",
     ["kong.db.strategies.postgres.connector"] = "kong/db/strategies/postgres/connector.lua",
     ["kong.db.strategies.postgres.tags"] = "kong/db/strategies/postgres/tags.lua",
+    ["kong.db.strategies.postgres.services"] = "kong/db/strategies/postgres/services.lua",
+    ["kong.db.strategies.postgres.plugins"] = "kong/db/strategies/postgres/plugins.lua",
     ["kong.db.strategies.off"] = "kong/db/strategies/off/init.lua",
     ["kong.db.strategies.off.connector"] = "kong/db/strategies/off/connector.lua",
     ["kong.db.strategies.off.tags"] = "kong/db/strategies/off/tags.lua",

kong/api/endpoints.lua

@@ -35,6 +35,7 @@ local ERRORS_HTTP_CODES = {
   [Errors.codes.INVALID_OPTIONS]         = 400,
   [Errors.codes.OPERATION_UNSUPPORTED]   = 405,
   [Errors.codes.FOREIGN_KEYS_UNRESOLVED] = 400,
+  [Errors.codes.REFERENCED_BY_OTHERS]    = 400,
 }
 
 local TAGS_AND_REGEX

kong/db/dao/ca_certificates.lua

@@ -0,0 +1,55 @@
+local certificate  = require "kong.runloop.certificate"
+local fmt = string.format
+
+local Ca_certificates = {}
+
+-- returns the first encountered entity element that is referencing the ca cert
+-- otherwise, returns nil, err
+function Ca_certificates:check_ca_reference(ca_id)
+  for _, entity in ipairs(certificate.get_ca_certificate_reference_entities()) do
+    local elements, err = self.db[entity]:select_by_ca_certificate(ca_id, 1)
+    if err then
+      local msg = fmt("failed to select %s by ca certificate %s: %s", entity, ca_id, err)
+      return nil, msg
+    end
+
+    if type(elements) == "table" and #elements > 0 then
+      return entity, elements[1]
+    end
+  end
+
+  local reference_plugins = certificate.get_ca_certificate_reference_plugins()
+  if reference_plugins and next(reference_plugins) then
+    local plugins, err = self.db.plugins:select_by_ca_certificate(ca_id, 1, reference_plugins)
+    if err then
+      local msg = fmt("failed to select plugins by ca_certificate %s: %s", ca_id, err)
+      return nil, msg
+    end
+
+    if type(plugins) == "table" and #plugins > 0 then
+      return "plugins", plugins[1]
+    end
+  end
+
+  return nil, nil
+end
+
+-- Overrides the default delete function to check the ca reference before deleting
+function Ca_certificates:delete(cert_pk, options)
+  local entity, element_or_err = self:check_ca_reference(cert_pk.id)
+  if entity then
+    local msg = fmt("ca certificate %s is still referenced by %s (id = %s)",
+                     cert_pk.id, entity, element_or_err.id)
+    local err_t = self.errors:referenced_by_others(msg)
+    return nil, tostring(err_t), err_t
+
+  elseif element_or_err then
+    local err_t = self.errors:database_error(element_or_err)
+    return nil, tostring(err_t), err_t
+  end
+
+  return self.super.delete(self, cert_pk, options)
+end
+
+
+return Ca_certificates

kong/db/dao/plugins.lua

@@ -371,5 +371,23 @@ function Plugins:get_handlers()
   return list
 end
 
+-- @ca_id: the id of ca certificate to be searched
+-- @limit: the maximum number of entities to return (must >= 0)
+-- @plugin_names: the plugin names to filter the entities (must be of type table, string or nil)
+-- @return an array of the plugin entity
+function Plugins:select_by_ca_certificate(ca_id, limit, plugin_names)
+  local param_type = type(plugin_names)
+  if param_type ~= "table" and param_type ~= "string" and param_type ~= "nil" then
+    return nil, "parameter `plugin_names` must be of type table, string, or nil"
+  end
+
+  local plugins, err = self.strategy:select_by_ca_certificate(ca_id, limit, plugin_names)
+  if err then
+    return nil, err
+  end
+
+  return self:rows_to_entities(plugins), nil
+end
+
 
 return Plugins

kong/db/dao/services.lua

@@ -0,0 +1,16 @@
+
+local Services = {}
+
+-- @ca_id: the id of ca certificate to be searched
+-- @limit: the maximum number of entities to return (must >= 0)
+-- @return an array of the service entity
+function Services:select_by_ca_certificate(ca_id, limit)
+  local services, err = self.strategy:select_by_ca_certificate(ca_id, limit)
+  if err then
+    return nil, err
+  end
+
+  return self:rows_to_entities(services), nil
+end
+
+return Services

kong/db/errors.lua

@@ -52,6 +52,7 @@ local ERRORS              = {
   INVALID_FOREIGN_KEY     = 16, -- foreign key is valid for matching a row
   INVALID_WORKSPACE       = 17, -- strategy reports a workspace error
   INVALID_UNIQUE_GLOBAL   = 18, -- unique field value is invalid for global query
+  REFERENCED_BY_OTHERS    = 19, -- still referenced by other entities
 }
 
 
@@ -77,6 +78,7 @@ local ERRORS_NAMES                 = {
   [ERRORS.INVALID_FOREIGN_KEY]     = "invalid foreign key",
   [ERRORS.INVALID_WORKSPACE]       = "invalid workspace",
   [ERRORS.INVALID_UNIQUE_GLOBAL]   = "invalid global query",
+  [ERRORS.REFERENCED_BY_OTHERS]    = "referenced by others",
 }
 
 
@@ -517,6 +519,15 @@ function _M:invalid_unique_global(name)
 end
 
 
+function _M:referenced_by_others(err)
+  if type(err) ~= "string" then
+    error("err must be a string", 2)
+  end
+
+  return new_err_t(self, ERRORS.REFERENCED_BY_OTHERS, err)
+end
+
+
 local flatten_errors
 do
   local function singular(noun)

kong/db/schema/entities/ca_certificates.lua

@@ -11,6 +11,7 @@ local CERT_TAG_LEN  = #CERT_TAG
 return {
   name        = "ca_certificates",
   primary_key = { "id" },
+  dao         = "kong.db.dao.ca_certificates",
 
   fields = {
     { id = typedefs.uuid, },

kong/db/schema/entities/services.lua

@@ -23,6 +23,7 @@ return {
   primary_key = { "id" },
   workspaceable = true,
   endpoint_key = "name",
+  dao = "kong.db.dao.services",
 
   fields = {
     { id                 = typedefs.uuid, },

kong/db/strategies/postgres/plugins.lua

@@ -0,0 +1,39 @@
+local kong = kong
+local fmt  = string.format
+local tb_insert = table.insert
+local tb_concat = table.concat
+
+local Plugins = {}
+
+function Plugins:select_by_ca_certificate(ca_id, limit, plugin_names)
+  local connector = kong.db.connector
+  local escape_literal = connector.escape_literal
+  local limit_condition = ""
+  if limit then
+    limit_condition = "LIMIT " .. escape_literal(connector, limit)
+  end
+
+  local name_condition = ""
+  local escaped_names = {}
+  if type(plugin_names) == "string" then
+    tb_insert(escaped_names, "name = " .. escape_literal(connector, plugin_names))
+  elseif type(plugin_names) == "table" then
+    for name, _ in pairs(plugin_names) do
+      tb_insert(escaped_names, "name = " .. escape_literal(connector, name))
+    end
+  end
+
+  if #escaped_names > 0 then
+    name_condition = "AND (" .. tb_concat(escaped_names, " OR ") .. ")"
+  end
+
+  local qs = fmt(
+    "SELECT * FROM plugins WHERE config->'ca_certificates' ? %s %s %s;",
+    escape_literal(connector, ca_id),
+    name_condition,
+    limit_condition)
+
+  return connector:query(qs)
+end
+
+return Plugins

kong/db/strategies/postgres/services.lua

@@ -0,0 +1,20 @@
+local kong = kong
+local fmt  = string.format
+
+local Services = {}
+
+function Services:select_by_ca_certificate(ca_id, limit)
+  local limit_condition = ""
+  if limit then
+    limit_condition = "LIMIT " .. kong.db.connector:escape_literal(limit)
+  end
+
+  local qs = fmt(
+    "SELECT * FROM services WHERE %s = ANY(ca_certificates) %s;",
+    kong.db.connector:escape_literal(ca_id),
+    limit_condition)
+
+  return kong.db.connector:query(qs)
+end
+
+return Services

kong/runloop/certificate.lua

@@ -2,6 +2,9 @@ local ngx_ssl = require "ngx.ssl"
 local pl_utils = require "pl.utils"
 local mlcache = require "kong.resty.mlcache"
 local new_tab = require "table.new"
+local constants = require "kong.constants"
+local utils = require "kong.tools.utils"
+local plugin_servers = require "kong.runloop.plugin_servers"
 local openssl_x509_store = require "resty.openssl.x509.store"
 local openssl_x509 = require "resty.openssl.x509"
 
@@ -19,6 +22,7 @@ local set_cert = ngx_ssl.set_cert
 local set_priv_key = ngx_ssl.set_priv_key
 local tb_concat   = table.concat
 local tb_sort   = table.sort
+local tb_insert   = table.insert
 local kong = kong
 local type = type
 local error = error
@@ -371,11 +375,106 @@ local function get_ca_certificate_store(ca_ids)
 end
 
 
+local function get_ca_certificate_store_for_plugin(ca_ids)
+  return kong.cache:get(ca_ids_cache_key(ca_ids),
+                        get_ca_store_opts, fetch_ca_certificates,
+                        ca_ids)
+end
+
+
+-- here we assume the field name is always `ca_certificates`
+local get_ca_certificate_reference_entities
+do
+  local function is_entity_referencing_ca_certificates(name)
+    local entity_schema = require("kong.db.schema.entities." .. name)
+    for _, field in ipairs(entity_schema.fields) do
+      if field.ca_certificates then
+        return true
+      end
+    end
+
+    return false
+  end
+
+  -- ordinary entities that reference ca certificates
+  -- For example: services
+  local CA_CERT_REFERENCE_ENTITIES
+  get_ca_certificate_reference_entities = function()
+    if not CA_CERT_REFERENCE_ENTITIES then
+      CA_CERT_REFERENCE_ENTITIES = {}
+      for _, entity_name in ipairs(constants.CORE_ENTITIES) do
+        local res = is_entity_referencing_ca_certificates(entity_name)
+        if res then
+          tb_insert(CA_CERT_REFERENCE_ENTITIES, entity_name)
+        end
+      end
+    end
+
+    return CA_CERT_REFERENCE_ENTITIES
+  end
+end
+
+
+-- here we assume the field name is always `ca_certificates`
+local get_ca_certificate_reference_plugins
+do
+  local function is_plugin_referencing_ca_certificates(name)
+    local plugin_schema = "kong.plugins." .. name .. ".schema"
+    local ok, schema = utils.load_module_if_exists(plugin_schema)
+    if not ok then
+      ok, schema = plugin_servers.load_schema(name)
+    end
+
+    if not ok then
+      return nil, "no configuration schema found for plugin: " .. name
+    end
+
+    for _, field in ipairs(schema.fields) do
+      if field.config then
+        for _, field in ipairs(field.config.fields) do
+          if field.ca_certificates then
+            return true
+          end
+        end
+      end
+    end
+
+    return false
+  end
+
+  -- loaded plugins that reference ca certificates
+  -- For example: mtls-auth
+  local CA_CERT_REFERENCE_PLUGINS
+  get_ca_certificate_reference_plugins = function()
+    if not CA_CERT_REFERENCE_PLUGINS then
+      CA_CERT_REFERENCE_PLUGINS = {}
+      local loaded_plugins = kong.configuration.loaded_plugins
+      for name, v in pairs(loaded_plugins) do
+        local res, err = is_plugin_referencing_ca_certificates(name)
+        if err then
+          return nil, err
+        end
+
+        if res then
+          CA_CERT_REFERENCE_PLUGINS[name] = true
+        end
+      end
+    end
+
+    return CA_CERT_REFERENCE_PLUGINS
+  end
+end
+
+
 return {
   init = init,
   find_certificate = find_certificate,
   produce_wild_snis = produce_wild_snis,
   execute = execute,
   get_certificate = get_certificate,
   get_ca_certificate_store = get_ca_certificate_store,
+  get_ca_certificate_store_for_plugin = get_ca_certificate_store_for_plugin,
+  ca_ids_cache_key = ca_ids_cache_key,
+  get_ca_certificate_reference_entities = get_ca_certificate_reference_entities,
+  get_ca_certificate_reference_plugins = get_ca_certificate_reference_plugins,
 }