Merge remote-tracking branch 'ce/release/2.8.x' into 2.8

CHANGELOG.md

@@ -83,6 +83,10 @@
 
 - **HTTP Log**: fix internal error during validating the schema if http_endpoint contains
   userinfo but headers is empty [#9574](https://github.com/Kong/kong/pull/9574)
+- Update the batch queues module so that queues no longer grow without bounds if
+  their consumers fail to process the entries.  Instead, old batches are now dropped
+  and an error is logged.
+  [#10247](https://github.com/Kong/kong/pull/10247)
 
 ##### CLI
 

CHANGELOG/unreleased/kong/11480.yaml

@@ -0,0 +1,7 @@
+message: Fix a problem that abnormal socket connection will be reused when querying Postgres database.
+type: bugfix
+scope: Core
+prs:
+  - 11480
+jiras:
+  - "FTI-5322"

changelog/unreleased/kong/bump_dns_stale_ttl.yml

@@ -0,0 +1,3 @@
+message: Bump `dns_stale_ttl` default to 1 hour so stale DNS record can be used for longer time in case of resolver downtime.
+type: performance
+scope: Configuration

changelog/unreleased/kong/fix_dns_blocking.yml

@@ -0,0 +1,3 @@
+message: Eliminate asynchronous timer in syncQuery() to prevent hang risk
+type: bugfix
+scope: Core

changelog/unreleased/kong/fix_dns_retry_and_timeout_handling.yml

@@ -0,0 +1,3 @@
+message: Update the DNS client to follow configured timeouts in a more predictable manner
+type: bugfix
+scope: Core

kong-2.8.4-0.rockspec

@@ -33,7 +33,7 @@ dependencies = {
   "luaxxhash >= 1.0",
   "lua-protobuf == 0.3.3",
   "lua-resty-worker-events == 1.0.0",
-  "lua-resty-healthcheck == 1.5.1",
+  "lua-resty-healthcheck == 1.5.3",
   "lua-resty-mlcache == 2.5.0",
   "lua-messagepack == 0.5.2",
   "lua-resty-openssl == 0.8.22",

kong.conf.default

@@ -1312,14 +1312,16 @@
                                  # property receives a value (in seconds), it
                                  # will override the TTL for all records.
 
-#dns_stale_ttl = 4               # Defines, in seconds, how long a record will
+#dns_stale_ttl = 3600            # Defines, in seconds, how long a record will
                                  # remain in cache past its TTL. This value
                                  # will be used while the new DNS record is
                                  # fetched in the background.
                                  # Stale data will be used from expiry of a
                                  # record until either the refresh query
                                  # completes, or the `dns_stale_ttl` number of
                                  # seconds have passed.
+                                 # This configuration enables Kong to be more
+                                 # resilient during resolver downtime.
 
 #dns_cache_size = 10000          # Defines the maximum allowed number of
                                  # DNS records stored in memory cache.
@@ -1560,3 +1562,14 @@
                                      # **Warning**: Certain variables, when made
                                      # available, may create opportunities to
                                      # escape the sandbox.
+
+#max_queued_batches = 100         # Maximum number of batches to keep on an internal
+                                  # plugin queue before dropping old batches.  This is
+                                  # meant as a global, last-resort control to prevent
+                                  # queues from consuming infinite memory.  When batches
+                                  # are being dropped, an error message
+                                  # "exceeded max_queued_batches (%d), dropping oldest"
+                                  # will be logged.  The error message will also include
+                                  # a string that identifies the plugin causing the
+                                  # problem.  Queues are used by the http-log, statsd,
+                                  # opentelemetry and datadog plugins.

kong/conf_loader/init.lua

@@ -663,6 +663,8 @@ local CONF_INFERENCES = {
   untrusted_lua = { enum = { "on", "off", "sandbox" } },
   untrusted_lua_sandbox_requires = { typ = "array" },
   untrusted_lua_sandbox_environment = { typ = "array" },
+
+  max_queued_batches = { typ = "number" },
 }
 
 

kong/db/strategies/postgres/connector.lua

@@ -497,6 +497,7 @@ function _mt:query(sql, operation)
     operation = "write"
   end
 
+  local conn, is_new_conn
   local res, err, partial, num_queries
 
   local ok
@@ -505,23 +506,36 @@ function _mt:query(sql, operation)
     return nil, "error acquiring query semaphore: " .. err
   end
 
-  local conn = self:get_stored_connection(operation)
-  if conn then
-    res, err, partial, num_queries = conn:query(sql)
-
-  else
-    local connection
+  conn = self:get_stored_connection(operation)
+  if not conn then
     local config = operation == "write" and self.config or self.config_ro
 
-    connection, err = connect(config)
-    if not connection then
+    conn, err = connect(config)
+    if not conn then
       self:release_query_semaphore_resource(operation)
       return nil, err
     end
+    is_new_conn = true
+  end
+
+  res, err, partial, num_queries = conn:query(sql)
 
-    res, err, partial, num_queries = connection:query(sql)
+  -- if err is string then either it is a SQL error
+  -- or it is a socket error, here we abort connections
+  -- that encounter errors instead of reusing them, for
+  -- safety reason
+  if err and type(err) == "string" then
+    ngx.log(ngx.DEBUG, "SQL query throw error: ", err, ", close connection")
+    local _, err = conn:disconnect()
+    if err then
+      -- We're at the end of the query - just logging if
+      -- we cannot cleanup the connection
+      ngx.log(ngx.ERR, "failed to disconnect: ", err)
+    end
+    self.store_connection(nil, operation)
 
-    setkeepalive(connection)
+  elseif is_new_conn then
+    setkeepalive(conn)
   end
 
   self:release_query_semaphore_resource(operation)

kong/init.lua

@@ -564,6 +564,12 @@ function Kong.init()
 
     if config.role ~= "control_plane" then
       assert(runloop.build_router("init"))
+
+      ok, err = runloop.set_init_versions_in_cache()
+      if not ok then
+        error("error setting initial versions for router and plugins iterator in cache: " ..
+              tostring(err))
+      end
     end
   end
 
@@ -638,12 +644,6 @@ function Kong.init_worker()
   end
   kong.core_cache = core_cache
 
-  ok, err = runloop.set_init_versions_in_cache()
-  if not ok then
-    stash_init_worker_error(err) -- 'err' fully formatted
-    return
-  end
-
   -- LEGACY
   singletons.cache          = cache
   singletons.core_cache     = core_cache
@@ -661,22 +661,31 @@ function Kong.init_worker()
     return
   end
 
-  if kong.configuration.role ~= "control_plane" then
+  local is_not_control_plane = kong.configuration.role ~= "control_plane"
+  if is_not_control_plane then
     ok, err = execute_cache_warmup(kong.configuration)
     if not ok then
       ngx_log(ngx_ERR, "failed to warm up the DB cache: " .. err)
     end
   end
 
-  runloop.init_worker.before()
-
-  -- run plugins init_worker context
   ok, err = runloop.update_plugins_iterator()
   if not ok then
     stash_init_worker_error("failed to build the plugins iterator: " .. err)
     return
   end
 
+  if is_not_control_plane then
+    ok, err = runloop.update_router()
+    if not ok then
+      stash_init_worker_error("failed to build the router: " .. err)
+      return
+    end
+  end
+
+  runloop.init_worker.before()
+
+  -- run plugins init_worker context
   local plugins_iterator = runloop.get_plugins_iterator()
   local errors = execute_init_worker_plugins_iterator(plugins_iterator, ctx)
   if errors then
@@ -689,7 +698,7 @@ function Kong.init_worker()
 
   runloop.init_worker.after()
 
-  if kong.configuration.role ~= "control_plane" then
+  if is_not_control_plane and ngx.worker.id() == 0 then
     plugin_servers.start()
   end
 

kong/plugins/http-log/handler.lua

@@ -170,7 +170,7 @@ function HttpLogHandler:log(conf)
     }
 
     local err
-    q, err = BatchQueue.new(process, opts)
+    q, err = BatchQueue.new("http-log", process, opts)
     if not q then
       kong.log.err("could not create queue: ", err)
       return