fix(vault): apply retry count maximum threshold for secret rotation

kong/pdk/vault.lua

@@ -51,6 +51,8 @@ local decode_json = cjson.decode
 
 local NEGATIVELY_CACHED_VALUE = "\0"
 local ROTATION_INTERVAL = tonumber(os.getenv("KONG_VAULT_ROTATION_INTERVAL"), 10) or 60
+local SECRETS_RETRY_KEY_PREFIX = "retry_count:"
+local SECRETS_RETRY_COUNT_THRESHOLD = tonumber(os.getenv("KONG_VAULT_SECRETS_RETRY_COUNT_THRESHOLD"), 10) or 60
 local DAO_MAX_TTL = constants.DATABASE.DAO_MAX_TTL
 
 
@@ -1238,6 +1240,10 @@ local function new(self)
   -- @treturn true|nil `true` after successfully rotating a secret, otherwise `nil`
   -- @treturn string|nil a string describing an error if there was one
   local function rotate_secret(old_cache_key, caching_strategy)
+    if old_cache_key:sub(1, #SECRETS_RETRY_KEY_PREFIX) == SECRETS_RETRY_KEY_PREFIX then
+      return true
+    end
+
     local reference, err = parse_cache_key(old_cache_key)
     if not reference then
       -- invalid cache keys are removed (in general should never happen)
@@ -1274,6 +1280,15 @@ local function new(self)
     -- we should refresh the secret at this point
     local ok, err = get_from_vault(reference, strategy, config, new_cache_key, parsed_reference)
     if not ok then
+      local retry_count = tonumber(SECRETS_CACHE:get(SECRETS_RETRY_KEY_PREFIX .. new_cache_key) or 0, 10)
+      if retry_count >= SECRETS_RETRY_COUNT_THRESHOLD then
+        SECRETS_CACHE:delete(new_cache_key)
+        SECRETS_CACHE:delete(SECRETS_RETRY_KEY_PREFIX .. new_cache_key)
+        return nil, fmt("could not retrieve value for reference %s (%s) after %d retries, removing from cache and stop rotation", reference, err, SECRETS_RETRY_COUNT_THRESHOLD)
+
+      else
+        SECRETS_CACHE:incr(SECRETS_RETRY_KEY_PREFIX .. new_cache_key, 1, 0)
+      end
       return nil, fmt("could not retrieve value for reference %s (%s)", reference, err)
     end
 

spec/02-integration/13-vaults/07-resurrect_spec.lua

@@ -133,6 +133,7 @@ describe("vault resurrect_ttl and rotation (#" .. strategy .. ") #" .. vault.nam
   lazy_setup(function()
     helpers.setenv("KONG_LUA_PATH_OVERRIDE", LUA_PATH)
     helpers.setenv("KONG_VAULT_ROTATION_INTERVAL", "1")
+    helpers.setenv("KONG_VAULT_SECRETS_RETRY_COUNT_THRESHOLD", "2")
 
     vault:setup()
     vault:create_secret(secret, "init")
@@ -225,9 +226,12 @@ describe("vault resurrect_ttl and rotation (#" .. strategy .. ") #" .. vault.nam
     vault:update_secret(secret, "old", { ttl = 2, resurrect_ttl = 2 })
     check_plugin_secret("old", 5)
     vault:delete_secret(secret)
+    helpers.clean_logfile()
     ngx.sleep(2.5)
     check_plugin_secret("old", 5)
     check_plugin_secret("", 5)
+
+    assert.logfile().has.line(fmt([[could not retrieve value for reference .*vault.*%s\/%s.* after 2 retries, removing from cache and stop rotation]], vault.prefix, secret), false, 4)
   end)
 end)