[backport -> release/3.8.x] fix(core): make deprecated shorthand fields take precedence when new value is null

changelog/unreleased/kong/fix-for-null-aware-shorthand.yml

@@ -0,0 +1,5 @@
+message: |
+  Changed the way deprecated shorthand fields are used with new fields.
+  If the new field contains null it allows for deprecated field to overwrite it if both are present in the request.
+type: bugfix
+scope: Core

kong/db/schema/init.lua

@@ -14,6 +14,7 @@ local deepcompare = require "pl.tablex".deepcompare
 
 local cycle_aware_deep_copy = require "kong.tools.table".cycle_aware_deep_copy
 local table_merge = require "kong.tools.table".table_merge
+local null_aware_table_merge = require "kong.tools.table".null_aware_table_merge
 local table_path = require "kong.tools.table".table_path
 local is_array = require "kong.tools.table".is_array
 
@@ -1752,9 +1753,14 @@ function Schema:process_auto_fields(data, context, nulls, opts)
             local deprecation = sdata.deprecation
             for k, v in pairs(new_values) do
               if type(v) == "table" then
-                data[k] = deprecation and table_merge(v, data[k])
-                                       or table_merge(data[k] or {}, v)
-              elseif not deprecation or data[k] == nil then
+                local source = {}
+                if data[k] and data[k] ~= null then
+                  source = data[k]
+                end
+                data[k] = deprecation and null_aware_table_merge(v, source)
+                                       or table_merge(source, v)
+
+              elseif not deprecation or (data[k] == nil or data[k] == null) then
                 data[k] = v
               end
             end

kong/tools/table.lua

@@ -45,6 +45,29 @@ function _M.table_merge(t1, t2)
 end
 
 
+--- Merges two table together but does not replace values from `t1` if `t2` for a given key has `ngx.null` value
+-- A new table is created with a non-recursive copy of the provided tables
+-- @param t1 The first table
+-- @param t2 The second table
+-- @return The (new) merged table
+function _M.null_aware_table_merge(t1, t2)
+  local res = {}
+  if t1 then
+    for k,v in pairs(t1) do
+      res[k] = v
+    end
+  end
+  if t2 then
+    for k,v in pairs(t2) do
+      if res[k] == nil or v ~= ngx.null then
+        res[k] = v
+      end
+    end
+  end
+  return res
+end
+
+
 --- Checks if a value exists in a table.
 -- @param arr The table to use
 -- @param val The value to check

spec/01-unit/01-db/01-schema/01-schema_spec.lua

@@ -4151,6 +4151,11 @@ describe("schema", function()
       local input = { username = "test1", name = "ignored", record = { x = "ignored" }, y = "test1" }
       local output, _ = TestSchema:process_auto_fields(input)
       assert.same({ name = "test1", record = { x = "test1" } }, output)
+
+      -- deprecated fields does take precedence if the new fields are null
+      local input = { username = "overwritten-1", name = ngx.null, record = { x = ngx.null }, y = "overwritten-2"  }
+      local output, _ = TestSchema:process_auto_fields(input)
+      assert.same({ name = "overwritten-1", record = { x = "overwritten-2" }  }, output)
     end)
 
     it("does not take precedence if deprecated", function()
@@ -4202,6 +4207,11 @@ describe("schema", function()
       local input = { username = "ignored", name = "test1", record = { x = "test1" }, y = "ignored"  }
       local output, _ = TestSchema:process_auto_fields(input)
       assert.same({ name = "test1", record = { x = "test1" }  }, output)
+
+      -- deprecated fields does take precedence if the new fields are null
+      local input = { username = "overwritten-1", name = ngx.null, record = { x = ngx.null }, y = "overwritten-2"  }
+      local output, _ = TestSchema:process_auto_fields(input)
+      assert.same({ name = "overwritten-1", record = { x = "overwritten-2" }  }, output)
     end)
 
     it("can produce multiple fields", function()