tests(plugins): add old version plugin compatibility test (#9077)

Test the old version plugin against the latest core code. We only focus on
whether there is any runtime error in the happy path. Because logic
changes from version to version, it is not feasible to test the behavior
of the plugin.

This test is not guaranteed to capture all interface changes, because it may not be able to cover all the dependent interfaces, and even if the interface is covered, it can't detect interface semantic changes or internal logic changes. These kinds of things still rely on the tests of the interfaces themselves to cover.

https://konghq.atlassian.net/browse/FTI-5923

.github/workflows/build_and_test.yml

@@ -36,6 +36,30 @@ env:
   RUNNER_COUNT: 7
 
 jobs:
+  metadata:
+    name: Metadata
+    runs-on: ubuntu-22.04
+    outputs:
+      old-kong-version: ${{ steps.old-kong-version.outputs.ref }}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Get Old Kong Version
+      id: old-kong-version
+      run: |
+        KONG_VERSION=$(bash scripts/grep-kong-version.sh)
+        major=$(echo "$KONG_VERSION" | cut -d. -f1)
+        minor=$(echo "$KONG_VERSION" | cut -d. -f2)
+        # if the minor version isn't 0, use the first release of the previous minor version;
+        # otherwise just leave it empty, so later the default branch or commit will be used.
+        if [ "$minor" -ne 0 ]; then
+          minor=$((minor - 1))
+          echo "ref=$major.$minor.0" >> $GITHUB_OUTPUT
+        else
+          echo "ref=" >> $GITHUB_OUTPUT
+        fi
+
   build:
     uses: ./.github/workflows/build.yml
     with:
@@ -137,7 +161,7 @@ jobs:
   busted-tests:
     name: Busted test runner ${{ matrix.runner }}
     runs-on: ubuntu-22.04
-    needs: [build,schedule]
+    needs: [metadata,build,schedule]
 
     strategy:
       fail-fast: false
@@ -185,6 +209,15 @@ jobs:
     - name: Checkout Kong source code
       uses: actions/checkout@v4
 
+    # used for plugin compatibility test
+    - name: Checkout old version Kong source code
+      uses: actions/checkout@v4
+      with:
+        path: kong-old
+        # if the minor version is 0, `ref` will default to ''
+        # which is same as in the previous step
+        ref: ${{ needs.metadata.outputs.old-kong-version }}
+
     - name: Lookup build cache
       id: cache-deps
       uses: actions/cache@v4
@@ -285,6 +318,7 @@ jobs:
         KONG_SPEC_TEST_GRPCBIN_PORT: "15002"
         KONG_SPEC_TEST_GRPCBIN_SSL_PORT: "15003"
         KONG_SPEC_TEST_OTELCOL_FILE_EXPORTER_PATH: ${{ github.workspace }}/tmp/otel/file_exporter.json
+        KONG_SPEC_TEST_OLD_VERSION_KONG_PATH: ${{ github.workspace }}/kong-old
         DD_ENV: ci
         DD_SERVICE: kong-ce-ci
         DD_CIVISIBILITY_MANUAL_API_ENABLED: 1

spec/03-plugins/03-http-log/05-old-plugin-compatibility_spec.lua

@@ -0,0 +1,86 @@
+-- This software is copyright Kong Inc. and its licensors.
+-- Use of the software is subject to the agreement between your organization
+-- and Kong Inc. If there is no such agreement, use is governed by and
+-- subject to the terms of the Kong Master Software License Agreement found
+-- at https://konghq.com/enterprisesoftwarelicense/.
+-- [ END OF LICENSE 0867164ffc95e54f04670b5169c09574bdbd9bba ]
+
+local helpers = require "spec.helpers"
+local fmt = string.format
+local plugin_name = "http-log"
+
+for _, strategy in helpers.all_strategies() do
+  describe(fmt("%s - old plugin compatibility [#%s]", plugin_name, strategy), function()
+    local bp, proxy_client, yaml_file
+    local recover_new_plugin
+
+    lazy_setup(function()
+      -- use the old version plugin
+      recover_new_plugin = helpers.use_old_plugin(plugin_name)
+
+      bp = helpers.get_db_utils(strategy == "off" and "postgres" or strategy, {
+        "routes",
+        "services",
+        "plugins",
+      })
+
+      local route = bp.routes:insert({ paths = { "/test" } })
+
+      bp.plugins:insert({
+        name = plugin_name,
+        route = { id = route.id },
+        config = {
+          http_endpoint = fmt("http://%s:%s/post_log/http", helpers.mock_upstream_host, helpers.mock_upstream_port),
+          custom_fields_by_lua = {
+            new_field = "return 123",
+            route = "return nil", -- unset route field
+          },
+        },
+      })
+
+      if strategy == "off" then
+        yaml_file = helpers.make_yaml_file()
+      end
+
+      assert(helpers.start_kong({
+        database   = strategy,
+        nginx_conf = "spec/fixtures/custom_nginx.template",
+        declarative_config = strategy == "off" and yaml_file or nil,
+        pg_host = strategy == "off" and "unknownhost.konghq.com" or nil,
+      }))
+    end)
+
+    lazy_teardown(function()
+      helpers.stop_kong()
+      -- recover the new version plugin
+      recover_new_plugin()
+    end)
+
+    before_each(function()
+      helpers.clean_logfile()
+      proxy_client = helpers.proxy_client()
+    end)
+
+    after_each(function()
+      if proxy_client then
+        proxy_client:close()
+      end
+    end)
+
+    it("should not throw exception when using old version plugin together with the new core", function()
+      local res = assert(proxy_client:send {
+        method = "GET",
+        path = "/test",
+      })
+      assert.not_same(500, res.status)
+
+      -- wait for the log handler to execute
+      ngx.sleep(5)
+
+      assert.logfile().has.no.line("[error]", true, 0)
+      assert.logfile().has.no.line("[alert]", true, 0)
+      assert.logfile().has.no.line("[crit]", true, 0)
+      assert.logfile().has.no.line("[emerg]", true, 0)
+    end)
+  end)
+end

spec/helpers.lua

@@ -36,6 +36,8 @@ local REDIS_SSL_PORT = tonumber(os.getenv("KONG_SPEC_TEST_REDIS_SSL_PORT") or 63
 local REDIS_SSL_SNI = os.getenv("KONG_SPEC_TEST_REDIS_SSL_SNI") or "test-redis.example.com"
 local TEST_COVERAGE_MODE = os.getenv("KONG_COVERAGE")
 local TEST_COVERAGE_TIMEOUT = 30
+-- consistent with path set in .github/workflows/build_and_test.yml and build/dockerfiles/deb.pongo.Dockerfile
+local OLD_VERSION_KONG_PATH = os.getenv("KONG_SPEC_TEST_OLD_VERSION_KONG_PATH") or "/usr/local/share/lua/5.1/kong/kong-old"
 local BLACKHOLE_HOST = "10.255.255.255"
 local KONG_VERSION = require("kong.meta")._VERSION
 local PLUGINS_LIST
@@ -4199,6 +4201,44 @@ do
   end
 end
 
+-- This function is used for plugin compatibility test.
+-- It will use the old version plugin by including the path of the old plugin
+-- at the first of LUA_PATH.
+-- The return value is a function which when called will recover the original
+-- LUA_PATH and remove the temporary directory if it exists.
+-- For an example of how to use it, please see:
+-- plugins-ee/rate-limiting-advanced/spec/06-old-plugin-compatibility_spec.lua
+-- spec/03-plugins/03-http-log/05-old-plugin-compatibility_spec.lua
+local function use_old_plugin(name)
+  assert(type(name) == "string", "must specify the plugin name")
+
+  local old_plugin_path
+  local temp_dir
+  if pl_path.exists(OLD_VERSION_KONG_PATH .. "/kong/plugins/" .. name) then
+    -- only include the path of the specified plugin into LUA_PATH
+    -- and keep the directory structure 'kong/plugins/...'
+    temp_dir = make_temp_dir()
+    old_plugin_path = temp_dir
+    local dest_dir = old_plugin_path .. "/kong/plugins"
+    assert(pl_dir.makepath(dest_dir), "failed to makepath " .. dest_dir)
+    assert(shell.run("cp -r " .. OLD_VERSION_KONG_PATH .. "/kong/plugins/" .. name .. " " .. dest_dir), "failed to copy the plugin directory")
+
+  else
+    error("the specified plugin " .. name .. " doesn't exist")
+  end
+
+  local origin_lua_path = os.getenv("LUA_PATH")
+  -- put the old plugin path at first
+  assert(setenv("LUA_PATH", old_plugin_path .. "/?.lua;" .. old_plugin_path .. "/?/init.lua;" .. origin_lua_path), "failed to set LUA_PATH env")
+
+  return function ()
+    setenv("LUA_PATH", origin_lua_path)
+    if temp_dir then
+      pl_dir.rmtree(temp_dir)
+    end
+  end
+end
+
 
 ----------------
 -- Variables/constants
@@ -4238,6 +4278,7 @@ end
 -- @field zipkin_port the port for Zipkin service, it can be set by env KONG_SPEC_TEST_ZIPKIN_PORT.
 -- @field otelcol_host The host for OpenTelemetry Collector service, it can be set by env KONG_SPEC_TEST_OTELCOL_HOST.
 -- @field otelcol_http_port the port for OpenTelemetry Collector service, it can be set by env KONG_SPEC_TEST_OTELCOL_HTTP_PORT.
+-- @field old_version_kong_path the path for the old version kong source code, it can be set by env KONG_SPEC_TEST_OLD_VERSION_KONG_PATH.
 -- @field otelcol_zpages_port the port for OpenTelemetry Collector Zpages service, it can be set by env KONG_SPEC_TEST_OTELCOL_ZPAGES_PORT.
 -- @field otelcol_file_exporter_path the path of for OpenTelemetry Collector's file exporter, it can be set by env KONG_SPEC_TEST_OTELCOL_FILE_EXPORTER_PATH.
 
@@ -4302,6 +4343,8 @@ end
 
   blackhole_host = BLACKHOLE_HOST,
 
+  old_version_kong_path = OLD_VERSION_KONG_PATH,
+
   -- Kong testing helpers
   execute = exec,
   dns_mock = dns_mock,
@@ -4372,6 +4415,9 @@ end
   get_grpc_target_port = get_grpc_target_port,
   generate_keys = generate_keys,
 
+  -- plugin compatibility test
+  use_old_plugin = use_old_plugin,
+
   -- Only use in CLI tests from spec/02-integration/01-cmd
   kill_all = function(prefix, timeout)
     local kill = require "kong.cmd.utils.kill"