feat(upgrade-tests): use local build + multiple "old" versions

.github/workflows/upgrade-tests.yml

@@ -25,45 +25,33 @@ concurrency:
   cancel-in-progress: true
 env:
   GH_TOKEN: ${{ github.token }}
+  BUILD_ROOT: ${{ github.workspace }}/bazel-bin/build
 
 jobs:
+  build:
+    uses: ./.github/workflows/build.yml
+    with:
+      relative-build-root: bazel-bin/build
+
   upgrade-test:
     name: Run migration tests
     runs-on: ubuntu-22.04
+    needs: build
 
     steps:
-      - name: Install Prerequisites
-        run: |
-          sudo apt-get -y update
-          sudo apt-get -y install ca-certificates curl gnupg lsb-release jq libyaml-dev net-tools
-          sudo mkdir -p /etc/apt/keyrings
-          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
-          echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-          sudo apt-get update
-          sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin
-
       - name: Clone Source Code
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: recursive
 
-      - name: Build Debian Package
-        run: |
-          make package/deb
-          mv bazel-bin/pkg/kong.amd64.deb .
-
-      - name: Build Docker Image
-        uses: docker/build-push-action@v5
+      - name: Lookup build cache
+        id: cache-deps
+        uses: actions/cache@v3
         with:
-          file: build/dockerfiles/deb.Dockerfile
-          context: .
-          push: false
-          tags: "kong-local/kong:latest"
-          build-args: |
-            KONG_BASE_IMAGE=ubuntu:22.04
-            KONG_ARTIFACT_PATH=./
+          path: ${{ env.BUILD_ROOT }}
+          key: ${{ needs.build.outputs.cache-key }}
 
       - name: Run Upgrade Tests
         run: |
-          bash ./scripts/upgrade-tests/test-upgrade-path.sh -i kong-local/kong:latest
+          bash ./scripts/upgrade-tests/test-upgrade-path.sh -i ${{ env.BUILD_ROOT }}/kong-dev-venv.sh

scripts/upgrade-tests/docker-compose.yml

@@ -13,33 +13,12 @@ services:
       timeout: 1s
       retries: 10
     environment:
-      KONG_PG_HOST: db_postgres
-      KONG_TEST_PG_HOST: db_postgres
+      KONG_PG_HOST: localhost
+      KONG_TEST_PG_HOST: localhost
     volumes:
       - ../../worktree/${OLD_KONG_VERSION}:/kong
     restart: on-failure
-    networks:
-      upgrade_tests:
-
-  kong_new:
-    image: ${NEW_KONG_IMAGE}
-    command: "tail -f /dev/null"
-    user: root
-    depends_on:
-      - db_postgres
-    healthcheck:
-      test: ["CMD", "true"]
-      interval: 1s
-      timeout: 1s
-      retries: 10
-    environment:
-      KONG_PG_HOST: db_postgres
-      KONG_TEST_PG_HOST: db_postgres
-    volumes:
-      - ../..:/kong
-    restart: on-failure
-    networks:
-      upgrade_tests:
+    network_mode: "host"
 
   db_postgres:
     image: postgres:9.5
@@ -55,8 +34,4 @@ services:
     restart: on-failure
     stdin_open: true
     tty: true
-    networks:
-      upgrade_tests:
-
-networks:
-  upgrade_tests:
+    network_mode: "host"

scripts/upgrade-tests/source-versions

@@ -0,0 +1,6 @@
+2.8.0
+3.1.0
+3.2.2
+3.3.0
+3.4.0
+3.5.0

scripts/upgrade-tests/test-upgrade-path.sh

@@ -2,10 +2,10 @@
 
 # This script runs the database upgrade tests from the
 # spec/05-migration directory.  It uses docker compose to stand up a
-# simple environment with postgres database server and
-# two Kong nodes.  One node contains the oldest supported version, the
-# other has the current version of Kong.  The testing is then done as
-# described in https://docs.google.com/document/d/1Df-iq5tNyuPj1UNG7bkhecisJFPswOfFqlOS3V4wXSc/edit?usp=sharing
+# simple environment with postgres database server and a Kong node.
+# The node contains the oldest supported version, the current version
+# of Kong is accessed via the local virtual environment. The testing is then
+# done as described in https://docs.google.com/document/d/1Df-iq5tNyuPj1UNG7bkhecisJFPswOfFqlOS3V4wXSc/edit?usp=sharing
 
 # Normally, the testing environment and the git worktree that is
 # required by this script are removed when the tests have run.  By
@@ -23,27 +23,14 @@ set -e
 
 trap "echo exiting because of error" 0
 
-function get_current_version() {
-    local image_tag=$1
-    local version_from_rockspec=$(perl -ne 'print "$1\n" if (/^\s*tag = "(.*)"/)' kong*.rockspec)
-    if docker pull $image_tag:$version_from_rockspec >/dev/null 2>/dev/null
-    then
-        echo $version_from_rockspec-ubuntu
-    else
-        echo master-ubuntu
-    fi
-}
-
-export OLD_KONG_VERSION=2.8.0
-export OLD_KONG_IMAGE=kong:$OLD_KONG_VERSION-ubuntu
-export NEW_KONG_IMAGE=kong/kong:$(get_current_version kong)
+export KONG_PG_HOST=localhost
+export KONG_TEST_PG_HOST=localhost
 
 function usage() {
     cat 1>&2 <<EOF
-usage: $0 [ -i <new-kong-image> ] [ <test> ... ]
+usage: $0 [ -i <venv-script> ] [ <test> ... ]
 
- <new-kong-image> must be the name of a kong image to use as the base image for the
-                  new kong version, based on this repository.
+ <venv-script> Script to source to set up Kong's virtual environment.
 EOF
 }
 
@@ -58,7 +45,7 @@ set -- $args
 while :; do
     case "$1" in
         -i)
-            export NEW_KONG_IMAGE=$2
+            venv_script=$2
             shift
             shift
             ;;
@@ -82,7 +69,6 @@ COMPOSE="docker compose -p $ENV_PREFIX -f scripts/upgrade-tests/docker-compose.y
 NETWORK_NAME=$ENV_PREFIX
 
 OLD_CONTAINER=$ENV_PREFIX-kong_old-1
-NEW_CONTAINER=$ENV_PREFIX-kong_new-1
 
 function prepare_container() {
     docker exec $1 apt-get update
@@ -92,16 +78,20 @@ function prepare_container() {
 }
 
 function build_containers() {
+    # Kong version >= 3.3 moved non Bazel-built dev setup to make dev-legacy
+    if (( $(echo "$OLD_KONG_VERSION" | sed 's/\.//g') >= 330 )); then
+        old_make_target="dev-legacy"
+    else
+        old_make_target="dev"
+    fi
+
     echo "Building containers"
 
     [ -d worktree/$OLD_KONG_VERSION ] || git worktree add worktree/$OLD_KONG_VERSION $OLD_KONG_VERSION
     $COMPOSE up --wait
     prepare_container $OLD_CONTAINER
-    prepare_container $NEW_CONTAINER
-    docker exec -w /kong $OLD_CONTAINER make dev CRYPTO_DIR=/usr/local/kong
-    # Kong version >= 3.3 moved non Bazel-built dev setup to make dev-legacy
-    docker exec -w /kong $NEW_CONTAINER make dev-legacy CRYPTO_DIR=/usr/local/kong
-    docker exec ${NEW_CONTAINER} ln -sf /kong/bin/kong /usr/local/bin/kong
+    docker exec -w /kong $OLD_CONTAINER make $old_make_target CRYPTO_DIR=/usr/local/kong
+    make dev-legacy CRYPTO_DIR=/usr/local/kong
 }
 
 function initialize_test_list() {
@@ -115,7 +105,7 @@ function initialize_test_list() {
 
         docker exec $OLD_CONTAINER kong migrations reset --yes || true
         docker exec $OLD_CONTAINER kong migrations bootstrap
-        docker exec $NEW_CONTAINER kong migrations status \
+        kong migrations status \
             | jq -r '.new_migrations | .[] | (.namespace | gsub("[.]"; "/")) as $namespace | .migrations[] | "\($namespace)/\(.)_spec.lua" | gsub("^kong"; "spec/05-migration")' \
             | sort > $all_tests_file
         ls 2>/dev/null $(cat $all_tests_file) \
@@ -158,7 +148,8 @@ function initialize_test_list() {
 
 function run_tests() {
     # Run the tests
-    BUSTED="env KONG_DATABASE=$1 KONG_DNS_RESOLVER= KONG_TEST_PG_DATABASE=kong /kong/bin/busted"
+    BUSTED_ENV="env KONG_DATABASE=$1 KONG_DNS_RESOLVER= KONG_TEST_PG_DATABASE=kong OLD_KONG_VERSION=$OLD_KONG_VERSION"
+
     shift
 
     set $TESTS
@@ -173,28 +164,44 @@ function run_tests() {
         echo Running $TEST
 
         echo ">> Setting up tests"
-        docker exec -w /upgrade-test $OLD_CONTAINER $BUSTED -t setup $TEST
+        docker exec -w /upgrade-test $OLD_CONTAINER $BUSTED_ENV /kong/bin/busted -t setup $TEST
         echo ">> Running migrations"
-        docker exec $NEW_CONTAINER kong migrations up
+        kong migrations up
         echo ">> Testing old_after_up,all_phases"
-        docker exec -w /upgrade-test $OLD_CONTAINER $BUSTED -t old_after_up,all_phases $TEST
+        docker exec -w /upgrade-test $OLD_CONTAINER $BUSTED_ENV /kong/bin/busted -t old_after_up,all_phases $TEST
         echo ">> Testing new_after_up,all_phases"
-        docker exec -w /kong $NEW_CONTAINER $BUSTED -t new_after_up,all_phases $TEST
+        $BUSTED_ENV bin/busted -t new_after_up,all_phases $TEST
         echo ">> Finishing migrations"
-        docker exec $NEW_CONTAINER kong migrations finish
+        kong migrations finish
         echo ">> Testing new_after_finish,all_phases"
-        docker exec -w /kong $NEW_CONTAINER $BUSTED -t new_after_finish,all_phases $TEST
+        $BUSTED_ENV bin/busted -t new_after_finish,all_phases $TEST
     done
 }
 
 function cleanup() {
-    git worktree remove worktree/$OLD_KONG_VERSION --force
+    sudo git worktree remove worktree/$OLD_KONG_VERSION --force
     $COMPOSE down
 }
 
-build_containers
-initialize_test_list
-run_tests postgres
-[ -z "$UPGRADE_ENV_PREFIX" ] && cleanup
+
+source $venv_script
+
+# Load supported "old" versions to run migration tests against
+old_versions=()
+mapfile -t old_versions < "scripts/upgrade-tests/source-versions"
+
+for old_version in "${old_versions[@]}"; do
+    export OLD_KONG_VERSION=$old_version
+    export OLD_KONG_IMAGE=kong:$OLD_KONG_VERSION-ubuntu
+
+    echo "Running tests using $OLD_KONG_VERSION as \"old version\" of Kong"
+
+    build_containers
+    initialize_test_list
+    run_tests postgres
+    [ -z "$UPGRADE_ENV_PREFIX" ] && cleanup
+done
+
+deactivate
 
 trap "" 0

spec/05-migration/plugins/http-log/migrations/001_280_to_300_spec.lua

@@ -8,7 +8,10 @@ local uh = require "spec.upgrade_helpers"
 -- to test the migration process. do not change it to use dynamic port.
 local HTTP_PORT = 29100
 
-describe("http-log plugin migration", function()
+local OLD_KONG_VERSION = os.getenv("OLD_KONG_VERSION")
+local handler = OLD_KONG_VERSION:sub(1,3) == "2.8" and describe or pending
+
+handler("http-log plugin migration", function()
     local mock
     lazy_setup(function()
       assert(uh.start_kong())

spec/05-migration/plugins/post-function/migrations/001_280_to_300_spec.lua

@@ -1,7 +1,12 @@
 
 local uh = require "spec/upgrade_helpers"
 
-describe("post-function plugin migration", function()
+
+local OLD_KONG_VERSION = os.getenv("OLD_KONG_VERSION")
+local handler = OLD_KONG_VERSION:sub(1,3) == "2.8" and describe or pending
+
+
+handler("post-function plugin migration", function()
 
     lazy_setup(function()
       assert(uh.start_kong())

spec/05-migration/plugins/pre-function/migrations/001_280_to_300_spec.lua

@@ -1,7 +1,10 @@
 
 local uh = require "spec/upgrade_helpers"
 
-describe("pre-function plugin migration", function()
+local OLD_KONG_VERSION = os.getenv("OLD_KONG_VERSION")
+local handler = OLD_KONG_VERSION:sub(1,3) == "2.8" and describe or pending
+
+handler("pre-function plugin migration", function()
 
     lazy_setup(function()
       assert(uh.start_kong())