Merge pull request #548 from groldan/build/acceptance_jdbcconfig

Add acceptance tests for the jdbcconfig catalog backend
geoserver · Oct 14, 2024 · 22348c9 · 22348c9
2 parents 41650d9 + b4bc6f4
commit 22348c9
Show file tree

Hide file tree

Showing 8 changed files with 153 additions and 61 deletions.
diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml
@@ -15,8 +15,12 @@ on:
       - "acceptance_tests/**"
       - "ci/**"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  build:
+  test:
     name: Build and Test Pull Request
     if: github.repository == 'geoserver/geoserver-cloud'
     runs-on: ubuntu-latest
@@ -26,58 +30,78 @@ jobs:
       uses: actions/checkout@v4
       with:
         submodules: recursive
-
-    - name: Setup Java
+    - name: Set up java
       uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: '21'
         cache: 'maven'
 
     - name: Validate source code formatting
-      run: make lint
-
-    - name: Build without tests
       run: |
-        make install
+        make lint
 
     - name: Test
       run: |
+        # `make test` runs mvn verify, which includes mvn package, so no need to run `package` separatedly for the upload step below
         make test
 
+    - name: Upload application jar files
+      uses: actions/upload-artifact@v4
+      with:
+        name: application-jars
+        path: ./**/target/*-bin.jar
+        retention-days: 1
+        compression-level: 0 # no compression
+
+  acceptance:
+    name: Acceptance Tests
+    if: github.repository == 'geoserver/geoserver-cloud'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    needs: test
+    strategy:
+      fail-fast: false
+      matrix:
+        catalog: [ 'datadir', 'pgconfig', 'jdbcconfig' ]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        submodules: recursive
+    - name: Set up java
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin'
+        java-version: '21'
+        cache: 'maven'
+
+    - name: Download application jar files
+      uses: actions/download-artifact@v4
+      with:
+        name: application-jars
+        path: .
+
+    - name: Copy config directory
+      # during the validate phase, the config/ dir is copied to each app's target/config
+      run: ./mvnw validate -ntp -T1C
+
     - name: Build images
-      run: |
-        make build-image
+      # REPACKAGE=false avoids to re-package the apps during build-image
+      run: REPACKAGE=false make build-image
 
     - name: Install CI dependencies
       run: python3 -m pip install --user --requirement=ci/requirements.txt
 
-    - name: Run acceptance tests datadir
+    - name: Run acceptance tests ${{ matrix.catalog }}
       run: |
-        make acceptance-tests-datadir
+        make acceptance-tests-${{ matrix.catalog }}
 
-    - name: Print docker compose logs datadir
-      run: (cd compose && c2cciutils-docker-logs)
+    - name: Print docker compose logs
       if: always()
+      run: (cd compose && c2cciutils-docker-logs)
 
-    - name: Cleanup acceptance tests datadir
-      run: |
-        make clean-acceptance-tests-datadir
-
-    # FIXME: fix pgconfig discrepancies before reactivating
-    # - name: Run acceptance tests pgconfig
-    #   run: |
-    #     make acceptance-tests-pgconfig
-
-    # - name: Print docker compose logs pgconfig
-    #   run: (cd compose && c2cciutils-docker-logs)
-    #   if: always()
-
-    # - name: Cleanup acceptance tests pgconfig
-    #   run: |
-    #     make clean-acceptance-tests-pgconfig
-
-    - name: Remove project jars from cached repository
+    - name: Cleanup acceptance tests
+      if: always()
       run: |
-        rm -rf ~/.m2/repository/org/geoserver
-        find ~/.m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {}
+        make clean-acceptance-tests-${{ matrix.catalog }}
diff --git a/Makefile b/Makefile
@@ -4,12 +4,6 @@ all: install test build-image
 TAG=$(shell mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
 
 COSIGN_PASSWORD := $(COSIGN_PASSWORD)
-COMPOSE_PGCONFIG_OPTIONS ?= -f compose.yml -f catalog-pgconfig.yml
-COMPOSE_DATADIR_OPTIONS ?= -f compose.yml -f catalog-datadir.yml
-COMPOSE_ACCEPTANCE_PGCONFIG_OPTIONS ?= --project-name gscloud-acceptance-pgconfig $(COMPOSE_PGCONFIG_OPTIONS) -f acceptance.yml
-COMPOSE_ACCEPTANCE_DATADIR_OPTIONS ?= --project-name gscloud-acceptance-datadir $(COMPOSE_DATADIR_OPTIONS) -f acceptance.yml
-UID=$(shell id -u)
-GID=$(shell id -g)
 
 REPACKAGE ?= true
 
@@ -19,23 +13,23 @@ clean:
 
 .PHONY: lint
 lint:
-	./mvnw fmt:check sortpom:verify -Dsort.verifyFailOn=strict -Dsort.verifyFail=stop -ntp
+	./mvnw fmt:check sortpom:verify -Dsort.verifyFailOn=strict -Dsort.verifyFail=stop -ntp -T1C
 
 .PHONY: format
 format:
-	./mvnw sortpom:sort fmt:format -ntp
+	./mvnw sortpom:sort fmt:format -ntp -T1C
 
 .PHONY: install
 install:
-	./mvnw clean install -DskipTests -ntp -T4 -U
+	./mvnw clean install -DskipTests -ntp -U -T1C
 
 .PHONY: package
 package:
-	./mvnw clean package -DskipTests -ntp -T4 -U
+	./mvnw clean package -DskipTests -ntp -U -T1C
 
 .PHONY: test
 test:
-	./mvnw verify -ntp -T4
+	./mvnw verify -ntp -T1C
 
 .PHONY: build-image
 build-image: build-base-images build-image-infrastructure build-image-geoserver
@@ -135,23 +129,30 @@ build-acceptance:
 	docker build --tag=acceptance:$(TAG) acceptance_tests
 
 .PHONY: acceptance-tests-pgconfig
-acceptance-tests-pgconfig:
 acceptance-tests-pgconfig: build-acceptance
-	(cd compose/ && TAG=$(TAG) GS_USER=$(UID):$(GID) docker compose $(COMPOSE_ACCEPTANCE_PGCONFIG_OPTIONS) up -d)
-	(cd compose/ && TAG=$(TAG) GS_USER=$(UID):$(GID) docker compose $(COMPOSE_ACCEPTANCE_PGCONFIG_OPTIONS) exec -T acceptance bash -c 'until [ -f /tmp/healthcheck ]; do echo "Waiting for /tmp/healthcheck to be available..."; sleep 5; done && pytest . -vvv --color=yes')
+	(cd compose/ && ./acceptance_pgconfig up -d)
+	(cd compose/ && ./acceptance_pgconfig exec -T acceptance bash -c 'until [ -f /tmp/healthcheck ]; do echo "Waiting for /tmp/healthcheck to be available..."; sleep 5; done && pytest . -vvv --color=yes')
 
 .PHONY: clean-acceptance-tests-pgconfig
 clean-acceptance-tests-pgconfig:
-	(cd compose/ && TAG=$(TAG) GS_USER=$(UID):$(GID) docker compose $(COMPOSE_ACCEPTANCE_PGCONFIG_OPTIONS) down -v)
+	(cd compose/ && ./acceptance_pgconfig down -v)
 
 .PHONY: acceptance-tests-datadir
-acceptance-tests-datadir:
 acceptance-tests-datadir: build-acceptance
-	(cd compose/ && TAG=$(TAG) GS_USER=$(UID):$(GID) docker compose $(COMPOSE_ACCEPTANCE_DATADIR_OPTIONS) up -d)
-	(cd compose/ && TAG=$(TAG) GS_USER=$(UID):$(GID) docker compose $(COMPOSE_ACCEPTANCE_DATADIR_OPTIONS) exec -T acceptance bash -c 'until [ -f /tmp/healthcheck ]; do echo "Waiting for /tmp/healthcheck to be available..."; sleep 5; done && pytest . -vvv --color=yes')
+	(cd compose/ && ./acceptance_datadir up -d)
+	(cd compose/ && ./acceptance_datadir exec -T acceptance bash -c 'until [ -f /tmp/healthcheck ]; do echo "Waiting for /tmp/healthcheck to be available..."; sleep 5; done && pytest . -vvv --color=yes')
 
 .PHONY: clean-acceptance-tests-datadir
 clean-acceptance-tests-datadir:
-	(cd compose/ && TAG=$(TAG) GS_USER=$(UID):$(GID) docker compose $(COMPOSE_ACCEPTANCE_DATADIR_OPTIONS) down -v)
+	(cd compose/ && ./acceptance_datadir down -v)
 	rm -rf compose/catalog-datadir/*
 	touch compose/catalog-datadir/.keep
+
+.PHONY: acceptance-tests-jdbcconfig
+acceptance-tests-jdbcconfig: build-acceptance
+	(cd compose/ && ./acceptance_jdbcconfig up -d)
+	(cd compose/ && ./acceptance_jdbcconfig exec -T acceptance bash -c 'until [ -f /tmp/healthcheck ]; do echo "Waiting for /tmp/healthcheck to be available..."; sleep 5; done && pytest . -vvv --color=yes')
+
+.PHONY: clean-acceptance-tests-jdbcconfig
+clean-acceptance-tests-jdbcconfig:
+	(cd compose/ && ./acceptance_jdbcconfig down -v)
diff --git a/acceptance_tests/entrypoint.py b/acceptance_tests/entrypoint.py
@@ -29,7 +29,6 @@ def timeout():
 endpoints = [
     f"{GEOSERVER_URL}/wms?SERVICE=WMS&REQUEST=GetCapabilities",
     f"{GEOSERVER_URL}/wfs?SERVICE=WFS&REQUEST=GetCapabilities",
-    f"{GEOSERVER_URL}/wps?SERVICE=WPS&REQUEST=GetCapabilities",
     f"{GEOSERVER_URL}/wcs?SERVICE=WCS&REQUEST=GetCapabilities",
     f"{GEOSERVER_URL}/ows?SERVICE=WMS&REQUEST=GetCapabilities",
     f"{GEOSERVER_URL}/gwc",

diff --git a/compose/acceptance_datadir b/compose/acceptance_datadir
@@ -0,0 +1,10 @@
+#/bin/bash
+
+UID=$(id -u)
+GID=$(id -g)
+
+GS_USER=$UID:$GID COMPOSE_PROJECT_NAME=gscloud-acceptance-datadir \
+docker compose \
+-f compose.yml \
+-f catalog-datadir.yml \
+-f acceptance.yml "$@"
diff --git a/compose/acceptance_jdbcconfig b/compose/acceptance_jdbcconfig
@@ -0,0 +1,7 @@
+#/bin/bash
+
+COMPOSE_PROJECT_NAME=gscloud-acceptance-jdbcconfig \
+docker compose \
+-f compose.yml \
+-f catalog-jdbcconfig.yml \
+-f acceptance.yml "$@"
diff --git a/compose/acceptance_pgconfig b/compose/acceptance_pgconfig
@@ -0,0 +1,7 @@
+#/bin/bash
+
+COMPOSE_PROJECT_NAME=gscloud-acceptance-pgconfig \
+docker compose \
+-f compose.yml \
+-f catalog-pgconfig.yml \
+-f acceptance.yml "$@"
diff --git a/compose/catalog-jdbcconfig.yml b/compose/catalog-jdbcconfig.yml
@@ -2,13 +2,22 @@ name: gscloud_dev_jdbcconfig
 
 volumes:
   jdbcconfig_data: # volume for postgresql data, used to store the geoserver config through jdbcconfig
+  datadir:
 
 #
 # Configures all geoserver services to use the postgresql database server with jdbcconfig as catalog backend.
 # Run with `docker compose -f compose.yml -f catalog-jdbcconfig.yml up -d`
 #
 
 services:
+  init-sample-data:
+    image: alpine:3.18.4
+    user: ${GS_USER}
+    command: sh -c "cd /opt/app/data_directory; if [ ! -f global.xml ]; then tar xfvz /tmp/datadir.tgz; fi"
+    volumes:
+      - datadir:/opt/app/data_directory
+      - ./catalog-datadir.tgz:/tmp/datadir.tgz
+
   database:
     # be sure geoserver.backend.jdbcconfig.initdb is set to true in application.yml at lease for the first app run
     image: postgres:13-alpine
@@ -33,50 +42,80 @@ services:
       JDBCCONFIG_URL: "${JDBCCONFIG_URL}"
       JDBCCONFIG_USERNAME: "${JDBCCONFIG_USERNAME}"
       JDBCCONFIG_PASSWORD: "${JDBCCONFIG_PASSWORD}"
+    volumes:
+      - datadir:/opt/app/data_directory
     depends_on:
-      - database
+      init-sample-data:
+        condition: service_completed_successfully
+      database:
+        condition: service_started
 
   wms:
     environment:
       SPRING_PROFILES_ACTIVE: "${GEOSERVER_DEFAULT_PROFILES},jdbcconfig"
       JDBCCONFIG_URL: "${JDBCCONFIG_URL}"
       JDBCCONFIG_USERNAME: "${JDBCCONFIG_USERNAME}"
       JDBCCONFIG_PASSWORD: "${JDBCCONFIG_PASSWORD}"
+    volumes:
+      - datadir:/opt/app/data_directory
     depends_on:
-      - database
+      init-sample-data:
+        condition: service_completed_successfully
+      database:
+        condition: service_started
 
   wcs:
     environment:
       SPRING_PROFILES_ACTIVE: "${GEOSERVER_DEFAULT_PROFILES},jdbcconfig"
       JDBCCONFIG_URL: "${JDBCCONFIG_URL}"
       JDBCCONFIG_USERNAME: "${JDBCCONFIG_USERNAME}"
       JDBCCONFIG_PASSWORD: "${JDBCCONFIG_PASSWORD}"
+    volumes:
+      - datadir:/opt/app/data_directory
     depends_on:
-      - database
+      init-sample-data:
+        condition: service_completed_successfully
+      database:
+        condition: service_started
 
   rest:
     environment:
       SPRING_PROFILES_ACTIVE: "${GEOSERVER_DEFAULT_PROFILES},jdbcconfig"
       JDBCCONFIG_URL: "${JDBCCONFIG_URL}"
       JDBCCONFIG_USERNAME: "${JDBCCONFIG_USERNAME}"
       JDBCCONFIG_PASSWORD: "${JDBCCONFIG_PASSWORD}"
+    volumes:
+      - datadir:/opt/app/data_directory
     depends_on:
-      - database
+      init-sample-data:
+        condition: service_completed_successfully
+      database:
+        condition: service_started
 
   webui:
     environment:
       SPRING_PROFILES_ACTIVE: "${GEOSERVER_DEFAULT_PROFILES},jdbcconfig"
       JDBCCONFIG_URL: "${JDBCCONFIG_URL}"
       JDBCCONFIG_USERNAME: "${JDBCCONFIG_USERNAME}"
       JDBCCONFIG_PASSWORD: "${JDBCCONFIG_PASSWORD}"
+    volumes:
+      - datadir:/opt/app/data_directory
     depends_on:
-      - database
+      init-sample-data:
+        condition: service_completed_successfully
+      database:
+        condition: service_started
 
   gwc:
     environment:
       SPRING_PROFILES_ACTIVE: "${GEOSERVER_DEFAULT_PROFILES},jdbcconfig"
       JDBCCONFIG_URL: "${JDBCCONFIG_URL}"
       JDBCCONFIG_USERNAME: "${JDBCCONFIG_USERNAME}"
       JDBCCONFIG_PASSWORD: "${JDBCCONFIG_PASSWORD}"
+    volumes:
+      - datadir:/opt/app/data_directory
     depends_on:
-      - database
+      init-sample-data:
+        condition: service_completed_successfully
+      database:
+        condition: service_started
diff --git a/compose/compose.yml b/compose/compose.yml
@@ -125,8 +125,8 @@ services:
     deploy:
       resources:
         limits:
-          cpus: '4.0'
-          memory: 1G
+          cpus: '8.0'
+          memory: 2G
 
   wfs:
     extends:
@@ -172,6 +172,11 @@ services:
     environment:
       JAVA_OPTS: "${JAVA_OPTS_GWC}"
     depends_on: *gs-dependencies
+    deploy:
+      resources:
+        limits:
+          cpus: '8.0'
+          memory: 2G
 
   rest:
     extends: