diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index fb3795631..5a1406717 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -15,8 +15,12 @@ on: - "acceptance_tests/**" - "ci/**" +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: - build: + test: name: Build and Test Pull Request if: github.repository == 'geoserver/geoserver-cloud' runs-on: ubuntu-latest @@ -26,8 +30,7 @@ jobs: uses: actions/checkout@v4 with: submodules: recursive - - - name: Setup Java + - name: Set up java uses: actions/setup-java@v4 with: distribution: 'temurin' @@ -35,49 +38,70 @@ jobs: cache: 'maven' - name: Validate source code formatting - run: make lint - - - name: Build without tests run: | - make install + make lint - name: Test run: | + # `make test` runs mvn verify, which includes mvn package, so no need to run `package` separatedly for the upload step below make test + - name: Upload application jar files + uses: actions/upload-artifact@v4 + with: + name: application-jars + path: ./**/target/*-bin.jar + retention-days: 1 + compression-level: 0 # no compression + + acceptance: + name: Acceptance Tests + if: github.repository == 'geoserver/geoserver-cloud' + runs-on: ubuntu-latest + timeout-minutes: 60 + needs: test + strategy: + fail-fast: false + matrix: + catalog: [ 'datadir', 'pgconfig', 'jdbcconfig' ] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + - name: Set up java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + cache: 'maven' + + - name: Download application jar files + uses: actions/download-artifact@v4 + with: + name: application-jars + path: . + + - name: Copy config directory + # during the validate phase, the config/ dir is copied to each app's target/config + run: ./mvnw validate -ntp -T1C + - name: Build images - run: | - make build-image + # REPACKAGE=false avoids to re-package the apps during build-image + run: REPACKAGE=false make build-image - name: Install CI dependencies run: python3 -m pip install --user --requirement=ci/requirements.txt - - name: Run acceptance tests datadir + - name: Run acceptance tests ${{ matrix.catalog }} run: | - make acceptance-tests-datadir + make acceptance-tests-${{ matrix.catalog }} - - name: Print docker compose logs datadir - run: (cd compose && c2cciutils-docker-logs) + - name: Print docker compose logs if: always() + run: (cd compose && c2cciutils-docker-logs) - - name: Cleanup acceptance tests datadir - run: | - make clean-acceptance-tests-datadir - - # FIXME: fix pgconfig discrepancies before reactivating - # - name: Run acceptance tests pgconfig - # run: | - # make acceptance-tests-pgconfig - - # - name: Print docker compose logs pgconfig - # run: (cd compose && c2cciutils-docker-logs) - # if: always() - - # - name: Cleanup acceptance tests pgconfig - # run: | - # make clean-acceptance-tests-pgconfig - - - name: Remove project jars from cached repository + - name: Cleanup acceptance tests + if: always() run: | - rm -rf ~/.m2/repository/org/geoserver - find ~/.m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {} + make clean-acceptance-tests-${{ matrix.catalog }} diff --git a/Makefile b/Makefile index 73fe49732..19c6312f4 100644 --- a/Makefile +++ b/Makefile @@ -4,12 +4,6 @@ all: install test build-image TAG=$(shell mvn help:evaluate -Dexpression=project.version -q -DforceStdout) COSIGN_PASSWORD := $(COSIGN_PASSWORD) -COMPOSE_PGCONFIG_OPTIONS ?= -f compose.yml -f catalog-pgconfig.yml -COMPOSE_DATADIR_OPTIONS ?= -f compose.yml -f catalog-datadir.yml -COMPOSE_ACCEPTANCE_PGCONFIG_OPTIONS ?= --project-name gscloud-acceptance-pgconfig $(COMPOSE_PGCONFIG_OPTIONS) -f acceptance.yml -COMPOSE_ACCEPTANCE_DATADIR_OPTIONS ?= --project-name gscloud-acceptance-datadir $(COMPOSE_DATADIR_OPTIONS) -f acceptance.yml -UID=$(shell id -u) -GID=$(shell id -g) REPACKAGE ?= true @@ -19,23 +13,23 @@ clean: .PHONY: lint lint: - ./mvnw fmt:check sortpom:verify -Dsort.verifyFailOn=strict -Dsort.verifyFail=stop -ntp + ./mvnw fmt:check sortpom:verify -Dsort.verifyFailOn=strict -Dsort.verifyFail=stop -ntp -T1C .PHONY: format format: - ./mvnw sortpom:sort fmt:format -ntp + ./mvnw sortpom:sort fmt:format -ntp -T1C .PHONY: install install: - ./mvnw clean install -DskipTests -ntp -T4 -U + ./mvnw clean install -DskipTests -ntp -U -T1C .PHONY: package package: - ./mvnw clean package -DskipTests -ntp -T4 -U + ./mvnw clean package -DskipTests -ntp -U -T1C .PHONY: test test: - ./mvnw verify -ntp -T4 + ./mvnw verify -ntp -T1C .PHONY: build-image build-image: build-base-images build-image-infrastructure build-image-geoserver @@ -135,23 +129,30 @@ build-acceptance: docker build --tag=acceptance:$(TAG) acceptance_tests .PHONY: acceptance-tests-pgconfig -acceptance-tests-pgconfig: acceptance-tests-pgconfig: build-acceptance - (cd compose/ && TAG=$(TAG) GS_USER=$(UID):$(GID) docker compose $(COMPOSE_ACCEPTANCE_PGCONFIG_OPTIONS) up -d) - (cd compose/ && TAG=$(TAG) GS_USER=$(UID):$(GID) docker compose $(COMPOSE_ACCEPTANCE_PGCONFIG_OPTIONS) exec -T acceptance bash -c 'until [ -f /tmp/healthcheck ]; do echo "Waiting for /tmp/healthcheck to be available..."; sleep 5; done && pytest . -vvv --color=yes') + (cd compose/ && ./acceptance_pgconfig up -d) + (cd compose/ && ./acceptance_pgconfig exec -T acceptance bash -c 'until [ -f /tmp/healthcheck ]; do echo "Waiting for /tmp/healthcheck to be available..."; sleep 5; done && pytest . -vvv --color=yes') .PHONY: clean-acceptance-tests-pgconfig clean-acceptance-tests-pgconfig: - (cd compose/ && TAG=$(TAG) GS_USER=$(UID):$(GID) docker compose $(COMPOSE_ACCEPTANCE_PGCONFIG_OPTIONS) down -v) + (cd compose/ && ./acceptance_pgconfig down -v) .PHONY: acceptance-tests-datadir -acceptance-tests-datadir: acceptance-tests-datadir: build-acceptance - (cd compose/ && TAG=$(TAG) GS_USER=$(UID):$(GID) docker compose $(COMPOSE_ACCEPTANCE_DATADIR_OPTIONS) up -d) - (cd compose/ && TAG=$(TAG) GS_USER=$(UID):$(GID) docker compose $(COMPOSE_ACCEPTANCE_DATADIR_OPTIONS) exec -T acceptance bash -c 'until [ -f /tmp/healthcheck ]; do echo "Waiting for /tmp/healthcheck to be available..."; sleep 5; done && pytest . -vvv --color=yes') + (cd compose/ && ./acceptance_datadir up -d) + (cd compose/ && ./acceptance_datadir exec -T acceptance bash -c 'until [ -f /tmp/healthcheck ]; do echo "Waiting for /tmp/healthcheck to be available..."; sleep 5; done && pytest . -vvv --color=yes') .PHONY: clean-acceptance-tests-datadir clean-acceptance-tests-datadir: - (cd compose/ && TAG=$(TAG) GS_USER=$(UID):$(GID) docker compose $(COMPOSE_ACCEPTANCE_DATADIR_OPTIONS) down -v) + (cd compose/ && ./acceptance_datadir down -v) rm -rf compose/catalog-datadir/* touch compose/catalog-datadir/.keep + +.PHONY: acceptance-tests-jdbcconfig +acceptance-tests-jdbcconfig: build-acceptance + (cd compose/ && ./acceptance_jdbcconfig up -d) + (cd compose/ && ./acceptance_jdbcconfig exec -T acceptance bash -c 'until [ -f /tmp/healthcheck ]; do echo "Waiting for /tmp/healthcheck to be available..."; sleep 5; done && pytest . -vvv --color=yes') + +.PHONY: clean-acceptance-tests-jdbcconfig +clean-acceptance-tests-jdbcconfig: + (cd compose/ && ./acceptance_jdbcconfig down -v) diff --git a/acceptance_tests/entrypoint.py b/acceptance_tests/entrypoint.py index 033418b38..69bf9c70e 100755 --- a/acceptance_tests/entrypoint.py +++ b/acceptance_tests/entrypoint.py @@ -29,7 +29,6 @@ def timeout(): endpoints = [ f"{GEOSERVER_URL}/wms?SERVICE=WMS&REQUEST=GetCapabilities", f"{GEOSERVER_URL}/wfs?SERVICE=WFS&REQUEST=GetCapabilities", - f"{GEOSERVER_URL}/wps?SERVICE=WPS&REQUEST=GetCapabilities", f"{GEOSERVER_URL}/wcs?SERVICE=WCS&REQUEST=GetCapabilities", f"{GEOSERVER_URL}/ows?SERVICE=WMS&REQUEST=GetCapabilities", f"{GEOSERVER_URL}/gwc", diff --git a/compose/acceptance_datadir b/compose/acceptance_datadir new file mode 100755 index 000000000..fa7323ca6 --- /dev/null +++ b/compose/acceptance_datadir @@ -0,0 +1,10 @@ +#/bin/bash + +UID=$(id -u) +GID=$(id -g) + +GS_USER=$UID:$GID COMPOSE_PROJECT_NAME=gscloud-acceptance-datadir \ +docker compose \ +-f compose.yml \ +-f catalog-datadir.yml \ +-f acceptance.yml "$@" diff --git a/compose/acceptance_jdbcconfig b/compose/acceptance_jdbcconfig new file mode 100755 index 000000000..753a0b560 --- /dev/null +++ b/compose/acceptance_jdbcconfig @@ -0,0 +1,7 @@ +#/bin/bash + +COMPOSE_PROJECT_NAME=gscloud-acceptance-jdbcconfig \ +docker compose \ +-f compose.yml \ +-f catalog-jdbcconfig.yml \ +-f acceptance.yml "$@" diff --git a/compose/acceptance_pgconfig b/compose/acceptance_pgconfig new file mode 100755 index 000000000..b7b09dcd4 --- /dev/null +++ b/compose/acceptance_pgconfig @@ -0,0 +1,7 @@ +#/bin/bash + +COMPOSE_PROJECT_NAME=gscloud-acceptance-pgconfig \ +docker compose \ +-f compose.yml \ +-f catalog-pgconfig.yml \ +-f acceptance.yml "$@" diff --git a/compose/catalog-jdbcconfig.yml b/compose/catalog-jdbcconfig.yml index 37fa0ed0a..09e1de831 100644 --- a/compose/catalog-jdbcconfig.yml +++ b/compose/catalog-jdbcconfig.yml @@ -2,6 +2,7 @@ name: gscloud_dev_jdbcconfig volumes: jdbcconfig_data: # volume for postgresql data, used to store the geoserver config through jdbcconfig + datadir: # # Configures all geoserver services to use the postgresql database server with jdbcconfig as catalog backend. @@ -9,6 +10,14 @@ volumes: # services: + init-sample-data: + image: alpine:3.18.4 + user: ${GS_USER} + command: sh -c "cd /opt/app/data_directory; if [ ! -f global.xml ]; then tar xfvz /tmp/datadir.tgz; fi" + volumes: + - datadir:/opt/app/data_directory + - ./catalog-datadir.tgz:/tmp/datadir.tgz + database: # be sure geoserver.backend.jdbcconfig.initdb is set to true in application.yml at lease for the first app run image: postgres:13-alpine @@ -33,8 +42,13 @@ services: JDBCCONFIG_URL: "${JDBCCONFIG_URL}" JDBCCONFIG_USERNAME: "${JDBCCONFIG_USERNAME}" JDBCCONFIG_PASSWORD: "${JDBCCONFIG_PASSWORD}" + volumes: + - datadir:/opt/app/data_directory depends_on: - - database + init-sample-data: + condition: service_completed_successfully + database: + condition: service_started wms: environment: @@ -42,8 +56,13 @@ services: JDBCCONFIG_URL: "${JDBCCONFIG_URL}" JDBCCONFIG_USERNAME: "${JDBCCONFIG_USERNAME}" JDBCCONFIG_PASSWORD: "${JDBCCONFIG_PASSWORD}" + volumes: + - datadir:/opt/app/data_directory depends_on: - - database + init-sample-data: + condition: service_completed_successfully + database: + condition: service_started wcs: environment: @@ -51,8 +70,13 @@ services: JDBCCONFIG_URL: "${JDBCCONFIG_URL}" JDBCCONFIG_USERNAME: "${JDBCCONFIG_USERNAME}" JDBCCONFIG_PASSWORD: "${JDBCCONFIG_PASSWORD}" + volumes: + - datadir:/opt/app/data_directory depends_on: - - database + init-sample-data: + condition: service_completed_successfully + database: + condition: service_started rest: environment: @@ -60,8 +84,13 @@ services: JDBCCONFIG_URL: "${JDBCCONFIG_URL}" JDBCCONFIG_USERNAME: "${JDBCCONFIG_USERNAME}" JDBCCONFIG_PASSWORD: "${JDBCCONFIG_PASSWORD}" + volumes: + - datadir:/opt/app/data_directory depends_on: - - database + init-sample-data: + condition: service_completed_successfully + database: + condition: service_started webui: environment: @@ -69,8 +98,13 @@ services: JDBCCONFIG_URL: "${JDBCCONFIG_URL}" JDBCCONFIG_USERNAME: "${JDBCCONFIG_USERNAME}" JDBCCONFIG_PASSWORD: "${JDBCCONFIG_PASSWORD}" + volumes: + - datadir:/opt/app/data_directory depends_on: - - database + init-sample-data: + condition: service_completed_successfully + database: + condition: service_started gwc: environment: @@ -78,5 +112,10 @@ services: JDBCCONFIG_URL: "${JDBCCONFIG_URL}" JDBCCONFIG_USERNAME: "${JDBCCONFIG_USERNAME}" JDBCCONFIG_PASSWORD: "${JDBCCONFIG_PASSWORD}" + volumes: + - datadir:/opt/app/data_directory depends_on: - - database + init-sample-data: + condition: service_completed_successfully + database: + condition: service_started diff --git a/compose/compose.yml b/compose/compose.yml index 1a272b5d4..fa73d14cf 100644 --- a/compose/compose.yml +++ b/compose/compose.yml @@ -125,8 +125,8 @@ services: deploy: resources: limits: - cpus: '4.0' - memory: 1G + cpus: '8.0' + memory: 2G wfs: extends: @@ -172,6 +172,11 @@ services: environment: JAVA_OPTS: "${JAVA_OPTS_GWC}" depends_on: *gs-dependencies + deploy: + resources: + limits: + cpus: '8.0' + memory: 2G rest: extends: