From 145e16d5da72fd74f8069e6041b613a32cd300e6 Mon Sep 17 00:00:00 2001 From: Admire Nyakudya Date: Sat, 19 Oct 2024 11:58:08 +0200 Subject: [PATCH 1/3] Upgrade to PG17 --- .example.env | 4 +- .github/workflows/build-latest.yaml | 32 ++++++++++-- .github/workflows/deploy-image.yaml | 51 +++++++++++++++++-- Dockerfile | 4 +- README.md | 12 ++--- docker-compose.build-timescale.yml | 1 - docker-compose.build.yml | 1 - docker-compose.yml | 1 - scenario_tests/collations/docker-compose.yml | 3 -- .../datadir_init/docker-compose-gs.yml | 2 +- .../datadir_init/docker-compose.yml | 2 +- scenario_tests/extensions/docker-compose.yml | 2 - .../init_scripts/docker-compose.yml | 2 +- .../logical_replication/docker-compose-gs.yml | 2 - .../logical_replication/docker-compose.yml | 2 - .../multiple_databases/docker-compose.yml | 2 - .../docker-compose-gs-promote.yml | 2 - .../docker-compose-gs.yml | 2 - .../docker-compose-root-promote.yml | 2 - .../streaming_replication/docker-compose.yml | 2 - scripts/env-data.sh | 2 +- 21 files changed, 89 insertions(+), 44 deletions(-) diff --git a/.example.env b/.example.env index c80e82e8..942a286c 100644 --- a/.example.env +++ b/.example.env @@ -13,8 +13,8 @@ LANG=en_US.UTF-8 # locale filter to include in the locale generator LANGS="en_US.UTF-8,id_ID.UTF-8" -POSTGRES_MAJOR_VERSION=16 +POSTGRES_MAJOR_VERSION=17 POSTGIS_MAJOR_VERSION=3 -POSTGIS_MINOR_RELEASE=4 +POSTGIS_MINOR_RELEASE=5 BUILD_TIMESCALE=false TIMESCALE_VERSION=2-2.14.2 diff --git a/.github/workflows/build-latest.yaml b/.github/workflows/build-latest.yaml index d839b237..a20dd6a7 100644 --- a/.github/workflows/build-latest.yaml +++ b/.github/workflows/build-latest.yaml @@ -21,7 +21,12 @@ jobs: build-docker-image: runs-on: ubuntu-latest timeout-minutes: 25 - if: github.actor != 'dependabot[bot]' + if: | + github.actor != 'dependabot[bot]' && + !( + contains(github.event.pull_request.title, '[skip-release]') || + contains(github.event.comment.body, '/skiprelease') + ) strategy: matrix: postgresMajorVersion: @@ -29,7 +34,7 @@ jobs: postgisMajorVersion: - 3 postgisMinorRelease: - - 4 + - 5 imageVersion: - imageDistro: debian imageDistroVersion: bookworm @@ -75,7 +80,12 @@ jobs: runs-on: ubuntu-latest needs: [build-docker-image] timeout-minutes: 20 - if: github.actor != 'dependabot[bot]' + if: | + github.actor != 'dependabot[bot]' && + !( + contains(github.event.pull_request.title, '[skip-release]') || + contains(github.event.comment.body, '/skiprelease') + ) strategy: matrix: scenario: @@ -106,7 +116,12 @@ jobs: scan_image: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.actor != 'dependabot[bot]' + if: | + github.actor != 'dependabot[bot]' && + !( + contains(github.event.pull_request.title, '[skip-release]') || + contains(github.event.comment.body, '/skiprelease') + ) needs: [build-docker-image, run-scenario-tests] steps: - uses: actions/checkout@v4 @@ -134,7 +149,14 @@ jobs: sarif_file: 'trivy-results.sarif' push-internal-pr-images: - if: github.event.pull_request.base.repo.url == github.event.pull_request.head.repo.url && github.actor != 'dependabot[bot]' + if: | + github.event_name == 'pull_request' && + github.event.pull_request.base.repo.url == github.event.pull_request.head.repo.url && + github.actor != 'dependabot[bot]' && + !( + contains(github.event.pull_request.title, '[skip-release]') || + contains(github.event.comment.body, '/skiprelease') + ) runs-on: ubuntu-latest needs: [ build-docker-image, run-scenario-tests ] steps: diff --git a/.github/workflows/deploy-image.yaml b/.github/workflows/deploy-image.yaml index 2002f468..2a7448f4 100644 --- a/.github/workflows/deploy-image.yaml +++ b/.github/workflows/deploy-image.yaml @@ -12,17 +12,22 @@ jobs: deploy-image: runs-on: ubuntu-latest timeout-minutes: 20 - if: github.actor != 'dependabot[bot]' + if: | + github.actor != 'dependabot[bot]' && + !( + contains(github.event.pull_request.title, '[skip-release]') || + contains(github.event.comment.body, '/skiprelease') + ) && github.event.workflow_run.conclusion == 'success' env: latest-ref: refs/heads/develop strategy: matrix: postgresMajorVersion: - - 16 + - 17 postgisMajorVersion: - 3 postgisMinorRelease: - - 4 + - 5 imageVersion: - imageDistro: debian imageDistroVersion: bookworm @@ -97,3 +102,43 @@ jobs: type=gha,scope=base cache-to: type=gha,scope=prod target: postgis-prod + + publish_release_artifacts: + runs-on: ubuntu-latest + timeout-minutes: 5 + if: | + github.actor != 'dependabot[bot]' && + !( + contains(github.event.pull_request.title, '[skip-release]') || + contains(github.event.comment.body, '/skiprelease') + ) && github.event.workflow_run.conclusion == 'success' + needs: [ deploy-image ] + strategy: + matrix: + postgresMajorVersion: + - 17 + postgisMajorVersion: + - 3 + postgisMinorRelease: + - 5 + steps: + - name: Checkout code + id: git_checkout + uses: actions/checkout@v3 + with: + ref: 'develop' + + - name: Get Current Date + id: current_date + run: echo "formatted=$(date -u +%Y.%m.%d)" >> $GITHUB_OUTPUT + + - name: Get Latest Commit Hash + id: latest_commit_hash + run: echo "commit=$(git rev-parse --short "$GITHUB_SHA")" >> $GITHUB_OUTPUT + + - name: publish_release + id: tag_releases + run: | + gh release create v${{ matrix.postgresMajorVersion }}.${{ matrix.postgisMajorVersion }}.${{ matrix.postgisMinorRelease }}--v${{ steps.current_date.outputs.formatted }}--${{ steps.latest_commit_hash.outputs.commit }} --notes ${{ steps.latest_commit_hash.outputs.commit }} --target develop --repo $GITHUB_REPOSITORY + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Dockerfile b/Dockerfile index 0695fe8a..134bac70 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,9 +72,9 @@ FROM postgis-base AS postgis-prod # Reset ARG for version ARG IMAGE_VERSION -ARG POSTGRES_MAJOR_VERSION=16 +ARG POSTGRES_MAJOR_VERSION=17 ARG POSTGIS_MAJOR_VERSION=3 -ARG POSTGIS_MINOR_RELEASE=4 +ARG POSTGIS_MINOR_RELEASE=5 # https://packagecloud.io/timescale/timescaledb ARG TIMESCALE_VERSION=2-2.11.2 ARG BUILD_TIMESCALE=false diff --git a/README.md b/README.md index c481501a..045e0783 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ The following convention is used for tagging the images we build: So for example: -``kartoza/postgis:14-3.1`` Provides PostgreSQL 14.0, PostGIS 3.1 +``kartoza/postgis:17-3.5`` Provides PostgreSQL 17.0, PostGIS 3.5 **Note:** We highly recommend that you use tagged versions because successive minor versions of `PostgreSQL` write their database clusters into different database directories - which will cause @@ -324,7 +324,7 @@ provided by the `timescaledb-tune`. Example, ```bash docker run -it --name timescale -e ACCEPT_TIMESCALE_TUNING=TRUE \ -e POSTGRES_MULTIPLE_EXTENSIONS=postgis,hstore,postgis_topology,postgis_raster,pgrouting,timescaledb \ - -e TIMESCALE_TUNING_PARAMS="-cpus=4" kartoza/postgis:14-3.1 + -e TIMESCALE_TUNING_PARAMS="-cpus=4" kartoza/postgis:17-3.5 ``` **Note:** `ACCEPT_TIMESCALE_TUNING` environment variable will overwrite all configurations based @@ -572,7 +572,7 @@ FORCE_SSL=TRUE The following example sets up a container with custom ssl private key and certificate: ```shell -docker run -p 25432:5432 -e FORCE_SSL=TRUE -e SSL_DIR="/etc/ssl_certificates" -e SSL_CERT_FILE='/etc/ssl_certificates/fullchain.pem' -e SSL_KEY_FILE='/etc/ssl_certificates/privkey.pem' -e SSL_CA_FILE='/etc/ssl_certificates/root.crt' -v /tmp/postgres/letsencrypt:/etc/ssl_certificates --name ssl -d kartoza/postgis:13-3.1 +docker run -p 25432:5432 -e FORCE_SSL=TRUE -e SSL_DIR="/etc/ssl_certificates" -e SSL_CERT_FILE='/etc/ssl_certificates/fullchain.pem' -e SSL_KEY_FILE='/etc/ssl_certificates/privkey.pem' -e SSL_CA_FILE='/etc/ssl_certificates/root.crt' -v /tmp/postgres/letsencrypt:/etc/ssl_certificates --name ssl -d kartoza/postgis:13-3.5 ``` The environment variable `SSL_DIR` allows a user to specify the location @@ -605,7 +605,7 @@ When setting up the database you need to define the following environment variab Example: ```shell -docker run -p 5432:5432 -e FORCE_SSL=TRUE -e SSL_CERT_FILE='/ssl_certificates/fullchain.pem' -e SSL_KEY_FILE='/ssl_certificates/privkey.pem' -e SSL_CA_FILE='/ssl_certificates/root.crt' --name ssl -d kartoza/postgis:13-3.1 +docker run -p 5432:5432 -e FORCE_SSL=TRUE -e SSL_CERT_FILE='/ssl_certificates/fullchain.pem' -e SSL_KEY_FILE='/ssl_certificates/privkey.pem' -e SSL_CA_FILE='/ssl_certificates/root.crt' --name ssl -d kartoza/postgis:13-3.5 ``` On the host machine where you need to connect to the database you also @@ -663,7 +663,7 @@ replication methods allowed are, Replication uses a dedicated user `REPLICATION_USER`. The role `${REPLICATION_USER}` uses the default group role `pg_read_all_data`. You can read more about this from the -[PostgreSQL documentation](https://www.postgresql.org/docs/14/predefined-roles.html) +[PostgreSQL documentation](https://www.postgresql.org/docs/17/predefined-roles.html) **Note:** When setting up replication you need to specify the password using the environment variable `REPLICATION_PASS`. If you do not specify it a random strong password will be generated. @@ -679,7 +679,7 @@ is accessed from the replicant database. When edits to the master layer are save automatically propagated to the replicant. Note also that the replicant is read-only. ```shell -docker run --name "streaming-replication" -e REPLICATION=true -e WAL_LEVEL='replica' -d -p 25432:5432 kartoza/postgis:14.3.2 +docker run --name "streaming-replication" -e REPLICATION=true -e WAL_LEVEL='replica' -d -p 25432:5432 kartoza/postgis:17.3.2 ``` **Note** If you do not pass the env variable `REPLICATION_PASS` a random password will be generated diff --git a/docker-compose.build-timescale.yml b/docker-compose.build-timescale.yml index 4225ab19..0bbe9330 100644 --- a/docker-compose.build-timescale.yml +++ b/docker-compose.build-timescale.yml @@ -1,5 +1,4 @@ # Used solely for docker-compose build -version: '3.9' services: postgis-base: image: kartoza/postgis:base-${DISTRO}-${IMAGE_VERSION}-${IMAGE_VARIANT} diff --git a/docker-compose.build.yml b/docker-compose.build.yml index 26d3ea95..029d9ea6 100644 --- a/docker-compose.build.yml +++ b/docker-compose.build.yml @@ -1,5 +1,4 @@ # Used solely for docker-compose build -version: '3.9' services: postgis-base: image: kartoza/postgis:base-${DISTRO}-${IMAGE_VERSION}-${IMAGE_VARIANT} diff --git a/docker-compose.yml b/docker-compose.yml index cf526fe4..ce2f3955 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,4 @@ # docker-compose build -version: '3.9' volumes: dbbackups: postgis-data: diff --git a/scenario_tests/collations/docker-compose.yml b/scenario_tests/collations/docker-compose.yml index 2fb399d0..d48ff984 100644 --- a/scenario_tests/collations/docker-compose.yml +++ b/scenario_tests/collations/docker-compose.yml @@ -1,6 +1,3 @@ - -version: '3.9' - volumes: pg-data-dir: new-pg-data-dir: diff --git a/scenario_tests/datadir_init/docker-compose-gs.yml b/scenario_tests/datadir_init/docker-compose-gs.yml index bb71d708..9817b7fa 100644 --- a/scenario_tests/datadir_init/docker-compose-gs.yml +++ b/scenario_tests/datadir_init/docker-compose-gs.yml @@ -1,4 +1,4 @@ -version: '3.9' + volumes: default-pg-data-dir: new-pg-data-dir: diff --git a/scenario_tests/datadir_init/docker-compose.yml b/scenario_tests/datadir_init/docker-compose.yml index 015ede98..deab20e9 100644 --- a/scenario_tests/datadir_init/docker-compose.yml +++ b/scenario_tests/datadir_init/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.9' + volumes: default-pg-data-dir: new-pg-data-dir: diff --git a/scenario_tests/extensions/docker-compose.yml b/scenario_tests/extensions/docker-compose.yml index 219544e1..060fcd90 100644 --- a/scenario_tests/extensions/docker-compose.yml +++ b/scenario_tests/extensions/docker-compose.yml @@ -1,6 +1,4 @@ -version: '3.9' - services: pg: image: 'kartoza/postgis:${TAG:-manual-build}' diff --git a/scenario_tests/init_scripts/docker-compose.yml b/scenario_tests/init_scripts/docker-compose.yml index d4626850..2ef67d89 100644 --- a/scenario_tests/init_scripts/docker-compose.yml +++ b/scenario_tests/init_scripts/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.9' + volumes: default-pg-data-dir-md5: new-pg-data-dir: diff --git a/scenario_tests/logical_replication/docker-compose-gs.yml b/scenario_tests/logical_replication/docker-compose-gs.yml index 835fc131..729b63fc 100644 --- a/scenario_tests/logical_replication/docker-compose-gs.yml +++ b/scenario_tests/logical_replication/docker-compose-gs.yml @@ -1,6 +1,4 @@ -version: '3.9' - volumes: pg-publisher-data-dir: pg-subscriber-data-dir: diff --git a/scenario_tests/logical_replication/docker-compose.yml b/scenario_tests/logical_replication/docker-compose.yml index aad31dd9..1bef2345 100644 --- a/scenario_tests/logical_replication/docker-compose.yml +++ b/scenario_tests/logical_replication/docker-compose.yml @@ -1,6 +1,4 @@ -version: '3.9' - volumes: pg-publisher-data-dir: pg-subscriber-data-dir: diff --git a/scenario_tests/multiple_databases/docker-compose.yml b/scenario_tests/multiple_databases/docker-compose.yml index 5ea63f5b..6977317e 100644 --- a/scenario_tests/multiple_databases/docker-compose.yml +++ b/scenario_tests/multiple_databases/docker-compose.yml @@ -1,6 +1,4 @@ -version: '3.9' - volumes: pg-db-data-dir: pg-db-schema-dir: diff --git a/scenario_tests/streaming_replication/docker-compose-gs-promote.yml b/scenario_tests/streaming_replication/docker-compose-gs-promote.yml index 494a3aba..42fe702e 100644 --- a/scenario_tests/streaming_replication/docker-compose-gs-promote.yml +++ b/scenario_tests/streaming_replication/docker-compose-gs-promote.yml @@ -1,6 +1,4 @@ -version: '2.1' - volumes: pg-master-data-dir: pg-node-data-dir: diff --git a/scenario_tests/streaming_replication/docker-compose-gs.yml b/scenario_tests/streaming_replication/docker-compose-gs.yml index 431a3d5d..b47e76e5 100644 --- a/scenario_tests/streaming_replication/docker-compose-gs.yml +++ b/scenario_tests/streaming_replication/docker-compose-gs.yml @@ -1,6 +1,4 @@ -version: '3.9' - volumes: pg-master-data-dir: pg-node-data-dir: diff --git a/scenario_tests/streaming_replication/docker-compose-root-promote.yml b/scenario_tests/streaming_replication/docker-compose-root-promote.yml index 470d3eec..c7c3589a 100644 --- a/scenario_tests/streaming_replication/docker-compose-root-promote.yml +++ b/scenario_tests/streaming_replication/docker-compose-root-promote.yml @@ -1,6 +1,4 @@ -version: '2.1' - volumes: pg-master-data-dir: pg-node-data-dir: diff --git a/scenario_tests/streaming_replication/docker-compose.yml b/scenario_tests/streaming_replication/docker-compose.yml index f483b467..1f5b38a9 100644 --- a/scenario_tests/streaming_replication/docker-compose.yml +++ b/scenario_tests/streaming_replication/docker-compose.yml @@ -1,6 +1,4 @@ -version: '3.9' - volumes: pg-master-data-dir: pg-node-data-dir: diff --git a/scripts/env-data.sh b/scripts/env-data.sh index 792fcd8f..8c1f0486 100644 --- a/scripts/env-data.sh +++ b/scripts/env-data.sh @@ -508,7 +508,7 @@ function configure_replication_permissions { } function streaming_replication { - until START_COMMAND "${PG_BASEBACKUP} -X stream -h ${REPLICATE_FROM} -p ${REPLICATE_PORT} -D ${DATADIR} -U ${REPLICATION_USER} -R -vP -w --label=gis_pg_custer" + until START_COMMAND "${PG_BASEBACKUP} -X stream -h ${REPLICATE_FROM} -p ${REPLICATE_PORT} -D ${DATADIR} -U ${REPLICATION_USER} --incremental -R -vP -w --label=gis_pg_custer" do echo -e "[Entrypoint] \e[1;31m Waiting for master to connect... \033[0m" sleep 1s From 6df08c994e419cbeba7948cf92f49881dc8edbe0 Mon Sep 17 00:00:00 2001 From: Admire Nyakudya Date: Sat, 19 Oct 2024 12:00:10 +0200 Subject: [PATCH 2/3] Fix version number in action --- .github/workflows/build-latest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-latest.yaml b/.github/workflows/build-latest.yaml index a20dd6a7..27afd9cd 100644 --- a/.github/workflows/build-latest.yaml +++ b/.github/workflows/build-latest.yaml @@ -30,7 +30,7 @@ jobs: strategy: matrix: postgresMajorVersion: - - 16 + - 17 postgisMajorVersion: - 3 postgisMinorRelease: From 795b5dc9f0e52d190a348e51d68c88eb372f264e Mon Sep 17 00:00:00 2001 From: Admire Nyakudya Date: Sat, 19 Oct 2024 12:29:07 +0200 Subject: [PATCH 3/3] remove incremental option --- scripts/env-data.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/env-data.sh b/scripts/env-data.sh index 8c1f0486..aa95f05e 100644 --- a/scripts/env-data.sh +++ b/scripts/env-data.sh @@ -508,7 +508,7 @@ function configure_replication_permissions { } function streaming_replication { - until START_COMMAND "${PG_BASEBACKUP} -X stream -h ${REPLICATE_FROM} -p ${REPLICATE_PORT} -D ${DATADIR} -U ${REPLICATION_USER} --incremental -R -vP -w --label=gis_pg_custer" + until START_COMMAND "${PG_BASEBACKUP} -X stream -h ${REPLICATE_FROM} -p ${REPLICATE_PORT} -D ${DATADIR} -U ${REPLICATION_USER} -R -vP -w --label=gis_pg_custer" do echo -e "[Entrypoint] \e[1;31m Waiting for master to connect... \033[0m" sleep 1s