fix(infra): container generation and workflows

During release v0.2.0, the container generation flow failed due to the new addition of pg16. This commit fixes that failure and makes the pg version in use by pgrx more explicit in GHA workflow configuration Signed-off-by: vados <[email protected]>
VADOSWARE · Dec 19, 2023 · fbc6f93 · fbc6f93
1 parent 8125d62
commit fbc6f93
Show file tree

Hide file tree

Showing 11 changed files with 106 additions and 109 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1 +1,3 @@
 target
+infra/docker
+secrets
diff --git a/.github/workflows/build-and-test-gnu/action.yaml b/.github/workflows/build-and-test-gnu/action.yaml
@@ -40,7 +40,7 @@ inputs:
   user:
     type: string
     default: idkit
-  pgrx-version:
+  pgrx-pg-version:
     type: string
     default: pg16
     decription: |
@@ -121,7 +121,7 @@ runs:
         CARGO_INCREMENTAL: ${{ inputs.cargo-env-incrmental }}
         CARGO_PROFILE: ${{ inputs.cargo-profile }}
         CARGO_TARGET_DIR: ${{ inputs.cargo-target-dir }}
-        PGRX_PG_VERSION: ${{ inputs.pgrx-version }}
+        PGRX_PG_VERSION: ${{ inputs.pgrx-pg-version }}
         PGRX_PKG_PATH_PREFIX: ${{ inputs.cargo-target-dir }}
         PKG_PG_VERSION: ${{ inputs.pg-version }}
         SCCACHE_DIR: ${{ inputs.sccache-dir }}
@@ -142,7 +142,7 @@ runs:
         CARGO_INCREMENTAL: ${{ inputs.cargo-env-incrmental }}
         CARGO_PROFILE: ${{ inputs.cargo-profile }}
         CARGO_TARGET_DIR: ${{ inputs.cargo-target-dir }}
-        PGRX_PG_VERSION: ${{ inputs.pgrx-version }}
+        PGRX_PG_VERSION: ${{ inputs.pgrx-pg-version }}
         PGRX_PKG_PATH_PREFIX: ${{ inputs.cargo-target-dir }}
         PKG_PG_VERSION: ${{ inputs.pg-version }}
         SCCACHE_DIR: ${{ inputs.sccache-dir }}
@@ -159,7 +159,7 @@ runs:
         CARGO_INCREMENTAL: ${{ inputs.cargo-env-incrmental }}
         CARGO_PROFILE: ${{ inputs.cargo-profile }}
         CARGO_TARGET_DIR: ${{ inputs.cargo-target-dir }}
-        PGRX_PG_VERSION: ${{ inputs.pgrx-version }}
+        PGRX_PG_VERSION: ${{ inputs.pgrx-pg-version }}
         PGRX_PKG_PATH_PREFIX: ${{ inputs.cargo-target-dir }}
         PKG_PG_VERSION: ${{ inputs.pg-version }}
         SCCACHE_DIR: ${{ inputs.sccache-dir }}
@@ -176,7 +176,7 @@ runs:
         CARGO_INCREMENTAL: ${{ inputs.cargo-env-incrmental }}
         CARGO_PROFILE: ${{ inputs.cargo-profile }}
         CARGO_TARGET_DIR: ${{ inputs.cargo-target-dir }}
-        PGRX_PG_VERSION: ${{ inputs.pgrx-version }}
+        PGRX_PG_VERSION: ${{ inputs.pgrx-pg-version }}
         PGRX_PKG_PATH_PREFIX: ${{ inputs.cargo-target-dir }}
         PKG_PG_VERSION: ${{ inputs.pg-version }}
         SCCACHE_DIR: ${{ inputs.sccache-dir }}
@@ -193,7 +193,7 @@ runs:
         CARGO_INCREMENTAL: ${{ inputs.cargo-env-incrmental }}
         CARGO_PROFILE: ${{ inputs.cargo-profile }}
         CARGO_TARGET_DIR: ${{ inputs.cargo-target-dir }}
-        PGRX_PG_VERSION: ${{ inputs.pgrx-version }}
+        PGRX_PG_VERSION: ${{ inputs.pgrx-pg-version }}
         PKG_PG_VERSION: ${{ inputs.pg-version }}
         SCCACHE_DIR: ${{ inputs.sccache-dir }}
         USER: ${{ inputs.user }}
@@ -214,7 +214,7 @@ runs:
         CARGO_INCREMENTAL: ${{ inputs.cargo-env-incrmental }}
         CARGO_PROFILE: ${{ inputs.cargo-profile }}
         CARGO_TARGET_DIR: ${{ inputs.cargo-target-dir }}
-        PGRX_PG_VERSION: ${{ inputs.pgrx-version }}
+        PGRX_PG_VERSION: ${{ inputs.pgrx-pg-version }}
         PGRX_PKG_PATH_PREFIX: ${{ inputs.cargo-target-dir }}
         PKG_PG_VERSION: ${{ inputs.pg-version }}
         PKG_TARBALL_SUFFIX: ${{ inputs.artifact-tarball-suffix }}

diff --git a/.github/workflows/build-rpm/action.yaml b/.github/workflows/build-rpm/action.yaml
@@ -19,7 +19,7 @@ inputs:
     default: x86_64
     decription: |
       Architecture to use while building the RPM
-  pgrx-version:
+  pgrx-pg-version:
     type: string
     default: pg16
     decription: |
@@ -53,23 +53,23 @@ runs:
     - name: Initialize cargo-pgrx
       shell: bash
       env:
-        PGRX_PG_VERSION: ${{ inputs.pgrx-version }}
+        PGRX_PG_VERSION: ${{ inputs.pgrx-pg-version }}
         PKG_PG_VERSION: ${{ inputs.pg-version }}
       run: |
         [[ -d /home/runner/.pgrx ]] || cargo pgrx init
 
     - name: Build RPM
       shell: bash
       env:
-        PGRX_PG_VERSION: ${{ inputs.pgrx-version }}
+        PGRX_PG_VERSION: ${{ inputs.pgrx-pg-version }}
         PKG_PG_VERSION: ${{ inputs.pg-version }}
       run: just package build-rpm
 
     - name: Get RPM output path
       id: rpm-output
       shell: bash
       env:
-        PGRX_PG_VERSION: ${{ inputs.pgrx-version }}
+        PGRX_PG_VERSION: ${{ inputs.pgrx-pg-version }}
         PKG_PG_VERSION: ${{ inputs.pg-version }}
       run: |
         echo path=$(just print-rpm-output-path) >> $GITHUB_OUTPUT

diff --git a/.github/workflows/container.yaml b/.github/workflows/container.yaml
@@ -2,6 +2,10 @@ name: container
 
 on:
   push:
+    branches:
+      - fix/infra/container-generation-and-workflows # TODO: remove
+      # Run on auto-generated release PRs
+      - prep-release-v[0-9]+\.[0-9]+\.[0-9]+
     tags:
       - v[0-9]+\.[0-9]+\.[0-9]+
 
@@ -21,23 +25,39 @@ jobs:
               arch: amd64
               pg_version: 15.5
               os_version: alpine3.18
+          - triple: x86_64-unknown-linux-musl
+            gh:
+              runner: ubuntu-22.04
+            container:
+              arch: amd64
+              pg_version: 16.1
+              os_version: alpine3.18
     steps:
       - uses: actions/checkout@v3
       - uses: docker/setup-qemu-action@v3
       - uses: taiki-e/install-action@v2
         with:
           tool: cargo-get,just
 
-      - name: registry login
+      - name: Registry login
         uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: build & push container image
-        run: just build-image push-image
+      - name: Build container image
+        run: just build-image
         env:
           CONTAINER_IMAGE_ARCH: ${{ matrix.config.container.arch }}
+          PGRX_PG_VERSION: ${{ matrix.config.container.pg_version }}
           POSTGRES_IMAGE_VERSION: ${{ matrix.config.container.pg_version }}
           POSTGRES_OS_IMAGE_VERSION: ${{ matrix.config.container.os_version }}
+
+      - name: Push pre-release image
+        if: ${{ startsWith(github.ref, 'refs/heads/prep-release') }}
+        run: just push-image-preprelease
+
+      - name: Push release image
+        if: ${{ startsWith(github.ref, 'refs/tags/v') }}
+        run: just push-image
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -16,11 +16,11 @@ jobs:
       matrix:
         config:
           - pgrx:
-              version: pg15
+              pg-version: pg15
             pg:
               version: 15.5
           - pgrx:
-              version: pg16
+              pg-version: pg16
             pg:
               version: 16.1
     steps:
@@ -30,7 +30,7 @@ jobs:
         uses: ./.github/workflows/build-rpm
         with:
           artifact-upload: true
-          pgrx-version: ${{ matrix.config.pgrx.version }}
+          pgrx-pg-version: ${{ matrix.config.pgrx.pg-version }}
           pg-version: ${{ matrix.config.pg.version }}
 
   build-zip-gnu:
@@ -41,11 +41,11 @@ jobs:
       matrix:
         config:
           - pgrx:
-              version: pg15
+              pg-version: pg15
             pg:
               version: 15.5
           - pgrx:
-              version: pg16
+              pg-version: pg16
             pg:
               version: 16.1
     steps:
@@ -55,8 +55,8 @@ jobs:
         uses: ./.github/workflows/build-and-test-gnu
         with:
           artifact-upload: true
-          artifact-tarball-suffix: "-${{ matrix.config.pgrx.version }}-gnu"
-          pgrx-version: ${{ matrix.config.pgrx.version }}
+          artifact-tarball-suffix: "-${{ matrix.config.pgrx.pg-version }}-gnu"
+          pgrx-pg-version: ${{ matrix.config.pgrx.pg-version }}
           pg-version: ${{ matrix.config.pg.version }}
 
   release:
@@ -80,7 +80,7 @@ jobs:
         uses: softprops/action-gh-release@v1
         with:
           prerelease: ${{ startsWith(github.ref, 'refs/heads/prep-release') }}
-          draft: true
+          draft: ${{ startsWith(github.ref, 'refs/heads/prep-release') }}
           # NOTE: while we have the CHANGELOG file, it always contains *all* changes,
           # so we will use the generated GitHub commits for now
           generate_release_notes: true

diff --git a/.gitignore b/.gitignore
@@ -7,3 +7,4 @@ secrets/docker/buildx/
 
 # Releases
 /pkg
+/pg_idkit-*-gnu.tar.gz
diff --git a/Justfile b/Justfile
@@ -1,5 +1,3 @@
-user := env_var('USER')
-
 docker := env_var_or_default("DOCKER", "docker")
 git := env_var_or_default("GIT", "git")
 tar := env_var_or_default("TAR", "tar")
@@ -27,11 +25,15 @@ changelog_file_path := absolute_path(justfile_directory() / "CHANGELOG")
 
 pkg_pg_version := env_var_or_default("PKG_PG_VERSION", "16.1")
 pkg_pg_config_path := env_var_or_default("PKG_PG_CONFIG_PATH", "~/.pgrx/" + pkg_pg_version + "/pgrx-install/bin/pg_config")
-pkg_tarball_suffix := env_var_or_default("PKG_TARBALL_SUFFIX", "-gnu")
+pkg_tarball_suffix := env_var_or_default("PKG_TARBALL_SUFFIX", "")
 
 pgrx_pg_version := env_var_or_default("PGRX_PG_VERSION", "pg16")
 pgrx_pkg_path_prefix := env_var_or_default("PGRX_PKG_PATH_PREFIX", "target")
-pgrx_pkg_output_dir := pgrx_pkg_path_prefix / "release" / "pg_idkit-" + pgrx_pg_version / "home" / user / ".pgrx" / pkg_pg_version / "pgrx-install"
+# If /root, 'home' does not appear in the generated prefix
+pkg_user_dir_prefix := if docker_build_user == "root" { docker_build_user } else { "home/" + docker_build_user }
+pgrx_pkg_output_dir := pgrx_pkg_path_prefix / "release" / "pg_idkit-" + pgrx_pg_version / pkg_user_dir_prefix / ".pgrx" / pkg_pg_version / "pgrx-install"
+
+docker_build_user := env_var_or_default('DOCKER_BUILD_USER', "root")
 
 default:
     {{just}} --list
@@ -108,8 +110,10 @@ build-watch: _check-tool-cargo _check-tool-cargo-watch
 build-test-watch: _check-tool-cargo _check-tool-cargo-watch
     {{cargo_watch}} -x "test $(CARGO_BUILD_FLAGS)" --watch src
 
-package:
+build-package:
     PGRX_IGNORE_RUST_VERSIONS=y {{cargo}} pgrx package --pg-config {{pkg_pg_config_path}}
+
+package: build-package
     mkdir -p pkg/pg_idkit-$({{just}} print-version)
     cp -r $({{just}} print-pkg-output-dir)/* pkg/pg_idkit-$({{just}} print-version)
     {{tar}} -C pkg -cvf pg_idkit-$(just print-version){{pkg_tarball_suffix}}.tar.gz  pg_idkit-$({{just}} print-version)
@@ -187,7 +191,7 @@ builder_gnu_image_name_full := env_var_or_default("BUILDER_IMAGE_NAME_FULL", bui
 
 # Build the docker image used in BUILDER
 build-builder-image:
-    {{docker}} build --build-arg USER={{user}} -f {{builder_gnu_dockerfile_path}} -t {{builder_gnu_image_name_full}} .
+    {{docker}} build -f {{builder_gnu_dockerfile_path}} -t {{builder_gnu_image_name_full}} .
 
 # Push the docker image used in BUILDER (to GitHub Container Registry)
 push-builder-image:
@@ -202,15 +206,15 @@ push-builder-image:
 #
 
 # Determine the Dockerfile to use when building the packaging utility base image
-base_pkg_dockerfile_path := "infra/docker/base-pkg-pg" + pkg_pg_version + "-" + pg_os_image_version + "-" + container_img_arch + ".Dockerfile"
+base_pkg_dockerfile_path := "infra/docker/base-pkg-" + pg_os_image_version + "-" + container_img_arch + ".Dockerfile"
 base_pkg_image_name := env_var_or_default("PKG_IMAGE_NAME", "ghcr.io/vadosware/pg_idkit/base-pkg")
 base_pkg_version := env_var_or_default("PKG_IMAGE_NAME", "0.1.x")
-base_pkg_image_tag := env_var_or_default("POSGRES_IMAGE_VERSION", base_pkg_version + "-" + "pg" + pg_image_version + "-" + pg_os_image_version + "-" + container_img_arch)
+base_pkg_image_tag := env_var_or_default("POSGRES_IMAGE_VERSION", base_pkg_version + "-" + pg_os_image_version + "-" + container_img_arch)
 base_pkg_image_name_full := env_var_or_default("PKG_IMAGE_NAME_FULL", base_pkg_image_name + ":" + base_pkg_image_tag)
 
 # Build the base image for packaging
 build-base-pkg-image:
-      {{docker}} build --build-arg USER={{user}} -f {{base_pkg_dockerfile_path}} . -t {{base_pkg_image_name_full}};
+      {{docker}} build --build-arg USER={{docker_build_user}} -f {{base_pkg_dockerfile_path}} . -t {{base_pkg_image_name_full}};
 
 # Push the base image for packaging
 push-base-pkg-image:
@@ -226,7 +230,11 @@ push-base-pkg-image:
 
 # Build the docker image for pg_idkit
 build-image:
-    {{docker}} build {{docker_platform_arg}} {{docker_progress_arg}} -f {{img_dockerfile_path}} -t {{pgidkit_image_name_full}} --build-arg USER={{user}} --build-arg PGIDKIT_REVISION={{revision}} --build-arg PGIDKIT_VERSION={{pgidkit_image_tag}} .
+    {{docker}} build {{docker_platform_arg}} {{docker_progress_arg}} -f {{img_dockerfile_path}} -t {{pgidkit_image_name_full}} --build-arg USER={{docker_build_user}} --build-arg PGIDKIT_REVISION={{revision}} --build-arg PGIDKIT_VERSION={{pgidkit_image_tag}} .
+
+# Push the pre-release docker image for pg_idkit
+push-image-prerelease:
+    {{docker}} push {{pgidkit_image_name_full}}-prerelease
 
 # Push the docker image for pg_idkit
 push-image:
@@ -246,7 +254,7 @@ rpm_scratch_location := "/tmp/pg_idkit/rpm/scratch"
 
 # Build an RPM distribution for pg_idkit
 build-rpm: _check-tool-strip _check-tool-cargo-generate-rpm
-    CARGO_FEATURES={{pgrx_pg_version}} {{just}} build-release
+    CARGO_FEATURES={{pgrx_pg_version}} {{just}} package
     {{strip}} -s {{pgrx_pkg_output_dir}}/lib/postgresql/pg_idkit.so
     mkdir -p {{rpm_scratch_location}}
     cp -r {{pgrx_pkg_output_dir}} {{rpm_scratch_location}}

diff --git a/...se-pkg-pg16.1-alpine3.18-amd64.Dockerfile → ...cker/base-pkg-alpine3.18-amd64.Dockerfile b/...se-pkg-pg16.1-alpine3.18-amd64.Dockerfile → ...cker/base-pkg-alpine3.18-amd64.Dockerfile
@@ -1,4 +1,4 @@
-FROM postgres:16.1-alpine3.18@sha256:b788d196db04847b17df664f4ae18062e712328d225b9ff75d4d7cd91a16c374 AS builder
+FROM alpine:3.19.0@sha256:13b7e62e8df80264dbb747995705a986aa530415763a6c58f84a3ca8af9a5bcd AS builder
 
 # Allow for overriding rust toolcahin version
 ARG RUST_TOOLCHAIN_VERSION=1.74
@@ -14,19 +14,25 @@ ENV PGRX_PG_VERSION=$PGRX_PG_VERSION
 ARG CARGO_FEATURES=pg16
 ENV CARGO_FEATURES=$CARGO_FEATURES
 
-ARG USER
-ENV USER=$USER
-
 # Install OS deps
 RUN apk add --no-cache \
     alpine-sdk \
-    coreutils \
+    bash \
+    bison \
     clang \
     clang-dev \
     clang-libs \
+    coreutils \
+    flex \
+    icu-dev \
+    linux-headers \
     musl-dev \
     openssl-dev \
-    rustup
+    perl \
+    readline \
+    readline-dev \
+    rustup \
+    zlib-dev
 
 # Install Rust & related deps
 RUN rustup-init -y --profile minimal --default-toolchain $RUST_TOOLCHAIN_VERSION
@@ -43,8 +49,7 @@ COPY . .
 
 # Initialize pgrx
 ENV PGRX_IGNORE_RUST_VERSIONS=y
-ENV PKG_PG_CONFIG_PATH=/usr/local/bin/pg_config
-RUN cargo pgrx init --pg16=$PKG_PG_CONFIG_PATH
+RUN just pgrx-init
 
 # Build the package
 RUN RUSTFLAGS="-Ctarget-feature=-crt-static" just build package
Original file line number	Diff line number	Diff line change
Expand Up		@@ -7,3 +7,4 @@ secrets/docker/buildx/

		# Releases
		/pkg
		/pg_idkit-*-gnu.tar.gz