From f9ea91ba6cb90c99bc14ddbf920f6e3b220fa85f Mon Sep 17 00:00:00 2001 From: Alex Huang Date: Mon, 16 Sep 2024 00:14:08 +0800 Subject: [PATCH] feat: Dynamically determine pgrx version in CI / Docker (#1655) --- .github/workflows/benchmark-pg_search.yml | 6 ++++- .github/workflows/lint-rust.yml | 6 ++++- .github/workflows/publish-pg_search.yml | 6 ++++- .github/workflows/test-pg_search.yml | 12 ++++++++-- docker/Dockerfile | 29 ++++++++++++++++------- 5 files changed, 45 insertions(+), 14 deletions(-) diff --git a/.github/workflows/benchmark-pg_search.yml b/.github/workflows/benchmark-pg_search.yml index 33fcfb0d18..26f85a8f68 100644 --- a/.github/workflows/benchmark-pg_search.yml +++ b/.github/workflows/benchmark-pg_search.yml @@ -52,10 +52,14 @@ jobs: sudo chown -R $(whoami) /usr/share/postgresql/${{ matrix.pg_version }}/ /usr/lib/postgresql/${{ matrix.pg_version }}/ /var/lib/postgresql/${{ matrix.pg_version }}/ echo "/usr/lib/postgresql/${{ matrix.pg_version }}/bin" >> $GITHUB_PATH + - name: Extract pgrx version + working-directory: pg_search/ + run: echo "PGRX_VERSION=$(cargo tree --depth 1 -i pgrx -p pg_search | head -n 1 | cut -f2 -dv)" >> $GITHUB_ENV + - name: Install pgrx & pg_search working-directory: pg_search/ run: | - cargo install -j $(nproc) --locked cargo-pgrx --version 0.12.4 + cargo install -j $(nproc) --locked cargo-pgrx --version ${{ env.PGRX_VERSION }} cargo pgrx init --pg${{ matrix.pg_version }}=/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config cargo pgrx install --pg-config="/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config" --release diff --git a/.github/workflows/lint-rust.yml b/.github/workflows/lint-rust.yml index 410ae567b7..8174d30847 100644 --- a/.github/workflows/lint-rust.yml +++ b/.github/workflows/lint-rust.yml @@ -36,8 +36,12 @@ jobs: sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' sudo apt-get update && sudo apt-get install -y postgresql-${{ matrix.pg_version }} postgresql-server-dev-${{ matrix.pg_version }} + - name: Extract pgrx version + working-directory: pg_search/ + run: echo "PGRX_VERSION=$(cargo tree --depth 1 -i pgrx -p pg_search | head -n 1 | cut -f2 -dv )" >> $GITHUB_ENV + - name: Install pgrx - run: cargo install --locked cargo-pgrx --version 0.12.4 + run: cargo install --locked cargo-pgrx --version ${{ env.PGRX_VERSION }} - name: Initialize pgrx for Current PostgreSQL Version run: cargo pgrx init --pg${{ matrix.pg_version }}=/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config diff --git a/.github/workflows/publish-pg_search.yml b/.github/workflows/publish-pg_search.yml index 093b772732..5cd87a9892 100644 --- a/.github/workflows/publish-pg_search.yml +++ b/.github/workflows/publish-pg_search.yml @@ -299,8 +299,12 @@ jobs: # Install PostgreSQL: sudo dnf install -y postgresql${{ matrix.pg_version }} postgresql${{ matrix.pg_version }}-server postgresql${{ matrix.pg_version }}-devel + - name: Extract pgrx version + working-directory: pg_search/ + run: echo "PGRX_VERSION=$(cargo tree --depth 1 -i pgrx -p pg_search | head -n 1 | cut -f2 -dv)" >> $GITHUB_ENV + - name: Install pgrx - run: cargo install --locked cargo-pgrx --version 0.12.4 + run: cargo install --locked cargo-pgrx --version ${{ env.PGRX_VERSION }} # Note: We need to specify bash as the shell to ensure that it doesn't default to /bin/sh on Debian, which doesn't support the `[[` syntax - name: Initialize pgrx for Current PostgreSQL Version diff --git a/.github/workflows/test-pg_search.yml b/.github/workflows/test-pg_search.yml index a7cefa590e..d9d854357d 100644 --- a/.github/workflows/test-pg_search.yml +++ b/.github/workflows/test-pg_search.yml @@ -121,10 +121,14 @@ jobs: sudo PG_CONFIG=/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config make -j sudo PG_CONFIG=/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config make install -j + - name: Extract pgrx version + working-directory: pg_search/ + run: echo "PGRX_VERSION=$(cargo tree --depth 1 -i pgrx -p pg_search | head -n 1 | cut -f2 -dv)" >> $GITHUB_ENV + - name: Install pgrx & llvm-tools-preview if: steps.check_skip.outputs.skip_remaining_steps != 'true' run: | - cargo install -j $(nproc) --locked cargo-pgrx --version 0.12.4 + cargo install -j $(nproc) --locked cargo-pgrx --version ${{ env.PGRX_VERSION }} rustup component add llvm-tools-preview cargo pgrx init "--pg${{ matrix.pg_version }}=/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config" @@ -279,10 +283,14 @@ jobs: cache-all-crates: true save-if: ${{ github.ref == 'refs/heads/dev' }} + - name: Extract pgrx version + working-directory: pg_search/ + run: echo "PGRX_VERSION=$(grep '^pgrx = ' Cargo.toml | sed -E 's/pgrx = "(.*)"/\1/')" >> $GITHUB_ENV + - name: Install pgrx & llvm-tools-preview if: steps.check_skip.outputs.skip_remaining_steps != 'true' run: | - cargo install -j $(nproc) --locked cargo-pgrx --version 0.12.4 + cargo install -j $(nproc) --locked cargo-pgrx --version ${{ env.PGRX_VERSION }} rustup component add llvm-tools-preview cargo pgrx init "--pg${{ matrix.pg_version }}=download" diff --git a/docker/Dockerfile b/docker/Dockerfile index 4c00231451..0354be2696 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,13 +9,10 @@ FROM postgres:${PG_VERSION_MAJOR}-bookworm AS builder ARG PG_VERSION_MAJOR=16 ARG RUST_VERSION=1.80.0 -ARG PGRX_VERSION=0.12.4 -ARG PGRX_VERSION_PG_VECTOR_SCALE=0.11.4 # Declare buildtime environment variables ENV PG_VERSION_MAJOR=${PG_VERSION_MAJOR} \ RUST_VERSION=${RUST_VERSION} \ - PGRX_VERSION=${PGRX_VERSION} \ PGRX_VERSION_PG_VECTOR_SCALE=${PGRX_VERSION_PG_VECTOR_SCALE} SHELL ["/bin/bash", "-o", "pipefail", "-c", "-e"] @@ -41,10 +38,18 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --defau ENV PATH="/root/.cargo/bin:$PATH" \ PGX_HOME=/usr/lib/postgresql/${PG_VERSION_MAJOR} +# Copy Cargo.toml because we need to extract the pgrx version +WORKDIR /tmp +COPY pg_search/Cargo.toml ./ + # Intall the pgrx version of pg_search and pg_analytics -RUN cargo install --locked cargo-pgrx --version "${PGRX_VERSION}" && \ +RUN PGRX_VERSION=$(sed -n 's/^pgrx *= *"=*\([0-9.]*\)"/\1/p' Cargo.toml) && \ + echo "PGRX_VERSION=$PGRX_VERSION" && \ + cargo install --locked cargo-pgrx --version "${PGRX_VERSION}" && \ cargo pgrx init "--pg${PG_VERSION_MAJOR}=/usr/lib/postgresql/${PG_VERSION_MAJOR}/bin/pg_config" +RUN rm -rf /tmp/Cargo.toml + ###################### # pg_search ###################### @@ -114,16 +119,22 @@ RUN export PG_CFLAGS="-Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-comp FROM builder AS builder-pgvectorscale -# Intall the pgrx version of pgvectorscale -RUN cargo install --locked cargo-pgrx --version "${PGRX_VERSION_PG_VECTOR_SCALE}" && \ - cargo pgrx init "--pg${PG_VERSION_MAJOR}=/usr/lib/postgresql/${PG_VERSION_MAJOR}/bin/pg_config" - ARG TARGETARCH -# Build the extension +SHELL ["/bin/bash", "-o", "pipefail", "-c", "-e"] + +# Clone the pgvectorscale repository WORKDIR /tmp RUN git clone --branch 0.3.0 https://github.com/timescale/pgvectorscale.git WORKDIR /tmp/pgvectorscale/pgvectorscale + +# Intall the pgrx version of pgvectorscale +RUN PGRX_VERSION=$(cargo tree --depth 1 -i pgrx -p pgvectorscale | head -n 1 | cut -f2 -dv) && \ + echo "PGRX_VERSION=$PGRX_VERSION" && \ + cargo install --locked cargo-pgrx --version "${PGRX_VERSION}" && \ + cargo pgrx init "--pg${PG_VERSION_MAJOR}=/usr/lib/postgresql/${PG_VERSION_MAJOR}/bin/pg_config" + +# Build the extension RUN if [ "$TARGETARCH" = "amd64" ]; then \ # Required for pgvectorscale to compile on x86_64/amd64 RUSTFLAGS="-C target-feature=+avx2,+fma" cargo pgrx package --pg-config "/usr/lib/postgresql/${PG_VERSION_MAJOR}/bin/pg_config"; \