From 345a0433396e28ff71a685f69328d66bab08672f Mon Sep 17 00:00:00 2001 From: Jacob Gillespie Date: Thu, 26 Oct 2023 16:50:39 +0100 Subject: [PATCH] ci: use Depot ephemeral registry for faster pulls (#18216) * ci: use Depot ephemeral registry for faster pulls * Install Depot CLI before pull --- .../actions/build-n-cache-image/action.yml | 14 ++++++--- .github/workflows/ci-e2e.yml | 31 ++++++++++++------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/.github/actions/build-n-cache-image/action.yml b/.github/actions/build-n-cache-image/action.yml index 03b5db99a1deb..a114e276025e3 100644 --- a/.github/actions/build-n-cache-image/action.yml +++ b/.github/actions/build-n-cache-image/action.yml @@ -4,15 +4,21 @@ inputs: actions-id-token-request-url: required: true description: "ACTIONS_ID_TOKEN_REQUEST_URL, issued by GitHub when permission 'id-token' is set to 'write'" - load: + save: required: false default: 'false' - description: Whether to load the image into local Docker after building it + description: Whether to save the image in the Depot ephemeral registry after building it outputs: tag: description: The tag of the image that was built value: ${{ steps.emit.outputs.tag }} + build-id: + description: The ID of the build + value: ${{ steps.build.outputs.build-id }} + unit-build-id: + description: The ID of the unit build + value: ${{ steps.build-unit.outputs.build-id }} runs: using: 'composite' @@ -30,10 +36,10 @@ runs: uses: depot/build-push-action@v1 with: buildx-fallback: false # buildx is so slow it's better to just fail - load: ${{ inputs.load }} tags: ${{ steps.emit.outputs.tag }} platforms: linux/amd64,linux/arm64 build-args: COMMIT_HASH=${{ github.sha }} + save: ${{ inputs.save }} env: ACTIONS_ID_TOKEN_REQUEST_URL: ${{ inputs.actions-id-token-request-url }} @@ -42,10 +48,10 @@ runs: uses: depot/build-push-action@v1 with: buildx-fallback: false # buildx is so slow it's better to just fail - load: ${{ inputs.load }} file: production-unit.Dockerfile tags: ${{ steps.emit.outputs.tag }} platforms: linux/amd64 build-args: COMMIT_HASH=${{ github.sha }} + save: ${{ inputs.save }} env: ACTIONS_ID_TOKEN_REQUEST_URL: ${{ inputs.actions-id-token-request-url }} diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml index 05162767a3d32..0ba4db2392761 100644 --- a/.github/workflows/ci-e2e.yml +++ b/.github/workflows/ci-e2e.yml @@ -76,15 +76,21 @@ jobs: permissions: contents: read id-token: write # allow issuing OIDC tokens for this workflow run + outputs: + tag: ${{ steps.build.outputs.tag }} + build-id: ${{ steps.build.outputs.build-id }} + unit-build-id: ${{ steps.build.outputs.unit-build-id }} steps: - name: Checkout if: needs.changes.outputs.shouldTriggerCypress == 'true' uses: actions/checkout@v3 - - name: Get Docker image cached in Depot + - name: Build the Docker image with Depot if: needs.changes.outputs.shouldTriggerCypress == 'true' # Build the container image in preparation for the E2E tests uses: ./.github/actions/build-n-cache-image + id: build with: + save: true actions-id-token-request-url: ${{ env.ACTIONS_ID_TOKEN_REQUEST_URL }} cypress: @@ -157,17 +163,20 @@ jobs: if: needs.changes.outputs.shouldTriggerCypress == 'true' run: ./bin/check_kafka_clickhouse_up + - name: Install Depot CLI + if: needs.changes.outputs.shouldTriggerCypress == 'true' + uses: depot/setup-action@v1 + - name: Get Docker image cached in Depot if: needs.changes.outputs.shouldTriggerCypress == 'true' - # We don't actually build the image here, because we use Depot, which acts as our cross-workflow cache. - # The build is first initiated in container-images-ci.yml, so by the time this runs, some layers already - # are cached, and the in-flight builds overall are deduplicated. According to Depot folks, this applies - # even if the builds _start_ concurrently! In short, only one build per commit push is ever executed. - uses: ./.github/actions/build-n-cache-image - id: docker-build + uses: depot/pull-action@v1 with: - actions-id-token-request-url: ${{ env.ACTIONS_ID_TOKEN_REQUEST_URL }} - load: true + # Use the production.Dockerfile image: + # build-id: ${{ needs.container.outputs.build-id }} + # Use the production-unit.Dockerfile image: + build-id: ${{ needs.container.outputs.unit-build-id }} + tags: | + ${{ needs.container.outputs.tag }} - name: Write .env # This step intentionally has no if, so that GH always considers the action as having run run: | @@ -199,8 +208,8 @@ jobs: run: | mkdir -p /tmp/logs - echo "Starting PostHog using the container image ${{ steps.docker-build.outputs.tag }}" - DOCKER_RUN="docker run --rm --network host --add-host kafka:127.0.0.1 --env-file .env ${{ steps.docker-build.outputs.tag }}" + echo "Starting PostHog using the container image ${{ needs.container.outputs.tag }}" + DOCKER_RUN="docker run --rm --network host --add-host kafka:127.0.0.1 --env-file .env ${{ needs.container.outputs.tag }}" $DOCKER_RUN ./bin/migrate $DOCKER_RUN python manage.py setup_dev