ci: Fix CI for forks

.github/actions/build-n-cache-image/action.yml

@@ -0,0 +1,37 @@
+name: Build and cache Docker image
+
+inputs:
+    actions-id-token-request-url:
+        required: true
+        description: "ACTIONS_ID_TOKEN_REQUEST_URL, issued by GitHub when permission 'id-token' is set to 'write'"
+    load:
+        required: false
+        default: 'false'
+        description: Whether to load the image into local Docker after building it
+
+outputs:
+    tag:
+        description: The tag of the image that was built
+        value: ${{ steps.emit.outputs.tag }}
+
+runs:
+    using: 'composite'
+    steps:
+        - name: Set up Depot CLI
+          uses: depot/setup-action@v1
+
+        - name: Emit image tag
+          id: emit
+          shell: bash
+          run: echo "tag=posthog/posthog:${{ github.sha }}" >> $GITHUB_OUTPUT
+
+        - name: Build image # We don't push this because we use Depot cache as the communication channel
+          id: build
+          uses: depot/build-push-action@v1
+          with:
+              buildx-fallback: false # buildx is so slow it's better to just fail
+              load: ${{ inputs.load }}
+              tags: ${{ steps.emit.outputs.tag }}
+              platforms: linux/amd64,linux/arm64
+          env:
+              ACTIONS_ID_TOKEN_REQUEST_URL: ${{ inputs.actions-id-token-request-url }}

.github/actions/run-backend-tests/action.yml

@@ -2,29 +2,28 @@
 # This is a composite action that packages our backend Django tests.
 # It is called by the `ci-backend.yml` job using a matrix.
 #
-name: Run Backend Django tests
+name: Run Django tests
 inputs:
     python-version:
         required: true
-        type: string
+        description: Python version, e.g. 3.10.10
     clickhouse-server-image:
         required: true
-        type: string
+        description: ClickHouse server image tag, e.g. clickhouse/clickhouse-server:latest
     segment:
         required: true
-        type: string
+        description: Either 'FOSS' or 'EE' segment
     concurrency:
         required: true
-        type: number
+        description: Count of concurrency groups
     group:
         required: true
-        type: number
+        description: Group number
     person-on-events:
         required: true
-        type: boolean
+        description: Whether testing with persons on events, true or false
     token:
         required: false
-        type: string
         description: GitHub token
 
 runs:

.github/workflows/automerge.yml

@@ -22,7 +22,7 @@ on:
 
 jobs:
     automerge:
-        name: Automerge
+        name: Automerge if requested
         runs-on: ubuntu-latest
         env:
             IS_POSTHOG_BOT_AVAILABLE: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN != '' }}

.github/workflows/ci-e2e.yml

@@ -32,66 +32,56 @@ jobs:
                         # NOTE: we are at risk of missing a dependency here. We could make
                         # the dependencies more clear if we separated the backend/frontend
                         # code completely
-                        - 'ee/**/*'
-                        - 'posthog/**/*'
-                        - 'hogvm/**/*'
-                        - 'bin/*.py'
+                        - 'ee/**'
+                        - 'posthog/**'
+                        - 'hogvm/**'
+                        - 'bin/*'
+                        - frontend/**/*
                         - requirements.txt
                         - requirements-dev.txt
-                        - mypy.ini
-                        - pytest.ini
                         - package.json
                         - pnpm-lock.yaml
                         # Make sure we run if someone is explicitly change the workflow
                         - .github/workflows/ci-e2e.yml
+                        - .github/actions/build-n-cache-inage/action.yml
                         # We use docker compose for tests, make sure we rerun on
                         # changes to docker-compose.dev.yml e.g. dependency
                         # version changes
                         - docker-compose.dev.yml
-                        - frontend/**/*
+                        - Dockerfile
 
     # Job that lists and chunks spec file names and caches node modules
-    cypress_prep:
+    chunks:
         needs: changes
         name: Cypress preparation
         runs-on: ubuntu-latest
         timeout-minutes: 30
         outputs:
-            specs: ${{ steps.set-specs.outputs.specs }}
+            chunks: ${{ steps.chunk.outputs.chunks }}
 
         steps:
-            - name: Wait for the container image to be ready
-              # these are required checks so, we can't skip entire sections
-              if: needs.changes.outputs.shouldTriggerCypress == 'true'
-              uses: lewagon/[email protected]
-              with:
-                  check-name: Build PostHog
-                  ref: ${{ github.event.pull_request.head.sha }}
-                  repo-token: ${{ secrets.GITHUB_TOKEN }}
-                  wait-interval: 10
-
-            - name: Checkout code
+            - name: Check out
               uses: actions/checkout@v3
 
-            - name: List cypress/e2e and produce a JSON array of the files, in chunks
-              id: set-specs
-              run: echo "specs=$(ls cypress/e2e/* | jq --slurp --raw-input -c 'split("\n")[:-1] | _nwise(3) | join("\n")' | jq --slurp -c .)" >> $GITHUB_OUTPUT
+            - name: Group spec files into chunks of three
+              id: chunk
+              run: echo "chunks=$(ls cypress/e2e/* | jq --slurp --raw-input -c 'split("\n")[:-1] | _nwise(3) | join("\n")' | jq --slurp -c .)" >> $GITHUB_OUTPUT
 
     cypress:
         name: Cypress E2E tests (${{ strategy.job-index }})
         runs-on: ubuntu-latest
         timeout-minutes: 30
-        needs: [cypress_prep, changes]
+        needs: [chunks, changes]
         permissions:
-            packages: read # allow pull from ghcr.io
+            id-token: write # allow issuing OIDC tokens for this workflow run
 
         strategy:
             # when one test fails, DO NOT cancel the other
             # containers, as there may be other spec failures
             # we want to know about.
             fail-fast: false
             matrix:
-                specs: ${{ fromJson(needs.cypress_prep.outputs.specs) }}
+                chunk: ${{ fromJson(needs.chunks.outputs.chunks) }}
 
         steps:
             - name: Checkout
@@ -147,7 +137,18 @@ jobs:
               if: needs.changes.outputs.shouldTriggerCypress == 'true'
               run: ./bin/check_kafka_clickhouse_up
 
-            - name: Setup env
+            - name: Get Docker image cached in Depot
+              # We don't actually build the image here, because we use Depot, which acts as our cross-workflow cache.
+              # The build is first initiated in container-images-ci.yml, so by the time this runs, some layers already
+              # are cached, and the in-flight builds overall are deduplicated. According to Depot folks, this applies
+              # even if the builds _start_ concurrently! In short, only one build per commit push is ever executed.
+              uses: ./.github/actions/build-n-cache-image
+              id: docker-build
+              with:
+                  actions-id-token-request-url: ${{ env.ACTIONS_ID_TOKEN_REQUEST_URL }}
+                  load: true
+
+            - name: Write .env
               run: |
                   cat <<EOT >> .env
                   SECRET_KEY=6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da
@@ -171,26 +172,14 @@ jobs:
                   GITHUB_ACTION_RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
                   EOT
 
-            - name: Lowercase GITHUB_REPOSITORY
-              id: lowercase
-              run: |
-                  echo "repository=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT"
-
-            - name: Get the PostHog container image of this PR
-              if: needs.changes.outputs.shouldTriggerCypress == 'true'
-              id: meta
-              uses: docker/metadata-action@v4
-              with:
-                  images: ghcr.io/${{ steps.lowercase.outputs.repository }}/posthog
-
             - name: Start PostHog
               # these are required checks so, we can't skip entire sections
               if: needs.changes.outputs.shouldTriggerCypress == 'true'
               run: |
                   mkdir -p /tmp/logs
 
-                  echo "Starting PostHog using the container image ${{ steps.meta.outputs.tags }}"
-                  DOCKER_RUN="docker run --rm --network host --add-host kafka:127.0.0.1 --env-file .env ${{ steps.meta.outputs.tags }}"
+                  echo "Starting PostHog using the container image ${{ steps.docker-build.outputs.tag }}"
+                  DOCKER_RUN="docker run --rm --network host --add-host kafka:127.0.0.1 --env-file .env ${{ steps.docker-build.outputs.tag }}"
 
                   $DOCKER_RUN ./bin/migrate
                   $DOCKER_RUN python manage.py setup_dev
@@ -216,7 +205,7 @@ jobs:
               with:
                   config-file: cypress.e2e.config.ts
                   config: retries=2
-                  spec: ${{ matrix.specs }}
+                  spec: ${{ matrix.chunk }}
                   install: false
 
             - name: Archive test screenshots

.github/workflows/container-images-cd.yml

@@ -28,7 +28,7 @@ jobs:
             packages: write # allow push to ghcr.io
 
         steps:
-            - name: Checkout code
+            - name: Check out
               uses: actions/checkout@v3
               with:
                   fetch-depth: 2
@@ -66,7 +66,6 @@ jobs:
               id: build
               uses: depot/build-push-action@v1
               with:
-                  project: x19jffd9zf # posthog
                   buildx-fallback: false # the fallback is so slow it's better to just fail
                   push: true
                   tags: posthog/posthog:latest,${{ steps.aws-ecr.outputs.registry }}/posthog-cloud:master

.github/workflows/container-images-ci.yml

@@ -1,13 +1,3 @@
-#
-# Make sure PostHog and PostHog Cloud container images can be built
-# successfully.
-#
-# - posthog_build: build and push the PostHog container image to the
-#   GitHub Container Registry
-#
-# - posthog_cloud_build: build the PostHog Cloud container image using
-#   as base image the container image from the previous step
-#
 name: Container Images CI
 
 on:
@@ -19,67 +9,49 @@ concurrency:
 
 jobs:
     posthog_build:
-        name: Build PostHog
+        name: Build Docker image
         runs-on: ubuntu-latest
         permissions:
             id-token: write # allow issuing OIDC tokens for this workflow run
             contents: read # allow at least reading the repo contents, add other permissions if necessary
-            packages: write # allow push to ghcr.io
-
-        outputs:
-            container_image_tags: ${{ steps.meta.outputs.tags }}
-            container_image_version: ${{ steps.meta.outputs.version }}
 
         steps:
-            - name: Checkout code
+            - name: Check out
               uses: actions/checkout@v3
 
-            - name: Lowercase GITHUB_REPOSITORY
-              id: lowercase
-              run: |
-                  echo "repository=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT"
-
-            - name: Docker meta
-              id: meta
-              uses: docker/metadata-action@v4
-              with:
-                  images: ghcr.io/${{ steps.lowercase.outputs.repository }}/posthog
-                  tags: |
-                      type=schedule
-                      type=ref,event=branch
-                      type=ref,event=tag
-                      type=ref,event=pr
-
-            - name: Set up Docker Buildx
-              uses: docker/setup-buildx-action@v2
-
-            - name: Set up QEMU
-              uses: docker/setup-qemu-action@v2
-
-            - name: Set up Depot CLI
-              uses: depot/setup-action@v1
-
-            - name: Login to GitHub Container Registry
-              uses: docker/login-action@v2
-              with:
-                  registry: ghcr.io
-                  username: ${{ github.repository_owner }}
-                  password: ${{ secrets.GITHUB_TOKEN }}
-
-            - name: Build container images
-              id: build
-              uses: depot/build-push-action@v1
+            - name: Build and cache Docker image in Depot
+              uses: ./.github/actions/build-n-cache-image
               with:
-                  project: x19jffd9zf # posthog
-                  buildx-fallback: false # the fallback is so slow it's better to just fail
-                  cache-from: type=gha # always pull the layers from GHA
-                  cache-to: type=gha,mode=max # always push the layers to GHA
-                  push: true
-                  tags: ${{ steps.meta.outputs.tags }}
-                  platforms: linux/amd64,linux/arm64
+                actions-id-token-request-url: ${{ env.ACTIONS_ID_TOKEN_REQUEST_URL }}
 
     deploy_preview:
+        name: Deploy preview environment
         uses: ./.github/workflows/pr-deploy.yml
         needs: [posthog_build]
         secrets: inherit
         if: ${{ contains(github.event.pull_request.labels.*.name, 'deploy') }}
+
+    lint:
+        name: Lint changed Dockerfiles
+        runs-on: ubuntu-latest
+        steps:
+            - name: Check out
+              uses: actions/checkout@v3
+              with:
+                  fetch-depth: 0
+
+            - name: Check if any Dockerfile has changed
+              id: changed-files
+              uses: tj-actions/changed-files@v34
+              with:
+                  files: |
+                      **/Dockerfile
+                      **/*.Dockerfile
+                      **/Dockerfile.*
+                  separator: ' '
+
+            - name: Lint changed Dockerfile(s) with Hadolint
+              uses: jbergstroem/hadolint-gh-action@v1
+              if: steps.changed-files.outputs.any_changed == 'true'
+              with:
+                  dockerfile: '${{ steps.changed-files.outputs.all_modified_files }}'