customer-data-pipeline.yml

#
# Build and test the Docker image for the CDP service found in the cdp/
# directory.
#
# This job is triggered by pushes to the master branch and by pull requests that
# touch the cdp/ directory.
#
# Once built we run the functional tests against the running image.

name: CDP CI

on:
    push:
        branches:
            - master
        paths:
            - cdp/**
            - .github/workflows/customer-data-pipeline.yml
    pull_request:
        branches:
            - master
        paths:
            - cdp/**
            - .github/workflows/customer-data-pipeline.yml

jobs:
    build:
        runs-on: ubuntu-latest
        steps:
            - uses: actions/checkout@v3
            - uses: docker/setup-buildx-action@v2
            - uses: docker/login-action@v2
              with:
                  registry: ghcr.io
                  username: ${{ github.actor }}
                  password: ${{ secrets.GITHUB_TOKEN }}

            - name: Lowercase GITHUB_REPOSITORY
              id: lowercase
              run: |
                  echo "repository=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT"

            - uses: docker/metadata-action@v4
              id: meta
              with:
                  images: ghcr.io/${{ steps.lowercase.outputs.repository }}/cdp

            # Make the image tags used for docker cache. We use this rather than
            # ${{ github.repository }} directly because the repository
            # organization name is has upper case characters, which are not
            # allowed in docker image names.
            - uses: docker/metadata-action@v4
              id: meta-cache
              with:
                  images: ghcr.io/${{ steps.lowercase.outputs.repository }}/cdp
                  tags: |
                      type=raw,value=cache

            - uses: docker/build-push-action@v4
              with:
                  context: cdp
                  file: cdp/Dockerfile
                  push: true
                  tags: ${{ steps.meta.outputs.tags }}
                  labels: ${{ steps.meta.outputs.labels }}
                  cache-from: type=registry,ref=${{ steps.meta-cache.outputs.tags }}
                  cache-to: type=registry,ref=${{ steps.meta-cache.outputs.tags }},mode=max

        # Output the image tags so that we can use them in the next job.
        outputs:
            tags: ${{ steps.meta.outputs.tags }}

    test:
        # Run the functional tests against the CDP service. We pull the image
        # from GHCR and run it locally. We need only the db service from the
        # main docker-compose.yml file, so we use the --services flag to only
        # start that service.
        runs-on: ubuntu-latest
        needs: build
        steps:
            - uses: actions/checkout@v3
            - uses: docker/setup-buildx-action@v2
            - uses: docker/login-action@v2
              with:
                  registry: ghcr.io
                  username: ${{ github.actor }}
                  password: ${{ secrets.GITHUB_TOKEN }}

            - name: Install pnpm
              uses: pnpm/action-setup@v2
              with:
                  version: 8.x.x

            - name: Setup node
              uses: actions/setup-node@v3
              with:
                  node-version: '18'
                  cache: 'pnpm'
                  cache-dependency-path: cdp/pnpm-lock.yaml

            - name: Install test dependencies
              working-directory: cdp
              run: |
                  pnpm install --frozen-lockfile

            - name: Start CDP
              working-directory: cdp
              run: |
                  mkdir -p /tmp/logs

                  docker compose -f ../docker-compose.dev.yml up -d db >> /tmp/logs/db.txt

                  # Wait for the db service to be ready, up to 30 seconds.
                  SECONDS=0
                  until docker compose -f ../docker-compose.dev.yml exec -T db pg_isready; do
                      if [ $SECONDS -gt 30 ]; then
                          echo "Timed out waiting for db service to be ready."
                          exit 1
                      fi
                      sleep 1
                  done

                  # Create a shell alias for the docker image we just built, using the tags output.
                  export SECRET_KEY=$(openssl rand -hex 32)
                  CDP_RUN="docker run -e SECRET_KEY=$SECRET_KEY -e DATABASE_URL=postgres://posthog:posthog@localhost:5432/posthog --rm --network=host ${{ needs.build.outputs.tags }}"

                  # Run the migrations.
                  $CDP_RUN sqlx migrate run

                  # Start the CDP service.
                  $CDP_RUN &> /tmp/logs/cdp.txt &

                  # Run the functional tests.
                  pnpm jest

            - name: Lowercase GITHUB_REPOSITORY
              id: lowercase
              run: |
                  echo "repository=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT"

            - name: Generate docker latest tag
              if: github.ref == 'refs/heads/master'
              uses: docker/metadata-action@v4
              id: meta
              with:
                  images: ghcr.io/${{ steps.lowercase.outputs.repository }}/cdp
                  tags: |
                      type=raw,value=latest

            - name: Push image as latest on master
              if: github.ref == 'refs/heads/master'
              run: |
                  docker tag ${{ needs.build.outputs.tags }} ${{ steps.meta.outputs.tags }}
                  docker push ${{ steps.meta.outputs.tags }}