ci-backend.yml

# This workflow runs all of our backend django tests.
#
# If these tests get too slow, look at increasing concurrency and re-timing the tests by manually dispatching
# .github/workflows/ci-backend-update-test-timing.yml action
name: Backend CI

on:
    push:
        branches:
            - master
    pull_request:
    workflow_dispatch:
        inputs:
            clickhouseServerVersion:
                description: ClickHouse server version. Leave blank for default
                type: string

env:
    SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only
    DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
    REDIS_URL: 'redis://localhost'
    CLICKHOUSE_HOST: 'localhost'
    CLICKHOUSE_SECURE: 'False'
    CLICKHOUSE_VERIFY: 'False'
    TEST: 1
    CLICKHOUSE_SERVER_IMAGE_VERSION: ${{ github.event.inputs.clickhouseServerVersion || '' }}
    OBJECT_STORAGE_ENABLED: 'True'
    OBJECT_STORAGE_ENDPOINT: 'http://localhost:19000'
    OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
    OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'

concurrency:
    group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
    # Allow query snapshot updates to finish, so that we don't need to wait for the whole stack to start all over again
    cancel-in-progress: false

jobs:
    # Job to decide if we should run backend ci
    # See https://github.com/dorny/paths-filter#conditional-execution for more details
    changes:
        runs-on: ubuntu-latest
        timeout-minutes: 5
        if: github.repository == 'PostHog/posthog'
        name: Determine need to run backend checks
        # Set job outputs to values from filter step
        outputs:
            backend: ${{ steps.filter.outputs.backend }}
        steps:
            # For pull requests it's not necessary to checkout the code, but we
            # also want this to run on master so we need to checkout
            - uses: actions/checkout@v3

            - uses: dorny/paths-filter@v2
              id: filter
              with:
                  filters: |
                      backend:
                        # Avoid running backend tests for irrelevant changes
                        # NOTE: we are at risk of missing a dependency here. We could make
                        # the dependencies more clear if we separated the backend/frontend
                        # code completely
                        - 'ee/**/*'
                        - 'posthog/**/*'
                        - 'bin/*.py'
                        - requirements.txt
                        - requirements-dev.txt
                        - mypy.ini
                        - pytest.ini
                        - frontend/src/queries/schema.json # Used for generating schema.py
                        # Make sure we run if someone is explicitly change the workflow
                        - .github/workflows/ci-backend.yml
                        - .github/workflows/backend-tests-action/action.yml
                        # We use docker compose for tests, make sure we rerun on
                        # changes to docker-compose.dev.yml e.g. dependency
                        # version changes
                        - docker-compose.dev.yml
                        - frontend/public/email/*

    backend-code-quality:
        needs: changes
        timeout-minutes: 10

        name: Python code quality checks
        runs-on: ubuntu-latest

        steps:
            - uses: actions/checkout@v3
              with:
                  fetch-depth: 1
                  path: 'current/'

            - name: Stop/Start stack with Docker Compose
              run: |
                  cd current
                  docker compose -f docker-compose.dev.yml down
                  docker compose -f docker-compose.dev.yml up -d

            - name: Set up Python
              uses: actions/setup-python@v4
              with:
                  python-version: 3.10.10
                  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

            - uses: syphar/restore-virtualenv@v1
              id: cache-backend-tests
              with:
                  custom_cache_key_element: v1-

            - uses: syphar/restore-pip-download-cache@v1
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'

            - name: Install SAML (python3-saml) dependencies
              run: |
                  sudo apt-get update
                  sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl

            - name: Install python dependencies
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
              run: |
                  cd current
                  python -m pip install -r requirements.txt -r requirements-dev.txt

            - name: Check for syntax errors, import sort, and code style violations
              run: |
                  cd current
                  ruff .

            - name: Check formatting
              run: |
                  cd current
                  black --exclude posthog/hogql/grammar --check .

            - name: Check static typing
              run: |
                  cd current
                  mypy -p posthog --exclude bin/migrate_kafka_data.py --exclude posthog/hogql/grammar/HogQLParser.py --exclude gunicorn.config.py

            - name: Check if "schema.py" is up to date
              run: |
                  cd current
                  npm run schema:build:python && git diff --exit-code

            - name: Check if antlr definitions are up to date
              run: |
                  # Installing a version of ant compatible with what we use in development from homebrew (4.13)
                  # "apt-get install antlr" would install 4.7 which is incompatible with our grammar.
                  export ANTLR_VERSION=4.13.0
                  # java version doesn't matter
                  sudo apt-get install default-jre
                  mkdir antlr
                  cd antlr
                  curl -o antlr.jar https://www.antlr.org/download/antlr-$ANTLR_VERSION-complete.jar
                  export PWD=`pwd`
                  echo '#!/bin/bash' > antlr
                  echo "java -jar $PWD/antlr.jar \$*" >> antlr
                  chmod +x antlr
                  export CLASSPATH=".:$PWD/antlr.jar:$CLASSPATH"
                  export PATH="$PWD:$PATH"

                  cd ../current
                  antlr | grep "Version"
                  npm run grammar:build && git diff --exit-code

    check-migrations:
        needs: changes
        if: needs.changes.outputs.backend == 'true'
        timeout-minutes: 5

        name: Validate Django migrations
        runs-on: ubuntu-latest

        steps:
            - uses: actions/checkout@v3

            - name: Stop/Start stack with Docker Compose
              run: |
                  docker compose -f docker-compose.dev.yml down
                  docker compose -f docker-compose.dev.yml up -d

            - name: Set up Python
              uses: actions/setup-python@v4
              with:
                  python-version: 3.10.10
                  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

            - uses: syphar/restore-virtualenv@v1
              id: cache-backend-tests
              with:
                  custom_cache_key_element: v1-

            - uses: syphar/restore-pip-download-cache@v1
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'

            - name: Install SAML (python3-saml) dependencies
              run: |
                  sudo apt-get update
                  sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl

            - name: Install python dependencies
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
              run: |
                  python -m pip install -r requirements.txt -r requirements-dev.txt

            - uses: actions/checkout@v3
              with:
                  ref: master

            - name: Run migrations up to master
              run: |
                  # We need to ensure we have requirements for the master branch
                  # now also, so we can run migrations up to master.
                  python -m pip install -r requirements.txt -r requirements-dev.txt
                  python manage.py migrate

            - uses: actions/checkout@v3

            - name: Check migrations
              run: |
                  python manage.py makemigrations --check --dry-run
                  git fetch origin master
                  # `git diff --name-only` returns a list of files that were changed - added OR deleted OR modified
                  # With `--name-status` we get the same, but including a column for status, respectively: A, D, M
                  # In this check we exclusively care about files that were
                  # added (A) in posthog/migrations/. We also want to ignore
                  # initial migrations (0001_*) as these are guaranteed to be
                  # run on initial setup where there is no data.
                  git diff --name-status origin/master..HEAD | grep "A\sposthog/migrations/" | awk '{print $2}' | grep -v migrations/0001_ | python manage.py test_migrations_are_safe

    django:
        needs: changes
        timeout-minutes: 30

        name: Django tests – ${{ matrix.segment }} (persons-on-events ${{ matrix.person-on-events && 'on' || 'off' }}), Py ${{ matrix.python-version }}, ${{ matrix.clickhouse-server-image }} (${{matrix.group}}/${{ matrix.concurrency }})
        runs-on: ubuntu-latest

        strategy:
            fail-fast: false
            matrix:
                python-version: ['3.10.10']
                clickhouse-server-image: ['clickhouse/clickhouse-server:22.8', 'clickhouse/clickhouse-server:23.4']
                segment: ['FOSS', 'EE']
                person-on-events: [false, true]
                # :NOTE: Keep concurrency and groups in sync
                concurrency: [5]
                group: [1, 2, 3, 4, 5]

        steps:
            # The first step is the only one that should run if `needs.changes.outputs.backend == 'false'`.
            # All the other ones should rely on `needs.changes.outputs.backend` directly or indirectly, so that they're
            # effectively skipped if backend code is unchanged. See https://github.com/PostHog/posthog/pull/15174.
            - uses: actions/checkout@v3
              with:
                  fetch-depth: 1
                  repository: ${{ github.event.pull_request.head.repo.full_name }}
                  ref: ${{ github.event.pull_request.head.ref }}
                  # Use PostHog Bot token when not on forks to enable proper snapshot updating
                  token: ${{ github.event.pull_request.head.repo.full_name == github.repository && secrets.POSTHOG_BOT_GITHUB_TOKEN || github.token }}

            - uses: ./.github/actions/run-backend-tests
              if: needs.changes.outputs.backend == 'true'
              with:
                  segment: ${{ matrix.segment }}
                  person-on-events: ${{ matrix.person-on-events }}
                  python-version: ${{ matrix.python-version }}
                  clickhouse-server-image: ${{ matrix.clickhouse-server-image }}
                  concurrency: ${{ matrix.concurrency }}
                  group: ${{ matrix.group }}
                  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

            - uses: EndBug/add-and-commit@v9
              # Skip on forks
              # Also skip for persons-on-events runs, as we want to ignore snapshots diverging there
              if: ${{ github.repository == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events }}
              with:
                  add: '["ee", "posthog/clickhouse/test/__snapshots__", "posthog/api/test/__snapshots__", "posthog/test/__snapshots__", "posthog/queries/", "posthog/migrations"]'
                  message: 'Update query snapshots'
                  pull: --rebase --autostash # Make sure we're up to date with other segments' updates
                  default_author: github_actions
                  github_token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

            - name: Check if any snapshot changes were left uncomitted
              id: changed-files
              if: ${{ github.repository == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events }}
              run: |
                  if [[ -z $(git status -s | grep -v ".test_durations" | tr -d "\n") ]]
                  then
                    echo 'files_found=false' >> $GITHUB_OUTPUT
                  else
                    echo 'diff=$(git status --porcelain)' >> $GITHUB_OUTPUT
                    echo 'files_found=true' >> $GITHUB_OUTPUT
                  fi

            - name: Fail CI if some snapshots have been updated but not committed
              if: steps.changed-files.outputs.files_found == 'true' && steps.add-and-commit.outcome == 'success'
              run: |
                  echo "${{ steps.changed-files.outputs.diff }}"
                  exit 1

            - name: Archive email renders
              uses: actions/upload-artifact@v3
              if: needs.changes.outputs.backend == 'true' && matrix.segment == 'FOSS' && matrix.person-on-events == false
              with:
                  name: email_renders
                  path: posthog/tasks/test/__emails__
                  retention-days: 5

    cloud:
        needs: changes
        timeout-minutes: 30
        # This workflow may be skipped as a whole (as opposed to the other ones), as it's optional on forks anyway
        if: github.repository == 'PostHog/posthog' && needs.changes.outputs.backend == 'true'

        name: Django tests – Cloud
        runs-on: ubuntu-latest
        steps:
            - name: Fetch posthog-cloud
              run: |
                  curl -u posthog-bot:${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }} -L https://github.com/posthog/posthog-cloud/tarball/master | tar --strip-components=1 -xz --

            - name: Checkout master
              uses: actions/checkout@v3
              with:
                  ref: 'master'
                  path: 'master/'

            - name: Link posthog-cloud at master
              run: |
                  cp -r multi_tenancy master/
                  cp -r messaging master/
                  cat multi_tenancy_settings.py > master/posthog/settings/cloud.py
                  cat requirements.txt >> master/requirements.txt

            - name: Stop/Start stack with Docker Compose
              run: |
                  docker compose -f master/docker-compose.dev.yml down
                  docker compose -f master/docker-compose.dev.yml up -d

            - name: Set up Python
              uses: actions/setup-python@v4
              with:
                  python-version: 3.10.10
                  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

            - uses: syphar/restore-virtualenv@v1
              id: cache-backend-tests

            - uses: syphar/restore-pip-download-cache@v1
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'

            - name: Install SAML (python3-saml) dependencies
              run: |
                  sudo apt-get update
                  sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl

            - name: Install python dependencies
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
              run: |
                  python -m pip install -r master/requirements.txt -r master/requirements-dev.txt

            - name: Wait for Clickhouse & Kafka
              run: master/bin/check_kafka_clickhouse_up

            # The 2-step migration process (first master, then current branch) verifies that it'll always
            # be possible to migrate to the new version without problems in production
            - name: Run migration on master branch
              run: |
                  python master/manage.py migrate

            - name: Checkout current branch
              uses: actions/checkout@v3
              with:
                  path: 'current/'

            - name: Install requirements.txt dependencies with pip at current branch
              run: |
                  cd current
                  python -m pip install -r requirements.txt -r requirements-dev.txt

            - name: Link posthog-cloud at current branch
              run: |
                  cp current/ee/conftest.py multi_tenancy/conftest.py
                  cp current/ee/conftest.py messaging/conftest.py
                  cp -r multi_tenancy current/
                  cp -r messaging current/
                  cat multi_tenancy_settings.py > current/posthog/settings/cloud.py
                  cat requirements.txt >> current/requirements.txt

            - name: Check migrations
              run: |
                  cd current
                  python manage.py makemigrations --check --dry-run
                  python manage.py migrate

            - name: Set up needed files
              run: |
                  cd current
                  mkdir -p frontend/dist
                  python manage.py collectstatic --noinput
                  touch frontend/dist/index.html
                  touch frontend/dist/layout.html
                  touch frontend/dist/exporter.html

            - name: Run cloud tests (posthog-cloud)
              run: |
                  source .env.template
                  cd current
                  pytest multi_tenancy messaging -m "not skip_on_multitenancy and not async_migrations" --durations=100 --durations-min=1.0

    async-migrations:
        name: Async migrations tests
        needs: changes
        if: needs.changes.outputs.backend == 'true'
        runs-on: ubuntu-latest
        steps:
            - name: 'Checkout repo'
              uses: actions/checkout@v3
              with:
                  fetch-depth: 1

            - name: Start stack with Docker Compose
              run: |
                  export CLICKHOUSE_SERVER_IMAGE_VERSION=${{ inputs.clickhouse-server-image-version }}
                  docker compose -f docker-compose.dev.yml down
                  docker compose -f docker-compose.dev.yml up -d

            - name: Set up Python
              uses: actions/setup-python@v4
              with:
                  python-version: 3.10.10
                  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

            - name: Install SAML (python3-saml) dependencies
              run: |
                  sudo apt-get update
                  sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl

            - uses: syphar/restore-virtualenv@v1
              id: cache-async-migrations-tests
              with:
                  custom_cache_key_element: v1-${{ inputs.cache-id }}

            - uses: syphar/restore-pip-download-cache@v1
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'

            - name: Install python dependencies
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
              shell: bash
              run: |
                  python -m pip install -r requirements.txt -r requirements-dev.txt

            - name: Add kafka host to /etc/hosts for kafka connectivity
              run: sudo echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts

            - name: Set up needed files
              run: |
                  mkdir -p frontend/dist
                  touch frontend/dist/index.html
                  touch frontend/dist/layout.html
                  touch frontend/dist/exporter.html

            - name: Wait for Clickhouse & Kafka
              run: bin/check_kafka_clickhouse_up

            - name: Run async migrations tests
              run: |
                  pytest -m "async_migrations"