From 50edb5b7fc689ed8a976ed8efb70925a29e8cc59 Mon Sep 17 00:00:00 2001 From: Kyle Galbraith Date: Tue, 19 Mar 2024 10:37:35 +0100 Subject: [PATCH] ci: Run backend and e2e cypress flow on depot runners (#20932) * run backend and e2e cypress flow on depot runners * change workflow name --- .github/workflows/ci-backend-depot.yml | 373 +++++++++++++++++++++++++ .github/workflows/ci-e2e-depot.yml | 278 ++++++++++++++++++ 2 files changed, 651 insertions(+) create mode 100644 .github/workflows/ci-backend-depot.yml create mode 100644 .github/workflows/ci-e2e-depot.yml diff --git a/.github/workflows/ci-backend-depot.yml b/.github/workflows/ci-backend-depot.yml new file mode 100644 index 0000000000000..743fef1edaed1 --- /dev/null +++ b/.github/workflows/ci-backend-depot.yml @@ -0,0 +1,373 @@ +# This workflow runs all of our backend django tests. +# +# If these tests get too slow, look at increasing concurrency and re-timing the tests by manually dispatching +# .github/workflows/ci-backend-update-test-timing.yml action +name: Backend CI (depot) + +on: + push: + branches: + - master + pull_request: + workflow_dispatch: + inputs: + clickhouseServerVersion: + description: ClickHouse server version. Leave blank for default + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + # This is so that the workflow run isn't canceled when a snapshot update is pushed within it by posthog-bot + # We do however cancel from container-images-ci.yml if a commit is pushed by someone OTHER than posthog-bot + cancel-in-progress: false + +env: + SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only + DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog' + REDIS_URL: 'redis://localhost' + CLICKHOUSE_HOST: 'localhost' + CLICKHOUSE_SECURE: 'False' + CLICKHOUSE_VERIFY: 'False' + TEST: 1 + CLICKHOUSE_SERVER_IMAGE_VERSION: ${{ github.event.inputs.clickhouseServerVersion || '' }} + OBJECT_STORAGE_ENABLED: 'True' + OBJECT_STORAGE_ENDPOINT: 'http://localhost:19000' + OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user' + OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password' + +jobs: + # Job to decide if we should run backend ci + # See https://github.com/dorny/paths-filter#conditional-execution for more details + changes: + runs-on: depot-ubuntu-latest + timeout-minutes: 5 + if: github.repository == 'PostHog/posthog' + name: Determine need to run backend checks + # Set job outputs to values from filter step + outputs: + backend: ${{ steps.filter.outputs.backend }} + steps: + # For pull requests it's not necessary to checkout the code, but we + # also want this to run on master so we need to checkout + - uses: actions/checkout@v3 + + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + backend: + # Avoid running backend tests for irrelevant changes + # NOTE: we are at risk of missing a dependency here. We could make + # the dependencies more clear if we separated the backend/frontend + # code completely + # really we should ignore ee/frontend/** but dorny doesn't support that + # - '!ee/frontend/**' + # including the negated rule appears to work + # but makes it always match because the checked file always isn't `ee/frontend/**` 🙈 + - 'ee/**/*' + - 'posthog/**/*' + - 'bin/*.py' + - requirements.txt + - requirements-dev.txt + - mypy.ini + - pytest.ini + - frontend/src/queries/schema.json # Used for generating schema.py + - plugin-transpiler/src # Used for transpiling plugins + # Make sure we run if someone is explicitly change the workflow + - .github/workflows/ci-backend.yml + - .github/actions/run-backend-tests/action.yml + # We use docker compose for tests, make sure we rerun on + # changes to docker-compose.dev.yml e.g. dependency + # version changes + - docker-compose.dev.yml + - frontend/public/email/* + # These scripts are used in the CI + - bin/check_temporal_up + - bin/check_kafka_clickhouse_up + + backend-code-quality: + needs: changes + timeout-minutes: 30 + + name: Python code quality checks + runs-on: depot-ubuntu-latest + + steps: + # If this run wasn't initiated by the bot (meaning: snapshot update) and we've determined + # there are backend changes, cancel previous runs + - uses: n1hility/cancel-previous-runs@v3 + if: github.actor != 'posthog-bot' && needs.changes.outputs.backend == 'true' + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.10.10 + cache: 'pip' + cache-dependency-path: '**/requirements*.txt' + token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }} + + - uses: syphar/restore-virtualenv@v1 + id: cache-backend-tests + with: + custom_cache_key_element: v2- + + - name: Install SAML (python3-saml) dependencies + run: | + sudo apt-get update + sudo apt-get install libxml2-dev libxmlsec1 libxmlsec1-dev libxmlsec1-openssl + + - name: Install Python dependencies + if: steps.cache-backend-tests.outputs.cache-hit != 'true' + run: | + python -m pip install -r requirements.txt -r requirements-dev.txt + + - name: Check for syntax errors, import sort, and code style violations + run: | + ruff check . + + - name: Check formatting + run: | + ruff format --exclude posthog/hogql/grammar --check --diff . + + - name: Add Problem Matcher + run: echo "::add-matcher::.github/mypy-problem-matcher.json" + + - name: Check static typing + run: | + mypy -p posthog | mypy-baseline filter + + - name: Check if "schema.py" is up to date + run: | + npm run schema:build:python && git diff --exit-code + + - name: Check if ANTLR definitions are up to date + run: | + cd .. + sudo apt-get install default-jre + mkdir antlr + cd antlr + curl -o antlr.jar https://www.antlr.org/download/antlr-$ANTLR_VERSION-complete.jar + export PWD=`pwd` + echo '#!/bin/bash' > antlr + echo "java -jar $PWD/antlr.jar \$*" >> antlr + chmod +x antlr + export CLASSPATH=".:$PWD/antlr.jar:$CLASSPATH" + export PATH="$PWD:$PATH" + + cd ../posthog + antlr | grep "Version" + npm run grammar:build && git diff --exit-code + env: + # Installing a version of ANTLR compatible with what's in Homebrew as of October 2023 (version 4.13), + # as apt-get is quite out of date. The same version must be set in hogql_parser/pyproject.toml + ANTLR_VERSION: '4.13.1' + + check-migrations: + needs: changes + if: needs.changes.outputs.backend == 'true' + timeout-minutes: 10 + + name: Validate Django migrations + runs-on: depot-ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Stop/Start stack with Docker Compose + run: | + docker compose -f docker-compose.dev.yml down + docker compose -f docker-compose.dev.yml up -d + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.10.10 + cache: 'pip' + cache-dependency-path: '**/requirements*.txt' + token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }} + + - uses: syphar/restore-virtualenv@v1 + id: cache-backend-tests + with: + custom_cache_key_element: v1- + + - name: Install SAML (python3-saml) dependencies + run: | + sudo apt-get update + sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl + + - name: Install python dependencies + if: steps.cache-backend-tests.outputs.cache-hit != 'true' + run: | + python -m pip install -r requirements.txt -r requirements-dev.txt + + - uses: actions/checkout@v3 + with: + ref: master + + - name: Run migrations up to master + run: | + # We need to ensure we have requirements for the master branch + # now also, so we can run migrations up to master. + python -m pip install -r requirements.txt -r requirements-dev.txt + python manage.py migrate + + - uses: actions/checkout@v3 + + - name: Check migrations + run: | + python manage.py makemigrations --check --dry-run + git fetch origin master + # `git diff --name-only` returns a list of files that were changed - added OR deleted OR modified + # With `--name-status` we get the same, but including a column for status, respectively: A, D, M + # In this check we exclusively care about files that were + # added (A) in posthog/migrations/. We also want to ignore + # initial migrations (0001_*) as these are guaranteed to be + # run on initial setup where there is no data. + git diff --name-status origin/master..HEAD | grep "A\sposthog/migrations/" | awk '{print $2}' | grep -v migrations/0001_ | python manage.py test_migrations_are_safe + + django: + needs: changes + timeout-minutes: 30 + + name: Django tests – ${{ matrix.segment }} (persons-on-events ${{ matrix.person-on-events && 'on' || 'off' }}), Py ${{ matrix.python-version }}, ${{ matrix.clickhouse-server-image }} (${{matrix.group}}/${{ matrix.concurrency }}) + runs-on: depot-ubuntu-latest + + strategy: + fail-fast: false + matrix: + python-version: ['3.10.10'] + clickhouse-server-image: ['clickhouse/clickhouse-server:23.11.2.11-alpine'] + segment: ['FOSS', 'EE'] + person-on-events: [false, true] + # :NOTE: Keep concurrency and groups in sync + concurrency: [5] + group: [1, 2, 3, 4, 5] + + steps: + # The first step is the only one that should run if `needs.changes.outputs.backend == 'false'`. + # All the other ones should rely on `needs.changes.outputs.backend` directly or indirectly, so that they're + # effectively skipped if backend code is unchanged. See https://github.com/PostHog/posthog/pull/15174. + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + # Use PostHog Bot token when not on forks to enable proper snapshot updating + token: ${{ github.event.pull_request.head.repo.full_name == github.repository && secrets.POSTHOG_BOT_GITHUB_TOKEN || github.token }} + + - uses: ./.github/actions/run-backend-tests + if: needs.changes.outputs.backend == 'true' + with: + segment: ${{ matrix.segment }} + person-on-events: ${{ matrix.person-on-events }} + python-version: ${{ matrix.python-version }} + clickhouse-server-image: ${{ matrix.clickhouse-server-image }} + concurrency: ${{ matrix.concurrency }} + group: ${{ matrix.group }} + token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }} + + - uses: EndBug/add-and-commit@v9 + # Skip on forks + # Also skip for persons-on-events runs, as we want to ignore snapshots diverging there + if: ${{ github.repository == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events }} + with: + add: '["ee", "./**/*.ambr", "posthog/queries/", "posthog/migrations", "posthog/tasks", "posthog/hogql/"]' + message: 'Update query snapshots' + pull: --rebase --autostash # Make sure we're up-to-date with other segments' updates + default_author: github_actions + github_token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }} + + - name: Check if any snapshot changes were left uncomitted + id: changed-files + if: ${{ github.repository == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events }} + run: | + if [[ -z $(git status -s | grep -v ".test_durations" | tr -d "\n") ]] + then + echo 'files_found=false' >> $GITHUB_OUTPUT + else + echo 'diff=$(git status --porcelain)' >> $GITHUB_OUTPUT + echo 'files_found=true' >> $GITHUB_OUTPUT + fi + + - name: Fail CI if some snapshots have been updated but not committed + if: steps.changed-files.outputs.files_found == 'true' && steps.add-and-commit.outcome == 'success' + run: | + echo "${{ steps.changed-files.outputs.diff }}" + exit 1 + + - name: Archive email renders + uses: actions/upload-artifact@v3 + if: needs.changes.outputs.backend == 'true' && matrix.segment == 'FOSS' && matrix.person-on-events == false + with: + name: email_renders + path: posthog/tasks/test/__emails__ + retention-days: 5 + + async-migrations: + name: Async migrations tests - ${{ matrix.clickhouse-server-image }} + needs: changes + strategy: + fail-fast: false + matrix: + clickhouse-server-image: ['clickhouse/clickhouse-server:23.11.2.11-alpine'] + if: needs.changes.outputs.backend == 'true' + runs-on: depot-ubuntu-latest + steps: + - name: 'Checkout repo' + uses: actions/checkout@v3 + with: + fetch-depth: 1 + + - name: Start stack with Docker Compose + run: | + export CLICKHOUSE_SERVER_IMAGE_VERSION=${{ matrix.clickhouse-server-image }} + docker compose -f docker-compose.dev.yml down + docker compose -f docker-compose.dev.yml up -d + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.10.10 + cache: 'pip' + cache-dependency-path: '**/requirements*.txt' + token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }} + + - uses: syphar/restore-virtualenv@v1 + id: cache-backend-tests + with: + custom_cache_key_element: v2- + + - name: Install SAML (python3-saml) dependencies + run: | + sudo apt-get update + sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl + + - name: Install python dependencies + if: steps.cache-backend-tests.outputs.cache-hit != 'true' + shell: bash + run: | + python -m pip install -r requirements.txt -r requirements-dev.txt + + - name: Add kafka host to /etc/hosts for kafka connectivity + run: sudo echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts + + - name: Set up needed files + run: | + mkdir -p frontend/dist + touch frontend/dist/index.html + touch frontend/dist/layout.html + touch frontend/dist/exporter.html + + - name: Wait for Clickhouse & Kafka + run: bin/check_kafka_clickhouse_up + + - name: Run async migrations tests + run: | + pytest -m "async_migrations" diff --git a/.github/workflows/ci-e2e-depot.yml b/.github/workflows/ci-e2e-depot.yml new file mode 100644 index 0000000000000..615238bf7d43b --- /dev/null +++ b/.github/workflows/ci-e2e-depot.yml @@ -0,0 +1,278 @@ +# +# This workflow runs CI E2E tests with Cypress. +# +# It relies on the container image built by 'container-images-ci.yml'. +# +name: E2E CI (depot) + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + changes: + runs-on: depot-ubuntu-latest + timeout-minutes: 5 + if: github.repository == 'PostHog/posthog' + name: Determine need to run E2E checks + # Set job outputs to values from filter step + outputs: + shouldTriggerCypress: ${{ steps.changes.outputs.shouldTriggerCypress }} + steps: + # For pull requests it's not necessary to check out the code + - uses: dorny/paths-filter@v2 + id: changes + with: + filters: | + shouldTriggerCypress: + # Avoid running E2E tests for irrelevant changes + # NOTE: we are at risk of missing a dependency here. We could make + # the dependencies more clear if we separated the backend/frontend + # code completely + - 'ee/**' + - 'posthog/**' + - 'hogvm/**' + - 'bin/*' + - frontend/**/* + - requirements.txt + - requirements-dev.txt + - package.json + - pnpm-lock.yaml + # Make sure we run if someone is explicitly change the workflow + - .github/workflows/ci-e2e.yml + - .github/actions/build-n-cache-image/action.yml + # We use docker compose for tests, make sure we rerun on + # changes to docker-compose.dev.yml e.g. dependency + # version changes + - docker-compose.dev.yml + - Dockerfile + - cypress/** + + # Job that lists and chunks spec file names and caches node modules + chunks: + needs: changes + name: Cypress preparation + runs-on: depot-ubuntu-latest + timeout-minutes: 5 + outputs: + chunks: ${{ steps.chunk.outputs.chunks }} + + steps: + - name: Check out + uses: actions/checkout@v3 + + - name: Group spec files into chunks of three + id: chunk + run: echo "chunks=$(ls cypress/e2e/* | jq --slurp --raw-input -c 'split("\n")[:-1] | _nwise(2) | join("\n")' | jq --slurp -c .)" >> $GITHUB_OUTPUT + + container: + name: Build and cache container image + runs-on: depot-ubuntu-latest + timeout-minutes: 60 + needs: [changes] + permissions: + contents: read + id-token: write # allow issuing OIDC tokens for this workflow run + outputs: + tag: ${{ steps.build.outputs.tag }} + build-id: ${{ steps.build.outputs.build-id }} + steps: + - name: Checkout + if: needs.changes.outputs.shouldTriggerCypress == 'true' + uses: actions/checkout@v3 + - name: Build the Docker image with Depot + if: needs.changes.outputs.shouldTriggerCypress == 'true' + # Build the container image in preparation for the E2E tests + uses: ./.github/actions/build-n-cache-image + id: build + with: + save: true + actions-id-token-request-url: ${{ env.ACTIONS_ID_TOKEN_REQUEST_URL }} + + cypress: + name: Cypress E2E tests (${{ strategy.job-index }}) + runs-on: depot-ubuntu-latest + timeout-minutes: 60 + needs: [chunks, changes, container] + permissions: + id-token: write # allow issuing OIDC tokens for this workflow run + + strategy: + # when one test fails, DO NOT cancel the other + # containers, as there may be other spec failures + # we want to know about. + fail-fast: false + matrix: + chunk: ${{ fromJson(needs.chunks.outputs.chunks) }} + + steps: + - name: Checkout + if: needs.changes.outputs.shouldTriggerCypress == 'true' + uses: actions/checkout@v3 + + - name: Install pnpm + if: needs.changes.outputs.shouldTriggerCypress == 'true' + uses: pnpm/action-setup@v2 + with: + version: 8.x.x + + - name: Set up Node.js + if: needs.changes.outputs.shouldTriggerCypress == 'true' + uses: actions/setup-node@v4 + with: + node-version: 18.12.1 + + - name: Get pnpm cache directory path + if: needs.changes.outputs.shouldTriggerCypress == 'true' + id: pnpm-cache-dir + run: echo "PNPM_STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT + + - name: Get cypress cache directory path + if: needs.changes.outputs.shouldTriggerCypress == 'true' + id: cypress-cache-dir + run: echo "CYPRESS_BIN_PATH=$(npx cypress cache path)" >> $GITHUB_OUTPUT + + - uses: actions/cache@v4 + if: needs.changes.outputs.shouldTriggerCypress == 'true' + id: pnpm-cache + with: + path: | + ${{ steps.pnpm-cache-dir.outputs.PNPM_STORE_PATH }} + ${{ steps.cypress-cache-dir.outputs.CYPRESS_BIN_PATH }} + key: ${{ runner.os }}-pnpm-cypress-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-cypress- + + - name: Install package.json dependencies with pnpm + if: needs.changes.outputs.shouldTriggerCypress == 'true' + run: pnpm install --frozen-lockfile + + - name: Stop/Start stack with Docker Compose + # these are required checks so, we can't skip entire sections + if: needs.changes.outputs.shouldTriggerCypress == 'true' + run: | + docker compose -f docker-compose.dev.yml down + docker compose -f docker-compose.dev.yml up -d + + - name: Wait for ClickHouse + # these are required checks so, we can't skip entire sections + if: needs.changes.outputs.shouldTriggerCypress == 'true' + run: ./bin/check_kafka_clickhouse_up + + - name: Install Depot CLI + if: needs.changes.outputs.shouldTriggerCypress == 'true' + uses: depot/setup-action@v1 + + - name: Get Docker image cached in Depot + if: needs.changes.outputs.shouldTriggerCypress == 'true' + uses: depot/pull-action@v1 + with: + build-id: ${{ needs.container.outputs.build-id }} + tags: ${{ needs.container.outputs.tag }} + + - name: Write .env # This step intentionally has no if, so that GH always considers the action as having run + run: | + cat <> .env + SECRET_KEY=6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da + REDIS_URL=redis://localhost + DATABASE_URL=postgres://posthog:posthog@localhost:5432/posthog + KAFKA_HOSTS=kafka:9092 + DISABLE_SECURE_SSL_REDIRECT=1 + SECURE_COOKIES=0 + OPT_OUT_CAPTURE=0 + E2E_TESTING=1 + SKIP_SERVICE_VERSION_REQUIREMENTS=1 + EMAIL_HOST=email.test.posthog.net + SITE_URL=http://localhost:8000 + NO_RESTART_LOOP=1 + CLICKHOUSE_SECURE=0 + OBJECT_STORAGE_ENABLED=1 + OBJECT_STORAGE_ENDPOINT=http://localhost:19000 + OBJECT_STORAGE_ACCESS_KEY_ID=object_storage_root_user + OBJECT_STORAGE_SECRET_ACCESS_KEY=object_storage_root_password + GITHUB_ACTION_RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + CELERY_METRICS_PORT=8999 + EOT + + - name: Start PostHog + # these are required checks so, we can't skip entire sections + if: needs.changes.outputs.shouldTriggerCypress == 'true' + run: | + mkdir -p /tmp/logs + + echo "Starting PostHog using the container image ${{ needs.container.outputs.tag }}" + DOCKER_RUN="docker run --rm --network host --add-host kafka:127.0.0.1 --env-file .env ${{ needs.container.outputs.tag }}" + + $DOCKER_RUN ./bin/migrate + $DOCKER_RUN python manage.py setup_dev + + # only starts the plugin server so that the "wait for PostHog" step passes + $DOCKER_RUN ./bin/docker-worker &> /tmp/logs/worker.txt & + $DOCKER_RUN ./bin/docker-server &> /tmp/logs/server.txt & + + - name: Wait for PostHog + # these are required checks so, we can't skip entire sections + if: needs.changes.outputs.shouldTriggerCypress == 'true' + # this action might be abandoned - but v1 doesn't point to latest of v1 (which it should) + # so pointing to v1.1.0 to remove warnings about node version with v1 + # todo check https://github.com/iFaxity/wait-on-action/releases for new releases + uses: iFaxity/wait-on-action@v1.1.0 + timeout-minutes: 3 + with: + verbose: true + log: true + resource: http://localhost:8000 + + - name: Cypress run + # these are required checks so, we can't skip entire sections + if: needs.changes.outputs.shouldTriggerCypress == 'true' + uses: cypress-io/github-action@v6 + with: + config-file: cypress.e2e.config.ts + config: retries=2 + spec: ${{ matrix.chunk }} + install: false + env: + E2E_TESTING: 1 + OPT_OUT_CAPTURE: 0 + GITHUB_ACTION_RUN_URL: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}' + + - name: Archive test screenshots + uses: actions/upload-artifact@v3 + with: + name: screenshots + path: cypress/screenshots + if: ${{ failure() }} + + - name: Archive test downloads + uses: actions/upload-artifact@v3 + with: + name: downloads + path: cypress/downloads + if: ${{ failure() }} + + - name: Archive test videos + uses: actions/upload-artifact@v3 + with: + name: videos + path: cypress/videos + if: ${{ failure() }} + + - name: Archive accessibility violations + if: needs.changes.outputs.shouldTriggerCypress == 'true' + uses: actions/upload-artifact@v3 + with: + name: accessibility-violations + path: '**/a11y/' + if-no-files-found: 'ignore' + + - name: Show logs on failure + # use artefact here, as I think the output will be too large for display in an action + uses: actions/upload-artifact@v3 + with: + name: logs-${{ strategy.job-index }} + path: /tmp/logs + if: ${{ failure() }}