wip: Kafka image upgrade #66729
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Plugin Server CI | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - master | |
| env: | |
| OBJECT_STORAGE_ENABLED: true | |
| OBJECT_STORAGE_ENDPOINT: 'http://localhost:19000' | |
| OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user' | |
| OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password' | |
| OBJECT_STORAGE_SESSION_RECORDING_FOLDER: 'session_recordings' | |
| OBJECT_STORAGE_BUCKET: 'posthog' | |
| # set the max buffer size small enough that the functional tests behave the same in CI as when running locally | |
| SESSION_RECORDING_MAX_BUFFER_SIZE_KB: 1024 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| # Job to decide if we should run plugin server ci | |
| # See https://github.com/dorny/paths-filter#conditional-execution for more details | |
| changes: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 5 | |
| name: Determine need to run plugin server checks | |
| outputs: | |
| plugin-server: ${{ steps.filter.outputs.plugin-server }} | |
| steps: | |
| # For pull requests it's not necessary to checkout the code, but we | |
| # also want this to run on master so we need to checkout | |
| - uses: actions/checkout@v3 | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| plugin-server: | |
| - .github/workflows/ci-plugin-server.yml | |
| - 'plugin-server/**' | |
| - 'posthog/clickhouse/migrations/**' | |
| - 'ee/migrations/**' | |
| - 'posthog/management/commands/setup_test_environment.py' | |
| - 'posthog/migrations/**' | |
| - 'posthog/plugins/**' | |
| - 'docker*.yml' | |
| - '*Dockerfile' | |
| code-quality: | |
| name: Code quality | |
| needs: changes | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| runs-on: ubuntu-24.04 | |
| defaults: | |
| run: | |
| working-directory: 'plugin-server' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@v4 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18.12.1 | |
| cache: pnpm | |
| - name: Install package.json dependencies with pnpm | |
| run: pnpm install --frozen-lockfile | |
| - name: Check formatting with prettier | |
| run: pnpm prettier:check | |
| - name: Lint with ESLint | |
| run: pnpm lint | |
| tests: | |
| name: Plugin Server Tests (${{matrix.shard}}) | |
| needs: changes | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: [1/3, 2/3, 3/3] | |
| env: | |
| REDIS_URL: 'redis://localhost' | |
| CLICKHOUSE_HOST: 'localhost' | |
| CLICKHOUSE_DATABASE: 'posthog_test' | |
| KAFKA_HOSTS: 'kafka:9092' | |
| steps: | |
| - name: Code check out | |
| # NOTE: We need this check on every step so that it still runs if skipped as we need it to suceed for the CI | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| uses: actions/checkout@v3 | |
| - name: Stop/Start stack with Docker Compose | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| run: | | |
| docker compose -f docker-compose.dev.yml down | |
| docker compose -f docker-compose.dev.yml up -d | |
| - name: Add Kafka to /etc/hosts | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| run: echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts | |
| - name: Set up Python | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11.9 | |
| cache: 'pip' | |
| cache-dependency-path: '**/requirements*.txt' | |
| token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }} | |
| # uv is a fast pip alternative: https://github.com/astral-sh/uv/ | |
| - run: pip install uv | |
| - name: Install rust | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| uses: dtolnay/[email protected] | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| rust/target | |
| key: ${{ runner.os }}-cargo-release-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Install sqlx-cli | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| working-directory: rust | |
| run: cargo install [email protected] --locked --no-default-features --features native-tls,postgres | |
| - name: Install SAML (python3-saml) dependencies | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl | |
| - name: Install python dependencies | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| run: | | |
| uv pip install --system -r requirements-dev.txt | |
| uv pip install --system -r requirements.txt | |
| - name: Install pnpm | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| uses: pnpm/action-setup@v4 | |
| - name: Set up Node.js | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18.12.1 | |
| cache: pnpm | |
| cache-dependency-path: plugin-server/pnpm-lock.yaml | |
| - name: Install package.json dependencies with pnpm | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| run: cd plugin-server && pnpm i | |
| - name: Wait for Clickhouse, Redis & Kafka | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| run: | | |
| docker compose -f docker-compose.dev.yml up kafka redis clickhouse -d --wait | |
| bin/check_kafka_clickhouse_up | |
| - name: Set up databases | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| env: | |
| TEST: 'true' | |
| SECRET_KEY: 'abcdef' # unsafe - for testing only | |
| DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog' | |
| run: cd plugin-server && pnpm setup:test | |
| - name: Test with Jest | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| env: | |
| # Below DB name has `test_` prepended, as that's how Django (ran above) creates the test DB | |
| DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/test_posthog' | |
| REDIS_URL: 'redis://localhost' | |
| NODE_OPTIONS: '--max_old_space_size=4096' | |
| run: cd plugin-server && pnpm test -- --runInBand --forceExit tests/ --shard=${{matrix.shard}} | |
| functional-tests: | |
| name: Functional tests | |
| needs: changes | |
| runs-on: ubuntu-24.04 | |
| env: | |
| REDIS_URL: 'redis://localhost' | |
| CLICKHOUSE_HOST: 'localhost' | |
| CLICKHOUSE_DATABASE: 'posthog_test' | |
| KAFKA_HOSTS: 'kafka:9092' | |
| DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog' | |
| RELOAD_PLUGIN_JITTER_MAX_MS: 0 | |
| ENCRYPTION_SALT_KEYS: '00beef0000beef0000beef0000beef00' | |
| steps: | |
| - name: Code check out | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| uses: actions/checkout@v3 | |
| - name: Stop/Start stack with Docker Compose | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| run: | | |
| docker compose -f docker-compose.dev.yml down | |
| docker compose -f docker-compose.dev.yml up -d | |
| - name: Add Kafka to /etc/hosts | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| run: echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts | |
| - name: Set up Python | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11.9 | |
| cache: 'pip' | |
| cache-dependency-path: '**/requirements*.txt' | |
| token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }} | |
| # uv is a fast pip alternative: https://github.com/astral-sh/uv/ | |
| - run: pip install uv | |
| - name: Install SAML (python3-saml) dependencies | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl | |
| - name: Install python dependencies | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| run: | | |
| uv pip install --system -r requirements-dev.txt | |
| uv pip install --system -r requirements.txt | |
| - name: Install pnpm | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| uses: pnpm/action-setup@v4 | |
| - name: Set up Node.js | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18.12.1 | |
| cache: pnpm | |
| cache-dependency-path: plugin-server/pnpm-lock.yaml | |
| - name: Install package.json dependencies with pnpm | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| run: | | |
| cd plugin-server | |
| pnpm install --frozen-lockfile | |
| pnpm build | |
| - name: Wait for Clickhouse, Redis & Kafka | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| run: | | |
| docker compose -f docker-compose.dev.yml up kafka redis clickhouse -d --wait | |
| bin/check_kafka_clickhouse_up | |
| - name: Set up databases | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| env: | |
| DEBUG: 'true' | |
| SECRET_KEY: 'abcdef' # unsafe - for testing only | |
| run: | | |
| ./manage.py migrate | |
| ./manage.py migrate_clickhouse | |
| - name: Run functional tests | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| run: | | |
| cd plugin-server | |
| ./bin/ci_functional_tests.sh | |
| - name: Upload coverage report | |
| uses: actions/upload-artifact@v3 | |
| if: needs.changes.outputs.plugin-server == 'true' | |
| with: | |
| name: functional-coverage | |
| if-no-files-found: warn | |
| retention-days: 1 | |
| path: 'plugin-server/coverage' |