merge master

.eslintrc.js

@@ -113,11 +113,7 @@ module.exports = {
                     {
                         name: 'dayjs',
                         message: 'Do not directly import dayjs. Only import the dayjs exported from lib/dayjs.',
-                    },
-                    {
-                        name: '@ant-design/icons',
-                        message: 'Please use icons from the @posthog/icons package instead',
-                    },
+                    }
                 ],
             },
         ],

.github/actions/run-backend-tests/action.yml

@@ -167,6 +167,12 @@ runs:
                   --durations=100 --durations-min=1.0 --store-durations \
                   $PYTEST_ARGS
 
+        # Uncomment this code to create an ssh-able console so you can debug issues with github actions
+        # (Consider changing the timeout in ci-backend.yml to have more time)
+        # - name: Setup tmate session
+        #   if: failure()
+        #   uses: mxschmitt/action-tmate@v3
+
         - name: Run /decide read replica tests
           id: run-decide-read-replica-tests
           if: ${{ inputs.segment == 'Core' && inputs.group == 1 && inputs.person-on-events != 'true' }}

.github/workflows/ci-backend.yml

@@ -224,6 +224,7 @@ jobs:
 
     django:
         needs: changes
+        # increase for tmate testing
         timeout-minutes: 30
 
         name: Django tests – ${{ matrix.segment }} (persons-on-events ${{ matrix.person-on-events && 'on' || 'off' }}), Py ${{ matrix.python-version }}, ${{ matrix.clickhouse-server-image }} (${{matrix.group}}/${{ matrix.concurrency }})

.github/workflows/ci-plugin-server.yml

@@ -57,7 +57,6 @@ jobs:
         defaults:
             run:
                 working-directory: 'plugin-server'
-
         steps:
             - uses: actions/checkout@v3
 
@@ -82,6 +81,7 @@ jobs:
     tests:
         name: Plugin Server Tests (${{matrix.shard}})
         needs: changes
+        if: needs.changes.outputs.plugin-server == 'true'
         runs-on: ubuntu-latest
 
         strategy:
@@ -97,21 +97,17 @@ jobs:
 
         steps:
             - name: Code check out
-              if: needs.changes.outputs.plugin-server == 'true'
               uses: actions/checkout@v3
 
             - name: Stop/Start stack with Docker Compose
-              if: needs.changes.outputs.plugin-server == 'true'
               run: |
                   docker compose -f docker-compose.dev.yml down
                   docker compose -f docker-compose.dev.yml up -d
 
             - name: Add Kafka to /etc/hosts
-              if: needs.changes.outputs.plugin-server == 'true'
               run: echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts
 
             - name: Set up Python
-              if: needs.changes.outputs.plugin-server == 'true'
               uses: actions/setup-python@v5
               with:
                   python-version: 3.11.9
@@ -122,50 +118,57 @@ jobs:
             # uv is a fast pip alternative: https://github.com/astral-sh/uv/
             - run: pip install uv
 
+            - name: Install rust
+              uses: dtolnay/[email protected]
+
+            - uses: actions/cache@v4
+              with:
+                  path: |
+                      ~/.cargo/registry
+                      ~/.cargo/git
+                      rust/target
+                  key: ${{ runner.os }}-cargo-release-${{ hashFiles('**/Cargo.lock') }}
+
+            - name: Install sqlx-cli
+              working-directory: rust
+              run: cargo install [email protected] --no-default-features --features native-tls,postgres
+
             - name: Install SAML (python3-saml) dependencies
-              if: needs.changes.outputs.plugin-server == 'true'
               run: |
                   sudo apt-get update
                   sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
 
             - name: Install python dependencies
-              if: needs.changes.outputs.plugin-server == 'true'
               run: |
                   uv pip install --system -r requirements-dev.txt
                   uv pip install --system -r requirements.txt
 
             - name: Install pnpm
-              if: needs.changes.outputs.plugin-server == 'true'
               uses: pnpm/action-setup@v4
 
             - name: Set up Node.js
-              if: needs.changes.outputs.plugin-server == 'true'
               uses: actions/setup-node@v4
               with:
                   node-version: 18.12.1
                   cache: pnpm
                   cache-dependency-path: plugin-server/pnpm-lock.yaml
 
             - name: Install package.json dependencies with pnpm
-              if: needs.changes.outputs.plugin-server == 'true'
               run: cd plugin-server && pnpm i
 
             - name: Wait for Clickhouse, Redis & Kafka
-              if: needs.changes.outputs.plugin-server == 'true'
               run: |
                   docker compose -f docker-compose.dev.yml up kafka redis clickhouse -d --wait
                   bin/check_kafka_clickhouse_up
 
             - name: Set up databases
-              if: needs.changes.outputs.plugin-server == 'true'
               env:
                   TEST: 'true'
                   SECRET_KEY: 'abcdef' # unsafe - for testing only
                   DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
               run: cd plugin-server && pnpm setup:test
 
             - name: Test with Jest
-              if: needs.changes.outputs.plugin-server == 'true'
               env:
                   # Below DB name has `test_` prepended, as that's how Django (ran above) creates the test DB
                   DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/test_posthog'

.github/workflows/container-images-cd.yml

@@ -132,7 +132,7 @@ jobs:
             - name: Check for changes that affect batch exports temporal worker
               id: check_changes_batch_exports_temporal_worker
               run: |
-                  echo "changed=$((git diff --name-only HEAD^ HEAD | grep -qE '^posthog/temporal/common|^posthog/temporal/batch_exports|^posthog/batch_exports/|^posthog/management/commands/start_temporal_worker.py$' && echo true) || echo false)" >> $GITHUB_OUTPUT
+                  echo "changed=$((git diff --name-only HEAD^ HEAD | grep -qE '^posthog/temporal/common|^posthog/temporal/batch_exports|^posthog/batch_exports/|^posthog/management/commands/start_temporal_worker.py$|^requirements.txt$' && echo true) || echo false)" >> $GITHUB_OUTPUT
 
             - name: Trigger Batch Exports Temporal Worker Cloud deployment
               if: steps.check_changes_batch_exports_temporal_worker.outputs.changed == 'true'
@@ -158,7 +158,7 @@ jobs:
             - name: Check for changes that affect general purpose temporal worker
               id: check_changes_general_purpose_temporal_worker
               run: |
-                  echo "changed=$((git diff --name-only HEAD^ HEAD | grep -qE '^posthog/temporal/common|^posthog/temporal/proxy_service|^posthog/management/commands/start_temporal_worker.py$' && echo true) || echo false)" >> $GITHUB_OUTPUT
+                  echo "changed=$((git diff --name-only HEAD^ HEAD | grep -qE '^posthog/temporal/common|^posthog/temporal/proxy_service|^posthog/management/commands/start_temporal_worker.py$|^requirements.txt$' && echo true) || echo false)" >> $GITHUB_OUTPUT
 
             - name: Trigger General Purpose Temporal Worker Cloud deployment
               if: steps.check_changes_general_purpose_temporal_worker.outputs.changed == 'true'
@@ -184,7 +184,7 @@ jobs:
             - name: Check for changes that affect data warehouse temporal worker
               id: check_changes_data_warehouse_temporal_worker
               run: |
-                  echo "changed=$((git diff --name-only HEAD^ HEAD | grep -qE '^posthog/temporal/common|^posthog/temporal/data_imports|^posthog/warehouse/|^posthog/management/commands/start_temporal_worker.py$' && echo true) || echo false)" >> $GITHUB_OUTPUT
+                  echo "changed=$((git diff --name-only HEAD^ HEAD | grep -qE '^posthog/temporal/common|^posthog/temporal/data_imports|^posthog/warehouse/|^posthog/management/commands/start_temporal_worker.py$|^requirements.txt$' && echo true) || echo false)" >> $GITHUB_OUTPUT
 
             - name: Trigger Data Warehouse Temporal Worker Cloud deployment
               if: steps.check_changes_data_warehouse_temporal_worker.outputs.changed == 'true'

.github/workflows/rust.yml

@@ -1,4 +1,4 @@
-name: Rust test CI
+name: Rust CI
 
 on:
     workflow_dispatch:
@@ -16,7 +16,7 @@ jobs:
         runs-on: ubuntu-latest
         timeout-minutes: 5
         if: github.repository == 'PostHog/posthog'
-        name: Determine need to run rust checks
+        name: Determine need to run Rust checks
         # Set job outputs to values from filter step
         outputs:
             rust: ${{ steps.filter.outputs.rust }}
@@ -40,7 +40,7 @@ jobs:
                         - 'ee/migrations/**'
 
     build:
-        name: Build rust services
+        name: Build Rust services
         needs: changes
         runs-on: depot-ubuntu-22.04-4
 
@@ -76,7 +76,7 @@ jobs:
               run: cargo build --all --locked --release && find target/release/ -maxdepth 1 -executable -type f | xargs strip
 
     test:
-        name: Test rust services
+        name: Test Rust services
         strategy:
             matrix:
                 package:
@@ -179,7 +179,7 @@ jobs:
                   echo "Cargo test completed"
 
     linting:
-        name: Lint rust services
+        name: Lint Rust services
         needs: changes
         runs-on: depot-ubuntu-22.04-4
 
@@ -225,7 +225,7 @@ jobs:
               run: cargo check --all-features
 
     shear:
-        name: Shear rust services
+        name: Shear Rust services
         needs: changes
         runs-on: depot-ubuntu-22.04-4
 

.storybook/main.ts

@@ -13,7 +13,7 @@ const config: StorybookConfig = {
         'storybook-addon-pseudo-states',
     ],
 
-    staticDirs: ['public'],
+    staticDirs: ['public', { from: '../frontend/public', to: '/static' }],
 
     webpackFinal: (config) => {
         const mainConfig = createEntry('main')

.vscode/launch.json

@@ -119,7 +119,8 @@
                 "WORKER_CONCURRENCY": "2",
                 "OBJECT_STORAGE_ENABLED": "True",
                 "HOG_HOOK_URL": "http://localhost:3300/hoghook",
-                "CDP_ASYNC_FUNCTIONS_RUSTY_HOOK_TEAMS": ""
+                "CDP_ASYNC_FUNCTIONS_RUSTY_HOOK_TEAMS": "",
+                "CDP_CYCLOTRON_ENABLED_TEAMS": "*"
             },
             "presentation": {
                 "group": "main"

README.md

@@ -27,6 +27,7 @@
 
 - Specify events manually, or use autocapture to get started quickly
 - Analyze data with ready-made visualizations, or do it yourself with SQL
+- Track website visitors separately with our GA4 alternative
 - Only capture properties on the people you want to track, save money when you don't
 - Gather insights by capturing session replays, console logs, and network monitoring
 - Improve your product with Experiments that automatically analyze performance
@@ -35,7 +36,7 @@
 - Connect to external services and manage data flows with PostHog CDP
 
 PostHog is available with hosting in the EU or US and is fully SOC 2 compliant. It's free to get started and comes with a generous monthly free tier:
-- 1 million product analytics events
+- 1 million events
 - 5k session replays
 - 1 million feature flag requests
 - 250 survey responses

bin/migrate

@@ -1,5 +1,11 @@
 #!/bin/bash
 set -e
+SCRIPT_DIR=$(dirname "$(readlink -f "$0")")
+
+# NOTE when running in docker, rust might not exist so we need to check for it
+if [ -d "$SCRIPT_DIR/../rust" ]; then
+    bash $SCRIPT_DIR/../rust/bin/migrate-cyclotron
+fi
 
 python manage.py migrate
 python manage.py migrate_clickhouse

bin/start-cyclotron

@@ -12,7 +12,7 @@ export RUST_LOG=${DEBUG:-debug}
 SQLX_QUERY_LEVEL=${SQLX_QUERY_LEVEL:-warn}
 export RUST_LOG=$RUST_LOG,sqlx::query=$SQLX_QUERY_LEVEL
 
-export DATABASE_URL=${DATABASE_URL:-postgres://posthog:posthog@localhost:5432/posthog}
+export DATABASE_URL=${CYCLOTRON_DATABASE_URL:-postgres://posthog:posthog@localhost:5432/cyclotron}
 export ALLOW_INTERNAL_IPS=${ALLOW_INTERNAL_IPS:-true}
 
 ./target/debug/cyclotron-fetch &

cypress/e2e/dashboard-shared.cy.ts

@@ -62,7 +62,6 @@ describe('Shared dashboard', () => {
 
         cy.get('.InsightCard').should('have.length', 6)
         // Make sure no element with text "There are no matching events for this query" exists
-        // TODO this was failing, it shouldn't be but YOLO
-        // cy.get('.insight-empty-state').should('not.exist')
+        cy.get('.insight-empty-state').should('not.exist')
     })
 })

docker-compose.dev-full.yml

@@ -47,6 +47,8 @@ services:
             - ./docker/clickhouse/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
             - ./docker/clickhouse/config.xml:/etc/clickhouse-server/config.xml
             - ./docker/clickhouse/users-dev.xml:/etc/clickhouse-server/users.xml
+            - ./docker/clickhouse/user_defined_function.xml:/etc/clickhouse-server/user_defined_function.xml
+            - ./posthog/user_scripts:/var/lib/clickhouse/user_scripts
         depends_on:
             - kafka
             - zookeeper

docker-compose.dev.yml

@@ -61,10 +61,15 @@ services:
             - '9440:9440'
             - '9009:9009'
         volumes:
+            # this new entrypoint file is to fix a bug detailed here https://github.com/ClickHouse/ClickHouse/pull/59991
+            # revert this when we upgrade clickhouse
+            - ./docker/clickhouse/entrypoint.sh:/entrypoint.sh
             - ./posthog/idl:/idl
             - ./docker/clickhouse/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
             - ./docker/clickhouse/config.xml:/etc/clickhouse-server/config.xml
             - ./docker/clickhouse/users-dev.xml:/etc/clickhouse-server/users.xml
+            - ./docker/clickhouse/user_defined_function.xml:/etc/clickhouse-server/user_defined_function.xml
+            - ./posthog/user_scripts:/var/lib/clickhouse/user_scripts
         extra_hosts:
             - 'host.docker.internal:host-gateway'
         depends_on:

docker/clickhouse/docker-entrypoint-initdb.d/init-db.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
 set -e
 
+apk add python3
 cp -r /idl/* /var/lib/clickhouse/format_schemas/