Merge branch 'master' into feat/e2e-via-celery

.dockerignore

@@ -39,3 +39,11 @@
 !test-runner-jest.config.js
 !test-runner-jest-environment.js
 !patches
+!rust
+rust/.env
+rust/.github
+rust/docker
+rust/target
+rust/cyclotron-node/dist
+rust/cyclotron-node/node_modules
+rust/cyclotron-node/index.node

.eslintrc.js

@@ -118,22 +118,6 @@ module.exports = {
                         name: '@ant-design/icons',
                         message: 'Please use icons from the @posthog/icons package instead',
                     },
-                    {
-                        name: 'antd',
-                        importNames: [
-                            'Card',
-                            'Col',
-                            'Row',
-                            'Alert',
-                            'Tooltip',
-                            'Progress',
-                            'Radio',
-                            'Divider',
-                            'Popconfirm',
-                            'Table',
-                        ],
-                        message: 'please use the Lemon equivalent instead',
-                    },
                 ],
             },
         ],

.github/actions/run-backend-tests/action.yml

@@ -202,7 +202,7 @@ runs:
 
         - name: Upload updated timing data as artifacts
           uses: actions/upload-artifact@v4
-          if: ${{ inputs.person-on-events != 'true' && inputs.clickhouse-server-image == 'clickhouse/clickhouse-server:23.12.5.81-alpine' }}
+          if: ${{ inputs.person-on-events != 'true' && inputs.clickhouse-server-image == 'clickhouse/clickhouse-server:23.12.6.19-alpine' }}
           with:
               name: timing_data-${{ inputs.segment }}-${{ inputs.group }}
               path: .test_durations

.github/workflows/build-and-deploy-prod.yml

@@ -18,7 +18,7 @@ jobs:
         if: github.repository == 'PostHog/posthog'
         steps:
             - name: Checkout master
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
             - name: Notify Sentry
               uses: getsentry/action-release@v1
               env:

.github/workflows/build-hogql-parser.yml

@@ -1,12 +1,6 @@
 name: Release hogql-parser
 
 on:
-    push:
-        branches:
-            - master
-        paths:
-            - hogql_parser/**
-            - .github/workflows/build-hogql-parser.yml
     pull_request:
         paths:
             - hogql_parser/**
@@ -19,7 +13,6 @@ concurrency:
 jobs:
     check-version:
         name: Check version legitimacy
-        if: github.repository == 'PostHog/posthog'
         runs-on: ubuntu-22.04
         outputs:
             parser-release-needed: ${{ steps.version.outputs.parser-release-needed }}
@@ -48,7 +41,7 @@ jobs:
                     if [[ "$published" != "$local" ]]; then
                       parser_release_needed='true'
                     else
-                      message_body="It looks like the code of \`hogql-parser\` has changed since last push, but its version stayed the same at $local. 👀\nMake sure to resolve this in \`hogql_parser/setup.py\` before merging!"
+                      message_body="It looks like the code of \`hogql-parser\` has changed since last push, but its version stayed the same at $local. 👀\nMake sure to resolve this in \`hogql_parser/setup.py\` before merging!${{ github.event.pull_request.head.repo.full_name != 'PostHog/posthog' && '\nThis needs to be performed on a branch created on the PostHog/posthog repo itself. A PostHog team member will assist with this.' || ''}}"
                       curl -s -u posthog-bot:${{ secrets.POSTHOG_BOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} -X POST -d "{ \"body\": \"$message_body\" }" "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments"
                     fi
                   fi
@@ -59,7 +52,8 @@ jobs:
         needs: check-version
         runs-on: ${{ matrix.os }}
         timeout-minutes: 30
-        if: ${{ needs.check-version.outputs.parser-release-needed == 'true' }}
+        if: needs.check-version.outputs.parser-release-needed == 'true' &&
+            github.event.pull_request.head.repo.full_name == 'PostHog/posthog'
         strategy:
             matrix:
                 # As of October 2023, GitHub doesn't have ARM Actions runners… and ARM emulation is insanely slow

.github/workflows/ci-backend-update-test-timing.yml

@@ -29,13 +29,13 @@ jobs:
                   group: 1
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
                   python-version: '3.11.9'
-                  clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.5.81-alpine'
+                  clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.6.19-alpine'
                   segment: 'FOSS'
                   person-on-events: false
 
             - name: Upload updated timing data as artifacts
               uses: actions/upload-artifact@v4
-              if: ${{ inputs.person-on-events != 'true' && inputs.clickhouse-server-image == 'clickhouse/clickhouse-server:23.12.5.81-alpine' }}
+              if: ${{ inputs.person-on-events != 'true' && inputs.clickhouse-server-image == 'clickhouse/clickhouse-server:23.12.6.19-alpine' }}
               with:
                   name: timing_data-${{ inputs.segment }}-${{ inputs.group }}
                   path: .test_durations

.github/workflows/ci-backend.yml

@@ -45,7 +45,6 @@ jobs:
     changes:
         runs-on: ubuntu-latest
         timeout-minutes: 5
-        if: github.repository == 'PostHog/posthog'
         name: Determine need to run backend checks
         # Set job outputs to values from filter step
         outputs:
@@ -234,7 +233,7 @@ jobs:
             fail-fast: false
             matrix:
                 python-version: ['3.11.9']
-                clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.5.81-alpine']
+                clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.6.19-alpine']
                 segment: ['Core']
                 person-on-events: [false, true]
                 # :NOTE: Keep concurrency and groups in sync
@@ -243,7 +242,7 @@ jobs:
                 include:
                     - segment: 'Temporal'
                       person-on-events: false
-                      clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.5.81-alpine'
+                      clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.6.19-alpine'
                       python-version: '3.11.9'
                       concurrency: 1
                       group: 1
@@ -274,7 +273,7 @@ jobs:
             - uses: EndBug/add-and-commit@v9
               # Skip on forks
               # Also skip for persons-on-events runs, as we want to ignore snapshots diverging there
-              if: ${{ github.repository == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events }}
+              if: ${{ github.event.pull_request.head.repo.full_name == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events }}
               with:
                   add: '["ee", "./**/*.ambr", "posthog/queries/", "posthog/migrations", "posthog/tasks", "posthog/hogql/"]'
                   message: 'Update query snapshots'
@@ -284,7 +283,7 @@ jobs:
 
             - name: Check if any snapshot changes were left uncomitted
               id: changed-files
-              if: ${{ github.repository == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events }}
+              if: ${{ github.event.pull_request.head.repo.full_name == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events }}
               run: |
                   if [[ -z $(git status -s | grep -v ".test_durations" | tr -d "\n") ]]
                   then
@@ -314,7 +313,7 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
-                clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.5.81-alpine']
+                clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.6.19-alpine']
         if: needs.changes.outputs.backend == 'true'
         runs-on: ubuntu-latest
         steps:
@@ -371,7 +370,11 @@ jobs:
         name: Calculate running time
         needs: [django, async-migrations]
         runs-on: ubuntu-latest
-        if: needs.changes.outputs.backend == 'true'
+        if: # Run on pull requests to PostHog/posthog + on PostHog/posthog outside of PRs - but never on forks
+            needs.changes.outputs.backend == 'true' &&
+            (
+            github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository
+            ) == 'PostHog/posthog'
         steps:
             - name: Calculate running time
               run: |
@@ -396,7 +399,6 @@ jobs:
                   echo running_time_run_id=${run_id} >> $GITHUB_ENV
                   echo running_time_run_started_at=${run_started_at} >> $GITHUB_ENV
             - name: Capture running time to PostHog
-              if: github.repository == 'PostHog/posthog'
               uses: PostHog/[email protected]
               with:
                   posthog-token: ${{secrets.POSTHOG_API_TOKEN}}

.github/workflows/ci-e2e.yml

@@ -16,7 +16,6 @@ jobs:
     changes:
         runs-on: ubuntu-latest
         timeout-minutes: 5
-        if: github.repository == 'PostHog/posthog'
         name: Determine need to run E2E checks
         # Set job outputs to values from filter step
         outputs:
@@ -191,7 +190,7 @@ jobs:
                   OBJECT_STORAGE_SECRET_ACCESS_KEY=object_storage_root_password
                   GITHUB_ACTION_RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
                   CELERY_METRICS_PORT=8999
-                  CLOUD_DEPLOYMENT=1
+                  CLOUD_DEPLOYMENT=E2E
                   EOT
 
             - name: Start PostHog
@@ -278,7 +277,8 @@ jobs:
         name: Calculate running time
         runs-on: ubuntu-latest
         needs: [cypress]
-        if: needs.changes.outputs.shouldTriggerCypress == 'true'
+        if: needs.changes.outputs.shouldTriggerCypress == 'true' &&
+            github.event.pull_request.head.repo.full_name == 'PostHog/posthog'
         steps:
             - name: Calculate running time
               run: |
@@ -302,8 +302,9 @@ jobs:
                   echo running_time_run_attempt=${run_attempt} >> $GITHUB_ENV
                   echo running_time_run_id=${run_id} >> $GITHUB_ENV
                   echo running_time_run_started_at=${run_started_at} >> $GITHUB_ENV
+
             - name: Capture running time to PostHog
-              if: github.repository == 'PostHog/posthog'
+              if: github.event.pull_request.head.repo.full_name == 'PostHog/posthog'
               uses: PostHog/[email protected]
               with:
                   posthog-token: ${{secrets.POSTHOG_API_TOKEN}}

.github/workflows/ci-frontend.yml

@@ -42,6 +42,8 @@ jobs:
                         - .eslintrc.js
                         - .prettier*
                         - babel.config.js
+                        - package.json
+                        - pnpm-lock.yaml
                         - jest.*.ts
                         - tsconfig.json
                         - tsconfig.*.json

.github/workflows/ci-hog.yml

@@ -22,8 +22,7 @@ jobs:
     changes:
         runs-on: ubuntu-latest
         timeout-minutes: 5
-        if: github.repository == 'PostHog/posthog'
-        name: Determine need to run hog checks
+        name: Determine need to run Hog checks
         # Set job outputs to values from filter step
         outputs:
             hog: ${{ steps.filter.outputs.hog }}
@@ -119,9 +118,15 @@ jobs:
                   antlr | grep "Version"
                   npm run grammar:build && git diff --exit-code
               env:
-                  # Installing a version of ANTLR compatible with what's in Homebrew as of October 2023 (version 4.13),
+                  # Installing a version of ANTLR compatible with what's in Homebrew as of August 2024 (version 4.13.2),
                   # as apt-get is quite out of date. The same version must be set in hogql_parser/pyproject.toml
-                  ANTLR_VERSION: '4.13.1'
+                  ANTLR_VERSION: '4.13.2'
+
+            - name: Check if STL bytecode is up to date
+              if: needs.changes.outputs.hog == 'true'
+              run: |
+                  python -m hogvm.stl.compile
+                  git diff --exit-code
 
             - name: Run HogVM Python tests
               if: needs.changes.outputs.hog == 'true'

.github/workflows/ci-plugin-server.yml

@@ -26,7 +26,6 @@ jobs:
     changes:
         runs-on: ubuntu-latest
         timeout-minutes: 5
-        if: github.repository == 'PostHog/posthog'
         name: Determine need to run plugin server checks
         outputs:
             plugin-server: ${{ steps.filter.outputs.plugin-server }}

.github/workflows/container-images-cd.yml

@@ -43,12 +43,6 @@ jobs:
             - name: Set up Depot CLI
               uses: depot/setup-action@v1
 
-            - name: Login to DockerHub
-              uses: docker/login-action@v3
-              with:
-                  username: posthog
-                  password: ${{ secrets.DOCKERHUB_TOKEN }}
-
             - name: Configure AWS credentials
               uses: aws-actions/configure-aws-credentials@v4
               with:
@@ -60,6 +54,12 @@ jobs:
               id: aws-ecr
               uses: aws-actions/amazon-ecr-login@v2
 
+            - name: Login to DockerHub
+              uses: docker/login-action@v2
+              with:
+                  username: posthog
+                  password: ${{ secrets.DOCKERHUB_TOKEN }}
+
             - name: Build and push container image
               id: build
               uses: depot/build-push-action@v1
@@ -99,7 +99,8 @@ jobs:
                         "release": "posthog",
                         "commit": ${{ toJson(github.event.head_commit) }},
                         "repository": ${{ toJson(github.repository) }},
-                        "labels": ${{ steps.labels.outputs.labels }}
+                        "labels": ${{ steps.labels.outputs.labels }},
+                        "timestamp": "${{ github.event.head_commit.timestamp }}"
                       }
 
             - name: Check for changes in plugins directory
@@ -124,7 +125,8 @@ jobs:
                         "release": "ingestion",
                         "commit": ${{ toJson(github.event.head_commit) }},
                         "repository": ${{ toJson(github.repository) }},
-                        "labels": ${{ toJson(steps.labels.outputs.labels) }}
+                        "labels": ${{ toJson(steps.labels.outputs.labels) }},
+                        "timestamp": "${{ github.event.head_commit.timestamp }}"
                       }
 
             - name: Check for changes that affect batch exports temporal worker
@@ -149,7 +151,8 @@ jobs:
                         "release": "temporal-worker",
                         "commit": ${{ toJson(github.event.head_commit) }},
                         "repository": ${{ toJson(github.repository) }},
-                        "labels": ${{ steps.labels.outputs.labels }}
+                        "labels": ${{ steps.labels.outputs.labels }},
+                        "timestamp": "${{ github.event.head_commit.timestamp }}"
                       }
 
             - name: Check for changes that affect general purpose temporal worker
@@ -174,7 +177,8 @@ jobs:
                         "release": "temporal-worker-general-purpose",
                         "commit": ${{ toJson(github.event.head_commit) }},
                         "repository": ${{ toJson(github.repository) }},
-                        "labels": ${{ steps.labels.outputs.labels }}
+                        "labels": ${{ steps.labels.outputs.labels }},
+                        "timestamp": "${{ github.event.head_commit.timestamp }}"
                       }
 
             - name: Check for changes that affect data warehouse temporal worker
@@ -199,5 +203,6 @@ jobs:
                         "release": "temporal-worker-data-warehouse",
                         "commit": ${{ toJson(github.event.head_commit) }},
                         "repository": ${{ toJson(github.repository) }},
-                        "labels": ${{ steps.labels.outputs.labels }}
+                        "labels": ${{ steps.labels.outputs.labels }},
+                        "timestamp": "${{ github.event.head_commit.timestamp }}"
                       }

.github/workflows/go.yml

@@ -0,0 +1,22 @@
+name: Go Test (for livestream service)
+
+on:
+    pull_request:
+        paths:
+            - 'livestream/**'
+
+jobs:
+    test:
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Checkout code
+              uses: actions/checkout@v2
+
+            - name: Set up Go
+              uses: actions/setup-go@v2
+              with:
+                  go-version: 1.22
+
+            - name: Run tests
+              run: cd livestream && go test -v

.github/workflows/livestream-docker-image.yml

@@ -84,5 +84,6 @@ jobs:
                         },
                         "release": "livestream",
                         "commit": ${{ toJson(github.event.head_commit) }},
-                        "repository": ${{ toJson(github.repository) }}
+                        "repository": ${{ toJson(github.repository) }},
+                        "timestamp": "${{ github.event.head_commit.timestamp }}"
                       }