Merge branch 'master' into funnel-actor-tests

.eslintrc.js

@@ -44,8 +44,6 @@ module.exports = {
         'react',
         'cypress',
         '@typescript-eslint',
-        'no-only-tests',
-        'jest',
         'compat',
         'posthog',
         'simple-import-sort',
@@ -60,7 +58,6 @@ module.exports = {
         'react/jsx-curly-brace-presence': ['error', { props: 'never', children: 'never', propElementValues: 'always' }],
         'no-console': ['error', { allow: ['warn', 'error'] }],
         'no-debugger': 'error',
-        'no-only-tests/no-only-tests': 'error',
         'simple-import-sort/imports': 'error',
         'simple-import-sort/exports': 'error',
         'react/prop-types': [0],
@@ -302,13 +299,25 @@ module.exports = {
                 node: true,
                 'jest/globals': true,
             },
+            "plugins": ["jest"],
+            "extends": ["plugin:jest/recommended"],
             globals: {
                 ...globals,
                 given: 'readonly',
             },
             rules: {
                 // The below complains needlessly about expect(api.createInvite).toHaveBeenCalledWith(...)
                 '@typescript-eslint/unbound-method': 'off',
+                // it doesn't know about expectLogic kea tests so isn't helpful
+                'jest/expect-expect': 'off',
+                // we import non-jest mocks from __mocks__ directories
+                'jest/no-mocks-import': 'off',
+                // I'll put expect whereever I want
+                'jest/no-standalone-expect': 'off',
+                // but it's helpful sometimes
+                'jest/no-export': 'off',
+                // also helpful sometimes, but not always
+                'jest/no-conditional-expect': 'warn'
             },
         },
         {

.github/actions/run-backend-tests/action.yml

@@ -60,7 +60,7 @@ runs:
           shell: bash
           id: hogql-parser-diff
           run: |
-              git fetch --no-tags --prune --depth=1 origin
+              git fetch --no-tags --prune --depth=1 origin master
               changed=$(git diff --quiet HEAD origin/master -- hogql_parser/ && echo "false" || echo "true")
               echo "changed=$changed" >> $GITHUB_OUTPUT
 
@@ -149,15 +149,15 @@ runs:
           env:
               PERSON_ON_EVENTS_V2_ENABLED: ${{ inputs.person-on-events }}
               GROUPS_ON_EVENTS_ENABLED: ${{ inputs.person-on-events }}
-              HOGQL_INSIGHTS_OVERRIDE: '1' # HogQL insights are fully rolled out in production
           shell: bash
           run: | # async_migrations covered in ci-async-migrations.yml
               pytest ${{
                   inputs.person-on-events == 'true'
                   && './posthog/clickhouse/ ./posthog/queries/ ./posthog/api/test/test_insight* ./posthog/api/test/dashboards/test_dashboard.py'
-                  || 'hogvm posthog'
+                  || 'posthog'
               }} ${{ inputs.person-on-events == 'true' && 'ee/clickhouse/' || 'ee/' }} -m "not async_migrations" \
                   --ignore=posthog/temporal \
+                  --ignore=hogvm/python/test \
                   --splits ${{ inputs.concurrency }} --group ${{ inputs.group }} \
                   --durations=100 --durations-min=1.0 --store-durations \
                   $PYTEST_ARGS

.github/pr-deploy/values.yaml.tmpl

@@ -12,8 +12,6 @@ env:
   value: "1"
 - name: COMMIT_SHA
   value: $COMMIT_SHA
-- name: PERSISTED_FEATURE_FLAGS
-  value: "hogql"
 
 siteUrl: https://$NAMESPACE.hedgehog-kitefin.ts.net
 tailscale:

.github/workflows/build-and-deploy-prod.yml

@@ -7,6 +7,8 @@ on:
     push:
         branches:
             - master
+        paths-ignore:
+            - 'rust/**'
 
 jobs:
     slack:

.github/workflows/build-hogql-parser.yml

@@ -84,11 +84,8 @@ jobs:
               if: matrix.os == 'ubuntu-22.04' # Only build the sdist once
               run: cd hogql_parser && python setup.py sdist
 
-            # uv is a fast pip alternative: https://github.com/astral-sh/uv/
-            - run: pip install uv
-
             - name: Install cibuildwheel
-              run: uv pip install --system cibuildwheel==2.16.*
+              run: pip install cibuildwheel==2.16.*
 
             - name: Build wheels
               run: cd hogql_parser && python -m cibuildwheel --output-dir dist

.github/workflows/ci-backend.yml

@@ -65,6 +65,7 @@ jobs:
                         # including the negated rule appears to work
                         # but makes it always match because the checked file always isn't `ee/frontend/**` 🙈
                         - 'ee/**/*'
+                        - 'hogvm/**/*'
                         - 'posthog/**/*'
                         - 'bin/*.py'
                         - requirements.txt
@@ -143,28 +144,6 @@ jobs:
               run: |
                   npm run schema:build:python && git diff --exit-code
 
-            - name: Check if ANTLR definitions are up to date
-              run: |
-                  cd ..
-                  sudo apt-get install default-jre
-                  mkdir antlr
-                  cd antlr
-                  curl -o antlr.jar https://www.antlr.org/download/antlr-$ANTLR_VERSION-complete.jar
-                  export PWD=`pwd`
-                  echo '#!/bin/bash' > antlr
-                  echo "java -jar $PWD/antlr.jar \$*" >> antlr
-                  chmod +x antlr
-                  export CLASSPATH=".:$PWD/antlr.jar:$CLASSPATH"
-                  export PATH="$PWD:$PATH"
-
-                  cd ../posthog
-                  antlr | grep "Version"
-                  npm run grammar:build && git diff --exit-code
-              env:
-                  # Installing a version of ANTLR compatible with what's in Homebrew as of October 2023 (version 4.13),
-                  # as apt-get is quite out of date. The same version must be set in hogql_parser/pyproject.toml
-                  ANTLR_VERSION: '4.13.1'
-
     check-migrations:
         needs: changes
         if: needs.changes.outputs.backend == 'true'

.github/workflows/ci-e2e.yml

@@ -34,7 +34,6 @@ jobs:
                         # code completely
                         - 'ee/**'
                         - 'posthog/**'
-                        - 'hogvm/**'
                         - 'bin/*'
                         - frontend/**/*
                         - requirements.txt

.github/workflows/ci-hog.yml

@@ -0,0 +1,127 @@
+# This workflow runs all of our backend django tests.
+#
+# If these tests get too slow, look at increasing concurrency and re-timing the tests by manually dispatching
+# .github/workflows/ci-backend-update-test-timing.yml action
+name: Hog CI
+
+on:
+    push:
+        branches:
+            - master
+    pull_request:
+
+jobs:
+    # Job to decide if we should run backend ci
+    # See https://github.com/dorny/paths-filter#conditional-execution for more details
+    changes:
+        runs-on: ubuntu-latest
+        timeout-minutes: 5
+        if: github.repository == 'PostHog/posthog'
+        name: Determine need to run hog checks
+        # Set job outputs to values from filter step
+        outputs:
+            hog: ${{ steps.filter.outputs.hog }}
+        steps:
+            # For pull requests it's not necessary to checkout the code, but we
+            # also want this to run on master so we need to checkout
+            - uses: actions/checkout@v3
+
+            - uses: dorny/paths-filter@v2
+              id: filter
+              with:
+                  filters: |
+                      hog:
+                        # Avoid running tests for irrelevant changes
+                        - 'hogvm/**/*'
+                        - 'posthog/hogql/**/*'
+                        - 'bin/hog'
+                        - 'bin/hoge'
+                        - requirements.txt
+                        - requirements-dev.txt
+                        - .github/workflows/ci-hog.yml
+
+    hog-tests:
+        needs: changes
+        timeout-minutes: 30
+
+        name: Hog tests
+        runs-on: ubuntu-latest
+
+        steps:
+            # If this run wasn't initiated by the bot (meaning: snapshot update) and we've determined
+            # there are backend changes, cancel previous runs
+            - uses: n1hility/cancel-previous-runs@v3
+              if: github.actor != 'posthog-bot' && needs.changes.outputs.hog == 'true'
+              with:
+                  token: ${{ secrets.GITHUB_TOKEN }}
+
+            - uses: actions/checkout@v3
+              with:
+                  fetch-depth: 1
+
+            - name: Set up Python
+              uses: actions/setup-python@v5
+              with:
+                  python-version: 3.10.10
+                  cache: 'pip'
+                  cache-dependency-path: '**/requirements*.txt'
+                  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
+
+            # uv is a fast pip alternative: https://github.com/astral-sh/uv/
+            - run: pip install uv
+
+            - name: Install SAML (python3-saml) dependencies
+              run: |
+                  sudo apt-get update
+                  sudo apt-get install libxml2-dev libxmlsec1 libxmlsec1-dev libxmlsec1-openssl
+
+            - name: Install Python dependencies
+              run: |
+                  uv pip install --system -r requirements.txt -r requirements-dev.txt
+
+            - name: Install pnpm
+              uses: pnpm/action-setup@v4
+
+            - name: Set up Node.js
+              uses: actions/setup-node@v4
+              with:
+                  node-version: 18.12.1
+
+            - name: Check if ANTLR definitions are up to date
+              run: |
+                  cd ..
+                  sudo apt-get install default-jre
+                  mkdir antlr
+                  cd antlr
+                  curl -o antlr.jar https://www.antlr.org/download/antlr-$ANTLR_VERSION-complete.jar
+                  export PWD=`pwd`
+                  echo '#!/bin/bash' > antlr
+                  echo "java -jar $PWD/antlr.jar \$*" >> antlr
+                  chmod +x antlr
+                  export CLASSPATH=".:$PWD/antlr.jar:$CLASSPATH"
+                  export PATH="$PWD:$PATH"
+
+                  cd ../posthog
+                  antlr | grep "Version"
+                  npm run grammar:build && git diff --exit-code
+              env:
+                  # Installing a version of ANTLR compatible with what's in Homebrew as of October 2023 (version 4.13),
+                  # as apt-get is quite out of date. The same version must be set in hogql_parser/pyproject.toml
+                  ANTLR_VERSION: '4.13.1'
+
+            - name: Run HogVM Python tests
+              run: |
+                  pytest hogvm
+
+            - name: Run HogVM TypeScript tests
+              run: |
+                  cd hogvm/typescript
+                  pnpm install --frozen-lockfile
+                  pnpm run test
+
+            - name: Run Hog tests
+              run: |
+                  cd hogvm/typescript
+                  pnpm run build 
+                  cd ../
+                  ./test.sh && git diff --exit-code

.github/workflows/container-images-cd.yml

@@ -13,6 +13,8 @@ on:
     push:
         branches:
             - master
+        paths-ignore:
+            - 'rust/**'
     workflow_dispatch:
 
 jobs:
@@ -120,17 +122,22 @@ jobs:
 
             - name: Trigger Batch Exports Temporal Worker Cloud deployment
               if: steps.check_changes_batch_exports_temporal_worker.outputs.changed == 'true'
-              uses: mvasigh/dispatch-action@main
+              uses: peter-evans/repository-dispatch@v3
               with:
                   token: ${{ steps.deployer.outputs.token }}
-                  repo: charts
-                  owner: PostHog
-                  event_type: temporal_worker_deploy
-                  message: |
+                  repository: PostHog/charts
+                  event-type: commit_state_update
+                  client-payload: |
                       {
-                        "image_tag": "${{ steps.build.outputs.digest }}",
-                        "worker_name": "temporal-worker",
-                        "context": ${{ toJson(github) }}
+                        "values": {
+                          "image": {
+                            "sha": "${{ steps.build.outputs.digest }}"
+                          }
+                        },
+                        "release": "temporal-worker",
+                        "commit": ${{ toJson(github.event.head_commit) }},
+                        "repository": ${{ toJson(github.repository) }},
+                        "labels": ${{ steps.labels.outputs.labels }}
                       }
 
             - name: Check for changes that affect general purpose temporal worker
@@ -140,17 +147,22 @@ jobs:
 
             - name: Trigger General Purpose Temporal Worker Cloud deployment
               if: steps.check_changes_general_purpose_temporal_worker.outputs.changed == 'true'
-              uses: mvasigh/dispatch-action@main
+              uses: peter-evans/repository-dispatch@v3
               with:
                   token: ${{ steps.deployer.outputs.token }}
-                  repo: charts
-                  owner: PostHog
-                  event_type: temporal_worker_deploy
-                  message: |
+                  repository: PostHog/charts
+                  event-type: commit_state_update
+                  client-payload: |
                       {
-                        "image_tag": "${{ steps.build.outputs.digest }}",
-                        "worker_name": "temporal-worker-general-purpose",
-                        "context": ${{ toJson(github) }}
+                        "values": {
+                          "image": {
+                            "sha": "${{ steps.build.outputs.digest }}"
+                          }
+                        },
+                        "release": "temporal-worker-general-purpose",
+                        "commit": ${{ toJson(github.event.head_commit) }},
+                        "repository": ${{ toJson(github.repository) }},
+                        "labels": ${{ steps.labels.outputs.labels }}
                       }
 
             - name: Check for changes that affect data warehouse temporal worker
@@ -160,15 +172,20 @@ jobs:
 
             - name: Trigger Data Warehouse Temporal Worker Cloud deployment
               if: steps.check_changes_data_warehouse_temporal_worker.outputs.changed == 'true'
-              uses: mvasigh/dispatch-action@main
+              uses: peter-evans/repository-dispatch@v3
               with:
                   token: ${{ steps.deployer.outputs.token }}
-                  repo: charts
-                  owner: PostHog
-                  event_type: temporal_worker_deploy
-                  message: |
+                  repository: PostHog/charts
+                  event-type: commit_state_update
+                  client-payload: |
                       {
-                        "image_tag": "${{ steps.build.outputs.digest }}",
-                        "worker_name": "temporal-worker-data-warehouse",
-                        "context": ${{ toJson(github) }}
+                        "values": {
+                          "image": {
+                            "sha": "${{ steps.build.outputs.digest }}"
+                          }
+                        },
+                        "release": "temporal-worker-data-warehouse",
+                        "commit": ${{ toJson(github.event.head_commit) }},
+                        "repository": ${{ toJson(github.repository) }},
+                        "labels": ${{ steps.labels.outputs.labels }}
                       }