Merge branch 'master' into dw-postgres-column-hints

.eslintrc.js

@@ -42,6 +42,7 @@ module.exports = {
     },
     plugins: [
         'react',
+        "react-hooks",
         'cypress',
         '@typescript-eslint',
         'compat',
@@ -51,6 +52,8 @@ module.exports = {
         'unused-imports',
     ],
     rules: {
+        "react-hooks/rules-of-hooks": "error",
+        "react-hooks/exhaustive-deps": "warn",
         // PyCharm always adds curly braces, I guess vscode doesn't, PR reviewers often complain they are present on props that don't need them
         // let's save the humans time and let the machines do the work
         // "never" means if the prop does not need the curly braces, they will be removed/errored

.github/actions/run-backend-tests/action.yml

@@ -31,11 +31,13 @@ runs:
     steps:
         # Pre-tests
 
+        # Copies the fully versioned UDF xml file for use in CI testing
         - name: Stop/Start stack with Docker Compose
           shell: bash
           run: |
               export CLICKHOUSE_SERVER_IMAGE=${{ inputs.clickhouse-server-image }}
               export DOCKER_REGISTRY_PREFIX="us-east1-docker.pkg.dev/posthog-301601/mirror/"
+              cp posthog/user_scripts/latest_user_defined_function.xml docker/clickhouse/user_defined_function.xml
               docker compose -f docker-compose.dev.yml down
               docker compose -f docker-compose.dev.yml up -d
 
@@ -208,7 +210,7 @@ runs:
 
         - name: Upload updated timing data as artifacts
           uses: actions/upload-artifact@v4
-          if: ${{ inputs.person-on-events != 'true' && inputs.clickhouse-server-image == 'clickhouse/clickhouse-server:23.12.6.19-alpine' }}
+          if: ${{ inputs.person-on-events != 'true' && inputs.clickhouse-server-image == 'clickhouse/clickhouse-server:23.12.6.19' }}
           with:
               name: timing_data-${{ inputs.segment }}-${{ inputs.group }}
               path: .test_durations

.github/workflows/ci-backend-update-test-timing.yml

@@ -29,13 +29,13 @@ jobs:
                   group: 1
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
                   python-version: '3.11.9'
-                  clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.6.19-alpine'
+                  clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.6.19'
                   segment: 'FOSS'
                   person-on-events: false
 
             - name: Upload updated timing data as artifacts
               uses: actions/upload-artifact@v4
-              if: ${{ inputs.person-on-events != 'true' && inputs.clickhouse-server-image == 'clickhouse/clickhouse-server:23.12.6.19-alpine' }}
+              if: ${{ inputs.person-on-events != 'true' && inputs.clickhouse-server-image == 'clickhouse/clickhouse-server:23.12.6.19' }}
               with:
                   name: timing_data-${{ inputs.segment }}-${{ inputs.group }}
                   path: .test_durations

.github/workflows/ci-backend.yml

@@ -234,7 +234,7 @@ jobs:
             fail-fast: false
             matrix:
                 python-version: ['3.11.9']
-                clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.6.19-alpine']
+                clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.6.19']
                 segment: ['Core']
                 person-on-events: [false, true]
                 # :NOTE: Keep concurrency and groups in sync
@@ -243,7 +243,7 @@ jobs:
                 include:
                     - segment: 'Temporal'
                       person-on-events: false
-                      clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.6.19-alpine'
+                      clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.6.19'
                       python-version: '3.11.9'
                       concurrency: 1
                       group: 1
@@ -314,7 +314,7 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
-                clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.6.19-alpine']
+                clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.6.19']
         if: needs.changes.outputs.backend == 'true'
         runs-on: ubuntu-latest
         steps:

.github/workflows/ci-e2e.yml

@@ -232,6 +232,9 @@ jobs:
                   config: retries=2
                   spec: ${{ matrix.chunk }}
                   install: false
+                  # We were seeing suprising crashes in headless mode
+                  # See https://github.com/cypress-io/cypress/issues/28893#issuecomment-1956480875
+                  headed: true
               env:
                   E2E_TESTING: 1
                   OPT_OUT_CAPTURE: 0

.github/workflows/report-pr-age.yml

@@ -28,14 +28,3 @@ jobs:
                   posthog-token: ${{secrets.POSTHOG_API_TOKEN}}
                   event: 'posthog-ci-pr-stats'
                   properties: '{"prAgeInSeconds": ${{ env.pr_age }}, "isRevert": ${{env.is_revert}}, "prTitle": "${{ github.event.pull_request.title}}", "prNumber": "${{ github.event.pull_request.number}}"  }'
-    report-pr-as-annotation:
-        name: Report PR to PostHog
-        runs-on: ubuntu-20.04
-        if: github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.merged == true
-        steps:
-            - name: Report PR to PostHog
-              uses: PostHog/[email protected]
-              with:
-                  posthog-token: ${{secrets.POSTHOG_PAUL_PERSONAL_API_KEY}}
-                  posthog-project-id: ${{secrets.POSTHOG_PROJECT_ID}}
-                  annotation-message: 'Merged PR #${{github.event.pull_request.number}} ${{github.event.pull_request.title}}'

.github/workflows/rust-docker-build.yml

@@ -31,6 +31,8 @@ jobs:
                       dockerfile: ./rust/Dockerfile
                     - image: property-defs-rs
                       dockerfile: ./rust/Dockerfile
+                    - image: cymbal
+                      dockerfile: ./rust/Dockerfile
         runs-on: depot-ubuntu-22.04-4
         permissions:
             id-token: write # allow issuing OIDC tokens for this workflow run
@@ -46,6 +48,7 @@ jobs:
             hook-janitor_digest: ${{ steps.digest.outputs.hook-janitor_digest }}
             hook-worker_digest: ${{ steps.digest.outputs.hook-worker_digest }}
             hook-migrator_digest: ${{ steps.digest.outputs.hook-migrator_digest }}
+            cymbal_digest: ${{ steps.digest.outputs.cymbal_digest }}
 
         defaults:
             run:
@@ -141,6 +144,10 @@ jobs:
                       values:
                           image:
                               sha: '${{ needs.build.outputs.property-defs-rs_digest }}'
+                    # - release: cymbal - disabled until a charts in place, for now we just build
+                    #   values:
+                    #       image:
+                    #           sha: '${{ needs.build.outputs.cymbal_digest }}'
                     - release: hoghooks
                       values:
                           api_image:

.gitignore

@@ -66,4 +66,6 @@ plugin-transpiler/dist
 .dlt
 *.db
 # Ignore any log files that happen to be present
-*.log
+*.log
+# pyright config (keep this until we have a standardized one)
+pyrightconfig.json

.storybook/test-runner.ts

@@ -1,5 +1,5 @@
 import { toMatchImageSnapshot } from 'jest-image-snapshot'
-import { getStoryContext, TestRunnerConfig, TestContext, waitForPageReady } from '@storybook/test-runner'
+import { getStoryContext, TestRunnerConfig, TestContext } from '@storybook/test-runner'
 import type { Locator, Page, LocatorScreenshotOptions } from '@playwright/test'
 import type { Mocks } from '~/mocks/utils'
 import { StoryContext } from '@storybook/csf'
@@ -154,24 +154,22 @@ async function expectStoryToMatchSnapshot(
         await Promise.all(waitForSelector.map((selector) => page.waitForSelector(selector)))
     }
 
-    // snapshot light theme
+    // Snapshot light theme
     await page.evaluate(() => {
         document.body.setAttribute('theme', 'light')
     })
 
-    // Wait for all images to load
     await waitForPageReady(page)
     await page.waitForFunction(() => Array.from(document.images).every((i: HTMLImageElement) => !!i.naturalWidth))
     await page.waitForTimeout(2000)
 
     await check(page, context, browser, 'light', storyContext.parameters?.testOptions?.snapshotTargetSelector)
 
-    // snapshot dark theme
+    // Snapshot dark theme
     await page.evaluate(() => {
         document.body.setAttribute('theme', 'dark')
     })
 
-    // Wait for all images to load
     await waitForPageReady(page)
     await page.waitForFunction(() => Array.from(document.images).every((i: HTMLImageElement) => !!i.naturalWidth))
     await page.waitForTimeout(100)
@@ -259,3 +257,16 @@ async function expectLocatorToMatchStorySnapshot(
         failureThresholdType: 'percent',
     })
 }
+
+/**
+ * Just like the `waitForPageReady` helper offered by Playwright - except we only wait for `networkidle` in CI,
+ * as it doesn't work with local Storybook (the live reload feature keeps up a long-running request, so we aren't idle).
+ */
+async function waitForPageReady(page: Page): Promise<void> {
+    await page.waitForLoadState("domcontentloaded");
+    await page.waitForLoadState("load");
+    if (process.env.CI) {
+        await page.waitForLoadState("networkidle");
+    }
+    await page.evaluate(() => document.fonts.ready);
+}