Merge branch 'master' into 3000-top-bar-system

.dockerignore

@@ -34,3 +34,7 @@
 !share/GeoLite2-City.mmdb
 !hogvm/python
 !unit.json
+!plugin-transpiler/src
+!plugin-transpiler/*.*
+!test-runner-jest.config.js
+!test-runner-jest-environment.js

.eslintrc.js

@@ -82,11 +82,15 @@ module.exports = {
                         name: 'dayjs',
                         message: 'Do not directly import dayjs. Only import the dayjs exported from lib/dayjs.',
                     },
+                    {
+                        name: '@ant-design/icons',
+                        message: 'Please use icons from the @posthog/icons package instead',
+                    },
                 ],
             },
         ],
         'react/forbid-dom-props': [
-            1,
+            'warn',
             {
                 forbid: [
                     {
@@ -98,7 +102,7 @@ module.exports = {
             },
         ],
         'posthog/warn-elements': [
-            1,
+            'warn',
             {
                 forbid: [
                     {
@@ -142,11 +146,15 @@ module.exports = {
                         element: 'LemonButtonWithDropdown',
                         message: 'use <LemonMenu> with a <LemonButton> child instead',
                     },
+                    {
+                        element: 'Tag',
+                        message: 'use <LemonTag> instead',
+                    },
                 ],
             },
         ],
         'react/forbid-elements': [
-            2,
+            'error',
             {
                 forbid: [
                     {
@@ -200,9 +208,10 @@ module.exports = {
                 ],
             },
         ],
-        'no-constant-condition': 0,
-        'no-prototype-builtins': 0,
-        'no-irregular-whitespace': 0,
+        'no-constant-binary-expression': 'error',
+        'no-constant-condition': 'off',
+        'no-prototype-builtins': 'off',
+        'no-irregular-whitespace': 'off',
     },
     overrides: [
         {

.github/ISSUE_TEMPLATE/bug_report.md

@@ -19,8 +19,8 @@ labels: bug
 
 ## Environment
 
-- [ ] PostHog Cloud US, project ID: [please provide from https://app.posthog.com/project/settings#project-id]
-- [ ] PostHog Cloud EU, project ID: [please provide from https://eu.posthog.com/project/settings#project-id]
+- [ ] PostHog Cloud US, project ID: [please provide from https://app.posthog.com/settings/project-details#variables]
+- [ ] PostHog Cloud EU, project ID: [please provide from https://eu.posthog.com/settings/project-details#variables]
 - [ ] PostHog Hobby self-hosted with `docker compose`, version/commit: [please provide]
 - [ ] PostHog self-hosted with Kubernetes (deprecated, see ["Sunsetting Kubernetes support"](https://posthog.com/blog/sunsetting-helm-support-posthog)), version/commit: [please provide]
 

.github/actions/run-backend-tests/action.yml

@@ -53,14 +53,33 @@ runs:
           shell: bash
           id: hogql-parser-diff
           run: |
-              changed=$(git diff --quiet HEAD master -- hogql_parser/ && echo "false" || echo "true")
-              echo "::set-output name=changed::$changed"
+              git fetch --no-tags --prune --depth=1 origin
+              changed=$(git diff --quiet HEAD origin/master -- hogql_parser/ && echo "false" || echo "true")
+              echo "changed=$changed" >> $GITHUB_OUTPUT
 
         - name: Install SAML (python3-saml) dependencies
           shell: bash
           run: |
               sudo apt-get update && sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
 
+        - name: Install pnpm
+          uses: pnpm/action-setup@v2
+          with:
+              version: 8.x.x
+
+        - name: Set up Node.js
+          uses: actions/setup-node@v3
+          with:
+              node-version: 18
+              cache: pnpm
+
+        - name: Install plugin-transpiler
+          shell: bash
+          run: |
+              cd plugin-transpiler
+              pnpm install
+              pnpm run build
+
         - uses: syphar/restore-virtualenv@v1
           id: cache-backend-tests
           with:

.github/workflows/benchmark.yml

@@ -18,7 +18,7 @@ jobs:
         environment: clickhouse-benchmarks
 
         # Benchmarks are expensive to run so we only run them (periodically) against master branch and for PRs labeled `performance`
-        if: ${{ github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'performance')  }}
+        if: ${{ github.repository == 'PostHog/posthog' && (github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'performance'))  }}
 
         env:
             DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'

.github/workflows/ci-backend.yml

@@ -68,6 +68,7 @@ jobs:
                         - mypy.ini
                         - pytest.ini
                         - frontend/src/queries/schema.json # Used for generating schema.py
+                        - plugin-transpiler/src # Used for transpiling plugins
                         # Make sure we run if someone is explicitly change the workflow
                         - .github/workflows/ci-backend.yml
                         - .github/actions/run-backend-tests/action.yml

.github/workflows/container-images-cd.yml

@@ -80,7 +80,7 @@ jobs:
                   push: true
                   file: production-unit.Dockerfile
                   tags: ${{ steps.aws-ecr.outputs.registry }}/posthog-cloud:unit
-                  platforms: linux/amd64
+                  platforms: linux/arm64,linux/amd64
                   build-args: COMMIT_HASH=${{ github.sha }}
 
             - name: get deployer token

.github/workflows/storybook-chromatic.yml

@@ -64,7 +64,6 @@ jobs:
             SHARD_COUNT: '2'
             CYPRESS_INSTALL_BINARY: '0'
             NODE_OPTIONS: --max-old-space-size=6144
-            JEST_IMAGE_SNAPSHOT_TRACK_OBSOLETE: '1' # Remove obsolete snapshots
             OPT_OUT_CAPTURE: 1
         outputs:
             # The below have to be manually listed unfortunately, as GitHub Actions doesn't allow matrix-dependent outputs
@@ -132,7 +131,7 @@ jobs:
                   retries=3
                   while [ $retries -gt 0 ]; do
                       pnpm exec http-server storybook-static --port 6006 --silent &
-                      if pnpm wait-on http://127.0.0.1:6006 --timeout 60; then
+                      if pnpm wait-on http://127.0.0.1:6006 --timeout 15; then
                           break
                       fi
                       retries=$((retries-1))
@@ -146,14 +145,7 @@ jobs:
                   # Update snapshots for PRs on the main repo, verify on forks, which don't have access to PostHog Bot
                   VARIANT: ${{ github.event.pull_request.head.repo.full_name == github.repository && 'update' || 'verify' }}
               run: |
-                  retries=3
-                  while [ $retries -gt 0 ]; do
-                      if pnpm test:visual-regression:stories:ci:$VARIANT --browsers ${{ matrix.browser }} --shard ${{ matrix.shard }}/$SHARD_COUNT; then
-                          break
-                      fi
-                      retries=$((retries-1))
-                      echo "Failed @storybook/test-runner, retrying... ($retries retries left)"
-                  done
+                  pnpm test:visual-regression:stories:ci:$VARIANT --browsers ${{ matrix.browser }} --shard ${{ matrix.shard }}/$SHARD_COUNT
 
             - name: Run @playwright/test (legacy, Chromium-only)
               if: matrix.browser == 'chromium' && matrix.shard == 1
@@ -163,6 +155,13 @@ jobs:
               run: |
                   pnpm test:visual-regression:legacy:ci:$VARIANT
 
+            - name: Archive failure screenshots
+              if: ${{ failure() }}
+              uses: actions/upload-artifact@v3
+              with:
+                  name: failure-screenshots-${{ matrix.browser }}
+                  path: frontend/__snapshots__/__failures__/
+
             - name: Count and optimize updated snapshots
               id: diff
               # Skip on forks

.gitignore

@@ -18,6 +18,7 @@ frontend/.cache/
 frontend/dist/
 frontend/types/
 frontend/__snapshots__/__diff_output__/
+frontend/__snapshots__/__failures__/
 *Type.ts
 frontend/pnpm-error.log
 frontend/tmp
@@ -55,3 +56,4 @@ gen/
 upgrade/
 hogvm/typescript/dist
 .wokeignore
+plugin-transpiler/dist

.storybook/test-runner.ts

@@ -48,36 +48,35 @@ declare module '@storybook/types' {
     }
 }
 
-const RETRY_TIMES = 5
+const RETRY_TIMES = 3
 const LOADER_SELECTORS = [
     '.ant-skeleton',
     '.Spinner',
     '.LemonSkeleton',
     '.LemonTableLoader',
+    '.Toastify__toast-container',
     '[aria-busy="true"]',
-    '[aria-label="Content is loading..."]',
     '.SessionRecordingPlayer--buffering',
     '.Lettermark--unknown',
 ]
 
 const customSnapshotsDir = `${process.cwd()}/frontend/__snapshots__`
 
-const TEST_TIMEOUT_MS = 10000
-const BROWSER_DEFAULT_TIMEOUT_MS = 9000 // Reduce the default timeout down from 30s, to pre-empt Jest timeouts
-const SCREENSHOT_TIMEOUT_MS = 9000
+const JEST_TIMEOUT_MS = 15000
+const PLAYWRIGHT_TIMEOUT_MS = 10000 // Must be shorter than JEST_TIMEOUT_MS
 
 module.exports = {
     setup() {
         expect.extend({ toMatchImageSnapshot })
         jest.retryTimes(RETRY_TIMES, { logErrorsBeforeRetry: true })
-        jest.setTimeout(TEST_TIMEOUT_MS)
+        jest.setTimeout(JEST_TIMEOUT_MS)
     },
     async postRender(page, context) {
         const browserContext = page.context()
         const storyContext = (await getStoryContext(page, context)) as StoryContext
         const { skip = false, snapshotBrowsers = ['chromium'] } = storyContext.parameters?.testOptions ?? {}
 
-        browserContext.setDefaultTimeout(BROWSER_DEFAULT_TIMEOUT_MS)
+        browserContext.setDefaultTimeout(PLAYWRIGHT_TIMEOUT_MS)
         if (!skip) {
             const currentBrowser = browserContext.browser()!.browserType().name() as SupportedBrowserName
             if (snapshotBrowsers.includes(currentBrowser)) {
@@ -202,7 +201,7 @@ async function expectLocatorToMatchStorySnapshot(
     browser: SupportedBrowserName,
     options?: LocatorScreenshotOptions
 ): Promise<void> {
-    const image = await locator.screenshot({ timeout: SCREENSHOT_TIMEOUT_MS, ...options })
+    const image = await locator.screenshot({ ...options })
     let customSnapshotIdentifier = context.id
     if (browser !== 'chromium') {
         customSnapshotIdentifier += `--${browser}`

.vscode/launch.json

@@ -70,7 +70,8 @@
                 "DATABASE_URL": "postgres://posthog:posthog@localhost:5432/posthog",
                 "SKIP_SERVICE_VERSION_REQUIREMENTS": "1",
                 "PRINT_SQL": "1",
-                "REPLAY_EVENTS_NEW_CONSUMER_RATIO": "1.0"
+                "REPLAY_EVENTS_NEW_CONSUMER_RATIO": "1.0",
+                "BILLING_SERVICE_URL": "https://billing.dev.posthog.dev"
             },
             "console": "integratedTerminal",
             "python": "${workspaceFolder}/env/bin/python",

Dockerfile.playwright

@@ -15,6 +15,6 @@ ENV CYPRESS_INSTALL_BINARY=0
 
 RUN pnpm install --frozen-lockfile
 
-COPY playwright.config.ts webpack.config.js babel.config.js tsconfig.json ./
+COPY playwright.config.ts webpack.config.js babel.config.js tsconfig.json test-runner-jest.config.js test-runner-jest-environment.js ./
 
 COPY .storybook/ .storybook/

bin/docker-server-unit

@@ -10,4 +10,6 @@ trap 'rm -rf "$PROMETHEUS_MULTIPROC_DIR"' EXIT
 export PROMETHEUS_METRICS_EXPORT_PORT=8001
 export STATSD_PORT=${STATSD_PORT:-8125}
 
-exec /usr/local/bin/docker-entrypoint.sh unitd --no-daemon
+# We need to run as --user root so that nginx unit can proxy the control socket for stats
+# However each application is run as "nobody"
+exec /usr/local/bin/docker-entrypoint.sh unitd --no-daemon --user root

bin/temporal-django-worker

@@ -4,6 +4,6 @@ set -e
 
 trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
 
-python3 manage.py start_temporal_worker
+python3 manage.py start_temporal_worker "$@"
 
 wait

bin/unit_metrics.py

@@ -0,0 +1,78 @@
+import http.client
+import json
+from prometheus_client import CollectorRegistry, Gauge, multiprocess, generate_latest
+
+UNIT_CONNECTIONS_ACCEPTED_TOTAL = Gauge(
+    "unit_connections_accepted_total",
+    "",
+    multiprocess_mode="livesum",
+)
+UNIT_CONNECTIONS_ACTIVE = Gauge(
+    "unit_connections_active",
+    "",
+    multiprocess_mode="livesum",
+)
+UNIT_CONNECTIONS_CLOSED = Gauge(
+    "unit_connections_closed",
+    "",
+    multiprocess_mode="livesum",
+)
+UNIT_CONNECTIONS_IDLE = Gauge(
+    "unit_connections_idle",
+    "",
+    multiprocess_mode="livesum",
+)
+UNIT_CONNECTIONS_TOTAL = Gauge(
+    "unit_requests_total",
+    "",
+    multiprocess_mode="livesum",
+)
+UNIT_PROCESSES_RUNNING_GAUGE = Gauge(
+    "unit_application_processes_running", "", multiprocess_mode="livesum", labelnames=["application"]
+)
+UNIT_PROCESSES_STARTING_GAUGE = Gauge(
+    "unit_application_processes_starting", "", multiprocess_mode="livesum", labelnames=["application"]
+)
+UNIT_PROCESSES_IDLE_GAUGE = Gauge(
+    "unit_application_processes_idle", "", multiprocess_mode="livesum", labelnames=["application"]
+)
+UNIT_REQUESTS_ACTIVE_GAUGE = Gauge(
+    "unit_application_requests_active", "", multiprocess_mode="livesum", labelnames=["application"]
+)
+
+
+def application(environ, start_response):
+    connection = http.client.HTTPConnection("localhost:8081")
+    connection.request("GET", "/status")
+    response = connection.getresponse()
+
+    statj = json.loads(response.read())
+    connection.close()
+
+    UNIT_CONNECTIONS_ACCEPTED_TOTAL.set(statj["connections"]["accepted"])
+    UNIT_CONNECTIONS_ACTIVE.set(statj["connections"]["active"])
+    UNIT_CONNECTIONS_IDLE.set(statj["connections"]["idle"])
+    UNIT_CONNECTIONS_CLOSED.set(statj["connections"]["closed"])
+    UNIT_CONNECTIONS_TOTAL.set(statj["requests"]["total"])
+
+    for application in statj["applications"].keys():
+        UNIT_PROCESSES_RUNNING_GAUGE.labels(application=application).set(
+            statj["applications"][application]["processes"]["running"]
+        )
+        UNIT_PROCESSES_STARTING_GAUGE.labels(application=application).set(
+            statj["applications"][application]["processes"]["starting"]
+        )
+        UNIT_PROCESSES_IDLE_GAUGE.labels(application=application).set(
+            statj["applications"][application]["processes"]["idle"]
+        )
+        UNIT_REQUESTS_ACTIVE_GAUGE.labels(application=application).set(
+            statj["applications"][application]["requests"]["active"]
+        )
+
+    start_response("200 OK", [("Content-Type", "text/plain")])
+    # Create the prometheus multi-process metric registry here
+    # This will aggregate metrics we send from the Django app
+    # We prepend our unit metrics here.
+    registry = CollectorRegistry()
+    multiprocess.MultiProcessCollector(registry)
+    yield generate_latest(registry)

cypress/e2e/a11y.cy.ts

@@ -16,12 +16,10 @@ describe('a11y', () => {
         'experiments',
         'events',
         'datamanagement',
-        'persons',
-        'cohorts',
-        'annotations',
+        'personsmanagement',
         'apps',
         'toolbarlaunch',
-        'projectsettings',
+        'settings',
     ]
 
     sidebarItems.forEach((sideBarItem) => {

cypress/e2e/annotations.cy.ts

@@ -1,10 +1,10 @@
 describe('Annotations', () => {
     beforeEach(() => {
-        cy.clickNavMenu('annotations')
+        cy.clickNavMenu('datamanagement')
+        cy.get('[data-attr=data-management-annotations-tab]').click()
     })
 
     it('Annotations loaded', () => {
-        cy.get('h1').should('contain', 'Annotations')
         cy.get('h2').should('contain', 'Create your first annotation')
         cy.get('[data-attr="product-introduction-docs-link"]').should('contain', 'Learn more about Annotations')
     })