merge master

.dockerignore

@@ -33,3 +33,4 @@
 !plugin-server/.prettierrc
 !share/GeoLite2-City.mmdb
 !hogvm/python
+!unit.json

.eslintrc.js

@@ -26,6 +26,7 @@ module.exports = {
         'plugin:eslint-comments/recommended',
         'plugin:storybook/recommended',
         'prettier',
+        'plugin:compat/recommended',
     ],
     globals,
     parser: '@typescript-eslint/parser',
@@ -36,7 +37,7 @@ module.exports = {
         ecmaVersion: 2018,
         sourceType: 'module',
     },
-    plugins: ['prettier', 'react', 'cypress', '@typescript-eslint', 'no-only-tests', 'jest'],
+    plugins: ['prettier', 'react', 'cypress', '@typescript-eslint', 'no-only-tests', 'jest', 'compat'],
     rules: {
         'no-console': ['error', { allow: ['warn', 'error'] }],
         'no-debugger': 'error',
@@ -111,10 +112,6 @@ module.exports = {
                         element: 'Divider',
                         message: 'use <LemonDivider> instead',
                     },
-                    {
-                        element: 'Typography',
-                        message: 'use utility classes instead',
-                    },
                     {
                         element: 'Card',
                         message: 'use utility classes instead',
@@ -123,10 +120,6 @@ module.exports = {
                         element: 'Button',
                         message: 'use <LemonButton> instead',
                     },
-                    {
-                        element: 'Input.TextArea',
-                        message: 'use <LemonTextArea> instead',
-                    },
                     {
                         element: 'Input',
                         message: 'use <LemonInput> instead',
@@ -147,10 +140,6 @@ module.exports = {
                         element: 'a',
                         message: 'use <Link> instead',
                     },
-                    {
-                        element: 'ReactMarkdown',
-                        message: 'use <LemonMarkdown> instead',
-                    },
                 ],
             },
         ],
@@ -182,6 +171,18 @@ module.exports = {
                         element: 'MonacoEditor',
                         message: 'use <CodeEditor> instead',
                     },
+                    {
+                        element: 'Typography',
+                        message: 'use utility classes instead',
+                    },
+                    {
+                        element: 'Input.TextArea',
+                        message: 'use <LemonTextArea> instead',
+                    },
+                    {
+                        element: 'ReactMarkdown',
+                        message: 'use <LemonMarkdown> instead',
+                    },
                 ],
             },
         ],

.github/actions/build-n-cache-image/action.yml

@@ -33,5 +33,19 @@ runs:
               load: ${{ inputs.load }}
               tags: ${{ steps.emit.outputs.tag }}
               platforms: linux/amd64,linux/arm64
+              build-args: COMMIT_HASH=${{ github.sha }}
+          env:
+              ACTIONS_ID_TOKEN_REQUEST_URL: ${{ inputs.actions-id-token-request-url }}
+
+        - name: Build unit image
+          id: build-unit
+          uses: depot/build-push-action@v1
+          with:
+              buildx-fallback: false # buildx is so slow it's better to just fail
+              load: ${{ inputs.load }}
+              file: production-unit.Dockerfile
+              tags: ${{ steps.emit.outputs.tag }}
+              platforms: linux/amd64
+              build-args: COMMIT_HASH=${{ github.sha }}
           env:
               ACTIONS_ID_TOKEN_REQUEST_URL: ${{ inputs.actions-id-token-request-url }}

.github/workflows/ci-backend.yml

@@ -169,7 +169,7 @@ jobs:
     check-migrations:
         needs: changes
         if: needs.changes.outputs.backend == 'true'
-        timeout-minutes: 5
+        timeout-minutes: 10
 
         name: Validate Django migrations
         runs-on: ubuntu-latest

.github/workflows/container-images-cd.yml

@@ -68,8 +68,20 @@ jobs:
               with:
                   buildx-fallback: false # the fallback is so slow it's better to just fail
                   push: true
-                  tags: posthog/posthog:latest,${{ steps.aws-ecr.outputs.registry }}/posthog-cloud:master
+                  tags: posthog/posthog:${{ github.sha }},posthog/posthog:latest,${{ steps.aws-ecr.outputs.registry }}/posthog-cloud:master
                   platforms: linux/arm64,linux/amd64
+                  build-args: COMMIT_HASH=${{ github.sha }}
+
+            - name: Build and push unit container image
+              id: build-unit
+              uses: depot/build-push-action@v1
+              with:
+                  buildx-fallback: false # the fallback is so slow it's better to just fail
+                  push: true
+                  file: production-unit.Dockerfile
+                  tags: ${{ steps.aws-ecr.outputs.registry }}/posthog-cloud:unit
+                  platforms: linux/amd64
+                  build-args: COMMIT_HASH=${{ github.sha }}
 
             - name: get deployer token
               id: deployer

.github/workflows/lint-new-pr.yml

@@ -2,12 +2,13 @@ name: Lint new PR
 
 on:
     pull_request:
-        types: [opened]
+        types: [opened, ready_for_review]
 
 jobs:
     check-description:
         name: Check that PR has description
         runs-on: ubuntu-20.04
+        if: github.event.pull_request.draft == false
 
         steps:
             - name: Check if PR is shame-worthy

.storybook/test-runner.ts

@@ -56,6 +56,7 @@ const LOADER_SELECTORS = [
     '.LemonSkeleton',
     '.LemonTableLoader',
     '[aria-busy="true"]',
+    '[aria-label="Content is loading..."]',
     '.SessionRecordingPlayer--buffering',
     '.Lettermark--unknown',
 ]

bin/copy-posthog-js

@@ -8,3 +8,4 @@ cp node_modules/posthog-js/dist/array.js* frontend/dist/
 cp node_modules/posthog-js/dist/array.full.js* frontend/dist/
 cp node_modules/posthog-js/dist/recorder.js* frontend/dist/
 cp node_modules/posthog-js/dist/recorder-v2.js* frontend/dist/
+cp node_modules/posthog-js/dist/surveys.js* frontend/dist/

bin/docker-server

@@ -20,6 +20,7 @@ exec gunicorn posthog.wsgi \
     --worker-tmp-dir /dev/shm \
     --workers=2 \
     --threads=8 \
+    --backlog=${GUNICORN_BACKLOG:-1000} \
     --worker-class=gthread \
     ${STATSD_HOST:+--statsd-host $STATSD_HOST:$STATSD_PORT} \
     --limit-request-line=16384 $@

bin/docker-server-unit

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# To ensure we are able to expose metrics from multiple processes, we need to
+# provide a directory for `prometheus_client` to store a shared registry.
+export PROMETHEUS_MULTIPROC_DIR=$(mktemp -d)
+chmod -R 777 $PROMETHEUS_MULTIPROC_DIR
+trap 'rm -rf "$PROMETHEUS_MULTIPROC_DIR"' EXIT
+
+export PROMETHEUS_METRICS_EXPORT_PORT=8001
+export STATSD_PORT=${STATSD_PORT:-8125}
+
+exec /usr/local/bin/docker-entrypoint.sh unitd --no-daemon

cypress/e2e/cohorts.cy.ts

@@ -52,5 +52,24 @@ describe('Cohorts', () => {
             cy.url().should('include', '/cohorts/')
             cy.get('[data-attr="cohort-name"]').should('have.value', 'Test Cohort')
         })
+
+        // back into cohort
+        cy.get('tbody').contains('Test Cohort').click()
+
+        // duplicate cohort (dynamic)
+        cy.get('[data-attr="more-button"]').click()
+        cy.get('.Popover__content').contains('Duplicate as dynamic cohort').click()
+        cy.get('.Toastify__toast-body').contains('View cohort').click()
+
+        // duplicate cohort (static)
+        cy.get('[data-attr="more-button"]').click()
+        cy.get('.Popover__content').contains('Duplicate as static cohort').click()
+        cy.get('.Toastify__toast-body').contains('View cohort').click()
+
+        // delete cohort
+        cy.get('[data-attr="more-button"]').click()
+        cy.get('.Popover__content').contains('Delete cohort').click()
+        cy.clickNavMenu('cohorts')
+        cy.get('tbody').should('not.have.text', 'Test Cohort (dynamic copy) (static copy)')
     })
 })

cypress/fixtures/api/user-enterprise.json

@@ -40,7 +40,7 @@
                 "timezone": "UTC"
             }
         ],
-        "available_features": ["zapier", "organizations_projects", "google_login", "dashboard_collaboration"]
+        "available_features": ["zapier", "organizations_projects", "social_sso", "dashboard_collaboration"]
     },
     "organizations": [{ "id": "01787e47-5558-0000-bae1-6bc513b42abe", "name": "Hogflix Movies" }]
 }

ee/api/test/test_team.py

@@ -11,6 +11,7 @@
 from posthog.models.organization import Organization, OrganizationMembership
 from posthog.models.team import Team
 from posthog.models.user import User
+from posthog.test.base import FuzzyInt
 
 
 class TestProjectEnterpriseAPI(APILicensedTest):
@@ -438,7 +439,7 @@ def test_list_teams_restricted_ones_hidden(self):
         projects_response = self.client.get(f"/api/projects/")
 
         # 9 (above):
-        with self.assertNumQueries(8):
+        with self.assertNumQueries(FuzzyInt(8, 9)):
             current_org_response = self.client.get(f"/api/organizations/{self.organization.id}/")
 
         self.assertEqual(projects_response.status_code, HTTP_200_OK)

ee/clickhouse/models/test/__snapshots__/test_cohort.ambr

@@ -20,7 +20,7 @@
      GROUP BY id
      HAVING max(is_deleted) = 0
      AND ((((has(['something'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$some_prop'), '^"|"$', ''))))
-           AND ((has(['something'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$another_prop'), '^"|"$', ''))))))) as person
+           AND ((has(['something'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$another_prop'), '^"|"$', '')))))) SETTINGS optimize_aggregation_in_order = 1) as person
   UNION ALL
   SELECT person_id,
          cohort_id,
@@ -54,7 +54,7 @@
             AND ((has(['something1'], replaceRegexpAll(JSONExtractRaw(properties, '$some_prop'), '^"|"$', '')))) )
      GROUP BY id
      HAVING max(is_deleted) = 0
-     AND ((has(['something1'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$some_prop'), '^"|"$', ''))))) as person
+     AND ((has(['something1'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$some_prop'), '^"|"$', '')))) SETTINGS optimize_aggregation_in_order = 1) as person
   UNION ALL
   SELECT person_id,
          cohort_id,
@@ -111,7 +111,7 @@
                   AND (((((NOT has(['something1'], replaceRegexpAll(JSONExtractRaw(properties, '$some_prop'), '^"|"$', ''))))))) )
            GROUP BY id
            HAVING max(is_deleted) = 0
-           AND (((((NOT has(['something1'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$some_prop'), '^"|"$', ''))))))))) person ON person.person_id = behavior_query.person_id
+           AND (((((NOT has(['something1'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$some_prop'), '^"|"$', ''))))))) SETTINGS optimize_aggregation_in_order = 1)) person ON person.person_id = behavior_query.person_id
      WHERE 1 = 1
        AND ((((performed_event_condition_15_level_level_0_level_0_level_0_0)))) ) as person
   UNION ALL
@@ -176,7 +176,7 @@
                               AND (((((NOT has(['something1'], replaceRegexpAll(JSONExtractRaw(properties, '$some_prop'), '^"|"$', ''))))))) )
                        GROUP BY id
                        HAVING max(is_deleted) = 0
-                       AND (((((NOT has(['something1'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$some_prop'), '^"|"$', ''))))))))) person ON person.person_id = behavior_query.person_id
+                       AND (((((NOT has(['something1'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, version), '$some_prop'), '^"|"$', ''))))))) SETTINGS optimize_aggregation_in_order = 1)) person ON person.person_id = behavior_query.person_id
                  WHERE 1 = 1
                    AND ((((performed_event_condition_17_level_level_0_level_0_level_0_0)))) ) ))
   '

ee/clickhouse/queries/experiments/secondary_experiment_result.py

@@ -3,10 +3,7 @@
 from zoneinfo import ZoneInfo
 
 from rest_framework.exceptions import ValidationError
-from ee.clickhouse.queries.experiments.trend_experiment_result import (
-    uses_count_per_property_value_aggregation,
-    uses_count_per_user_aggregation,
-)
+from ee.clickhouse.queries.experiments.trend_experiment_result import uses_math_aggregation_by_user_or_property_value
 
 from posthog.constants import INSIGHT_FUNNELS, INSIGHT_TRENDS, TRENDS_CUMULATIVE
 from posthog.models.feature_flag import FeatureFlag
@@ -59,7 +56,7 @@ def __init__(
 
         self.team = team
         if query_filter.insight == INSIGHT_TRENDS and not (
-            uses_count_per_user_aggregation(query_filter) or uses_count_per_property_value_aggregation(query_filter)
+            uses_math_aggregation_by_user_or_property_value(query_filter)
         ):
             query_filter = query_filter.shallow_clone({"display": TRENDS_CUMULATIVE})
 
@@ -99,9 +96,7 @@ def get_trend_count_data_for_variants(self, insight_results) -> Dict[str, float]
             count = result["count"]
             breakdown_value = result["breakdown_value"]
 
-            if uses_count_per_user_aggregation(self.query_filter) or uses_count_per_property_value_aggregation(
-                self.query_filter
-            ):
+            if uses_math_aggregation_by_user_or_property_value(self.query_filter):
                 count = result["count"] / len(result.get("data", [0]))
 
             if breakdown_value in self.variants:

ee/clickhouse/queries/experiments/trend_experiment_result.py

@@ -25,7 +25,7 @@
 from posthog.models.filters.filter import Filter
 from posthog.models.team import Team
 from posthog.queries.trends.trends import Trends
-from posthog.queries.trends.util import COUNT_PER_ACTOR_MATH_FUNCTIONS, PROPERTY_MATH_FUNCTIONS
+from posthog.queries.trends.util import ALL_SUPPORTED_MATH_FUNCTIONS
 
 Probability = float
 
@@ -41,16 +41,18 @@ class Variant:
     absolute_exposure: int
 
 
-def uses_count_per_user_aggregation(filter: Filter):
+def uses_math_aggregation_by_user_or_property_value(filter: Filter):
+    # sync with frontend: https://github.com/PostHog/posthog/blob/master/frontend/src/scenes/experiments/experimentLogic.tsx#L662
+    # the selector experimentCountPerUserMath
+
     entities = filter.entities
-    count_per_actor_keys = COUNT_PER_ACTOR_MATH_FUNCTIONS.keys()
-    return any(entity.math in count_per_actor_keys for entity in entities)
+    math_keys = ALL_SUPPORTED_MATH_FUNCTIONS
 
+    # 'sum' doesn't need special handling, we can have custom exposure for sum filters
+    if "sum" in math_keys:
+        math_keys.remove("sum")
 
-def uses_count_per_property_value_aggregation(filter: Filter):
-    entities = filter.entities
-    count_per_prop_value_keys = PROPERTY_MATH_FUNCTIONS.keys()
-    return any(entity.math in count_per_prop_value_keys for entity in entities)
+    return any(entity.math in math_keys for entity in entities)
 
 
 class ClickhouseTrendExperimentResult:
@@ -89,14 +91,11 @@ def __init__(
                 experiment_end_date.astimezone(ZoneInfo(team.timezone)) if experiment_end_date else None
             )
 
-        count_per_user_aggregation = uses_count_per_user_aggregation(filter)
-        count_per_property_value_aggregation = uses_count_per_property_value_aggregation(filter)
+        uses_math_aggregation = uses_math_aggregation_by_user_or_property_value(filter)
 
         query_filter = filter.shallow_clone(
             {
-                "display": TRENDS_CUMULATIVE
-                if not (count_per_user_aggregation or count_per_property_value_aggregation)
-                else TRENDS_LINEAR,
+                "display": TRENDS_CUMULATIVE if not uses_math_aggregation else TRENDS_LINEAR,
                 "date_from": start_date_in_project_timezone,
                 "date_to": end_date_in_project_timezone,
                 "explicit_date": True,
@@ -107,7 +106,7 @@ def __init__(
             }
         )
 
-        if count_per_user_aggregation or count_per_property_value_aggregation:
+        if uses_math_aggregation:
             # A trend experiment can have only one metric, so take the first one to calculate exposure
             # We copy the entity to avoid mutating the original filter
             entity = query_filter.shallow_clone({}).entities[0]
@@ -213,9 +212,7 @@ def get_variants(self, insight_results, exposure_results):
         exposure_counts = {}
         exposure_ratios = {}
 
-        if uses_count_per_user_aggregation(self.query_filter) or uses_count_per_property_value_aggregation(
-            self.query_filter
-        ):
+        if uses_math_aggregation_by_user_or_property_value(self.query_filter):
             filtered_exposure_results = [
                 result for result in exposure_results if result["action"]["math"] == UNIQUE_USERS
             ]