Merge branch 'master' into feat/hogql-sample

.github/workflows/ci-backend.yml

@@ -272,9 +272,9 @@ jobs:
               # Also skip for persons-on-events runs, as we want to ignore snapshots diverging there
               if: ${{ github.repository == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events }}
               with:
-                  add: '["ee", "posthog/clickhouse/test/__snapshots__", "posthog/api/test/__snapshots__", "posthog/test/__snapshots__", "posthog/queries/", "posthog/migrations", "posthog/tasks", "posthog/hogql/"]'
+                  add: '["ee", "./**/*.ambr", "posthog/queries/", "posthog/migrations", "posthog/tasks", "posthog/hogql/"]'
                   message: 'Update query snapshots'
-                  pull: --rebase --autostash # Make sure we're up to date with other segments' updates
+                  pull: --rebase --autostash # Make sure we're up-to-date with other segments' updates
                   default_author: github_actions
                   github_token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
 

.github/workflows/container-images-cd.yml

@@ -122,3 +122,21 @@ jobs:
                       {
                         "image_tag": "${{ steps.build.outputs.digest }}"
                       }
+
+            - name: Check for changes that affect temporal worker
+              id: check_changes_temporal_worker
+              run: |
+                  echo "::set-output name=changed::$(git diff --name-only HEAD^ HEAD | grep -E '^posthog/temporal/|^posthog/batch_exports/|^posthog/management/commands/start_temporal_worker.py$' || true)"
+
+            - name: Trigger Temporal Worker Cloud deployment
+              if: steps.check_changes_temporal_worker.outputs.changed != ''
+              uses: mvasigh/dispatch-action@main
+              with:
+                  token: ${{ steps.deployer.outputs.token }}
+                  repo: charts
+                  owner: PostHog
+                  event_type: temporal_worker_deploy
+                  message: |
+                      {
+                        "image_tag": "${{ steps.build.outputs.digest }}"
+                      }

.github/workflows/storybook-chromatic.yml

@@ -129,8 +129,15 @@ jobs:
 
             - name: Serve Storybook in the background
               run: |
-                  pnpm exec http-server storybook-static --port 6006 --silent &
-                  pnpm wait-on http://127.0.0.1:6006 --timeout 60 # Wait for the server to be ready
+                  retries=3
+                  while [ $retries -gt 0 ]; do
+                      pnpm exec http-server storybook-static --port 6006 --silent &
+                      if pnpm wait-on http://127.0.0.1:6006 --timeout 60; then
+                          break
+                      fi
+                      retries=$((retries-1))
+                      echo "Failed to serve Storybook, retrying... ($retries retries left)"
+                  done
 
             - name: Run @storybook/test-runner
               env:
@@ -139,7 +146,14 @@ jobs:
                   # Update snapshots for PRs on the main repo, verify on forks, which don't have access to PostHog Bot
                   VARIANT: ${{ github.event.pull_request.head.repo.full_name == github.repository && 'update' || 'verify' }}
               run: |
-                  pnpm test:visual-regression:stories:ci:$VARIANT --browsers ${{ matrix.browser }} --shard ${{ matrix.shard }}/$SHARD_COUNT
+                  retries=3
+                  while [ $retries -gt 0 ]; do
+                      if pnpm test:visual-regression:stories:ci:$VARIANT --browsers ${{ matrix.browser }} --shard ${{ matrix.shard }}/$SHARD_COUNT; then
+                          break
+                      fi
+                      retries=$((retries-1))
+                      echo "Failed @storybook/test-runner, retrying... ($retries retries left)"
+                  done
 
             - name: Run @playwright/test (legacy, Chromium-only)
               if: matrix.browser == 'chromium' && matrix.shard == 1

cypress/e2e/insights-navigation.cy.ts

@@ -80,6 +80,7 @@ describe('Insights', () => {
 
         it('can open event explorer as an insight', () => {
             cy.clickNavMenu('events')
+            cy.get('[data-attr="data-table-export-menu"]').click()
             cy.get('[data-attr="open-json-editor-button"]').click()
             cy.get('[data-attr="insight-json-tab"]').should('exist')
         })

cypress/e2e/surveys.cy.ts

@@ -15,6 +15,7 @@ describe('Surveys', () => {
 
         // go to create a new survey
         cy.get('[data-attr="create-survey"]').click()
+        cy.get('[data-attr="new-blank-survey"]').click()
 
         cy.get('[data-attr="survey-name"]').type(name)
 
@@ -41,7 +42,7 @@ describe('Surveys', () => {
     it('shows survey disabled banner when surveys disabled', () => {
         cy.get('div.LemonBanner.LemonBanner--warning.mb-2').should(
             'contain',
-            'Survey popups are currently disabled for this project'
+            'Survey popovers are currently disabled for this project'
         )
         cy.get('div.LemonBanner.LemonBanner--warning.mb-2').contains('Configure').click()
 
@@ -65,7 +66,7 @@ describe('Surveys', () => {
         // now lemon banner should be back
         cy.get('div.LemonBanner.LemonBanner--warning.mb-2').should(
             'contain',
-            'Survey popups are currently disabled for this project'
+            'Survey popovers are currently disabled for this project'
         )
     })
 
@@ -76,6 +77,7 @@ describe('Surveys', () => {
 
         // click via top right button
         cy.get('[data-attr="new-survey"]').click()
+        cy.get('[data-attr="new-blank-survey"]').click()
 
         // select "add filter" and "property"
         cy.get('[data-attr="survey-name"]').type(name)
@@ -90,7 +92,7 @@ describe('Surveys', () => {
 
         cy.get('[id="scenes.surveys.surveyLogic.new.survey.questions.0.scale"]')
             .invoke('html')
-            .should('include', '1 - 10')
+            .should('include', '0 - 10')
 
         cy.get('[id="scenes.surveys.surveyLogic.new.survey.questions.0.upperBoundLabel"]').should(
             'have.value',
@@ -114,6 +116,9 @@ describe('Surveys', () => {
         cy.get('[data-attr="survey-preview"]').find('form').find('.ratings-number').should('have.length', 5)
 
         // add targeting filters
+        cy.get('.LemonCollapsePanel').contains('Targeting').click()
+        cy.contains('All users').click()
+        cy.get('.Popover__content').contains('Users who match').click()
         cy.contains('Add user targeting').click()
 
         // select the first property
@@ -156,6 +161,7 @@ describe('Surveys', () => {
         cy.get('.Popover__content').contains('Edit').click()
 
         // remove user targeting properties
+        cy.get('.LemonCollapsePanel').contains('Targeting').click()
         cy.contains('Remove all user properties').click()
 
         // save
@@ -176,6 +182,7 @@ describe('Surveys', () => {
     it('Delete survey', () => {
         cy.get('h1').should('contain', 'Surveys')
         cy.get('[data-attr=new-survey]').click()
+        cy.get('[data-attr=new-blank-survey]').click()
         cy.get('[data-attr=survey-name]').focus().type(name).should('have.value', name)
         cy.get('[data-attr=save-survey]').first().click()
 

cypress/support/e2e.ts

@@ -30,6 +30,7 @@ beforeEach(() => {
             decideResponse({
                 // set feature flags here e.g.
                 // 'toolbar-launch-side-action': true,
+                'surveys-new-creation-flow': true,
                 'surveys-results-visualizations': true,
                 'auto-redirect': true,
                 notebooks: true,

ee/api/test/__snapshots__/test_organization_resource_access.ambr

@@ -39,12 +39,13 @@
          "posthog_organization"."is_member_join_email_enabled",
          "posthog_organization"."enforce_2fa",
          "posthog_organization"."customer_id",
-         "posthog_organization"."available_features",
          "posthog_organization"."available_product_features",
          "posthog_organization"."usage",
+         "posthog_organization"."never_drop_data",
          "posthog_organization"."setup_section_2_completed",
          "posthog_organization"."personalization",
-         "posthog_organization"."domain_whitelist"
+         "posthog_organization"."domain_whitelist",
+         "posthog_organization"."available_features"
   FROM "posthog_organization"
   WHERE "posthog_organization"."id" = '00000000-0000-0000-0000-000000000000'::uuid
   LIMIT 21 /*controller='organization_resource_access-list',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/resource_access/%3F%24'*/
@@ -62,12 +63,13 @@
          "posthog_organization"."is_member_join_email_enabled",
          "posthog_organization"."enforce_2fa",
          "posthog_organization"."customer_id",
-         "posthog_organization"."available_features",
          "posthog_organization"."available_product_features",
          "posthog_organization"."usage",
+         "posthog_organization"."never_drop_data",
          "posthog_organization"."setup_section_2_completed",
          "posthog_organization"."personalization",
-         "posthog_organization"."domain_whitelist"
+         "posthog_organization"."domain_whitelist",
+         "posthog_organization"."available_features"
   FROM "posthog_organization"
   WHERE "posthog_organization"."id" = '00000000-0000-0000-0000-000000000000'::uuid
   LIMIT 21 /*controller='organization_resource_access-list',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/resource_access/%3F%24'*/
@@ -143,12 +145,13 @@
          "posthog_organization"."is_member_join_email_enabled",
          "posthog_organization"."enforce_2fa",
          "posthog_organization"."customer_id",
-         "posthog_organization"."available_features",
          "posthog_organization"."available_product_features",
          "posthog_organization"."usage",
+         "posthog_organization"."never_drop_data",
          "posthog_organization"."setup_section_2_completed",
          "posthog_organization"."personalization",
-         "posthog_organization"."domain_whitelist"
+         "posthog_organization"."domain_whitelist",
+         "posthog_organization"."available_features"
   FROM "posthog_organization"
   WHERE "posthog_organization"."id" = '00000000-0000-0000-0000-000000000000'::uuid
   LIMIT 21 /*controller='organization_resource_access-list',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/resource_access/%3F%24'*/
@@ -250,12 +253,13 @@
          "posthog_organization"."is_member_join_email_enabled",
          "posthog_organization"."enforce_2fa",
          "posthog_organization"."customer_id",
-         "posthog_organization"."available_features",
          "posthog_organization"."available_product_features",
          "posthog_organization"."usage",
+         "posthog_organization"."never_drop_data",
          "posthog_organization"."setup_section_2_completed",
          "posthog_organization"."personalization",
-         "posthog_organization"."domain_whitelist"
+         "posthog_organization"."domain_whitelist",
+         "posthog_organization"."available_features"
   FROM "posthog_organization"
   WHERE "posthog_organization"."id" = '00000000-0000-0000-0000-000000000000'::uuid
   LIMIT 21 /*controller='organization_resource_access-list',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/resource_access/%3F%24'*/

ee/billing/billing_manager.py

@@ -248,6 +248,11 @@ def update_org_details(self, organization: Organization, billing_status: Billing
             organization.available_product_features = data["available_product_features"]
             org_modified = True
 
+        never_drop_data = data.get("never_drop_data", None)
+        if never_drop_data != organization.never_drop_data:
+            organization.never_drop_data = never_drop_data
+            org_modified = True
+
         if org_modified:
             organization.save()
 

ee/billing/quota_limiting.py

@@ -14,9 +14,9 @@
 from posthog.models.team.team import Team
 from posthog.redis import get_client
 from posthog.tasks.usage_report import (
+    convert_team_usage_rows_to_dict,
     get_teams_with_billable_event_count_in_period,
     get_teams_with_recording_count_in_period,
-    convert_team_usage_rows_to_dict,
 )
 from posthog.utils import get_current_day
 
@@ -80,6 +80,9 @@ def org_quota_limited_until(organization: Organization, resource: QuotaResource)
     is_quota_limited = usage + todays_usage >= limit + OVERAGE_BUFFER[resource]
     billing_period_end = round(dateutil.parser.isoparse(organization.usage["period"][1]).timestamp())
 
+    if is_quota_limited and organization.never_drop_data:
+        return None
+
     if is_quota_limited and billing_period_end:
         return billing_period_end
 

ee/billing/test/test_quota_limiting.py

@@ -1,7 +1,6 @@
 import time
 from uuid import uuid4
 
-
 from dateutil.relativedelta import relativedelta
 from django.utils import timezone
 from django.utils.timezone import now
@@ -185,6 +184,23 @@ def test_org_quota_limited_until(self):
         self.organization.usage["recordings"]["usage"] = 1100  # Over limit + buffer
         assert org_quota_limited_until(self.organization, QuotaResource.RECORDINGS) == 1612137599
 
+    def test_over_quota_but_not_dropped_org(self):
+        self.organization.usage = None
+        assert org_quota_limited_until(self.organization, QuotaResource.EVENTS) is None
+
+        self.organization.usage = {
+            "events": {"usage": 100, "limit": 90},
+            "recordings": {"usage": 100, "limit": 90},
+            "period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
+        }
+        self.organization.never_drop_data = True
+
+        assert org_quota_limited_until(self.organization, QuotaResource.EVENTS) is None
+        assert org_quota_limited_until(self.organization, QuotaResource.RECORDINGS) is None
+
+        # reset for subsequent tests
+        self.organization.never_drop_data = False
+
     def test_sync_org_quota_limits(self):
         with freeze_time("2021-01-01T12:59:59Z"):
             other_team = create_team(organization=self.organization)

ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiment_secondary_results.ambr

@@ -1,6 +1,6 @@
 # name: ClickhouseTestExperimentSecondaryResults.test_basic_secondary_metric_results
   '
-  /* user_id:128 celery:posthog.celery.sync_insight_caching_state */
+  /* user_id:126 celery:posthog.celery.sync_insight_caching_state */
   SELECT team_id,
          date_diff('second', max(timestamp), now()) AS age
   FROM events

ee/clickhouse/views/test/test_clickhouse_experiments.py

@@ -119,7 +119,6 @@ def test_creating_updating_basic_experiment(self):
         self.assertEqual(experiment.end_date.strftime("%Y-%m-%dT%H:%M"), end_date)
 
     def test_adding_behavioral_cohort_filter_to_experiment_fails(self):
-
         cohort = Cohort.objects.create(
             team=self.team,
             filters={
@@ -739,7 +738,6 @@ def test_creating_experiment_with_group_aggregation_parameter(self):
         self.assertEqual(created_ff.filters["aggregation_group_type_index"], 0)
 
     def test_used_in_experiment_is_populated_correctly_for_feature_flag_list(self) -> None:
-
         ff_key = "a-b-test"
         response = self.client.post(
             f"/api/projects/{self.team.id}/experiments/",
@@ -945,7 +943,6 @@ def test_create_experiment_updates_feature_flag_cache(self):
 class ClickhouseTestFunnelExperimentResults(ClickhouseTestMixin, APILicensedTest):
     @snapshot_clickhouse_queries
     def test_experiment_flow_with_event_results(self):
-
         journeys_for(
             {
                 "person1": [
@@ -1031,7 +1028,6 @@ def test_experiment_flow_with_event_results(self):
 
     @snapshot_clickhouse_queries
     def test_experiment_flow_with_event_results_with_hogql_aggregation(self):
-
         journeys_for(
             {
                 "person1": [
@@ -1150,7 +1146,6 @@ def test_experiment_flow_with_event_results_with_hogql_aggregation(self):
         self.assertAlmostEqual(response_data["expected_loss"], 1, places=2)
 
     def test_experiment_flow_with_event_results_cached(self):
-
         journeys_for(
             {
                 "person1": [
@@ -1248,7 +1243,6 @@ def test_experiment_flow_with_event_results_cached(self):
 
     @snapshot_clickhouse_queries
     def test_experiment_flow_with_event_results_and_events_out_of_time_range_timezones(self):
-
         journeys_for(
             {
                 "person1": [

ee/session_recordings/session_recording_extensions.py

@@ -25,6 +25,7 @@
 
 MINIMUM_AGE_FOR_RECORDING = timedelta(hours=24)
 
+
 # TODO rename this...
 def save_recording_with_new_content(recording: SessionRecording, content: str) -> str:
     if not settings.OBJECT_STORAGE_ENABLED:

ee/tasks/subscriptions/__init__.py

@@ -3,7 +3,7 @@
 
 import structlog
 from prometheus_client import Counter
-from sentry_sdk import capture_exception
+from sentry_sdk import capture_exception, capture_message
 
 from ee.tasks.subscriptions.email_subscriptions import send_email_subscription_report
 from ee.tasks.subscriptions.slack_subscriptions import send_slack_subscription_report
@@ -41,11 +41,15 @@ def _deliver_subscription_report(
             # Same value as before so nothing to do
             return
 
+    insights, assets = generate_assets(subscription)
+
+    if not assets:
+        capture_message("No assets are in this subscription", tags={"subscription_id": subscription.id})
+        return
+
     if subscription.target_type == "email":
         SUBSCRIPTION_QUEUED.labels(destination="email").inc()
 
-        insights, assets = generate_assets(subscription)
-
         # Send emails
         emails = subscription.target_value.split(",")
         if is_new_subscription_target:
@@ -77,7 +81,6 @@ def _deliver_subscription_report(
     elif subscription.target_type == "slack":
         SUBSCRIPTION_QUEUED.labels(destination="slack").inc()
 
-        insights, assets = generate_assets(subscription)
         try:
             send_slack_subscription_report(
                 subscription, assets, total_asset_count=len(insights), is_new_subscription=is_new_subscription_target

ee/tasks/subscriptions/subscription_utils.py

@@ -44,9 +44,12 @@ def generate_assets(
         ]
         ExportedAsset.objects.bulk_create(assets)
 
+        if not assets:
+            return insights, assets
+
         # Wait for all assets to be exported
         tasks = [exporter.export_asset.si(asset.id) for asset in assets]
-        # run them one after the other so we don't exhaust celery workers
+        # run them one after the other, so we don't exhaust celery workers
         parallel_job = chain(*tasks).apply_async()
 
         wait_for_parallel_celery_group(