feat: add field to never drop data for a customer

ee/api/test/__snapshots__/test_organization_resource_access.ambr

@@ -39,12 +39,13 @@
          "posthog_organization"."is_member_join_email_enabled",
          "posthog_organization"."enforce_2fa",
          "posthog_organization"."customer_id",
-         "posthog_organization"."available_features",
          "posthog_organization"."available_product_features",
          "posthog_organization"."usage",
+         "posthog_organization"."never_drop_data",
          "posthog_organization"."setup_section_2_completed",
          "posthog_organization"."personalization",
-         "posthog_organization"."domain_whitelist"
+         "posthog_organization"."domain_whitelist",
+         "posthog_organization"."available_features"
   FROM "posthog_organization"
   WHERE "posthog_organization"."id" = '00000000-0000-0000-0000-000000000000'::uuid
   LIMIT 21 /*controller='organization_resource_access-list',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/resource_access/%3F%24'*/
@@ -62,12 +63,13 @@
          "posthog_organization"."is_member_join_email_enabled",
          "posthog_organization"."enforce_2fa",
          "posthog_organization"."customer_id",
-         "posthog_organization"."available_features",
          "posthog_organization"."available_product_features",
          "posthog_organization"."usage",
+         "posthog_organization"."never_drop_data",
          "posthog_organization"."setup_section_2_completed",
          "posthog_organization"."personalization",
-         "posthog_organization"."domain_whitelist"
+         "posthog_organization"."domain_whitelist",
+         "posthog_organization"."available_features"
   FROM "posthog_organization"
   WHERE "posthog_organization"."id" = '00000000-0000-0000-0000-000000000000'::uuid
   LIMIT 21 /*controller='organization_resource_access-list',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/resource_access/%3F%24'*/
@@ -143,12 +145,13 @@
          "posthog_organization"."is_member_join_email_enabled",
          "posthog_organization"."enforce_2fa",
          "posthog_organization"."customer_id",
-         "posthog_organization"."available_features",
          "posthog_organization"."available_product_features",
          "posthog_organization"."usage",
+         "posthog_organization"."never_drop_data",
          "posthog_organization"."setup_section_2_completed",
          "posthog_organization"."personalization",
-         "posthog_organization"."domain_whitelist"
+         "posthog_organization"."domain_whitelist",
+         "posthog_organization"."available_features"
   FROM "posthog_organization"
   WHERE "posthog_organization"."id" = '00000000-0000-0000-0000-000000000000'::uuid
   LIMIT 21 /*controller='organization_resource_access-list',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/resource_access/%3F%24'*/
@@ -250,12 +253,13 @@
          "posthog_organization"."is_member_join_email_enabled",
          "posthog_organization"."enforce_2fa",
          "posthog_organization"."customer_id",
-         "posthog_organization"."available_features",
          "posthog_organization"."available_product_features",
          "posthog_organization"."usage",
+         "posthog_organization"."never_drop_data",
          "posthog_organization"."setup_section_2_completed",
          "posthog_organization"."personalization",
-         "posthog_organization"."domain_whitelist"
+         "posthog_organization"."domain_whitelist",
+         "posthog_organization"."available_features"
   FROM "posthog_organization"
   WHERE "posthog_organization"."id" = '00000000-0000-0000-0000-000000000000'::uuid
   LIMIT 21 /*controller='organization_resource_access-list',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/resource_access/%3F%24'*/

ee/billing/billing_manager.py

@@ -248,6 +248,11 @@ def update_org_details(self, organization: Organization, billing_status: Billing
             organization.available_product_features = data["available_product_features"]
             org_modified = True
 
+        never_drop_data = data.get("never_drop_data", None)
+        if never_drop_data != organization.never_drop_data:
+            organization.never_drop_data = never_drop_data
+            org_modified = True
+
         if org_modified:
             organization.save()
 

ee/billing/quota_limiting.py

@@ -4,23 +4,28 @@
 from typing import Dict, List, Mapping, Optional, Sequence, TypedDict, cast
 
 import dateutil.parser
+from django.core.cache import cache
 from django.db.models import Q
 from django.utils import timezone
 from sentry_sdk import capture_exception
 
 from posthog.cache_utils import cache_for
+from posthog.email import is_email_available
 from posthog.event_usage import report_organization_action
 from posthog.models.organization import Organization, OrganizationUsageInfo
 from posthog.models.team.team import Team
 from posthog.redis import get_client
+from posthog.tasks.email import send_over_quota_but_not_dropped_email_to_cs
 from posthog.tasks.usage_report import (
+    convert_team_usage_rows_to_dict,
     get_teams_with_billable_event_count_in_period,
     get_teams_with_recording_count_in_period,
-    convert_team_usage_rows_to_dict,
 )
 from posthog.utils import get_current_day
 
 QUOTA_LIMITER_CACHE_KEY = "@posthog/quota-limits/"
+QUOTA_OVERAGE_NO_DROP_EMAILED_CACHE_KEY = "@posthog/quota-overage-no-drop-emailed/"
+QUOTA_OVERAGE_NO_DROP_EMAILED_CACHE_TIMEOUT = 60 * 60 * 24 * 7  # 7 days, 604800 seconds
 
 
 class QuotaResource(Enum):
@@ -60,6 +65,19 @@ def list_limited_team_tokens(resource: QuotaResource) -> List[str]:
     return [x.decode("utf-8") for x in results]
 
 
+def add_quota_overage_no_drop_cache_item(organization_id: int) -> None:
+    now = timezone.now()
+    cache.set(
+        f"{QUOTA_OVERAGE_NO_DROP_EMAILED_CACHE_KEY}{organization_id}",
+        now,
+        timeout=QUOTA_OVERAGE_NO_DROP_EMAILED_CACHE_TIMEOUT,
+    )
+
+
+def get_quota_overage_no_drop_cache_item(organization_id: int) -> bool:
+    return cache.get(f"{QUOTA_OVERAGE_NO_DROP_EMAILED_CACHE_KEY}{organization_id}", False)
+
+
 class UsageCounters(TypedDict):
     events: int
     recordings: int
@@ -80,6 +98,13 @@ def org_quota_limited_until(organization: Organization, resource: QuotaResource)
     is_quota_limited = usage + todays_usage >= limit + OVERAGE_BUFFER[resource]
     billing_period_end = round(dateutil.parser.isoparse(organization.usage["period"][1]).timestamp())
 
+    if is_quota_limited and organization.never_drop_data:
+        if not get_quota_overage_no_drop_cache_item(organization.id):
+            add_quota_overage_no_drop_cache_item(organization.id)
+            if is_email_available():
+                send_over_quota_but_not_dropped_email_to_cs.delay(organization.id)
+        return None
+
     if is_quota_limited and billing_period_end:
         return billing_period_end
 

ee/billing/test/test_quota_limiting.py

@@ -1,8 +1,9 @@
 import time
+from unittest.mock import patch
 from uuid import uuid4
 
-
 from dateutil.relativedelta import relativedelta
+from django.core.cache import cache
 from django.utils import timezone
 from django.utils.timezone import now
 from freezegun import freeze_time
@@ -185,6 +186,51 @@ def test_org_quota_limited_until(self):
         self.organization.usage["recordings"]["usage"] = 1100  # Over limit + buffer
         assert org_quota_limited_until(self.organization, QuotaResource.RECORDINGS) == 1612137599
 
+    @patch("ee.billing.quota_limiting.is_email_available", return_value=True)
+    @patch("ee.billing.quota_limiting.send_over_quota_but_not_dropped_email_to_cs.delay")
+    def test_over_quota_but_not_dropped_org(self, mock_email, mock_is_email_available):
+        self.organization.usage = None
+        assert org_quota_limited_until(self.organization, QuotaResource.EVENTS) is None
+
+        self.organization.usage = {
+            "events": {"usage": 100, "limit": 90},
+            "recordings": {"usage": 100, "limit": 90},
+            "period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
+        }
+        self.organization.never_drop_data = True
+
+        assert org_quota_limited_until(self.organization, QuotaResource.EVENTS) is None
+        assert org_quota_limited_until(self.organization, QuotaResource.RECORDINGS) is None
+
+        mock_is_email_available.assert_called_once()
+        mock_email.assert_called_once_with(self.organization.id)
+
+        self.organization.never_drop_data = False
+
+    @patch("ee.billing.quota_limiting.is_email_available", return_value=True)
+    @patch("ee.billing.quota_limiting.send_over_quota_but_not_dropped_email_to_cs.delay")
+    def test_over_quota_but_not_dropped_org_already_emailed(self, mock_email, mock_is_email_available):
+        self.organization.usage = None
+        assert org_quota_limited_until(self.organization, QuotaResource.EVENTS) is None
+        cache.set(
+            f"@posthog/quota-overage-no-drop-emailed/{self.organization.id}", timezone.now(), timeout=60 * 60 * 24 * 7
+        )
+
+        self.organization.usage = {
+            "events": {"usage": 100, "limit": 90},
+            "recordings": {"usage": 100, "limit": 90},
+            "period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"],
+        }
+        self.organization.never_drop_data = True
+
+        assert org_quota_limited_until(self.organization, QuotaResource.EVENTS) is None
+        assert org_quota_limited_until(self.organization, QuotaResource.RECORDINGS) is None
+
+        mock_is_email_available.assert_not_called()
+        mock_email.assert_not_called()
+
+        self.organization.never_drop_data = False
+
     def test_sync_org_quota_limits(self):
         with freeze_time("2021-01-01T12:59:59Z"):
             other_team = create_team(organization=self.organization)

ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiment_secondary_results.ambr

@@ -1,6 +1,6 @@
 # name: ClickhouseTestExperimentSecondaryResults.test_basic_secondary_metric_results
   '
-  /* user_id:51 celery:posthog.celery.sync_insight_caching_state */
+  /* user_id:53 celery:posthog.celery.sync_insight_caching_state */
   SELECT team_id,
          date_diff('second', max(timestamp), now()) AS age
   FROM events

ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiments.ambr

@@ -1,6 +1,6 @@
 # name: ClickhouseTestFunnelExperimentResults.test_experiment_flow_with_event_results
   '
-  /* user_id:58 celery:posthog.celery.sync_insight_caching_state */
+  /* user_id:60 celery:posthog.celery.sync_insight_caching_state */
   SELECT team_id,
          date_diff('second', max(timestamp), now()) AS age
   FROM events
@@ -138,7 +138,7 @@
 ---
 # name: ClickhouseTestFunnelExperimentResults.test_experiment_flow_with_event_results_and_events_out_of_time_range_timezones
   '
-  /* user_id:59 celery:posthog.celery.sync_insight_caching_state */
+  /* user_id:61 celery:posthog.celery.sync_insight_caching_state */
   SELECT team_id,
          date_diff('second', max(timestamp), now()) AS age
   FROM events
@@ -276,7 +276,7 @@
 ---
 # name: ClickhouseTestFunnelExperimentResults.test_experiment_flow_with_event_results_for_three_test_variants
   '
-  /* user_id:61 celery:posthog.celery.sync_insight_caching_state */
+  /* user_id:63 celery:posthog.celery.sync_insight_caching_state */
   SELECT team_id,
          date_diff('second', max(timestamp), now()) AS age
   FROM events
@@ -414,7 +414,7 @@
 ---
 # name: ClickhouseTestFunnelExperimentResults.test_experiment_flow_with_event_results_with_hogql_aggregation
   '
-  /* user_id:62 celery:posthog.celery.sync_insight_caching_state */
+  /* user_id:64 celery:posthog.celery.sync_insight_caching_state */
   SELECT team_id,
          date_diff('second', max(timestamp), now()) AS age
   FROM events
@@ -552,7 +552,7 @@
 ---
 # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results
   '
-  /* user_id:65 celery:posthog.celery.sync_insight_caching_state */
+  /* user_id:67 celery:posthog.celery.sync_insight_caching_state */
   SELECT team_id,
          date_diff('second', max(timestamp), now()) AS age
   FROM events
@@ -749,7 +749,7 @@
 ---
 # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_for_three_test_variants
   '
-  /* user_id:66 celery:posthog.celery.sync_insight_caching_state */
+  /* user_id:68 celery:posthog.celery.sync_insight_caching_state */
   SELECT team_id,
          date_diff('second', max(timestamp), now()) AS age
   FROM events
@@ -892,7 +892,7 @@
 ---
 # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_out_of_timerange_timezone
   '
-  /* user_id:68 celery:posthog.celery.sync_insight_caching_state */
+  /* user_id:70 celery:posthog.celery.sync_insight_caching_state */
   SELECT team_id,
          date_diff('second', max(timestamp), now()) AS age
   FROM events
@@ -1089,7 +1089,7 @@
 ---
 # name: ClickhouseTestTrendExperimentResults.test_experiment_flow_with_event_results_with_hogql_filter
   '
-  /* user_id:70 celery:posthog.celery.sync_insight_caching_state */
+  /* user_id:72 celery:posthog.celery.sync_insight_caching_state */
   SELECT team_id,
          date_diff('second', max(timestamp), now()) AS age
   FROM events

frontend/src/scenes/billing/billingLogic.ts

@@ -167,7 +167,9 @@ export const billingLogic = kea<billingLogicType>([
                     return {
                         status: 'error',
                         title: 'Usage limit exceeded',
-                        message: `You have exceeded the usage limit for ${productOverLimit.name}. Please upgrade your plan or data loss may occur.`,
+                        message: `You have exceeded the usage limit for ${productOverLimit.name}. Please ${
+                            productOverLimit.subscribed ? 'increase your billing limit' : 'upgrade your plan'
+                        } or data loss may occur.`,
                     }
                 }
 

latest_migrations.manifest

@@ -5,7 +5,7 @@ contenttypes: 0002_remove_content_type_name
 ee: 0015_add_verified_properties
 otp_static: 0002_throttling
 otp_totp: 0002_auto_20190420_0723
-posthog: 0348_alter_datawarehousetable_format
+posthog: 0349_organization_never_drop_data
 sessions: 0001_initial
 social_django: 0010_uid_db_index
 two_factor: 0007_auto_20201201_1019

posthog/api/test/__snapshots__/test_action.ambr

@@ -238,12 +238,13 @@
          "posthog_organization"."is_member_join_email_enabled",
          "posthog_organization"."enforce_2fa",
          "posthog_organization"."customer_id",
-         "posthog_organization"."available_features",
          "posthog_organization"."available_product_features",
          "posthog_organization"."usage",
+         "posthog_organization"."never_drop_data",
          "posthog_organization"."setup_section_2_completed",
          "posthog_organization"."personalization",
-         "posthog_organization"."domain_whitelist"
+         "posthog_organization"."domain_whitelist",
+         "posthog_organization"."available_features"
   FROM "posthog_organizationmembership"
   INNER JOIN "posthog_organization" ON ("posthog_organizationmembership"."organization_id" = "posthog_organization"."id")
   WHERE "posthog_organizationmembership"."user_id" = 2 /*controller='project_actions-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/actions/%3F%24'*/
@@ -261,12 +262,13 @@
          "posthog_organization"."is_member_join_email_enabled",
          "posthog_organization"."enforce_2fa",
          "posthog_organization"."customer_id",
-         "posthog_organization"."available_features",
          "posthog_organization"."available_product_features",
          "posthog_organization"."usage",
+         "posthog_organization"."never_drop_data",
          "posthog_organization"."setup_section_2_completed",
          "posthog_organization"."personalization",
-         "posthog_organization"."domain_whitelist"
+         "posthog_organization"."domain_whitelist",
+         "posthog_organization"."available_features"
   FROM "posthog_organization"
   WHERE "posthog_organization"."id" = '00000000-0000-0000-0000-000000000000'::uuid
   LIMIT 21 /*controller='project_actions-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/actions/%3F%24'*/
@@ -366,12 +368,13 @@
          "posthog_organization"."is_member_join_email_enabled",
          "posthog_organization"."enforce_2fa",
          "posthog_organization"."customer_id",
-         "posthog_organization"."available_features",
          "posthog_organization"."available_product_features",
          "posthog_organization"."usage",
+         "posthog_organization"."never_drop_data",
          "posthog_organization"."setup_section_2_completed",
          "posthog_organization"."personalization",
-         "posthog_organization"."domain_whitelist"
+         "posthog_organization"."domain_whitelist",
+         "posthog_organization"."available_features"
   FROM "posthog_organizationmembership"
   INNER JOIN "posthog_organization" ON ("posthog_organizationmembership"."organization_id" = "posthog_organization"."id")
   WHERE "posthog_organizationmembership"."user_id" = 2 /*controller='project_actions-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/actions/%3F%24'*/
@@ -400,12 +403,13 @@
          "posthog_organization"."is_member_join_email_enabled",
          "posthog_organization"."enforce_2fa",
          "posthog_organization"."customer_id",
-         "posthog_organization"."available_features",
          "posthog_organization"."available_product_features",
          "posthog_organization"."usage",
+         "posthog_organization"."never_drop_data",
          "posthog_organization"."setup_section_2_completed",
          "posthog_organization"."personalization",
-         "posthog_organization"."domain_whitelist"
+         "posthog_organization"."domain_whitelist",
+         "posthog_organization"."available_features"
   FROM "posthog_organization"
   WHERE "posthog_organization"."id" = '00000000-0000-0000-0000-000000000000'::uuid
   LIMIT 21 /*controller='project_actions-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/actions/%3F%24'*/
@@ -554,12 +558,13 @@
          "posthog_organization"."is_member_join_email_enabled",
          "posthog_organization"."enforce_2fa",
          "posthog_organization"."customer_id",
-         "posthog_organization"."available_features",
          "posthog_organization"."available_product_features",
          "posthog_organization"."usage",
+         "posthog_organization"."never_drop_data",
          "posthog_organization"."setup_section_2_completed",
          "posthog_organization"."personalization",
-         "posthog_organization"."domain_whitelist"
+         "posthog_organization"."domain_whitelist",
+         "posthog_organization"."available_features"
   FROM "posthog_organizationmembership"
   INNER JOIN "posthog_organization" ON ("posthog_organizationmembership"."organization_id" = "posthog_organization"."id")
   WHERE "posthog_organizationmembership"."user_id" = 2 /*controller='project_actions-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/actions/%3F%24'*/
@@ -577,12 +582,13 @@
          "posthog_organization"."is_member_join_email_enabled",
          "posthog_organization"."enforce_2fa",
          "posthog_organization"."customer_id",
-         "posthog_organization"."available_features",
          "posthog_organization"."available_product_features",
          "posthog_organization"."usage",
+         "posthog_organization"."never_drop_data",
          "posthog_organization"."setup_section_2_completed",
          "posthog_organization"."personalization",
-         "posthog_organization"."domain_whitelist"
+         "posthog_organization"."domain_whitelist",
+         "posthog_organization"."available_features"
   FROM "posthog_organization"
   WHERE "posthog_organization"."id" = '00000000-0000-0000-0000-000000000000'::uuid
   LIMIT 21 /*controller='project_actions-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/actions/%3F%24'*/