diff --git a/ee/api/test/test_billing.py b/ee/api/test/test_billing.py index 87838d0b39dcc..88addd2d7f416 100644 --- a/ee/api/test/test_billing.py +++ b/ee/api/test/test_billing.py @@ -43,6 +43,7 @@ def create_missing_billing_customer(**kwargs) -> CustomerInfo: usage_summary={ "events": {"limit": None, "usage": 0}, "recordings": {"limit": None, "usage": 0}, + "rows_synced": {"limit": None, "usage": 0}, }, free_trial_until=None, available_features=[], @@ -96,6 +97,7 @@ def create_billing_customer(**kwargs) -> CustomerInfo: usage_summary={ "events": {"limit": None, "usage": 0}, "recordings": {"limit": None, "usage": 0}, + "rows_synced": {"limit": None, "usage": 0}, }, free_trial_until=None, ) @@ -292,6 +294,7 @@ def mock_implementation(url: str, headers: Any = None, params: Any = None) -> Ma "usage_summary": { "events": {"limit": None, "usage": 0}, "recordings": {"limit": None, "usage": 0}, + "rows_synced": {"limit": None, "usage": 0}, }, "free_trial_until": None, } @@ -363,6 +366,7 @@ def mock_implementation(url: str, headers: Any = None, params: Any = None) -> Ma "usage_summary": { "events": {"limit": None, "usage": 0}, "recordings": {"limit": None, "usage": 0}, + "rows_synced": {"limit": None, "usage": 0}, }, "free_trial_until": None, "current_total_amount_usd": "0.00", @@ -521,6 +525,11 @@ def mock_implementation(url: str, headers: Any = None, params: Any = None) -> Ma "todays_usage": 0, "usage": 0, }, + "rows_synced": { + "limit": None, + "todays_usage": 0, + "usage": 0, + }, "period": ["2022-10-07T11:12:48", "2022-11-07T11:12:48"], } @@ -556,6 +565,11 @@ def mock_implementation_missing_customer(url: str, headers: Any = None, params: "todays_usage": 0, "usage": 0, }, + "rows_synced": { + "limit": None, + "todays_usage": 0, + "usage": 0, + }, "period": ["2022-10-07T11:12:48", "2022-11-07T11:12:48"], } assert self.organization.customer_id == "cus_123" @@ -613,5 +627,6 @@ def mock_implementation(url: str, headers: Any = None, params: Any = None) -> Ma assert self.organization.usage == { "events": {"limit": None, "usage": 0, "todays_usage": 0}, "recordings": {"limit": None, "usage": 0, "todays_usage": 0}, + "rows_synced": {"limit": None, "usage": 0, "todays_usage": 0}, "period": ["2022-10-07T11:12:48", "2022-11-07T11:12:48"], } diff --git a/ee/billing/billing_manager.py b/ee/billing/billing_manager.py index c626083460ef4..a906f6796603e 100644 --- a/ee/billing/billing_manager.py +++ b/ee/billing/billing_manager.py @@ -225,6 +225,7 @@ def update_org_details(self, organization: Organization, billing_status: Billing usage_info = OrganizationUsageInfo( events=usage_summary["events"], recordings=usage_summary["recordings"], + rows_synced=usage_summary["rows_synced"], period=[ data["billing_period"]["current_period_start"], data["billing_period"]["current_period_end"], diff --git a/ee/billing/quota_limiting.py b/ee/billing/quota_limiting.py index ae6eefcc0b77a..0a6860d62769c 100644 --- a/ee/billing/quota_limiting.py +++ b/ee/billing/quota_limiting.py @@ -17,6 +17,7 @@ convert_team_usage_rows_to_dict, get_teams_with_billable_event_count_in_period, get_teams_with_recording_count_in_period, + get_teams_with_rows_synced_in_period, ) from posthog.utils import get_current_day @@ -26,11 +27,13 @@ class QuotaResource(Enum): EVENTS = "events" RECORDINGS = "recordings" + ROWS_SYNCED = "rows_synced" OVERAGE_BUFFER = { QuotaResource.EVENTS: 0, QuotaResource.RECORDINGS: 1000, + QuotaResource.ROWS_SYNCED: 0, } @@ -53,7 +56,7 @@ def remove_limited_team_tokens(resource: QuotaResource, tokens: List[str]) -> No @cache_for(timedelta(seconds=30), background_refresh=True) -def list_limited_team_tokens(resource: QuotaResource) -> List[str]: +def list_limited_team_attributes(resource: QuotaResource) -> List[str]: now = timezone.now() redis_client = get_client() results = redis_client.zrangebyscore(f"{QUOTA_LIMITER_CACHE_KEY}{resource.value}", min=now.timestamp(), max="+inf") @@ -63,6 +66,7 @@ def list_limited_team_tokens(resource: QuotaResource) -> List[str]: class UsageCounters(TypedDict): events: int recordings: int + rows_synced: int def org_quota_limited_until(organization: Organization, resource: QuotaResource) -> Optional[int]: @@ -93,19 +97,34 @@ def sync_org_quota_limits(organization: Organization): if not organization.usage: return None - team_tokens: List[str] = [x for x in list(organization.teams.values_list("api_token", flat=True)) if x] - - if not team_tokens: - capture_exception(Exception(f"quota_limiting: No team tokens found for organization: {organization.id}")) - return - - for resource in [QuotaResource.EVENTS, QuotaResource.RECORDINGS]: + for resource in [QuotaResource.EVENTS, QuotaResource.RECORDINGS, QuotaResource.ROWS_SYNCED]: + team_attributes = get_team_attribute_by_quota_resource(organization, resource) quota_limited_until = org_quota_limited_until(organization, resource) if quota_limited_until: - add_limited_team_tokens(resource, {x: quota_limited_until for x in team_tokens}) + add_limited_team_tokens(resource, {x: quota_limited_until for x in team_attributes}) else: - remove_limited_team_tokens(resource, team_tokens) + remove_limited_team_tokens(resource, team_attributes) + + +def get_team_attribute_by_quota_resource(organization: Organization, resource: QuotaResource): + if resource in [QuotaResource.EVENTS, QuotaResource.RECORDINGS]: + team_tokens: List[str] = [x for x in list(organization.teams.values_list("api_token", flat=True)) if x] + + if not team_tokens: + capture_exception(Exception(f"quota_limiting: No team tokens found for organization: {organization.id}")) + return + + return team_tokens + + if resource == QuotaResource.ROWS_SYNCED: + team_ids: List[str] = [x for x in list(organization.teams.values_list("id", flat=True)) if x] + + if not team_ids: + capture_exception(Exception(f"quota_limiting: No team ids found for organization: {organization.id}")) + return + + return team_ids def set_org_usage_summary( @@ -125,7 +144,7 @@ def set_org_usage_summary( new_usage = copy.deepcopy(new_usage) - for field in ["events", "recordings"]: + for field in ["events", "recordings", "rows_synced"]: resource_usage = new_usage[field] # type: ignore if todays_usage: @@ -155,6 +174,9 @@ def update_all_org_billing_quotas(dry_run: bool = False) -> Dict[str, Dict[str, teams_with_recording_count_in_period=convert_team_usage_rows_to_dict( get_teams_with_recording_count_in_period(period_start, period_end) ), + teams_with_rows_synced_in_period=convert_team_usage_rows_to_dict( + get_teams_with_rows_synced_in_period(period_start, period_end) + ), ) teams: Sequence[Team] = list( @@ -171,6 +193,7 @@ def update_all_org_billing_quotas(dry_run: bool = False) -> Dict[str, Dict[str, team_report = UsageCounters( events=all_data["teams_with_event_count_in_period"].get(team.id, 0), recordings=all_data["teams_with_recording_count_in_period"].get(team.id, 0), + rows_synced=all_data["teams_with_rows_synced_in_period"].get(team.id, 0), ) org_id = str(team.organization.id) @@ -183,7 +206,7 @@ def update_all_org_billing_quotas(dry_run: bool = False) -> Dict[str, Dict[str, for field in team_report: org_report[field] += team_report[field] # type: ignore - quota_limited_orgs: Dict[str, Dict[str, int]] = {"events": {}, "recordings": {}} + quota_limited_orgs: Dict[str, Dict[str, int]] = {"events": {}, "recordings": {}, "rows_synced": {}} # We find all orgs that should be rate limited for org_id, todays_report in todays_usage_report.items(): @@ -195,7 +218,7 @@ def update_all_org_billing_quotas(dry_run: bool = False) -> Dict[str, Dict[str, if set_org_usage_summary(org, todays_usage=todays_report): org.save(update_fields=["usage"]) - for field in ["events", "recordings"]: + for field in ["events", "recordings", "rows_synced"]: quota_limited_until = org_quota_limited_until(org, QuotaResource(field)) if quota_limited_until: @@ -207,12 +230,13 @@ def update_all_org_billing_quotas(dry_run: bool = False) -> Dict[str, Dict[str, previously_quota_limited_team_tokens: Dict[str, Dict[str, int]] = { "events": {}, "recordings": {}, + "rows_synced": {}, } for field in quota_limited_orgs: - previously_quota_limited_team_tokens[field] = list_limited_team_tokens(QuotaResource(field)) + previously_quota_limited_team_tokens[field] = list_limited_team_attributes(QuotaResource(field)) - quota_limited_teams: Dict[str, Dict[str, int]] = {"events": {}, "recordings": {}} + quota_limited_teams: Dict[str, Dict[str, int]] = {"events": {}, "recordings": {}, "rows_synced": {}} # Convert the org ids to team tokens for team in teams: @@ -233,6 +257,7 @@ def update_all_org_billing_quotas(dry_run: bool = False) -> Dict[str, Dict[str, properties = { "quota_limited_events": quota_limited_orgs["events"].get(org_id, None), "quota_limited_recordings": quota_limited_orgs["events"].get(org_id, None), + "quota_limited_rows_synced": quota_limited_orgs["rows_synced"].get(org_id, None), } report_organization_action( diff --git a/ee/billing/test/test_quota_limiting.py b/ee/billing/test/test_quota_limiting.py index 3bdc70a06df9e..b8e68c235b2c5 100644 --- a/ee/billing/test/test_quota_limiting.py +++ b/ee/billing/test/test_quota_limiting.py @@ -9,7 +9,7 @@ from ee.billing.quota_limiting import ( QUOTA_LIMITER_CACHE_KEY, QuotaResource, - list_limited_team_tokens, + list_limited_team_attributes, org_quota_limited_until, replace_limited_team_tokens, set_org_usage_summary, @@ -47,15 +47,18 @@ def test_billing_rate_limit_not_set_if_missing_org_usage(self) -> None: result = update_all_org_billing_quotas() assert result["events"] == {} assert result["recordings"] == {} + assert result["rows_synced"] == {} assert self.redis_client.zrange(f"{QUOTA_LIMITER_CACHE_KEY}events", 0, -1) == [] assert self.redis_client.zrange(f"{QUOTA_LIMITER_CACHE_KEY}recordings", 0, -1) == [] + assert self.redis_client.zrange(f"{QUOTA_LIMITER_CACHE_KEY}rows_synced", 0, -1) == [] def test_billing_rate_limit(self) -> None: with self.settings(USE_TZ=False): self.organization.usage = { "events": {"usage": 99, "limit": 100}, "recordings": {"usage": 1, "limit": 100}, + "rows_synced": {"usage": 5, "limit": 100}, "period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"], } self.organization.save() @@ -77,16 +80,19 @@ def test_billing_rate_limit(self) -> None: org_id = str(self.organization.id) assert result["events"] == {org_id: 1612137599} assert result["recordings"] == {} + assert result["rows_synced"] == {} assert self.redis_client.zrange(f"{QUOTA_LIMITER_CACHE_KEY}events", 0, -1) == [ self.team.api_token.encode("UTF-8") ] assert self.redis_client.zrange(f"{QUOTA_LIMITER_CACHE_KEY}recordings", 0, -1) == [] + assert self.redis_client.zrange(f"{QUOTA_LIMITER_CACHE_KEY}rows_synced", 0, -1) == [] self.organization.refresh_from_db() assert self.organization.usage == { "events": {"usage": 99, "limit": 100, "todays_usage": 10}, "recordings": {"usage": 1, "limit": 100, "todays_usage": 0}, + "rows_synced": {"usage": 5, "limit": 100, "todays_usage": 0}, "period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"], } @@ -94,6 +100,7 @@ def test_set_org_usage_summary_updates_correctly(self): self.organization.usage = { "events": {"usage": 99, "limit": 100}, "recordings": {"usage": 1, "limit": 100}, + "rows_synced": {"usage": 5, "limit": 100}, "period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"], } self.organization.save() @@ -101,6 +108,7 @@ def test_set_org_usage_summary_updates_correctly(self): new_usage = dict( events={"usage": 100, "limit": 100}, recordings={"usage": 2, "limit": 100}, + rows_synced={"usage": 6, "limit": 100}, period=[ "2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z", @@ -112,6 +120,7 @@ def test_set_org_usage_summary_updates_correctly(self): assert self.organization.usage == { "events": {"usage": 100, "limit": 100, "todays_usage": 0}, "recordings": {"usage": 2, "limit": 100, "todays_usage": 0}, + "rows_synced": {"usage": 6, "limit": 100, "todays_usage": 0}, "period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"], } @@ -119,6 +128,7 @@ def test_set_org_usage_summary_does_nothing_if_the_same(self): self.organization.usage = { "events": {"usage": 99, "limit": 100, "todays_usage": 10}, "recordings": {"usage": 1, "limit": 100, "todays_usage": 11}, + "rows_synced": {"usage": 5, "limit": 100, "todays_usage": 11}, "period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"], } self.organization.save() @@ -126,6 +136,7 @@ def test_set_org_usage_summary_does_nothing_if_the_same(self): new_usage = dict( events={"usage": 99, "limit": 100}, recordings={"usage": 1, "limit": 100}, + rows_synced={"usage": 5, "limit": 100}, period=[ "2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z", @@ -137,6 +148,7 @@ def test_set_org_usage_summary_does_nothing_if_the_same(self): assert self.organization.usage == { "events": {"usage": 99, "limit": 100, "todays_usage": 10}, "recordings": {"usage": 1, "limit": 100, "todays_usage": 11}, + "rows_synced": {"usage": 5, "limit": 100, "todays_usage": 11}, "period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"], } @@ -144,15 +156,19 @@ def test_set_org_usage_summary_updates_todays_usage(self): self.organization.usage = { "events": {"usage": 99, "limit": 100, "todays_usage": 10}, "recordings": {"usage": 1, "limit": 100, "todays_usage": 11}, + "rows_synced": {"usage": 5, "limit": 100, "todays_usage": 11}, "period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"], } self.organization.save() - assert set_org_usage_summary(self.organization, todays_usage={"events": 20, "recordings": 21}) + assert set_org_usage_summary( + self.organization, todays_usage={"events": 20, "recordings": 21, "rows_synced": 21} + ) assert self.organization.usage == { "events": {"usage": 99, "limit": 100, "todays_usage": 20}, "recordings": {"usage": 1, "limit": 100, "todays_usage": 21}, + "rows_synced": {"usage": 5, "limit": 100, "todays_usage": 21}, "period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"], } @@ -163,6 +179,7 @@ def test_org_quota_limited_until(self): self.organization.usage = { "events": {"usage": 99, "limit": 100}, "recordings": {"usage": 1, "limit": 100}, + "rows_synced": {"usage": 99, "limit": 100}, "period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"], } @@ -184,6 +201,11 @@ def test_org_quota_limited_until(self): self.organization.usage["recordings"]["usage"] = 1100 # Over limit + buffer assert org_quota_limited_until(self.organization, QuotaResource.RECORDINGS) == 1612137599 + assert org_quota_limited_until(self.organization, QuotaResource.ROWS_SYNCED) is None + + self.organization.usage["rows_synced"]["usage"] = 101 + assert org_quota_limited_until(self.organization, QuotaResource.ROWS_SYNCED) == 1612137599 + def test_over_quota_but_not_dropped_org(self): self.organization.usage = None assert org_quota_limited_until(self.organization, QuotaResource.EVENTS) is None @@ -191,12 +213,14 @@ def test_over_quota_but_not_dropped_org(self): self.organization.usage = { "events": {"usage": 100, "limit": 90}, "recordings": {"usage": 100, "limit": 90}, + "rows_synced": {"usage": 100, "limit": 90}, "period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"], } self.organization.never_drop_data = True assert org_quota_limited_until(self.organization, QuotaResource.EVENTS) is None assert org_quota_limited_until(self.organization, QuotaResource.RECORDINGS) is None + assert org_quota_limited_until(self.organization, QuotaResource.ROWS_SYNCED) is None # reset for subsequent tests self.organization.never_drop_data = False @@ -208,21 +232,32 @@ def test_sync_org_quota_limits(self): now = timezone.now().timestamp() replace_limited_team_tokens(QuotaResource.EVENTS, {"1234": now + 10000}) + replace_limited_team_tokens(QuotaResource.ROWS_SYNCED, {"1337": now + 10000}) self.organization.usage = { "events": {"usage": 99, "limit": 100}, "recordings": {"usage": 1, "limit": 100}, + "rows_synced": {"usage": 35, "limit": 100}, "period": ["2021-01-01T00:00:00Z", "2021-01-31T23:59:59Z"], } sync_org_quota_limits(self.organization) - assert list_limited_team_tokens(QuotaResource.EVENTS) == ["1234"] + assert list_limited_team_attributes(QuotaResource.EVENTS) == ["1234"] + assert list_limited_team_attributes(QuotaResource.ROWS_SYNCED) == ["1337"] self.organization.usage["events"]["usage"] = 120 + self.organization.usage["rows_synced"]["usage"] = 120 sync_org_quota_limits(self.organization) - assert sorted(list_limited_team_tokens(QuotaResource.EVENTS)) == sorted( + assert sorted(list_limited_team_attributes(QuotaResource.EVENTS)) == sorted( ["1234", self.team.api_token, other_team.api_token] ) + # rows_synced uses teams, not tokens + assert sorted(list_limited_team_attributes(QuotaResource.ROWS_SYNCED)) == sorted( + ["1337", str(self.team.pk), str(other_team.pk)] + ) + self.organization.usage["events"]["usage"] = 80 + self.organization.usage["rows_synced"]["usage"] = 36 sync_org_quota_limits(self.organization) - assert sorted(list_limited_team_tokens(QuotaResource.EVENTS)) == sorted(["1234"]) + assert sorted(list_limited_team_attributes(QuotaResource.EVENTS)) == sorted(["1234"]) + assert sorted(list_limited_team_attributes(QuotaResource.ROWS_SYNCED)) == sorted(["1337"]) diff --git a/latest_migrations.manifest b/latest_migrations.manifest index 83decc1fa7bd1..7ad1758c3c617 100644 --- a/latest_migrations.manifest +++ b/latest_migrations.manifest @@ -5,7 +5,7 @@ contenttypes: 0002_remove_content_type_name ee: 0015_add_verified_properties otp_static: 0002_throttling otp_totp: 0002_auto_20190420_0723 -posthog: 0363_add_replay_payload_capture_config +posthog: 0364_team_external_data_workspace_rows sessions: 0001_initial social_django: 0010_uid_db_index two_factor: 0007_auto_20201201_1019 diff --git a/posthog/api/capture.py b/posthog/api/capture.py index a7d72f9ca1f3e..ac954a3b8d6d2 100644 --- a/posthog/api/capture.py +++ b/posthog/api/capture.py @@ -262,11 +262,11 @@ def drop_events_over_quota(token: str, events: List[Any]) -> List[Any]: if not settings.EE_AVAILABLE: return events - from ee.billing.quota_limiting import QuotaResource, list_limited_team_tokens + from ee.billing.quota_limiting import QuotaResource, list_limited_team_attributes results = [] - limited_tokens_events = list_limited_team_tokens(QuotaResource.EVENTS) - limited_tokens_recordings = list_limited_team_tokens(QuotaResource.RECORDINGS) + limited_tokens_events = list_limited_team_attributes(QuotaResource.EVENTS) + limited_tokens_recordings = list_limited_team_attributes(QuotaResource.RECORDINGS) for event in events: if event.get("event") in SESSION_RECORDING_EVENT_NAMES: diff --git a/posthog/api/test/__snapshots__/test_action.ambr b/posthog/api/test/__snapshots__/test_action.ambr index 66a6b7ac1190e..e09dabc5bf688 100644 --- a/posthog/api/test/__snapshots__/test_action.ambr +++ b/posthog/api/test/__snapshots__/test_action.ambr @@ -71,7 +71,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_actions-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/actions/%3F%24'*/ @@ -226,7 +227,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_actions-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/actions/%3F%24'*/ @@ -552,7 +554,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_actions-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/actions/%3F%24'*/ diff --git a/posthog/api/test/__snapshots__/test_annotation.ambr b/posthog/api/test/__snapshots__/test_annotation.ambr index 50d15b6145259..1373bf5f4060b 100644 --- a/posthog/api/test/__snapshots__/test_annotation.ambr +++ b/posthog/api/test/__snapshots__/test_annotation.ambr @@ -71,7 +71,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ @@ -150,7 +151,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ @@ -474,7 +476,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_annotations-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/annotations/%3F%24'*/ diff --git a/posthog/api/test/__snapshots__/test_decide.ambr b/posthog/api/test/__snapshots__/test_decide.ambr index c11bcf8af7d75..655c44eff5ab9 100644 --- a/posthog/api/test/__snapshots__/test_decide.ambr +++ b/posthog/api/test/__snapshots__/test_decide.ambr @@ -81,7 +81,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='team-detail',route='api/projects/%28%3FP%3Cid%3E%5B%5E/.%5D%2B%29/%3F%24'*/ @@ -306,7 +307,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -461,7 +463,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -604,7 +607,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."api_token" = 'token123' LIMIT 21 diff --git a/posthog/api/test/__snapshots__/test_early_access_feature.ambr b/posthog/api/test/__snapshots__/test_early_access_feature.ambr index 5328b0eaa8e62..d7908693f9cb2 100644 --- a/posthog/api/test/__snapshots__/test_early_access_feature.ambr +++ b/posthog/api/test/__snapshots__/test_early_access_feature.ambr @@ -49,7 +49,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -174,7 +175,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."api_token" = 'token123' LIMIT 21 /*controller='posthog.api.early_access_feature.early_access_features',route='%5Eapi/early_access_features/%3F%28%3F%3A%5B%3F%23%5D.%2A%29%3F%24'*/ diff --git a/posthog/api/test/__snapshots__/test_element.ambr b/posthog/api/test/__snapshots__/test_element.ambr index 2f1e429bac578..97b03322b3e2d 100644 --- a/posthog/api/test/__snapshots__/test_element.ambr +++ b/posthog/api/test/__snapshots__/test_element.ambr @@ -78,7 +78,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='element-stats',route='api/element/stats/%3F%24'*/ diff --git a/posthog/api/test/__snapshots__/test_insight.ambr b/posthog/api/test/__snapshots__/test_insight.ambr index 0f8b2b5457332..4d3b882aa5b51 100644 --- a/posthog/api/test/__snapshots__/test_insight.ambr +++ b/posthog/api/test/__snapshots__/test_insight.ambr @@ -669,7 +669,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -719,7 +720,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -852,7 +854,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -1075,7 +1078,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -1225,6 +1229,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_user"."id", "posthog_user"."password", "posthog_user"."last_login", @@ -1352,6 +1357,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_organization"."id", "posthog_organization"."name", "posthog_organization"."slug", @@ -1460,6 +1466,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_organization"."id", "posthog_organization"."name", "posthog_organization"."slug", @@ -1595,7 +1602,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -1687,7 +1695,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -1771,7 +1780,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -1828,7 +1838,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ diff --git a/posthog/api/test/__snapshots__/test_organization_feature_flag.ambr b/posthog/api/test/__snapshots__/test_organization_feature_flag.ambr index b0b05656ef376..e2b852a604b20 100644 --- a/posthog/api/test/__snapshots__/test_organization_feature_flag.ambr +++ b/posthog/api/test/__snapshots__/test_organization_feature_flag.ambr @@ -125,7 +125,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='organization_feature_flags-copy-flags',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/feature_flags/copy_flags/%3F%24'*/ @@ -221,7 +222,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='organization_feature_flags-copy-flags',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/feature_flags/copy_flags/%3F%24'*/ @@ -313,7 +315,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='organization_feature_flags-copy-flags',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/feature_flags/copy_flags/%3F%24'*/ @@ -515,7 +518,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='organization_feature_flags-copy-flags',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/feature_flags/copy_flags/%3F%24'*/ @@ -648,7 +652,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='organization_feature_flags-copy-flags',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/feature_flags/copy_flags/%3F%24'*/ @@ -787,7 +792,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='organization_feature_flags-copy-flags',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/feature_flags/copy_flags/%3F%24'*/ @@ -879,7 +885,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='organization_feature_flags-copy-flags',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/feature_flags/copy_flags/%3F%24'*/ @@ -1085,7 +1092,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='organization_feature_flags-copy-flags',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/feature_flags/copy_flags/%3F%24'*/ @@ -1211,7 +1219,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='organization_feature_flags-copy-flags',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/feature_flags/copy_flags/%3F%24'*/ @@ -1268,7 +1277,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='organization_feature_flags-copy-flags',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/feature_flags/copy_flags/%3F%24'*/ @@ -1414,7 +1424,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='organization_feature_flags-copy-flags',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/feature_flags/copy_flags/%3F%24'*/ @@ -1675,7 +1686,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid /*controller='organization_feature_flags-detail',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/feature_flags/%28%3FP%3Cfeature_flag_key%3E%5B%5E/.%5D%2B%29/%3F%24'*/ ' diff --git a/posthog/api/test/__snapshots__/test_preflight.ambr b/posthog/api/test/__snapshots__/test_preflight.ambr index 2d2cb9a03cbfe..dcd94e83ea36c 100644 --- a/posthog/api/test/__snapshots__/test_preflight.ambr +++ b/posthog/api/test/__snapshots__/test_preflight.ambr @@ -89,7 +89,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='posthog.views.preflight_check',route='%5E_preflight/%3F%28%3F%3A%5B%3F%23%5D.%2A%29%3F%24'*/ diff --git a/posthog/api/test/__snapshots__/test_survey.ambr b/posthog/api/test/__snapshots__/test_survey.ambr index 4536dfb45977e..1d5a134d8111f 100644 --- a/posthog/api/test/__snapshots__/test_survey.ambr +++ b/posthog/api/test/__snapshots__/test_survey.ambr @@ -150,7 +150,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."api_token" = 'token123' LIMIT 21 /*controller='posthog.api.survey.surveys',route='%5Eapi/surveys/%3F%28%3F%3A%5B%3F%23%5D.%2A%29%3F%24'*/ diff --git a/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr b/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr index c5475d4515b21..08769c536599c 100644 --- a/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr +++ b/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr @@ -71,7 +71,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -196,7 +197,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_dashboards-detail',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%28%3FP%3Cpk%3E%5B%5E/.%5D%2B%29/%3F%24'*/ @@ -324,6 +326,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_organization"."id", "posthog_organization"."name", "posthog_organization"."slug", @@ -535,6 +538,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_user"."id", "posthog_user"."password", "posthog_user"."last_login", @@ -701,6 +705,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_organization"."id", "posthog_organization"."name", "posthog_organization"."slug", @@ -879,6 +884,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_user"."id", "posthog_user"."password", "posthog_user"."last_login", @@ -1043,6 +1049,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_organization"."id", "posthog_organization"."name", "posthog_organization"."slug", @@ -1281,7 +1288,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -1338,7 +1346,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -1486,7 +1495,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -1603,7 +1613,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -1660,7 +1671,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -1806,7 +1818,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -1939,7 +1952,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -2192,7 +2206,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -2432,7 +2447,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/ @@ -2568,6 +2584,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_organization"."id", "posthog_organization"."name", "posthog_organization"."slug", @@ -2705,7 +2722,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -2817,7 +2835,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/ @@ -2923,7 +2942,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -3073,7 +3093,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -3170,6 +3191,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_user"."id", "posthog_user"."password", "posthog_user"."last_login", @@ -3292,7 +3314,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -3408,7 +3431,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -3549,7 +3573,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -3862,7 +3887,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/ @@ -4020,7 +4046,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -4154,7 +4181,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -4246,7 +4274,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -4407,7 +4436,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -4464,7 +4494,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -4580,7 +4611,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -4737,7 +4769,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -5155,7 +5188,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -5296,7 +5330,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -5388,7 +5423,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -5511,7 +5547,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/ @@ -5595,7 +5632,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -5652,7 +5690,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -5768,7 +5807,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -5915,7 +5955,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -6078,6 +6119,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_user"."id", "posthog_user"."password", "posthog_user"."last_login", @@ -6477,7 +6519,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_dashboards-detail',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%28%3FP%3Cpk%3E%5B%5E/.%5D%2B%29/%3F%24'*/ @@ -6634,6 +6677,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_organization"."id", "posthog_organization"."name", "posthog_organization"."slug", @@ -6815,6 +6859,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_user"."id", "posthog_user"."password", "posthog_user"."last_login", @@ -6981,6 +7026,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_organization"."id", "posthog_organization"."name", "posthog_organization"."slug", @@ -7112,7 +7158,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -7209,6 +7256,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_user"."id", "posthog_user"."password", "posthog_user"."last_login", @@ -7373,6 +7421,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_organization"."id", "posthog_organization"."name", "posthog_organization"."slug", @@ -8005,7 +8054,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -8273,7 +8323,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -8434,7 +8485,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -8491,7 +8543,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -8607,7 +8660,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -8764,7 +8818,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -8887,7 +8942,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -9015,7 +9071,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -9162,7 +9219,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -9471,7 +9529,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/ @@ -9618,7 +9677,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -9722,7 +9782,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ @@ -9859,6 +9920,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_user"."id", "posthog_user"."password", "posthog_user"."last_login", @@ -9977,7 +10039,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_dashboards-detail',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%28%3FP%3Cpk%3E%5B%5E/.%5D%2B%29/%3F%24'*/ @@ -10116,6 +10179,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_organization"."id", "posthog_organization"."name", "posthog_organization"."slug", @@ -10297,6 +10361,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_user"."id", "posthog_user"."password", "posthog_user"."last_login", @@ -10446,7 +10511,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/ @@ -10557,6 +10623,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_user"."id", "posthog_user"."password", "posthog_user"."last_login", @@ -10709,7 +10776,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/ @@ -10894,7 +10962,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/ @@ -11005,6 +11074,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_user"."id", "posthog_user"."password", "posthog_user"."last_login", @@ -11157,7 +11227,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/ @@ -11304,6 +11375,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_organization"."id", "posthog_organization"."name", "posthog_organization"."slug", @@ -11531,7 +11603,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/ diff --git a/posthog/api/test/notebooks/__snapshots__/test_notebook.ambr b/posthog/api/test/notebooks/__snapshots__/test_notebook.ambr index fc373eefb7a43..32ff35e826dd3 100644 --- a/posthog/api/test/notebooks/__snapshots__/test_notebook.ambr +++ b/posthog/api/test/notebooks/__snapshots__/test_notebook.ambr @@ -71,7 +71,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_notebooks-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/notebooks/%3F%24'*/ @@ -168,7 +169,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_notebooks-detail',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/notebooks/%28%3FP%3Cshort_id%3E%5B%5E/.%5D%2B%29/%3F%24'*/ @@ -225,7 +227,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_notebooks-detail',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/notebooks/%28%3FP%3Cshort_id%3E%5B%5E/.%5D%2B%29/%3F%24'*/ @@ -334,7 +337,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_notebooks-all-activity',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/notebooks/activity/%3F%24'*/ @@ -546,7 +550,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_notebooks-detail',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/notebooks/%28%3FP%3Cshort_id%3E%5B%5E/.%5D%2B%29/%3F%24'*/ @@ -657,6 +662,7 @@ "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at", "posthog_user"."id", "posthog_user"."password", "posthog_user"."last_login", @@ -763,7 +769,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_notebooks-detail',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/notebooks/%28%3FP%3Cshort_id%3E%5B%5E/.%5D%2B%29/%3F%24'*/ diff --git a/posthog/celery.py b/posthog/celery.py index a7b62848bfab3..46b1e3b402f71 100644 --- a/posthog/celery.py +++ b/posthog/celery.py @@ -27,7 +27,8 @@ from posthog.cloud_utils import is_cloud from posthog.metrics import pushed_metrics_registry from posthog.redis import get_client -from posthog.utils import get_crontab, get_instance_region +from posthog.utils import get_crontab +from posthog.ph_client import get_ph_client # set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "posthog.settings") @@ -333,6 +334,13 @@ def setup_periodic_tasks(sender: Celery, **kwargs): name="sync datawarehouse sources that have settled in s3 bucket", ) + # Every 30 minutes try to retrieve and calculate total rows synced in period + sender.add_periodic_task( + crontab(minute="*/30"), + calculate_external_data_rows_synced.s(), + name="calculate external data rows synced", + ) + # Set up clickhouse query instrumentation @task_prerun.connect @@ -903,29 +911,10 @@ def debug_task(self): @app.task(ignore_result=True) def calculate_decide_usage() -> None: from django.db.models import Q - from posthoganalytics import Posthog - from posthog.models import Team from posthog.models.feature_flag.flag_analytics import capture_team_decide_usage - if not is_cloud(): - return - - # send EU data to EU, US data to US - api_key = None - host = None - region = get_instance_region() - if region == "EU": - api_key = "phc_dZ4GK1LRjhB97XozMSkEwPXx7OVANaJEwLErkY1phUF" - host = "https://eu.posthog.com" - elif region == "US": - api_key = "sTMFPsFhdP1Ssg" - host = "https://app.posthog.com" - - if not api_key: - return - - ph_client = Posthog(api_key, host=host) + ph_client = get_ph_client() for team in Team.objects.select_related("organization").exclude( Q(organization__for_internal_metrics=True) | Q(is_demo=True) @@ -935,6 +924,22 @@ def calculate_decide_usage() -> None: ph_client.shutdown() +@app.task(ignore_result=True) +def calculate_external_data_rows_synced() -> None: + from django.db.models import Q + from posthog.models import Team + from posthog.tasks.warehouse import ( + capture_workspace_rows_synced_by_team, + check_external_data_source_billing_limit_by_team, + ) + + for team in Team.objects.select_related("organization").exclude( + Q(organization__for_internal_metrics=True) | Q(is_demo=True) | Q(external_data_workspace_id__isnull=True) + ): + capture_workspace_rows_synced_by_team.delay(team.pk) + check_external_data_source_billing_limit_by_team.delay(team.pk) + + @app.task(ignore_result=True) def find_flags_with_enriched_analytics(): from datetime import datetime, timedelta @@ -1092,7 +1097,7 @@ def ee_persist_finished_recordings(): @app.task(ignore_result=True) def sync_datawarehouse_sources(): try: - from posthog.warehouse.sync_resource import sync_resources + from posthog.tasks.warehouse import sync_resources except ImportError: pass else: diff --git a/posthog/migrations/0364_team_external_data_workspace_rows.py b/posthog/migrations/0364_team_external_data_workspace_rows.py new file mode 100644 index 0000000000000..ec9478becd1c9 --- /dev/null +++ b/posthog/migrations/0364_team_external_data_workspace_rows.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.19 on 2023-11-07 20:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("posthog", "0363_add_replay_payload_capture_config"), + ] + + operations = [ + migrations.AddField( + model_name="team", + name="external_data_workspace_last_synced_at", + field=models.DateTimeField(blank=True, null=True), + ) + ] diff --git a/posthog/models/filters/test/__snapshots__/test_filter.ambr b/posthog/models/filters/test/__snapshots__/test_filter.ambr index 648679a965912..c236a5d28a6fa 100644 --- a/posthog/models/filters/test/__snapshots__/test_filter.ambr +++ b/posthog/models/filters/test/__snapshots__/test_filter.ambr @@ -49,7 +49,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -106,7 +107,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -163,7 +165,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -220,7 +223,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -277,7 +281,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 diff --git a/posthog/models/organization.py b/posthog/models/organization.py index 4e1c7af79838c..869ba9f0f6e75 100644 --- a/posthog/models/organization.py +++ b/posthog/models/organization.py @@ -53,6 +53,7 @@ class OrganizationUsageResource(TypedDict): class OrganizationUsageInfo(TypedDict): events: Optional[OrganizationUsageResource] recordings: Optional[OrganizationUsageResource] + rows_synced: Optional[OrganizationUsageResource] period: Optional[List[str]] diff --git a/posthog/models/team/team.py b/posthog/models/team/team.py index 2f5654e0f039a..d03799ad2343b 100644 --- a/posthog/models/team/team.py +++ b/posthog/models/team/team.py @@ -247,6 +247,7 @@ def aggregate_users_by_distinct_id(self) -> bool: event_properties_with_usage: models.JSONField = models.JSONField(default=list, blank=True) event_properties_numerical: models.JSONField = models.JSONField(default=list, blank=True) external_data_workspace_id: models.CharField = models.CharField(max_length=400, null=True, blank=True) + external_data_workspace_last_synced_at: models.DateTimeField = models.DateTimeField(null=True, blank=True) objects: TeamManager = TeamManager() diff --git a/posthog/ph_client.py b/posthog/ph_client.py new file mode 100644 index 0000000000000..e81161a59d470 --- /dev/null +++ b/posthog/ph_client.py @@ -0,0 +1,27 @@ +from posthog.utils import get_instance_region +from posthog.cloud_utils import is_cloud + + +def get_ph_client(): + from posthoganalytics import Posthog + + if not is_cloud(): + return + + # send EU data to EU, US data to US + api_key = None + host = None + region = get_instance_region() + if region == "EU": + api_key = "phc_dZ4GK1LRjhB97XozMSkEwPXx7OVANaJEwLErkY1phUF" + host = "https://eu.posthog.com" + elif region == "US": + api_key = "sTMFPsFhdP1Ssg" + host = "https://app.posthog.com" + + if not api_key: + return + + ph_client = Posthog(api_key, host=host) + + return ph_client diff --git a/posthog/session_recordings/test/__snapshots__/test_session_recordings.ambr b/posthog/session_recordings/test/__snapshots__/test_session_recordings.ambr index 883b68b6b75f1..0afe5e0ac247d 100644 --- a/posthog/session_recordings/test/__snapshots__/test_session_recordings.ambr +++ b/posthog/session_recordings/test/__snapshots__/test_session_recordings.ambr @@ -49,7 +49,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -106,7 +107,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -163,7 +165,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -220,7 +223,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -277,7 +281,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -356,7 +361,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ @@ -487,7 +493,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -681,7 +688,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -753,7 +761,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -810,7 +819,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -867,7 +877,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -924,7 +935,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -981,7 +993,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -1038,7 +1051,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -1117,7 +1131,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ @@ -1185,7 +1200,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -1264,7 +1280,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ @@ -1540,7 +1557,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -1619,7 +1637,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ @@ -1908,7 +1927,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -1987,7 +2007,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ @@ -2234,7 +2255,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -2321,7 +2343,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -2400,7 +2423,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ @@ -2700,7 +2724,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -2779,7 +2804,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ @@ -2829,7 +2855,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ @@ -3545,7 +3572,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -3624,7 +3652,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ @@ -3892,7 +3921,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -3982,7 +4012,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ @@ -4252,7 +4283,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -4331,7 +4363,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ @@ -4614,7 +4647,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -4693,7 +4727,8 @@ "posthog_team"."extra_settings", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 /*controller='project_session_recordings-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/session_recordings/%3F%24'*/ diff --git a/posthog/tasks/test/__snapshots__/test_usage_report.ambr b/posthog/tasks/test/__snapshots__/test_usage_report.ambr index 74f71be82a5cc..92f4167485468 100644 --- a/posthog/tasks/test/__snapshots__/test_usage_report.ambr +++ b/posthog/tasks/test/__snapshots__/test_usage_report.ambr @@ -255,6 +255,24 @@ GROUP BY team_id ' --- +# name: TestFeatureFlagsUsageReport.test_usage_report_decide_requests.24 + ' + + SELECT team, + sum(rows_synced) + FROM + (SELECT JSONExtractString(properties, 'job_id') AS job_id, + distinct_id AS team, + any(JSONExtractInt(properties, 'count')) AS rows_synced + FROM events + WHERE team_id = 2 + AND event = 'external data sync job' + AND parseDateTimeBestEffort(JSONExtractString(properties, 'start_time')) BETWEEN '2022-01-10 00:00:00' AND '2022-01-10 23:59:59' + GROUP BY job_id, + team) + GROUP BY team + ' +--- # name: TestFeatureFlagsUsageReport.test_usage_report_decide_requests.3 ' diff --git a/posthog/tasks/test/test_usage_report.py b/posthog/tasks/test/test_usage_report.py index 715c3829855d2..79a7ab46b8ab2 100644 --- a/posthog/tasks/test/test_usage_report.py +++ b/posthog/tasks/test/test_usage_report.py @@ -399,6 +399,7 @@ def _test_usage_report(self) -> List[dict]: "event_explorer_api_bytes_read": 0, "event_explorer_api_rows_read": 0, "event_explorer_api_duration_ms": 0, + "rows_synced_in_period": 0, "date": "2022-01-09", "organization_id": str(self.organization.id), "organization_name": "Test", @@ -440,6 +441,7 @@ def _test_usage_report(self) -> List[dict]: "event_explorer_api_bytes_read": 0, "event_explorer_api_rows_read": 0, "event_explorer_api_duration_ms": 0, + "rows_synced_in_period": 0, }, str(self.org_1_team_2.id): { "event_count_lifetime": 11, @@ -475,6 +477,7 @@ def _test_usage_report(self) -> List[dict]: "event_explorer_api_bytes_read": 0, "event_explorer_api_rows_read": 0, "event_explorer_api_duration_ms": 0, + "rows_synced_in_period": 0, }, }, }, @@ -533,6 +536,7 @@ def _test_usage_report(self) -> List[dict]: "event_explorer_api_bytes_read": 0, "event_explorer_api_rows_read": 0, "event_explorer_api_duration_ms": 0, + "rows_synced_in_period": 0, "date": "2022-01-09", "organization_id": str(self.org_2.id), "organization_name": "Org 2", @@ -574,6 +578,7 @@ def _test_usage_report(self) -> List[dict]: "event_explorer_api_bytes_read": 0, "event_explorer_api_rows_read": 0, "event_explorer_api_duration_ms": 0, + "rows_synced_in_period": 0, } }, }, @@ -980,6 +985,95 @@ def test_usage_report_survey_responses(self, billing_task_mock: MagicMock, posth assert org_2_report["teams"]["5"]["survey_responses_count_in_month"] == 7 +@freeze_time("2022-01-10T00:01:00Z") +class TestExternalDataSyncUsageReport(ClickhouseDestroyTablesMixin, TestCase, ClickhouseTestMixin): + def setUp(self) -> None: + Team.objects.all().delete() + return super().setUp() + + def _setup_teams(self) -> None: + self.analytics_org = Organization.objects.create(name="PostHog") + self.org_1 = Organization.objects.create(name="Org 1") + self.org_2 = Organization.objects.create(name="Org 2") + + self.analytics_team = Team.objects.create(pk=2, organization=self.analytics_org, name="Analytics") + + self.org_1_team_1 = Team.objects.create(pk=3, organization=self.org_1, name="Team 1 org 1") + self.org_1_team_2 = Team.objects.create(pk=4, organization=self.org_1, name="Team 2 org 1") + self.org_2_team_3 = Team.objects.create(pk=5, organization=self.org_2, name="Team 3 org 2") + + @patch("posthog.tasks.usage_report.Client") + @patch("posthog.tasks.usage_report.send_report_to_billing_service") + def test_external_data_rows_synced_response( + self, billing_task_mock: MagicMock, posthog_capture_mock: MagicMock + ) -> None: + self._setup_teams() + + for i in range(5): + start_time = (now() - relativedelta(hours=i)).strftime("%Y-%m-%dT%H:%M:%SZ") + _create_event( + distinct_id="3", + event="external data sync job", + properties={ + "count": 10, + "job_id": 10924, + "start_time": start_time, + }, + timestamp=now() - relativedelta(hours=i), + team=self.analytics_team, + ) + # identical job id should be deduped and not counted + _create_event( + distinct_id="3", + event="external data sync job", + properties={ + "count": 10, + "job_id": 10924, + "start_time": start_time, + }, + timestamp=now() - relativedelta(hours=i, minutes=i), + team=self.analytics_team, + ) + + for i in range(5): + _create_event( + distinct_id="4", + event="external data sync job", + properties={ + "count": 10, + "job_id": 10924, + "start_time": (now() - relativedelta(hours=i)).strftime("%Y-%m-%dT%H:%M:%SZ"), + }, + timestamp=now() - relativedelta(hours=i), + team=self.analytics_team, + ) + + flush_persons_and_events() + + period = get_previous_day(at=now() + relativedelta(days=1)) + period_start, period_end = period + all_reports = _get_all_org_reports(period_start, period_end) + + assert len(all_reports) == 3 + + org_1_report = _get_full_org_usage_report_as_dict( + _get_full_org_usage_report(all_reports[str(self.org_1.id)], get_instance_metadata(period)) + ) + + org_2_report = _get_full_org_usage_report_as_dict( + _get_full_org_usage_report(all_reports[str(self.org_2.id)], get_instance_metadata(period)) + ) + + assert org_1_report["organization_name"] == "Org 1" + assert org_1_report["rows_synced_in_period"] == 20 + + assert org_1_report["teams"]["3"]["rows_synced_in_period"] == 10 + assert org_1_report["teams"]["4"]["rows_synced_in_period"] == 10 + + assert org_2_report["organization_name"] == "Org 2" + assert org_2_report["rows_synced_in_period"] == 0 + + class SendUsageTest(LicensedTestMixin, ClickhouseDestroyTablesMixin, APIBaseTest): def setUp(self) -> None: super().setUp() @@ -1039,6 +1133,10 @@ def _usage_report_response(self) -> Any: "usage": 1000, "limit": None, }, + "rows_synced": { + "usage": 1000, + "limit": None, + }, }, } } @@ -1185,6 +1283,7 @@ def test_org_usage_updated_correctly(self, mock_post: MagicMock, mock_client: Ma assert self.team.organization.usage == { "events": {"limit": None, "usage": 10000, "todays_usage": 0}, "recordings": {"limit": None, "usage": 1000, "todays_usage": 0}, + "rows_synced": {"limit": None, "usage": 1000, "todays_usage": 0}, "period": ["2021-10-01T00:00:00Z", "2021-10-31T00:00:00Z"], } diff --git a/posthog/tasks/test/test_warehouse.py b/posthog/tasks/test/test_warehouse.py new file mode 100644 index 0000000000000..20b669b754995 --- /dev/null +++ b/posthog/tasks/test/test_warehouse.py @@ -0,0 +1,167 @@ +from posthog.test.base import APIBaseTest +import datetime +from unittest.mock import patch, MagicMock +from posthog.tasks.warehouse import ( + _traverse_jobs_by_field, + capture_workspace_rows_synced_by_team, + check_external_data_source_billing_limit_by_team, +) +from posthog.warehouse.models import ExternalDataSource +from freezegun import freeze_time + + +class TestWarehouse(APIBaseTest): + @patch("posthog.tasks.warehouse.send_request") + @freeze_time("2023-11-07") + def test_traverse_jobs_by_field(self, send_request_mock: MagicMock) -> None: + send_request_mock.return_value = { + "data": [ + { + "jobId": 5827835, + "status": "succeeded", + "jobType": "sync", + "startTime": "2023-11-07T16:50:49Z", + "connectionId": "fake", + "lastUpdatedAt": "2023-11-07T16:52:54Z", + "duration": "PT2M5S", + "rowsSynced": 93353, + }, + { + "jobId": 5783573, + "status": "succeeded", + "jobType": "sync", + "startTime": "2023-11-05T18:32:41Z", + "connectionId": "fake-2", + "lastUpdatedAt": "2023-11-05T18:35:11Z", + "duration": "PT2M30S", + "rowsSynced": 97747, + }, + ] + } + mock_capture = MagicMock() + response = _traverse_jobs_by_field(mock_capture, self.team, "fake-url", "rowsSynced") + + self.assertEqual( + response, + [ + {"count": 93353, "startTime": "2023-11-07T16:50:49Z"}, + {"count": 97747, "startTime": "2023-11-05T18:32:41Z"}, + ], + ) + + self.assertEqual(mock_capture.capture.call_count, 2) + mock_capture.capture.assert_called_with( + self.team.pk, + "external data sync job", + { + "count": 97747, + "workspace_id": self.team.external_data_workspace_id, + "team_id": self.team.pk, + "team_uuid": self.team.uuid, + "startTime": "2023-11-05T18:32:41Z", + "job_id": "5783573", + }, + ) + + @patch("posthog.tasks.warehouse._traverse_jobs_by_field") + @patch("posthog.tasks.warehouse.get_ph_client") + @freeze_time("2023-11-07") + def test_capture_workspace_rows_synced_by_team( + self, mock_capture: MagicMock, traverse_jobs_mock: MagicMock + ) -> None: + traverse_jobs_mock.return_value = [ + {"count": 97747, "startTime": "2023-11-05T18:32:41Z"}, + {"count": 93353, "startTime": "2023-11-07T16:50:49Z"}, + ] + + capture_workspace_rows_synced_by_team(self.team.pk) + + self.team.refresh_from_db() + self.assertEqual( + self.team.external_data_workspace_last_synced_at, + datetime.datetime(2023, 11, 7, 16, 50, 49, tzinfo=datetime.timezone.utc), + ) + + @patch("posthog.tasks.warehouse._traverse_jobs_by_field") + @patch("posthog.tasks.warehouse.get_ph_client") + @freeze_time("2023-11-07") + def test_capture_workspace_rows_synced_by_team_month_cutoff( + self, mock_capture: MagicMock, traverse_jobs_mock: MagicMock + ) -> None: + # external_data_workspace_last_synced_at unset + traverse_jobs_mock.return_value = [ + {"count": 93353, "startTime": "2023-11-07T16:50:49Z"}, + ] + + capture_workspace_rows_synced_by_team(self.team.pk) + + self.team.refresh_from_db() + self.assertEqual( + self.team.external_data_workspace_last_synced_at, + datetime.datetime(2023, 11, 7, 16, 50, 49, tzinfo=datetime.timezone.utc), + ) + + @patch("posthog.tasks.warehouse._traverse_jobs_by_field") + @patch("posthog.tasks.warehouse.get_ph_client") + @freeze_time("2023-11-07") + def test_capture_workspace_rows_synced_by_team_month_cutoff_field_set( + self, mock_capture: MagicMock, traverse_jobs_mock: MagicMock + ) -> None: + self.team.external_data_workspace_last_synced_at = datetime.datetime( + 2023, 10, 29, 18, 32, 41, tzinfo=datetime.timezone.utc + ) + self.team.save() + traverse_jobs_mock.return_value = [ + {"count": 97747, "startTime": "2023-10-30T18:32:41Z"}, + {"count": 93353, "startTime": "2023-11-07T16:50:49Z"}, + ] + + capture_workspace_rows_synced_by_team(self.team.pk) + + self.team.refresh_from_db() + self.assertEqual( + self.team.external_data_workspace_last_synced_at, + datetime.datetime(2023, 11, 7, 16, 50, 49, tzinfo=datetime.timezone.utc), + ) + + @patch("posthog.warehouse.external_data_source.connection.send_request") + @patch("ee.billing.quota_limiting.list_limited_team_attributes") + def test_external_data_source_billing_limit_deactivate( + self, usage_limit_mock: MagicMock, send_request_mock: MagicMock + ) -> None: + usage_limit_mock.return_value = [self.team.pk] + + external_source = ExternalDataSource.objects.create( + source_id="test_id", + connection_id="fake connectino_id", + destination_id="fake destination_id", + team=self.team, + status="running", + source_type="Stripe", + ) + + check_external_data_source_billing_limit_by_team(self.team.pk) + + external_source.refresh_from_db() + self.assertEqual(external_source.status, "inactive") + + @patch("posthog.warehouse.external_data_source.connection.send_request") + @patch("ee.billing.quota_limiting.list_limited_team_attributes") + def test_external_data_source_billing_limit_activate( + self, usage_limit_mock: MagicMock, send_request_mock: MagicMock + ) -> None: + usage_limit_mock.return_value = [] + + external_source = ExternalDataSource.objects.create( + source_id="test_id", + connection_id="fake connectino_id", + destination_id="fake destination_id", + team=self.team, + status="inactive", + source_type="Stripe", + ) + + check_external_data_source_billing_limit_by_team(self.team.pk) + + external_source.refresh_from_db() + self.assertEqual(external_source.status, "running") diff --git a/posthog/tasks/usage_report.py b/posthog/tasks/usage_report.py index a9a06ecbff7c5..9ffbc39227331 100644 --- a/posthog/tasks/usage_report.py +++ b/posthog/tasks/usage_report.py @@ -110,6 +110,8 @@ class UsageReportCounters: # Surveys survey_responses_count_in_period: int survey_responses_count_in_month: int + # Data Warehouse + rows_synced_in_period: int # Instance metadata to be included in oveall report @@ -591,6 +593,34 @@ def get_teams_with_survey_responses_count_in_period( return results +@timed_log() +@retry(tries=QUERY_RETRIES, delay=QUERY_RETRY_DELAY, backoff=QUERY_RETRY_BACKOFF) +def get_teams_with_rows_synced_in_period(begin: datetime, end: datetime) -> List[Tuple[int, int]]: + team_to_query = 1 if get_instance_region() == "EU" else 2 + + # dedup by job id incase there were duplicates sent + results = sync_execute( + """ + SELECT team, sum(rows_synced) FROM ( + SELECT JSONExtractString(properties, 'job_id') AS job_id, distinct_id AS team, any(JSONExtractInt(properties, 'count')) AS rows_synced + FROM events + WHERE team_id = %(team_to_query)s AND event = 'external data sync job' AND parseDateTimeBestEffort(JSONExtractString(properties, 'start_time')) BETWEEN %(begin)s AND %(end)s + GROUP BY job_id, team + ) + GROUP BY team + """, + { + "begin": begin, + "end": end, + "team_to_query": team_to_query, + }, + workload=Workload.OFFLINE, + settings=CH_BILLING_SETTINGS, + ) + + return results + + @app.task(ignore_result=True, max_retries=0) def capture_report( capture_event_name: str, @@ -784,6 +814,7 @@ def _get_all_usage_data(period_start: datetime, period_end: datetime) -> Dict[st teams_with_survey_responses_count_in_month=get_teams_with_survey_responses_count_in_period( period_start.replace(day=1), period_end ), + teams_with_rows_synced_in_period=get_teams_with_rows_synced_in_period(period_start, period_end), ) @@ -854,6 +885,7 @@ def _get_team_report(all_data: Dict[str, Any], team: Team) -> UsageReportCounter event_explorer_api_duration_ms=all_data["teams_with_event_explorer_api_duration_ms"].get(team.id, 0), survey_responses_count_in_period=all_data["teams_with_survey_responses_count_in_period"].get(team.id, 0), survey_responses_count_in_month=all_data["teams_with_survey_responses_count_in_month"].get(team.id, 0), + rows_synced_in_period=all_data["teams_with_rows_synced_in_period"].get(team.id, 0), ) diff --git a/posthog/tasks/warehouse.py b/posthog/tasks/warehouse.py new file mode 100644 index 0000000000000..2450251830c59 --- /dev/null +++ b/posthog/tasks/warehouse.py @@ -0,0 +1,167 @@ +from django.conf import settings +import datetime +from posthog.models import Team +from posthog.warehouse.external_data_source.client import send_request +from posthog.warehouse.models.external_data_source import ExternalDataSource +from posthog.warehouse.models import DataWarehouseCredential, DataWarehouseTable +from posthog.warehouse.external_data_source.connection import retrieve_sync +from urllib.parse import urlencode +from posthog.ph_client import get_ph_client +from typing import Any, Dict, List, TYPE_CHECKING +from posthog.celery import app +import structlog + +logger = structlog.get_logger(__name__) + +AIRBYTE_JOBS_URL = "https://api.airbyte.com/v1/jobs" +DEFAULT_DATE_TIME = datetime.datetime(2023, 11, 7, tzinfo=datetime.timezone.utc) + +if TYPE_CHECKING: + from posthoganalytics import Posthog + + +def sync_resources() -> None: + resources = ExternalDataSource.objects.filter(are_tables_created=False, status__in=["running", "error"]) + + for resource in resources: + sync_resource.delay(resource.pk) + + +@app.task(ignore_result=True) +def sync_resource(resource_id: str) -> None: + resource = ExternalDataSource.objects.get(pk=resource_id) + + try: + job = retrieve_sync(resource.connection_id) + except Exception as e: + logger.exception("Data Warehouse: Sync Resource failed with an unexpected exception.", exc_info=e) + resource.status = "error" + resource.save() + return + + if job is None: + logger.error(f"Data Warehouse: No jobs found for connection: {resource.connection_id}") + resource.status = "error" + resource.save() + return + + if job["status"] == "succeeded": + resource = ExternalDataSource.objects.get(pk=resource_id) + credential, _ = DataWarehouseCredential.objects.get_or_create( + team_id=resource.team.pk, + access_key=settings.AIRBYTE_BUCKET_KEY, + access_secret=settings.AIRBYTE_BUCKET_SECRET, + ) + + data = { + "credential": credential, + "name": "stripe_customers", + "format": "Parquet", + "url_pattern": f"https://{settings.AIRBYTE_BUCKET_DOMAIN}/airbyte/{resource.team.pk}/customers/*.parquet", + "team_id": resource.team.pk, + } + + table = DataWarehouseTable(**data) + try: + table.columns = table.get_columns() + except Exception as e: + logger.exception( + f"Data Warehouse: Sync Resource failed with an unexpected exception for connection: {resource.connection_id}", + exc_info=e, + ) + else: + table.save() + + resource.are_tables_created = True + resource.status = job["status"] + resource.save() + + else: + resource.status = job["status"] + resource.save() + + +DEFAULT_USAGE_LIMIT = 1000000 +ROWS_PER_DOLLAR = 66666 # 1 million rows per $15 + + +@app.task(ignore_result=True, max_retries=2) +def check_external_data_source_billing_limit_by_team(team_id: int) -> None: + from posthog.warehouse.external_data_source.connection import deactivate_connection_by_id, activate_connection_by_id + from ee.billing.quota_limiting import list_limited_team_attributes, QuotaResource + + limited_teams_rows_synced = list_limited_team_attributes(QuotaResource.ROWS_SYNCED) + + team = Team.objects.get(pk=team_id) + all_active_connections = ExternalDataSource.objects.filter(team=team, status__in=["running", "succeeded"]) + all_inactive_connections = ExternalDataSource.objects.filter(team=team, status="inactive") + + # TODO: consider more boundaries + if team_id in limited_teams_rows_synced: + for connection in all_active_connections: + deactivate_connection_by_id(connection.connection_id) + connection.status = "inactive" + connection.save() + else: + for connection in all_inactive_connections: + activate_connection_by_id(connection.connection_id) + connection.status = "running" + connection.save() + + +@app.task(ignore_result=True, max_retries=2) +def capture_workspace_rows_synced_by_team(team_id: int) -> None: + ph_client = get_ph_client() + team = Team.objects.get(pk=team_id) + now = datetime.datetime.now(datetime.timezone.utc) + begin = team.external_data_workspace_last_synced_at or DEFAULT_DATE_TIME + + params = { + "workspaceIds": team.external_data_workspace_id, + "limit": 100, + "offset": 0, + "status": "succeeded", + "orderBy": "createdAt|ASC", + "updatedAtStart": begin.strftime("%Y-%m-%dT%H:%M:%SZ"), + "updatedAtEnd": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + result_totals = _traverse_jobs_by_field(ph_client, team, AIRBYTE_JOBS_URL + "?" + urlencode(params), "rowsSynced") + + # TODO: check assumption that ordering is possible with API + team.external_data_workspace_last_synced_at = result_totals[-1]["startTime"] if result_totals else now + team.save() + + ph_client.shutdown() + + +def _traverse_jobs_by_field( + ph_client: "Posthog", team: Team, url: str, field: str, acc: List[Dict[str, Any]] = [] +) -> List[Dict[str, Any]]: + response = send_request(url, method="GET") + response_data = response.get("data", []) + response_next = response.get("next", None) + + for job in response_data: + acc.append( + { + "count": job[field], + "startTime": job["startTime"], + } + ) + ph_client.capture( + team.pk, + "external data sync job", + { + "count": job[field], + "workspace_id": team.external_data_workspace_id, + "team_id": team.pk, + "team_uuid": team.uuid, + "startTime": job["startTime"], + "job_id": str(job["jobId"]), + }, + ) + + if response_next: + return _traverse_jobs_by_field(ph_client, team, response_next, field, acc) + + return acc diff --git a/posthog/test/__snapshots__/test_feature_flag.ambr b/posthog/test/__snapshots__/test_feature_flag.ambr index d714f9a077c0d..a34d2ba545ae0 100644 --- a/posthog/test/__snapshots__/test_feature_flag.ambr +++ b/posthog/test/__snapshots__/test_feature_flag.ambr @@ -140,7 +140,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 @@ -525,7 +526,8 @@ "posthog_team"."event_properties", "posthog_team"."event_properties_with_usage", "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id" + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" WHERE "posthog_team"."id" = 2 LIMIT 21 diff --git a/posthog/warehouse/api/external_data_source.py b/posthog/warehouse/api/external_data_source.py index 3bfcc64e497d1..7510ec26cd02b 100644 --- a/posthog/warehouse/api/external_data_source.py +++ b/posthog/warehouse/api/external_data_source.py @@ -10,7 +10,7 @@ from posthog.warehouse.external_data_source.source import StripeSourcePayload, create_stripe_source, delete_source from posthog.warehouse.external_data_source.connection import create_connection, start_sync from posthog.warehouse.external_data_source.destination import create_destination, delete_destination -from posthog.warehouse.sync_resource import sync_resource +from posthog.tasks.warehouse import sync_resource from posthog.api.routing import StructuredViewSetMixin from rest_framework.decorators import action diff --git a/posthog/warehouse/external_data_source/connection.py b/posthog/warehouse/external_data_source/connection.py index fc89f22abb65b..9a37222f9d8d4 100644 --- a/posthog/warehouse/external_data_source/connection.py +++ b/posthog/warehouse/external_data_source/connection.py @@ -37,6 +37,22 @@ def create_connection(source_id: str, destination_id: str) -> ExternalDataConnec ) +def activate_connection_by_id(connection_id: str): + update_connection_status_by_id(connection_id, "active") + + +def deactivate_connection_by_id(connection_id: str): + update_connection_status_by_id(connection_id, "inactive") + + +def update_connection_status_by_id(connection_id: str, status: str): + connection_id_url = f"{AIRBYTE_CONNECTION_URL}/{connection_id}" + + payload = {"status": status} + + send_request(connection_id_url, method="PATCH", payload=payload) + + def update_connection_stream(connection_id: str): connection_id_url = f"{AIRBYTE_CONNECTION_URL}/{connection_id}" diff --git a/posthog/warehouse/external_data_source/workspace.py b/posthog/warehouse/external_data_source/workspace.py index e92c07fc888cd..ceb8ed50ac33f 100644 --- a/posthog/warehouse/external_data_source/workspace.py +++ b/posthog/warehouse/external_data_source/workspace.py @@ -1,6 +1,7 @@ from posthog.models import Team from posthog.warehouse.external_data_source.client import send_request from django.conf import settings +import datetime AIRBYTE_WORKSPACE_URL = "https://api.airbyte.com/v1/workspaces" @@ -23,6 +24,8 @@ def get_or_create_workspace(team_id: int): if not team.external_data_workspace_id: workspace_id = create_workspace(team_id) team.external_data_workspace_id = workspace_id + # start tracking from now + team.external_data_workspace_last_synced_at = datetime.datetime.now(datetime.timezone.utc) team.save() return team.external_data_workspace_id diff --git a/posthog/warehouse/sync_resource.py b/posthog/warehouse/sync_resource.py deleted file mode 100644 index 3072bf43986d9..0000000000000 --- a/posthog/warehouse/sync_resource.py +++ /dev/null @@ -1,69 +0,0 @@ -from posthog.warehouse.models.external_data_source import ExternalDataSource -from posthog.warehouse.models import DataWarehouseCredential, DataWarehouseTable -from posthog.warehouse.external_data_source.connection import retrieve_sync -from posthog.celery import app - -from django.conf import settings -import structlog - -logger = structlog.get_logger(__name__) - - -def sync_resources(): - resources = ExternalDataSource.objects.filter(are_tables_created=False, status__in=["running", "error"]) - - for resource in resources: - sync_resource.delay(resource.pk) - - -@app.task(ignore_result=True) -def sync_resource(resource_id): - resource = ExternalDataSource.objects.get(pk=resource_id) - - try: - job = retrieve_sync(resource.connection_id) - except Exception as e: - logger.exception("Data Warehouse: Sync Resource failed with an unexpected exception.", exc_info=e) - resource.status = "error" - resource.save() - return - - if job is None: - logger.error(f"Data Warehouse: No jobs found for connection: {resource.connection_id}") - resource.status = "error" - resource.save() - return - - if job["status"] == "succeeded": - resource = ExternalDataSource.objects.get(pk=resource_id) - credential, _ = DataWarehouseCredential.objects.get_or_create( - team_id=resource.team.pk, - access_key=settings.AIRBYTE_BUCKET_KEY, - access_secret=settings.AIRBYTE_BUCKET_SECRET, - ) - - data = { - "credential": credential, - "name": "stripe_customers", - "format": "Parquet", - "url_pattern": f"https://{settings.AIRBYTE_BUCKET_DOMAIN}/airbyte/{resource.team.pk}/customers/*.parquet", - "team_id": resource.team.pk, - } - - table = DataWarehouseTable(**data) - try: - table.columns = table.get_columns() - except Exception as e: - logger.exception( - f"Data Warehouse: Sync Resource failed with an unexpected exception for connection: {resource.connection_id}", - exc_info=e, - ) - else: - table.save() - - resource.are_tables_created = True - resource.status = job["status"] - resource.save() - else: - resource.status = job["status"] - resource.save()