diff --git a/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts b/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts index 9781c1e8e2bbd..c85c57d67a883 100644 --- a/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts +++ b/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts @@ -134,9 +134,11 @@ export const definitionPopoverLogic = kea([ } if (!('distinct_id_field' in item)) { - const idField = Object.values(warehouseItem.fields).find((n) => n.name === 'id') - if (idField) { - warehouseItem['distinct_id_field'] = idField.name + const distinctIdField = + Object.values(warehouseItem.fields).find((n) => n.name === 'distinct_id') ?? + Object.values(warehouseItem.fields).find((n) => n.name === 'id') + if (distinctIdField) { + warehouseItem['distinct_id_field'] = distinctIdField.name } } diff --git a/frontend/src/scenes/web-analytics/webAnalyticsLogic.tsx b/frontend/src/scenes/web-analytics/webAnalyticsLogic.tsx index 207f8f99f0247..d5d93400466d3 100644 --- a/frontend/src/scenes/web-analytics/webAnalyticsLogic.tsx +++ b/frontend/src/scenes/web-analytics/webAnalyticsLogic.tsx @@ -216,6 +216,9 @@ const getDashboardItemId = (section: TileId, tab: string | undefined, isModal?: // pretend to be a new-AdHoc to get the correct behaviour elsewhere return `new-AdHoc.web-analytics.${section}.${tab || 'default'}.${isModal ? 'modal' : 'default'}` } + +const teamId = window.POSTHOG_APP_CONTEXT?.current_team?.id +const persistConfig = { persist: true, prefix: `${teamId}__` } export const webAnalyticsLogic = kea([ path(['scenes', 'webAnalytics', 'webAnalyticsSceneLogic']), connect(() => ({ @@ -280,7 +283,7 @@ export const webAnalyticsLogic = kea([ reducers({ webAnalyticsFilters: [ initialWebAnalyticsFilter, - { persist: true }, + persistConfig, { setWebAnalyticsFilters: (_, { webAnalyticsFilters }) => webAnalyticsFilters, togglePropertyFilter: (oldPropertyFilters, { key, value, type }): WebAnalyticsPropertyFilters => { @@ -352,7 +355,7 @@ export const webAnalyticsLogic = kea([ ], _graphsTab: [ null as string | null, - { persist: true }, + persistConfig, { setGraphsTab: (_, { tab }) => tab, togglePropertyFilter: (oldTab, { tabChange }) => tabChange?.graphsTab || oldTab, @@ -360,7 +363,7 @@ export const webAnalyticsLogic = kea([ ], _sourceTab: [ null as string | null, - { persist: true }, + persistConfig, { setSourceTab: (_, { tab }) => tab, togglePropertyFilter: (oldTab, { tabChange }) => tabChange?.sourceTab || oldTab, @@ -368,7 +371,7 @@ export const webAnalyticsLogic = kea([ ], _deviceTab: [ null as string | null, - { persist: true }, + persistConfig, { setDeviceTab: (_, { tab }) => tab, togglePropertyFilter: (oldTab, { tabChange }) => tabChange?.deviceTab || oldTab, @@ -376,7 +379,7 @@ export const webAnalyticsLogic = kea([ ], _pathTab: [ null as string | null, - { persist: true }, + persistConfig, { setPathTab: (_, { tab }) => tab, togglePropertyFilter: (oldTab, { tabChange }) => tabChange?.pathTab || oldTab, @@ -384,7 +387,7 @@ export const webAnalyticsLogic = kea([ ], _geographyTab: [ null as string | null, - { persist: true }, + persistConfig, { setGeographyTab: (_, { tab }) => tab, togglePropertyFilter: (oldTab, { tabChange }) => tabChange?.geographyTab || oldTab, @@ -392,7 +395,7 @@ export const webAnalyticsLogic = kea([ ], isPathCleaningEnabled: [ null as boolean | null, - { persist: true }, + persistConfig, { setIsPathCleaningEnabled: (_, { isPathCleaningEnabled }) => isPathCleaningEnabled, }, @@ -413,7 +416,7 @@ export const webAnalyticsLogic = kea([ dateTo: initialDateTo, interval: initialInterval, }, - { persist: true }, + persistConfig, { setDates: (_, { dateTo, dateFrom }) => ({ dateTo, @@ -443,21 +446,21 @@ export const webAnalyticsLogic = kea([ ], shouldFilterTestAccounts: [ false as boolean, - { persist: true }, + persistConfig, { setShouldFilterTestAccounts: (_, { shouldFilterTestAccounts }) => shouldFilterTestAccounts, }, ], shouldStripQueryParams: [ false as boolean, - { persist: true }, + persistConfig, { setShouldStripQueryParams: (_, { shouldStripQueryParams }) => shouldStripQueryParams, }, ], conversionGoal: [ null as WebAnalyticsConversionGoal | null, - { persist: true }, + persistConfig, { setConversionGoal: (_, { conversionGoal }) => conversionGoal, }, diff --git a/posthog/api/proxy_record.py b/posthog/api/proxy_record.py index 8c8f9d488f07b..552835aa9562f 100644 --- a/posthog/api/proxy_record.py +++ b/posthog/api/proxy_record.py @@ -25,6 +25,20 @@ def generate_target_cname(organization_id, domain) -> str: return f"{digest}.{settings.PROXY_BASE_CNAME}" +def _capture_proxy_event(request, record: ProxyRecord, event_type: str) -> None: + organization = Organization.objects.get(id=record.organization_id) + posthoganalytics.capture( + request.user.distinct_id, + f"managed reverse proxy {event_type}", + properties={ + "proxy_record_id": record.id, + "domain": record.domain, + "target_cname": record.target_cname, + }, + groups=groups(organization), + ) + + class ProxyRecordSerializer(serializers.ModelSerializer): class Meta: model = ProxyRecord @@ -79,17 +93,7 @@ def create(self, request, *args, **kwargs): ) serializer = self.get_serializer(record) - organization = Organization.objects.get(id=record.organization_id) - posthoganalytics.capture( - request.user.distinct_id, - "managed reverse proxy created", - properties={ - "proxy_record_id": record.id, - "domain": record.domain, - "target_cname": record.target_cname, - }, - groups=groups(organization), - ) + _capture_proxy_event(request, record, "created") return Response(serializer.data) def destroy(self, request, *args, pk=None, **kwargs): @@ -120,6 +124,8 @@ def destroy(self, request, *args, pk=None, **kwargs): record.status = ProxyRecord.Status.DELETING record.save() + _capture_proxy_event(request, record, "deleted") + return Response( {"success": True}, status=status.HTTP_200_OK, diff --git a/posthog/api/test/__snapshots__/test_action.ambr b/posthog/api/test/__snapshots__/test_action.ambr index 2b453c3a24b20..8ac1823a033c1 100644 --- a/posthog/api/test/__snapshots__/test_action.ambr +++ b/posthog/api/test/__snapshots__/test_action.ambr @@ -156,12 +156,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '99' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '99' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -471,12 +471,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '99' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '99' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -667,12 +667,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '99' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '99' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL diff --git a/posthog/api/test/__snapshots__/test_annotation.ambr b/posthog/api/test/__snapshots__/test_annotation.ambr index 457607fc74fc8..9340e03a2a4d8 100644 --- a/posthog/api/test/__snapshots__/test_annotation.ambr +++ b/posthog/api/test/__snapshots__/test_annotation.ambr @@ -144,12 +144,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '107' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '107' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -454,12 +454,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '107' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '107' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -669,12 +669,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '107' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '107' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL diff --git a/posthog/api/test/__snapshots__/test_decide.ambr b/posthog/api/test/__snapshots__/test_decide.ambr index beb108518df40..a1f1ebcced5c8 100644 --- a/posthog/api/test/__snapshots__/test_decide.ambr +++ b/posthog/api/test/__snapshots__/test_decide.ambr @@ -754,12 +754,12 @@ INNER JOIN "posthog_team" ON ("ee_accesscontrol"."team_id" = "posthog_team"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '253' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '253' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL diff --git a/posthog/api/test/__snapshots__/test_element.ambr b/posthog/api/test/__snapshots__/test_element.ambr index 414a8a1831062..e3ce7d60cebca 100644 --- a/posthog/api/test/__snapshots__/test_element.ambr +++ b/posthog/api/test/__snapshots__/test_element.ambr @@ -151,12 +151,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '272' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '272' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL diff --git a/posthog/api/test/__snapshots__/test_feature_flag.ambr b/posthog/api/test/__snapshots__/test_feature_flag.ambr index 93dfa76ea2cdb..b51af7a796f7d 100644 --- a/posthog/api/test/__snapshots__/test_feature_flag.ambr +++ b/posthog/api/test/__snapshots__/test_feature_flag.ambr @@ -2001,12 +2001,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '313' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '313' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -2021,12 +2021,12 @@ AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'feature_flag' - AND "ee_accesscontrol"."resource_id" = '130' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'feature_flag' - AND "ee_accesscontrol"."resource_id" = '130' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999)) ''' diff --git a/posthog/api/test/__snapshots__/test_insight.ambr b/posthog/api/test/__snapshots__/test_insight.ambr index cea3c1e49dcab..01390b5f4b341 100644 --- a/posthog/api/test/__snapshots__/test_insight.ambr +++ b/posthog/api/test/__snapshots__/test_insight.ambr @@ -1380,12 +1380,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '447' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '447' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -1493,12 +1493,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '447' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '447' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL diff --git a/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr b/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr index 7d964fa88087f..dfd916657a89b 100644 --- a/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr +++ b/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr @@ -332,12 +332,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -352,12 +352,12 @@ AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '1' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '1' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999)) ''' @@ -1057,12 +1057,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -2298,12 +2298,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -2402,12 +2402,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -2530,52 +2530,52 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '55' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '55' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '56' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '56' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '57' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '57' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '58' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '58' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '59' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '59' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999)) ''' @@ -3459,12 +3459,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -4634,12 +4634,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -4654,12 +4654,12 @@ AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '60' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '60' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999)) ''' @@ -5590,12 +5590,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -6140,12 +6140,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -6220,12 +6220,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -7580,12 +7580,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -8592,12 +8592,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -9328,12 +9328,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -9348,12 +9348,12 @@ AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '67' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '67' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999)) ''' @@ -10348,12 +10348,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -10898,12 +10898,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -10978,12 +10978,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '76' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -11186,22 +11186,22 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '69' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '69' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '70' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'dashboard' - AND "ee_accesscontrol"."resource_id" = '70' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999)) ''' diff --git a/posthog/api/test/notebooks/__snapshots__/test_notebook.ambr b/posthog/api/test/notebooks/__snapshots__/test_notebook.ambr index 6527545701a76..f585776717839 100644 --- a/posthog/api/test/notebooks/__snapshots__/test_notebook.ambr +++ b/posthog/api/test/notebooks/__snapshots__/test_notebook.ambr @@ -110,12 +110,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '83' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '83' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -552,12 +552,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '83' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '83' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -719,12 +719,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '83' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '83' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL diff --git a/posthog/session_recordings/test/__snapshots__/test_session_recordings.ambr b/posthog/session_recordings/test/__snapshots__/test_session_recordings.ambr index c893026d31baf..88a534a569646 100644 --- a/posthog/session_recordings/test/__snapshots__/test_session_recordings.ambr +++ b/posthog/session_recordings/test/__snapshots__/test_session_recordings.ambr @@ -640,12 +640,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '444' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '444' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -1690,12 +1690,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -2445,12 +2445,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -3136,12 +3136,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -3890,12 +3890,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -4608,12 +4608,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -5408,12 +5408,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -5673,12 +5673,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -6107,12 +6107,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -6573,12 +6573,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -7267,12 +7267,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL @@ -8018,12 +8018,12 @@ LEFT OUTER JOIN "posthog_organizationmembership" ON ("ee_accesscontrol"."organization_member_id" = "posthog_organizationmembership"."id") WHERE (("ee_accesscontrol"."organization_member_id" IS NULL AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("posthog_organizationmembership"."user_id" = 99999 AND "ee_accesscontrol"."resource" = 'project' - AND "ee_accesscontrol"."resource_id" = '451' + AND "ee_accesscontrol"."resource_id" = '99999' AND "ee_accesscontrol"."role_id" IS NULL AND "ee_accesscontrol"."team_id" = 99999) OR ("ee_accesscontrol"."organization_member_id" IS NULL diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py index 34450437c6dcd..33daed600cebf 100644 --- a/posthog/settings/temporal.py +++ b/posthog/settings/temporal.py @@ -16,9 +16,15 @@ MAX_CONCURRENT_ACTIVITIES: int | None = get_from_env("MAX_CONCURRENT_ACTIVITIES", None, optional=True, type_cast=int) BATCH_EXPORT_S3_UPLOAD_CHUNK_SIZE_BYTES: int = 1024 * 1024 * 50 # 50MB +BATCH_EXPORT_S3_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES: int = get_from_env( + "BATCH_EXPORT_S3_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES", 0, type_cast=int +) BATCH_EXPORT_SNOWFLAKE_UPLOAD_CHUNK_SIZE_BYTES: int = 1024 * 1024 * 100 # 100MB BATCH_EXPORT_POSTGRES_UPLOAD_CHUNK_SIZE_BYTES: int = 1024 * 1024 * 50 # 50MB BATCH_EXPORT_BIGQUERY_UPLOAD_CHUNK_SIZE_BYTES: int = 1024 * 1024 * 100 # 100MB +BATCH_EXPORT_BIGQUERY_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES: int = get_from_env( + "BATCH_EXPORT_BIGQUERY_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES", 0, type_cast=int +) BATCH_EXPORT_HTTP_UPLOAD_CHUNK_SIZE_BYTES: int = 1024 * 1024 * 50 # 50MB BATCH_EXPORT_HTTP_BATCH_SIZE: int = 5000 BATCH_EXPORT_BUFFER_QUEUE_MAX_SIZE_BYTES: int = 1024 * 1024 * 300 # 300MB diff --git a/posthog/temporal/batch_exports/bigquery_batch_export.py b/posthog/temporal/batch_exports/bigquery_batch_export.py index 3774f983754af..ae5a7f58733c9 100644 --- a/posthog/temporal/batch_exports/bigquery_batch_export.py +++ b/posthog/temporal/batch_exports/bigquery_batch_export.py @@ -454,7 +454,7 @@ async def insert_into_bigquery_activity(inputs: BigQueryInsertInputs) -> Records data_interval_end = dt.datetime.fromisoformat(inputs.data_interval_end) full_range = (data_interval_start, data_interval_end) - queue = RecordBatchQueue() + queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_BIGQUERY_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES) producer = Producer(clickhouse_client=client) producer_task = producer.start( queue=queue, diff --git a/posthog/temporal/batch_exports/s3_batch_export.py b/posthog/temporal/batch_exports/s3_batch_export.py index 41383c8e17114..62071fa866363 100644 --- a/posthog/temporal/batch_exports/s3_batch_export.py +++ b/posthog/temporal/batch_exports/s3_batch_export.py @@ -1,4 +1,5 @@ import asyncio +import collections.abc import contextlib import dataclasses import datetime as dt @@ -6,7 +7,6 @@ import json import posixpath import typing -import collections.abc import aioboto3 import botocore.exceptions @@ -30,36 +30,62 @@ default_fields, execute_batch_export_insert_activity, get_data_interval, - iter_model_records, start_batch_export_run, wait_for_delta_past_data_interval_end, ) -from posthog.temporal.batch_exports.metrics import ( - get_bytes_exported_metric, - get_rows_exported_metric, +from posthog.temporal.batch_exports.heartbeat import ( + BatchExportRangeHeartbeatDetails, + DateRange, + HeartbeatParseError, + should_resume_from_activity_heartbeat, +) +from posthog.temporal.batch_exports.spmc import ( + Consumer, + Producer, + RecordBatchQueue, + run_consumer_loop, + wait_for_schema_or_producer, ) from posthog.temporal.batch_exports.temporary_file import ( BatchExportTemporaryFile, - BatchExportWriter, - FlushCallable, - JSONLBatchExportWriter, - ParquetBatchExportWriter, - UnsupportedFileFormatError, + WriterFormat, ) from posthog.temporal.batch_exports.utils import ( - apeek_first_and_rewind, - cast_record_batch_json_columns, set_status_to_running_task, ) from posthog.temporal.common.clickhouse import get_client from posthog.temporal.common.heartbeat import Heartbeater from posthog.temporal.common.logger import bind_temporal_worker_logger -from posthog.temporal.batch_exports.heartbeat import ( - BatchExportRangeHeartbeatDetails, - DateRange, - HeartbeatParseError, - should_resume_from_activity_heartbeat, -) + +NON_RETRYABLE_ERROR_TYPES = [ + # S3 parameter validation failed. + "ParamValidationError", + # This error usually indicates credentials are incorrect or permissions are missing. + "ClientError", + # An S3 bucket doesn't exist. + "NoSuchBucket", + # Couldn't connect to custom S3 endpoint + "EndpointConnectionError", + # Input contained an empty S3 endpoint URL + "EmptyS3EndpointURLError", + # User provided an invalid S3 key + "InvalidS3Key", + # All consumers failed with non-retryable errors. + "RecordBatchConsumerNonRetryableExceptionGroup", +] + +FILE_FORMAT_EXTENSIONS = { + "Parquet": "parquet", + "JSONLines": "jsonl", +} + +COMPRESSION_EXTENSIONS = { + "gzip": "gz", + "snappy": "sz", + "brotli": "br", + "ztsd": "zst", + "lz4": "lz4", +} def get_allowed_template_variables(inputs) -> dict[str, str]: @@ -78,20 +104,6 @@ def get_allowed_template_variables(inputs) -> dict[str, str]: } -FILE_FORMAT_EXTENSIONS = { - "Parquet": "parquet", - "JSONLines": "jsonl", -} - -COMPRESSION_EXTENSIONS = { - "gzip": "gz", - "snappy": "sz", - "brotli": "br", - "ztsd": "zst", - "lz4": "lz4", -} - - def get_s3_key(inputs) -> str: """Return an S3 key given S3InsertInputs.""" template_variables = get_allowed_template_variables(inputs) @@ -199,6 +211,7 @@ def __init__( self.kms_key_id = kms_key_id self.upload_id: str | None = None self.parts: list[Part] = [] + self.pending_parts: list[Part] = [] if self.endpoint_url == "": raise EmptyS3EndpointURLError() @@ -214,7 +227,7 @@ def to_state(self) -> S3MultiPartUploadState: @property def part_number(self): """Return the current part number.""" - return len(self.parts) + return len(self.parts) + len(self.pending_parts) def is_upload_in_progress(self) -> bool: """Whether this S3MultiPartUpload is in progress or not.""" @@ -311,6 +324,8 @@ async def upload_part( ): """Upload a part of this multi-part upload.""" next_part_number = self.part_number + 1 + part = {"PartNumber": next_part_number, "ETag": ""} + self.pending_parts.append(part) if rewind is True: body.rewind() @@ -335,7 +350,9 @@ async def upload_part( finally: reader.detach() # BufferedReader closes the file otherwise. - self.parts.append({"PartNumber": next_part_number, "ETag": etag}) + self.pending_parts.pop(self.pending_parts.index(part)) + part["ETag"] = etag + self.parts.append(part) async def upload_part_retryable( self, @@ -441,6 +458,52 @@ def append_upload_state(self, upload_state: S3MultiPartUploadState): self.upload_state.parts.append(part) +class S3Consumer(Consumer): + def __init__( + self, + heartbeater: Heartbeater, + heartbeat_details: S3HeartbeatDetails, + data_interval_start: dt.datetime | str | None, + s3_upload: S3MultiPartUpload, + ): + super().__init__(heartbeater, heartbeat_details, data_interval_start) + self.heartbeat_details: S3HeartbeatDetails = heartbeat_details + self.s3_upload = s3_upload + + async def flush( + self, + batch_export_file: BatchExportTemporaryFile, + records_since_last_flush: int, + bytes_since_last_flush: int, + flush_counter: int, + last_date_range: DateRange, + is_last: bool, + error: Exception | None, + ): + if error is not None: + await self.logger.adebug("Error while writing part %d", self.s3_upload.part_number + 1, exc_info=error) + await self.logger.awarning( + "An error was detected while writing part %d. Partial part will not be uploaded in case it can be retried.", + self.s3_upload.part_number + 1, + ) + return + + await self.logger.adebug( + "Uploading part %s containing %s records with size %s bytes", + self.s3_upload.part_number + 1, + records_since_last_flush, + bytes_since_last_flush, + ) + + await self.s3_upload.upload_part(batch_export_file) + + self.rows_exported_counter.add(records_since_last_flush) + self.bytes_exported_counter.add(bytes_since_last_flush) + + self.heartbeat_details.track_done_range(last_date_range, self.data_interval_start) + self.heartbeat_details.append_upload_state(self.s3_upload.to_state()) + + @dataclasses.dataclass class S3InsertInputs: """Inputs for S3 exports.""" @@ -576,143 +639,85 @@ async def insert_into_s3_activity(inputs: S3InsertInputs) -> RecordsCompleted: raise ConnectionError("Cannot establish connection to ClickHouse") s3_upload, details = await initialize_and_resume_multipart_upload(inputs) - - # TODO: Switch to single-producer multiple consumer done_ranges: list[DateRange] = details.done_ranges - if done_ranges: - data_interval_start: str | None = done_ranges[-1][1].isoformat() - else: - data_interval_start = inputs.data_interval_start model: BatchExportModel | BatchExportSchema | None = None if inputs.batch_export_schema is None and "batch_export_model" in { field.name for field in dataclasses.fields(inputs) }: model = inputs.batch_export_model + if model is not None: + model_name = model.name + extra_query_parameters = model.schema["values"] if model.schema is not None else None + fields = model.schema["fields"] if model.schema is not None else None + else: + model_name = "events" + extra_query_parameters = None + fields = None else: model = inputs.batch_export_schema + model_name = "custom" + extra_query_parameters = model["values"] if model is not None else {} + fields = model["fields"] if model is not None else None - record_iterator = iter_model_records( - model=model, - client=client, + data_interval_start = ( + dt.datetime.fromisoformat(inputs.data_interval_start) if inputs.data_interval_start else None + ) + data_interval_end = dt.datetime.fromisoformat(inputs.data_interval_end) + full_range = (data_interval_start, data_interval_end) + + queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_S3_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES) + producer = Producer(clickhouse_client=client) + producer_task = producer.start( + queue=queue, + model_name=model_name, + is_backfill=inputs.is_backfill, team_id=inputs.team_id, - interval_start=data_interval_start, - interval_end=inputs.data_interval_end, + full_range=full_range, + done_ranges=done_ranges, + fields=fields, + destination_default_fields=s3_default_fields(), exclude_events=inputs.exclude_events, include_events=inputs.include_events, - is_backfill=inputs.is_backfill, - destination_default_fields=s3_default_fields(), + extra_query_parameters=extra_query_parameters, ) - - first_record_batch, record_iterator = await apeek_first_and_rewind(record_iterator) - records_completed = 0 - if first_record_batch is None: - return records_completed - async with s3_upload as s3_upload: + record_batch_schema = await wait_for_schema_or_producer(queue, producer_task) + if record_batch_schema is None: + return records_completed - async def flush_to_s3( - local_results_file, - records_since_last_flush: int, - bytes_since_last_flush: int, - flush_counter: int, - last_date_range: DateRange, - last: bool, - error: Exception | None, - ): - if error is not None: - await logger.adebug("Error while writing part %d", s3_upload.part_number + 1, exc_info=error) - await logger.awarning( - "An error was detected while writing part %d. Partial part will not be uploaded in case it can be retried.", - s3_upload.part_number + 1, - ) - return - - await logger.adebug( - "Uploading %s part %s containing %s records with size %s bytes", - "last " if last else "", - s3_upload.part_number + 1, - records_since_last_flush, - bytes_since_last_flush, - ) - - await s3_upload.upload_part(local_results_file) - - rows_exported.add(records_since_last_flush) - bytes_exported.add(bytes_since_last_flush) - - details.track_done_range(last_date_range, data_interval_start) - details.append_upload_state(s3_upload.to_state()) - heartbeater.set_from_heartbeat_details(details) - - first_record_batch = cast_record_batch_json_columns(first_record_batch) - column_names = first_record_batch.column_names - column_names.pop(column_names.index("_inserted_at")) - - schema = pa.schema( - # NOTE: For some reason, some batches set non-nullable fields as non-nullable, whereas other - # record batches have them as nullable. - # Until we figure it out, we set all fields to nullable. There are some fields we know - # are not nullable, but I'm opting for the more flexible option until we out why schemas differ - # between batches. - [field.with_nullable(True) for field in first_record_batch.select(column_names).schema] - ) + record_batch_schema = pa.schema( + # NOTE: For some reason, some batches set non-nullable fields as non-nullable, whereas other + # record batches have them as nullable. + # Until we figure it out, we set all fields to nullable. There are some fields we know + # are not nullable, but I'm opting for the more flexible option until we out why schemas differ + # between batches. + [field.with_nullable(True) for field in record_batch_schema if field.name != "_inserted_at"] + ) - writer = get_batch_export_writer( - inputs, - flush_callable=flush_to_s3, + async with s3_upload as s3_upload: + records_completed = await run_consumer_loop( + queue=queue, + consumer_cls=S3Consumer, + producer_task=producer_task, + heartbeater=heartbeater, + heartbeat_details=details, + data_interval_end=data_interval_end, + data_interval_start=data_interval_start, + schema=record_batch_schema, + writer_format=WriterFormat.from_str(inputs.file_format, "S3"), max_bytes=settings.BATCH_EXPORT_S3_UPLOAD_CHUNK_SIZE_BYTES, - schema=schema, + s3_upload=s3_upload, + writer_file_kwargs={"compression": inputs.compression}, + non_retryable_error_types=NON_RETRYABLE_ERROR_TYPES, ) - async with writer.open_temporary_file(): - rows_exported = get_rows_exported_metric() - bytes_exported = get_bytes_exported_metric() - - async for record_batch in record_iterator: - record_batch = cast_record_batch_json_columns(record_batch) - - await writer.write_record_batch(record_batch) - - details.complete_done_ranges(inputs.data_interval_end) - heartbeater.set_from_heartbeat_details(details) - - records_completed = writer.records_total await s3_upload.complete() return records_completed -def get_batch_export_writer( - inputs: S3InsertInputs, flush_callable: FlushCallable, max_bytes: int, schema: pa.Schema | None = None -) -> BatchExportWriter: - """Return the `BatchExportWriter` corresponding to configured `file_format`. - - Raises: - UnsupportedFileFormatError: If no writer exists for given `file_format`. - """ - writer: BatchExportWriter - - if inputs.file_format == "Parquet": - writer = ParquetBatchExportWriter( - max_bytes=max_bytes, - flush_callable=flush_callable, - compression=inputs.compression, - schema=schema, - ) - elif inputs.file_format == "JSONLines": - writer = JSONLBatchExportWriter( - max_bytes=max_bytes, - flush_callable=flush_callable, - compression=inputs.compression, - ) - else: - raise UnsupportedFileFormatError(inputs.file_format, "S3") - - return writer - - @workflow.defn(name="s3-export", failure_exception_types=[workflow.NondeterminismError]) class S3BatchExportWorkflow(PostHogWorkflow): """A Temporal Workflow to export ClickHouse data into S3. @@ -789,19 +794,6 @@ async def run(self, inputs: S3BatchExportInputs): insert_into_s3_activity, insert_inputs, interval=inputs.interval, - non_retryable_error_types=[ - # S3 parameter validation failed. - "ParamValidationError", - # This error usually indicates credentials are incorrect or permissions are missing. - "ClientError", - # An S3 bucket doesn't exist. - "NoSuchBucket", - # Couldn't connect to custom S3 endpoint - "EndpointConnectionError", - # Input contained an empty S3 endpoint URL - "EmptyS3EndpointURLError", - # User provided an invalid S3 key - "InvalidS3Key", - ], + non_retryable_error_types=NON_RETRYABLE_ERROR_TYPES, finish_inputs=finish_inputs, ) diff --git a/posthog/temporal/batch_exports/spmc.py b/posthog/temporal/batch_exports/spmc.py index 23a1737f24de6..34a503646a3e4 100644 --- a/posthog/temporal/batch_exports/spmc.py +++ b/posthog/temporal/batch_exports/spmc.py @@ -16,6 +16,7 @@ from posthog.temporal.batch_exports.sql import ( SELECT_FROM_EVENTS_VIEW, SELECT_FROM_EVENTS_VIEW_BACKFILL, + SELECT_FROM_EVENTS_VIEW_RECENT, SELECT_FROM_EVENTS_VIEW_UNBOUNDED, SELECT_FROM_PERSONS_VIEW, SELECT_FROM_PERSONS_VIEW_BACKFILL, @@ -362,6 +363,8 @@ def consumer_done_callback(task: asyncio.Task): consumer_number += 1 while not consumer.flush_start_event.is_set() and not consumer_task.done(): + # Block until we either start flushing or we are done consuming. + # Flush start should always happen first unless the consumer task fails. await asyncio.sleep(0) if consumer_task.done(): @@ -472,7 +475,7 @@ def start( done_ranges: list[tuple[dt.datetime, dt.datetime]], fields: list[BatchExportField] | None = None, destination_default_fields: list[BatchExportField] | None = None, - use_latest_schema: bool = True, + use_latest_schema: bool = False, **parameters, ) -> asyncio.Task: if fields is None: @@ -503,7 +506,16 @@ def start( else: parameters["include_events"] = [] - if str(team_id) in settings.UNCONSTRAINED_TIMESTAMP_TEAM_IDS: + start_at, end_at = full_range + + if start_at: + is_5_min_batch_export = (end_at - start_at) == dt.timedelta(seconds=300) + else: + is_5_min_batch_export = False + + if is_5_min_batch_export and not is_backfill: + query_template = SELECT_FROM_EVENTS_VIEW_RECENT + elif str(team_id) in settings.UNCONSTRAINED_TIMESTAMP_TEAM_IDS: query_template = SELECT_FROM_EVENTS_VIEW_UNBOUNDED elif is_backfill: query_template = SELECT_FROM_EVENTS_VIEW_BACKFILL diff --git a/posthog/temporal/batch_exports/sql.py b/posthog/temporal/batch_exports/sql.py index 921cb8f437287..7cb3922268ead 100644 --- a/posthog/temporal/batch_exports/sql.py +++ b/posthog/temporal/batch_exports/sql.py @@ -114,6 +114,26 @@ """ ) +SELECT_FROM_EVENTS_VIEW_RECENT = Template( + """ +SELECT + $fields +FROM + events_batch_export_recent( + team_id={team_id}, + interval_start={interval_start}, + interval_end={interval_end}, + include_events={include_events}::Array(String), + exclude_events={exclude_events}::Array(String) + ) AS events +FORMAT ArrowStream +SETTINGS + -- This is half of configured MAX_MEMORY_USAGE for batch exports. + max_bytes_before_external_sort=50000000000, + max_replica_delay_for_distributed_queries=1 +""" +) + SELECT_FROM_EVENTS_VIEW_UNBOUNDED = Template( """ SELECT diff --git a/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py b/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py index d5d8d26b40373..979869c4b31d7 100644 --- a/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py +++ b/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py @@ -28,10 +28,10 @@ ) from posthog.temporal.batch_exports.s3_batch_export import ( FILE_FORMAT_EXTENSIONS, - S3HeartbeatDetails, IntermittentUploadPartTimeoutError, S3BatchExportInputs, S3BatchExportWorkflow, + S3HeartbeatDetails, S3InsertInputs, S3MultiPartUpload, get_s3_key, @@ -1589,7 +1589,7 @@ def __init__(self, *args, **kwargs): assert run.records_completed is None assert ( run.latest_error - == "IntermittentUploadPartTimeoutError: An intermittent `RequestTimeout` was raised while attempting to upload part 1" + == "RecordBatchConsumerRetryableExceptionGroup: At least one unhandled retryable errors in a RecordBatch consumer TaskGroup (1 sub-exception)" ) run = runs[1] diff --git a/posthog/test/base.py b/posthog/test/base.py index c3ff194e0c146..43dcc0e130964 100644 --- a/posthog/test/base.py +++ b/posthog/test/base.py @@ -263,6 +263,15 @@ def clean_varying_query_parts(query, replace_all_numbers): "SELECT distinct_id, 1 as value", query, ) + + # rbac has some varying IDs we can replace + # e.g. AND "ee_accesscontrol"."resource_id" = '450' + query = re.sub( + r"\"resource_id\" = '\d+'", + "\"resource_id\" = '99999'", + query, + ) + return query