diff --git a/frontend/src/loadPostHogJS.tsx b/frontend/src/loadPostHogJS.tsx index 2c5a3285ef509..badabf1105246 100644 --- a/frontend/src/loadPostHogJS.tsx +++ b/frontend/src/loadPostHogJS.tsx @@ -29,16 +29,38 @@ export function loadPostHogJS(): void { bootstrap: window.POSTHOG_USER_IDENTITY_WITH_FLAGS ? window.POSTHOG_USER_IDENTITY_WITH_FLAGS : {}, opt_in_site_apps: true, api_transport: 'fetch', - loaded: (posthog) => { - if (posthog.sessionRecording) { - posthog.sessionRecording._forceAllowLocalhostNetworkCapture = true + loaded: (loadedInstance) => { + if (loadedInstance.sessionRecording) { + loadedInstance.sessionRecording._forceAllowLocalhostNetworkCapture = true } if (window.IMPERSONATED_SESSION) { - posthog.opt_out_capturing() + loadedInstance.sessionManager?.resetSessionId() + loadedInstance.opt_out_capturing() } else { - posthog.opt_in_capturing() + loadedInstance.opt_in_capturing() } + + const Cypress = (window as any).Cypress + + if (Cypress) { + Object.entries(Cypress.env()).forEach(([key, value]) => { + if (key.startsWith('POSTHOG_PROPERTY_')) { + loadedInstance.register_for_session({ + [key.replace('POSTHOG_PROPERTY_', 'E2E_TESTING_').toLowerCase()]: value, + }) + } + }) + } + + // This is a helpful flag to set to automatically reset the recording session on load for testing multiple recordings + const shouldResetSessionOnLoad = loadedInstance.getFeatureFlag(FEATURE_FLAGS.SESSION_RESET_ON_LOAD) + if (shouldResetSessionOnLoad) { + loadedInstance.sessionManager?.resetSessionId() + } + + // Make sure we have access to the object in window for debugging + window.posthog = loadedInstance }, scroll_root_selector: ['main', 'html'], autocapture: { @@ -52,26 +74,6 @@ export function loadPostHogJS(): void { : undefined, }) ) - - const Cypress = (window as any).Cypress - - if (Cypress) { - Object.entries(Cypress.env()).forEach(([key, value]) => { - if (key.startsWith('POSTHOG_PROPERTY_')) { - posthog.register_for_session({ - [key.replace('POSTHOG_PROPERTY_', 'E2E_TESTING_').toLowerCase()]: value, - }) - } - }) - } - - // This is a helpful flag to set to automatically reset the recording session on load for testing multiple recordings - const shouldResetSessionOnLoad = posthog.getFeatureFlag(FEATURE_FLAGS.SESSION_RESET_ON_LOAD) - if (shouldResetSessionOnLoad) { - posthog.sessionManager?.resetSessionId() - } - // Make sure we have access to the object in window for debugging - window.posthog = posthog } else { posthog.init('fake token', { autocapture: false, diff --git a/frontend/src/scenes/pipeline/hogfunctions/filters/HogFunctionFilters.tsx b/frontend/src/scenes/pipeline/hogfunctions/filters/HogFunctionFilters.tsx index 9dee4c767cc5e..681e63a6239b2 100644 --- a/frontend/src/scenes/pipeline/hogfunctions/filters/HogFunctionFilters.tsx +++ b/frontend/src/scenes/pipeline/hogfunctions/filters/HogFunctionFilters.tsx @@ -161,6 +161,10 @@ export function HogFunctionFilters(): JSX.Element { value: '{person.id}', label: 'Run once per person per interval', }, + { + value: '{concat(person.id, event.event)}', + label: 'Run once per person per event name per interval', + }, ]} value={value?.hash ?? null} onChange={(val) => diff --git a/plugin-server/tests/cdp/examples.ts b/plugin-server/tests/cdp/examples.ts index fafafce472e3d..f92dd9ed4f97a 100644 --- a/plugin-server/tests/cdp/examples.ts +++ b/plugin-server/tests/cdp/examples.ts @@ -708,8 +708,16 @@ export const HOG_MASK_EXAMPLES: Record> person: { masking: { ttl: 30, - hash: '{person.uuid}', - bytecode: ['_h', 32, 'uuid', 32, 'person', 1, 2], + hash: '{person.id}', + bytecode: ['_h', 32, 'id', 32, 'person', 1, 2], + threshold: null, + }, + }, + personAndEvent: { + masking: { + ttl: 30, + hash: '{concat(person.id, event.event)}', + bytecode: ['_H', 1, 32, 'id', 32, 'person', 1, 2, 32, 'event', 32, 'event', 1, 2, 2, 'concat', 2], threshold: null, }, }, diff --git a/plugin-server/tests/cdp/hog-masker.test.ts b/plugin-server/tests/cdp/hog-masker.test.ts index df05043290e63..a3c7a955574db 100644 --- a/plugin-server/tests/cdp/hog-masker.test.ts +++ b/plugin-server/tests/cdp/hog-masker.test.ts @@ -118,6 +118,7 @@ describe('HogMasker', () => { describe('ttl', () => { let hogFunctionPerson: HogFunctionType let hogFunctionAll: HogFunctionType + let hogFunctionPersonAndEvent: HogFunctionType beforeEach(() => { hogFunctionPerson = createHogFunction({ @@ -127,6 +128,13 @@ describe('HogMasker', () => { }, }) + hogFunctionPersonAndEvent = createHogFunction({ + masking: { + ...HOG_MASK_EXAMPLES.personAndEvent.masking!, + ttl: 1, + }, + }) + hogFunctionAll = createHogFunction({ masking: { ...HOG_MASK_EXAMPLES.all.masking!, @@ -145,20 +153,33 @@ describe('HogMasker', () => { }) it('should mask with custom hog hash', async () => { - const globalsPerson1 = createHogExecutionGlobals({ person: { uuid: '1' } as any }) - const globalsPerson2 = createHogExecutionGlobals({ person: { uuid: '2' } as any }) + const globals1 = createHogExecutionGlobals({ + person: { id: '1' } as any, + event: { event: '$pageview' } as any, + }) + const globals2 = createHogExecutionGlobals({ + person: { id: '2' } as any, + event: { event: '$autocapture' } as any, + }) + const globals3 = createHogExecutionGlobals({ + person: { id: '2' } as any, + event: { event: '$pageview' } as any, + }) const invocations = [ - createInvocation(hogFunctionPerson, globalsPerson1), - createInvocation(hogFunctionAll, globalsPerson1), - createInvocation(hogFunctionPerson, globalsPerson2), - createInvocation(hogFunctionAll, globalsPerson2), + createInvocation(hogFunctionPerson, globals1), + createInvocation(hogFunctionAll, globals1), + createInvocation(hogFunctionPersonAndEvent, globals1), + createInvocation(hogFunctionPerson, globals2), + createInvocation(hogFunctionAll, globals2), + createInvocation(hogFunctionPersonAndEvent, globals2), + createInvocation(hogFunctionPersonAndEvent, globals3), ] const res = await masker.filterByMasking(invocations) expect(res.masked.length).toEqual(1) - expect(res.notMasked.length).toEqual(3) + expect(res.notMasked.length).toEqual(6) const res2 = await masker.filterByMasking(invocations) - expect(res2.masked.length).toEqual(4) + expect(res2.masked.length).toEqual(7) expect(res2.notMasked.length).toEqual(0) }) diff --git a/posthog/api/test/__snapshots__/test_api_docs.ambr b/posthog/api/test/__snapshots__/test_api_docs.ambr index 7c98b589bbd18..044ee43f1983e 100644 --- a/posthog/api/test/__snapshots__/test_api_docs.ambr +++ b/posthog/api/test/__snapshots__/test_api_docs.ambr @@ -1,139 +1,139 @@ # serializer version: 1 # name: TestAPIDocsSchema.test_api_docs_generation_warnings_snapshot list([ - '/home/runner/work/posthog/posthog/posthog/api/plugin_log_entry.py: Warning [PluginLogEntryViewSet]: could not derive type of path parameter "plugin_config_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', + '', + '/home/runner/work/posthog/posthog/ee/api/dashboard_collaborator.py: Warning [DashboardCollaboratorViewSet]: could not derive type of path parameter "project_id" because model "ee.models.dashboard_privilege.DashboardPrivilege" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/ee/api/explicit_team_member.py: Warning [ExplicitTeamMemberViewSet]: could not derive type of path parameter "project_id" because model "ee.models.explicit_team_membership.ExplicitTeamMembership" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/ee/api/feature_flag_role_access.py: Warning [FeatureFlagRoleAccessViewSet]: could not derive type of path parameter "project_id" because model "ee.models.feature_flag_role_access.FeatureFlagRoleAccess" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/ee/api/rbac/role.py: Warning [RoleMembershipViewSet]: could not derive type of path parameter "organization_id" because model "ee.models.rbac.role.RoleMembership" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/ee/api/rbac/role.py: Warning [RoleViewSet > RoleSerializer]: unable to resolve type hint for function "get_associated_flags". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/ee/api/rbac/role.py: Warning [RoleViewSet > RoleSerializer]: unable to resolve type hint for function "get_members". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/ee/api/subscription.py: Warning [SubscriptionViewSet > SubscriptionSerializer]: unable to resolve type hint for function "summary". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/ee/api/subscription.py: Warning [SubscriptionViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.subscription.Subscription" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/ee/clickhouse/views/experiment_holdouts.py: Warning [ExperimentHoldoutViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.experiment.ExperimentHoldout" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/ee/clickhouse/views/experiments.py: Warning [EnterpriseExperimentsViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.experiment.Experiment" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/ee/clickhouse/views/groups.py: Warning [GroupsTypesViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.group_type_mapping.GroupTypeMapping" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/ee/clickhouse/views/groups.py: Warning [GroupsViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.group.group.Group" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/ee/clickhouse/views/insights.py: Warning [EnterpriseInsightsViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.insight.Insight" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/ee/clickhouse/views/person.py: Warning [EnterprisePersonViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.person.person.Person" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/ee/session_recordings/session_recording_playlist.py: Warning [SessionRecordingPlaylistViewSet]: could not derive type of path parameter "project_id" because model "posthog.session_recordings.models.session_recording_playlist.SessionRecordingPlaylist" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/ee/session_recordings/session_recording_playlist.py: Warning [SessionRecordingPlaylistViewSet]: could not derive type of path parameter "session_recording_id" because model "posthog.session_recordings.models.session_recording_playlist.SessionRecordingPlaylist" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/action.py: Warning [ActionViewSet > ActionSerializer]: unable to resolve type hint for function "get_creation_context". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/api/action.py: Warning [ActionViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.action.action.Action" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/activity_log.py: Warning [ActivityLogViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.activity_logging.activity_log.ActivityLog" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/annotation.py: Warning [AnnotationsViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.annotation.Annotation" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', "/home/runner/work/posthog/posthog/posthog/api/app_metrics.py: Error [AppMetricsViewSet]: exception raised while getting serializer. Hint: Is get_serializer_class() returning None or is get_queryset() not working without a request? Ignoring the view for now. (Exception: 'AppMetricsViewSet' should either include a `serializer_class` attribute, or override the `get_serializer_class()` method.)", - '/home/runner/work/posthog/posthog/posthog/api/app_metrics.py: Warning [AppMetricsViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.plugin.PluginConfig" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/posthog/api/app_metrics.py: Error [HistoricalExportsAppMetricsViewSet]: unable to guess serializer. This is graceful fallback handling for APIViews. Consider using GenericAPIView as view base class, if view is under your control. Either way you may want to add a serializer_class (or method). Ignoring view for now.', - '/home/runner/work/posthog/posthog/posthog/api/app_metrics.py: Warning [HistoricalExportsAppMetricsViewSet]: could not derive type of path parameter "project_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/app_metrics.py: Warning [HistoricalExportsAppMetricsViewSet]: could not derive type of path parameter "plugin_config_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/app_metrics.py: Warning [AppMetricsViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.plugin.PluginConfig" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/posthog/api/app_metrics.py: Warning [HistoricalExportsAppMetricsViewSet]: could not derive type of path parameter "id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/batch_exports/http.py: Warning [BatchExportViewSet]: could not derive type of path parameter "project_id" because model "posthog.batch_exports.models.BatchExport" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/batch_exports/http.py: Warning [BatchExportViewSet > BatchExportSerializer]: could not resolve serializer field "HogQLSelectQueryField(required=False)". Defaulting to "string"', - '/home/runner/work/posthog/posthog/posthog/batch_exports/http.py: Warning [BatchExportRunViewSet]: could not derive type of path parameter "project_id" because model "posthog.batch_exports.models.BatchExportRun" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/app_metrics.py: Warning [HistoricalExportsAppMetricsViewSet]: could not derive type of path parameter "plugin_config_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/app_metrics.py: Warning [HistoricalExportsAppMetricsViewSet]: could not derive type of path parameter "project_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/cohort.py: Warning [CohortViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.cohort.cohort.Cohort" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/posthog/api/dashboards/dashboard.py: Warning [DashboardsViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.dashboard.Dashboard" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/ee/api/dashboard_collaborator.py: Warning [DashboardCollaboratorViewSet]: could not derive type of path parameter "project_id" because model "ee.models.dashboard_privilege.DashboardPrivilege" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/sharing.py: Warning [SharingConfigurationViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.sharing_configuration.SharingConfiguration" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/event.py: Warning [EventViewSet]: could not derive type of path parameter "project_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/models/event/util.py: Warning [EventViewSet > ClickhouseEventSerializer]: unable to resolve type hint for function "get_id". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/models/event/util.py: Warning [EventViewSet > ClickhouseEventSerializer]: unable to resolve type hint for function "get_distinct_id". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/models/event/util.py: Warning [EventViewSet > ClickhouseEventSerializer]: unable to resolve type hint for function "get_properties". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/models/event/util.py: Warning [EventViewSet > ClickhouseEventSerializer]: unable to resolve type hint for function "get_event". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/models/event/util.py: Warning [EventViewSet > ClickhouseEventSerializer]: unable to resolve type hint for function "get_timestamp". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/models/event/util.py: Warning [EventViewSet > ClickhouseEventSerializer]: unable to resolve type hint for function "get_person". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/models/event/util.py: Warning [EventViewSet > ClickhouseEventSerializer]: unable to resolve type hint for function "get_elements". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/models/event/util.py: Warning [EventViewSet > ClickhouseEventSerializer]: unable to resolve type hint for function "get_elements_chain". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/api/dashboards/dashboard_templates.py: Warning [DashboardTemplateViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.dashboard_templates.DashboardTemplate" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/early_access_feature.py: Warning [EarlyAccessFeatureViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.early_access_feature.EarlyAccessFeature" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/posthog/api/event.py: Warning [EventViewSet]: could not derive type of path parameter "id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/ee/api/explicit_team_member.py: Warning [ExplicitTeamMemberViewSet]: could not derive type of path parameter "project_id" because model "ee.models.explicit_team_membership.ExplicitTeamMembership" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/exports.py: Warning [ExportedAssetViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.exported_asset.ExportedAsset" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/exports.py: Warning [ExportedAssetViewSet > ExportedAssetSerializer]: unable to resolve type hint for function "has_content". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/api/event.py: Warning [EventViewSet]: could not derive type of path parameter "project_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', + "/home/runner/work/posthog/posthog/posthog/api/event_definition.py: Error [EventDefinitionViewSet]: exception raised while getting serializer. Hint: Is get_serializer_class() returning None or is get_queryset() not working without a request? Ignoring the view for now. (Exception: 'AnonymousUser' object has no attribute 'organization')", + '/home/runner/work/posthog/posthog/posthog/api/event_definition.py: Warning [EventDefinitionViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.event_definition.EventDefinition" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/posthog/api/exports.py: Warning [ExportedAssetViewSet > ExportedAssetSerializer]: unable to resolve type hint for function "filename". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/ee/clickhouse/views/groups.py: Warning [GroupsViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.group.group.Group" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/ee/clickhouse/views/insights.py: Warning [EnterpriseInsightsViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.insight.Insight" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/insight.py: Warning [EnterpriseInsightsViewSet > InsightSerializer]: unable to resolve type hint for function "get_last_refresh". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/api/exports.py: Warning [ExportedAssetViewSet > ExportedAssetSerializer]: unable to resolve type hint for function "has_content". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/api/exports.py: Warning [ExportedAssetViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.exported_asset.ExportedAsset" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/feature_flag.py: Warning [FeatureFlagViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.feature_flag.feature_flag.FeatureFlag" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/posthog/api/insight.py: Warning [EnterpriseInsightsViewSet > InsightSerializer]: unable to resolve type hint for function "get_cache_target_age". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/api/insight.py: Warning [EnterpriseInsightsViewSet > InsightSerializer]: unable to resolve type hint for function "get_next_allowed_client_refresh". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/api/insight.py: Warning [EnterpriseInsightsViewSet > InsightSerializer]: unable to resolve type hint for function "get_result". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/api/insight.py: Warning [EnterpriseInsightsViewSet > InsightSerializer]: unable to resolve type hint for function "get_hasMore". Consider using a type hint or @extend_schema_field. Defaulting to string.', '/home/runner/work/posthog/posthog/posthog/api/insight.py: Warning [EnterpriseInsightsViewSet > InsightSerializer]: unable to resolve type hint for function "get_columns". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/api/insight.py: Warning [EnterpriseInsightsViewSet > InsightSerializer]: unable to resolve type hint for function "get_timezone". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/api/insight.py: Warning [EnterpriseInsightsViewSet > InsightSerializer]: unable to resolve type hint for function "get_hasMore". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/api/insight.py: Warning [EnterpriseInsightsViewSet > InsightSerializer]: unable to resolve type hint for function "get_hogql". Consider using a type hint or @extend_schema_field. Defaulting to string.', '/home/runner/work/posthog/posthog/posthog/api/insight.py: Warning [EnterpriseInsightsViewSet > InsightSerializer]: unable to resolve type hint for function "get_is_cached". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/api/insight.py: Warning [EnterpriseInsightsViewSet > InsightSerializer]: unable to resolve type hint for function "get_last_refresh". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/api/insight.py: Warning [EnterpriseInsightsViewSet > InsightSerializer]: unable to resolve type hint for function "get_next_allowed_client_refresh". Consider using a type hint or @extend_schema_field. Defaulting to string.', '/home/runner/work/posthog/posthog/posthog/api/insight.py: Warning [EnterpriseInsightsViewSet > InsightSerializer]: unable to resolve type hint for function "get_query_status". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/api/insight.py: Warning [EnterpriseInsightsViewSet > InsightSerializer]: unable to resolve type hint for function "get_hogql". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/api/insight.py: Warning [EnterpriseInsightsViewSet > InsightSerializer]: unable to resolve type hint for function "get_result". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/api/insight.py: Warning [EnterpriseInsightsViewSet > InsightSerializer]: unable to resolve type hint for function "get_timezone". Consider using a type hint or @extend_schema_field. Defaulting to string.', '/home/runner/work/posthog/posthog/posthog/api/insight.py: Warning [EnterpriseInsightsViewSet > InsightSerializer]: unable to resolve type hint for function "get_types". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/ee/clickhouse/views/person.py: Warning [EnterprisePersonViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.person.person.Person" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/plugin.py: Warning [PipelineDestinationsConfigsViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.plugin.PluginConfig" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/notebook.py: Warning [NotebookViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.notebook.notebook.Notebook" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/organization.py: Warning [OrganizationViewSet > OrganizationSerializer]: unable to resolve type hint for function "get_member_count". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/api/organization.py: Warning [OrganizationViewSet > OrganizationSerializer]: unable to resolve type hint for function "get_metadata". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/api/person.py: Warning [SessionRecordingViewSet > SessionRecordingSerializer > MinimalPersonSerializer]: unable to resolve type hint for function "get_distinct_ids". Consider using a type hint or @extend_schema_field. Defaulting to string.', '/home/runner/work/posthog/posthog/posthog/api/plugin.py: Warning [PipelineDestinationsConfigsViewSet > PluginConfigSerializer]: unable to resolve type hint for function "get_config". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/api/plugin.py: Warning [PipelineDestinationsConfigsViewSet > PluginConfigSerializer]: unable to resolve type hint for function "get_delivery_rate_24h". Consider using a type hint or @extend_schema_field. Defaulting to string.', '/home/runner/work/posthog/posthog/posthog/api/plugin.py: Warning [PipelineDestinationsConfigsViewSet > PluginConfigSerializer]: unable to resolve type hint for function "get_error". Consider using a type hint or @extend_schema_field. Defaulting to string.', '/home/runner/work/posthog/posthog/posthog/api/plugin.py: Warning [PipelineDestinationsConfigsViewSet > PluginConfigSerializer]: unable to resolve type hint for function "get_plugin_info". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/api/plugin.py: Warning [PipelineDestinationsConfigsViewSet > PluginConfigSerializer]: unable to resolve type hint for function "get_delivery_rate_24h". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/api/plugin.py: Warning [PipelineDestinationsConfigsViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.plugin.PluginConfig" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/plugin.py: Warning [PipelineDestinationsViewSet > PluginSerializer]: unable to resolve type hint for function "get_hog_function_migration_available". Consider using a type hint or @extend_schema_field. Defaulting to string.', '/home/runner/work/posthog/posthog/posthog/api/plugin.py: Warning [PipelineFrontendAppsConfigsViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.plugin.PluginConfig" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/posthog/api/plugin.py: Warning [PipelineImportAppsConfigsViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.plugin.PluginConfig" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/posthog/api/plugin.py: Warning [PipelineTransformationsConfigsViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.plugin.PluginConfig" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/posthog/api/plugin.py: Warning [PluginConfigViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.plugin.PluginConfig" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/query.py: Warning [QueryViewSet]: could not derive type of path parameter "project_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', - '/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pydantic/_internal/_model_construction.py: Warning [QueryViewSet > ModelMetaclass]: Encountered 2 components with identical names "Person" and different classes and . This will very likely result in an incorrect schema. Try renaming one.', - '/home/runner/work/posthog/posthog/posthog/api/query.py: Warning [QueryViewSet]: could not derive type of path parameter "id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/query.py: Error [QueryViewSet]: unable to guess serializer. This is graceful fallback handling for APIViews. Consider using GenericAPIView as view base class, if view is under your control. Either way you may want to add a serializer_class (or method). Ignoring view for now.', - '/home/runner/work/posthog/posthog/posthog/session_recordings/session_recording_api.py: Warning [SessionRecordingViewSet]: could not derive type of path parameter "project_id" because model "posthog.session_recordings.models.session_recording.SessionRecording" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/person.py: Warning [SessionRecordingViewSet > SessionRecordingSerializer > MinimalPersonSerializer]: unable to resolve type hint for function "get_distinct_ids". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/session_recordings/session_recording_api.py: Warning [SessionRecordingViewSet > SessionRecordingSerializer]: unable to resolve type hint for function "storage". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/api/session.py: Warning [SessionViewSet]: could not derive type of path parameter "project_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/ee/api/subscription.py: Warning [SubscriptionViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.subscription.Subscription" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/ee/api/subscription.py: Warning [SubscriptionViewSet > SubscriptionSerializer]: unable to resolve type hint for function "summary". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/api/organization.py: Warning [OrganizationViewSet > OrganizationSerializer]: unable to resolve type hint for function "get_metadata". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/api/organization.py: Warning [OrganizationViewSet > OrganizationSerializer]: unable to resolve type hint for function "get_member_count". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/batch_exports/http.py: Warning [BatchExportOrganizationViewSet]: could not derive type of path parameter "organization_id" because model "posthog.batch_exports.models.BatchExport" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/plugin.py: Warning [PipelineDestinationsViewSet > PluginSerializer]: unable to resolve type hint for function "get_hog_function_migration_available". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/api/project.py: Warning [ProjectViewSet > ProjectBackwardCompatSerializer]: could not resolve field on model with path "person_on_events_querying_enabled". This is likely a custom field that does some unknown magic. Maybe consider annotating the field/property? Defaulting to "string". (Exception: Project has no field named \'person_on_events_querying_enabled\')', + '/home/runner/work/posthog/posthog/posthog/api/plugin_log_entry.py: Warning [PluginLogEntryViewSet]: could not derive type of path parameter "plugin_config_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/posthog/api/project.py: Warning [ProjectViewSet > ProjectBackwardCompatSerializer]: could not resolve field on model with path "default_modifiers". This is likely a custom field that does some unknown magic. Maybe consider annotating the field/property? Defaulting to "string". (Exception: Project has no field named \'default_modifiers\')', + '/home/runner/work/posthog/posthog/posthog/api/project.py: Warning [ProjectViewSet > ProjectBackwardCompatSerializer]: could not resolve field on model with path "person_on_events_querying_enabled". This is likely a custom field that does some unknown magic. Maybe consider annotating the field/property? Defaulting to "string". (Exception: Project has no field named \'person_on_events_querying_enabled\')', '/home/runner/work/posthog/posthog/posthog/api/project.py: Warning [ProjectViewSet > ProjectBackwardCompatSerializer]: unable to resolve type hint for function "get_product_intents". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/api/proxy_record.py: Warning [ProxyRecordViewset]: could not derive type of path parameter "organization_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/proxy_record.py: Warning [ProxyRecordViewset]: could not derive type of path parameter "id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/ee/api/rbac/role.py: Warning [RoleViewSet > RoleSerializer]: unable to resolve type hint for function "get_members". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/ee/api/rbac/role.py: Warning [RoleViewSet > RoleSerializer]: unable to resolve type hint for function "get_associated_flags". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/ee/api/rbac/role.py: Warning [RoleMembershipViewSet]: could not derive type of path parameter "organization_id" because model "ee.models.rbac.role.RoleMembership" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/action.py: Warning [ActionViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.action.action.Action" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/action.py: Warning [ActionViewSet > ActionSerializer]: unable to resolve type hint for function "get_creation_context". Consider using a type hint or @extend_schema_field. Defaulting to string.', - '/home/runner/work/posthog/posthog/posthog/api/activity_log.py: Warning [ActivityLogViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.activity_logging.activity_log.ActivityLog" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/annotation.py: Warning [AnnotationsViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.annotation.Annotation" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/cohort.py: Warning [CohortViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.cohort.cohort.Cohort" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/dashboards/dashboard_templates.py: Warning [DashboardTemplateViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.dashboard_templates.DashboardTemplate" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/early_access_feature.py: Warning [EarlyAccessFeatureViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.early_access_feature.EarlyAccessFeature" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/team.py: Warning [TeamViewSet > TeamSerializer]: unable to resolve type hint for function "get_product_intents". Consider using a type hint or @extend_schema_field. Defaulting to string.', - "/home/runner/work/posthog/posthog/posthog/api/event_definition.py: Error [EventDefinitionViewSet]: exception raised while getting serializer. Hint: Is get_serializer_class() returning None or is get_queryset() not working without a request? Ignoring the view for now. (Exception: 'AnonymousUser' object has no attribute 'organization')", - '/home/runner/work/posthog/posthog/posthog/api/event_definition.py: Warning [EventDefinitionViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.event_definition.EventDefinition" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/ee/clickhouse/views/experiment_holdouts.py: Warning [ExperimentHoldoutViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.experiment.ExperimentHoldout" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/ee/clickhouse/views/experiments.py: Warning [EnterpriseExperimentsViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.experiment.Experiment" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/feature_flag.py: Warning [FeatureFlagViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.feature_flag.feature_flag.FeatureFlag" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/ee/api/feature_flag_role_access.py: Warning [FeatureFlagRoleAccessViewSet]: could not derive type of path parameter "project_id" because model "ee.models.feature_flag_role_access.FeatureFlagRoleAccess" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/ee/clickhouse/views/groups.py: Warning [GroupsTypesViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.group_type_mapping.GroupTypeMapping" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/notebook.py: Warning [NotebookViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.notebook.notebook.Notebook" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', "/home/runner/work/posthog/posthog/posthog/api/property_definition.py: Error [PropertyDefinitionViewSet]: exception raised while getting serializer. Hint: Is get_serializer_class() returning None or is get_queryset() not working without a request? Ignoring the view for now. (Exception: 'AnonymousUser' object has no attribute 'organization')", '/home/runner/work/posthog/posthog/posthog/api/property_definition.py: Warning [PropertyDefinitionViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.property_definition.PropertyDefinition" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/ee/session_recordings/session_recording_playlist.py: Warning [SessionRecordingPlaylistViewSet]: could not derive type of path parameter "project_id" because model "posthog.session_recordings.models.session_recording_playlist.SessionRecordingPlaylist" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/ee/session_recordings/session_recording_playlist.py: Warning [SessionRecordingPlaylistViewSet]: could not derive type of path parameter "session_recording_id" because model "posthog.session_recordings.models.session_recording_playlist.SessionRecordingPlaylist" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', - '/home/runner/work/posthog/posthog/posthog/api/survey.py: Warning [SurveyViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.feedback.survey.Survey" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/proxy_record.py: Warning [ProxyRecordViewset]: could not derive type of path parameter "id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/proxy_record.py: Warning [ProxyRecordViewset]: could not derive type of path parameter "organization_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/query.py: Error [QueryViewSet]: unable to guess serializer. This is graceful fallback handling for APIViews. Consider using GenericAPIView as view base class, if view is under your control. Either way you may want to add a serializer_class (or method). Ignoring view for now.', + '/home/runner/work/posthog/posthog/posthog/api/query.py: Warning [QueryViewSet]: could not derive type of path parameter "id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/query.py: Warning [QueryViewSet]: could not derive type of path parameter "project_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/session.py: Warning [SessionViewSet]: could not derive type of path parameter "project_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/sharing.py: Warning [SharingConfigurationViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.sharing_configuration.SharingConfiguration" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/posthog/api/survey.py: Warning [SurveyViewSet > SurveySerializer]: unable to resolve type hint for function "get_conditions". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/api/survey.py: Warning [SurveyViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.feedback.survey.Survey" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/api/team.py: Warning [TeamViewSet > TeamSerializer]: unable to resolve type hint for function "get_product_intents". Consider using a type hint or @extend_schema_field. Defaulting to string.', '/home/runner/work/posthog/posthog/posthog/api/web_experiment.py: Warning [WebExperimentViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.web_experiment.WebExperiment" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/batch_exports/http.py: Warning [BatchExportOrganizationViewSet]: could not derive type of path parameter "organization_id" because model "posthog.batch_exports.models.BatchExport" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/batch_exports/http.py: Warning [BatchExportRunViewSet]: could not derive type of path parameter "project_id" because model "posthog.batch_exports.models.BatchExportRun" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/batch_exports/http.py: Warning [BatchExportViewSet > BatchExportSerializer]: could not resolve serializer field "HogQLSelectQueryField(required=False)". Defaulting to "string"', + '/home/runner/work/posthog/posthog/posthog/batch_exports/http.py: Warning [BatchExportViewSet]: could not derive type of path parameter "project_id" because model "posthog.batch_exports.models.BatchExport" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/home/runner/work/posthog/posthog/posthog/models/event/util.py: Warning [EventViewSet > ClickhouseEventSerializer]: unable to resolve type hint for function "get_distinct_id". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/models/event/util.py: Warning [EventViewSet > ClickhouseEventSerializer]: unable to resolve type hint for function "get_elements". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/models/event/util.py: Warning [EventViewSet > ClickhouseEventSerializer]: unable to resolve type hint for function "get_elements_chain". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/models/event/util.py: Warning [EventViewSet > ClickhouseEventSerializer]: unable to resolve type hint for function "get_event". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/models/event/util.py: Warning [EventViewSet > ClickhouseEventSerializer]: unable to resolve type hint for function "get_id". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/models/event/util.py: Warning [EventViewSet > ClickhouseEventSerializer]: unable to resolve type hint for function "get_person". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/models/event/util.py: Warning [EventViewSet > ClickhouseEventSerializer]: unable to resolve type hint for function "get_properties". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/models/event/util.py: Warning [EventViewSet > ClickhouseEventSerializer]: unable to resolve type hint for function "get_timestamp". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/session_recordings/session_recording_api.py: Warning [SessionRecordingViewSet > SessionRecordingSerializer]: unable to resolve type hint for function "storage". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/session_recordings/session_recording_api.py: Warning [SessionRecordingViewSet]: could not derive type of path parameter "project_id" because model "posthog.session_recordings.models.session_recording.SessionRecording" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', + '/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pydantic/_internal/_model_construction.py: Warning [QueryViewSet > ModelMetaclass]: Encountered 2 components with identical names "Person" and different classes and . This will very likely result in an incorrect schema. Try renaming one.', + 'Warning: encountered multiple names for the same choice set (EffectivePrivilegeLevelEnum). This may be unwanted even though the generated schema is technically correct. Add an entry to ENUM_NAME_OVERRIDES to fix the naming.', 'Warning: encountered multiple names for the same choice set (HrefMatchingEnum). This may be unwanted even though the generated schema is technically correct. Add an entry to ENUM_NAME_OVERRIDES to fix the naming.', - 'Warning: enum naming encountered a non-optimally resolvable collision for fields named "kind". The same name has been used for multiple choice sets in multiple components. The collision was resolved with "Kind496Enum". add an entry to ENUM_NAME_OVERRIDES to fix the naming.', - 'Warning: enum naming encountered a non-optimally resolvable collision for fields named "kind". The same name has been used for multiple choice sets in multiple components. The collision was resolved with "KindCfaEnum". add an entry to ENUM_NAME_OVERRIDES to fix the naming.', + 'Warning: encountered multiple names for the same choice set (MembershipLevelEnum). This may be unwanted even though the generated schema is technically correct. Add an entry to ENUM_NAME_OVERRIDES to fix the naming.', 'Warning: enum naming encountered a non-optimally resolvable collision for fields named "kind". The same name has been used for multiple choice sets in multiple components. The collision was resolved with "Kind069Enum". add an entry to ENUM_NAME_OVERRIDES to fix the naming.', 'Warning: enum naming encountered a non-optimally resolvable collision for fields named "kind". The same name has been used for multiple choice sets in multiple components. The collision was resolved with "Kind0ddEnum". add an entry to ENUM_NAME_OVERRIDES to fix the naming.', + 'Warning: enum naming encountered a non-optimally resolvable collision for fields named "kind". The same name has been used for multiple choice sets in multiple components. The collision was resolved with "Kind496Enum". add an entry to ENUM_NAME_OVERRIDES to fix the naming.', + 'Warning: enum naming encountered a non-optimally resolvable collision for fields named "kind". The same name has been used for multiple choice sets in multiple components. The collision was resolved with "KindCfaEnum". add an entry to ENUM_NAME_OVERRIDES to fix the naming.', 'Warning: enum naming encountered a non-optimally resolvable collision for fields named "type". The same name has been used for multiple choice sets in multiple components. The collision was resolved with "TypeF73Enum". add an entry to ENUM_NAME_OVERRIDES to fix the naming.', - 'Warning: encountered multiple names for the same choice set (EffectivePrivilegeLevelEnum). This may be unwanted even though the generated schema is technically correct. Add an entry to ENUM_NAME_OVERRIDES to fix the naming.', - 'Warning: encountered multiple names for the same choice set (MembershipLevelEnum). This may be unwanted even though the generated schema is technically correct. Add an entry to ENUM_NAME_OVERRIDES to fix the naming.', - 'Warning: operationId "environments_app_metrics_historical_exports_retrieve" has collisions [(\'/api/environments/{project_id}/app_metrics/{plugin_config_id}/historical_exports/\', \'get\'), (\'/api/environments/{project_id}/app_metrics/{plugin_config_id}/historical_exports/{id}/\', \'get\')]. resolving with numeral suffixes.', - 'Warning: operationId "environments_insights_activity_retrieve" has collisions [(\'/api/environments/{project_id}/insights/{id}/activity/\', \'get\'), (\'/api/environments/{project_id}/insights/activity/\', \'get\')]. resolving with numeral suffixes.', 'Warning: operationId "Funnels" has collisions [(\'/api/environments/{project_id}/insights/funnel/\', \'post\'), (\'/api/projects/{project_id}/insights/funnel/\', \'post\')]. resolving with numeral suffixes.', 'Warning: operationId "Trends" has collisions [(\'/api/environments/{project_id}/insights/trend/\', \'post\'), (\'/api/projects/{project_id}/insights/trend/\', \'post\')]. resolving with numeral suffixes.', - 'Warning: operationId "environments_persons_activity_retrieve" has collisions [(\'/api/environments/{project_id}/persons/{id}/activity/\', \'get\'), (\'/api/environments/{project_id}/persons/activity/\', \'get\')]. resolving with numeral suffixes.', - 'Warning: operationId "list" has collisions [(\'/api/organizations/\', \'get\'), (\'/api/organizations/{organization_id}/projects/\', \'get\')]. resolving with numeral suffixes.', - 'Warning: operationId "create" has collisions [(\'/api/organizations/\', \'post\'), (\'/api/organizations/{organization_id}/projects/\', \'post\')]. resolving with numeral suffixes.', - 'Warning: operationId "retrieve" has collisions [(\'/api/organizations/{id}/\', \'get\'), (\'/api/organizations/{organization_id}/projects/{id}/\', \'get\')]. resolving with numeral suffixes.', - 'Warning: operationId "update" has collisions [(\'/api/organizations/{id}/\', \'put\'), (\'/api/organizations/{organization_id}/projects/{id}/\', \'put\')]. resolving with numeral suffixes.', - 'Warning: operationId "partial_update" has collisions [(\'/api/organizations/{id}/\', \'patch\'), (\'/api/organizations/{organization_id}/projects/{id}/\', \'patch\')]. resolving with numeral suffixes.', - 'Warning: operationId "destroy" has collisions [(\'/api/organizations/{id}/\', \'delete\'), (\'/api/organizations/{organization_id}/projects/{id}/\', \'delete\')]. resolving with numeral suffixes.', - 'Warning: operationId "batch_exports_list" has collisions [(\'/api/organizations/{organization_id}/batch_exports/\', \'get\'), (\'/api/projects/{project_id}/batch_exports/\', \'get\')]. resolving with numeral suffixes.', + 'Warning: operationId "app_metrics_historical_exports_retrieve" has collisions [(\'/api/projects/{project_id}/app_metrics/{plugin_config_id}/historical_exports/\', \'get\'), (\'/api/projects/{project_id}/app_metrics/{plugin_config_id}/historical_exports/{id}/\', \'get\')]. resolving with numeral suffixes.', + 'Warning: operationId "batch_exports_backfill_create" has collisions [(\'/api/organizations/{organization_id}/batch_exports/{id}/backfill/\', \'post\'), (\'/api/projects/{project_id}/batch_exports/{id}/backfill/\', \'post\')]. resolving with numeral suffixes.', 'Warning: operationId "batch_exports_create" has collisions [(\'/api/organizations/{organization_id}/batch_exports/\', \'post\'), (\'/api/projects/{project_id}/batch_exports/\', \'post\')]. resolving with numeral suffixes.', - 'Warning: operationId "batch_exports_retrieve" has collisions [(\'/api/organizations/{organization_id}/batch_exports/{id}/\', \'get\'), (\'/api/projects/{project_id}/batch_exports/{id}/\', \'get\')]. resolving with numeral suffixes.', - 'Warning: operationId "batch_exports_update" has collisions [(\'/api/organizations/{organization_id}/batch_exports/{id}/\', \'put\'), (\'/api/projects/{project_id}/batch_exports/{id}/\', \'put\')]. resolving with numeral suffixes.', - 'Warning: operationId "batch_exports_partial_update" has collisions [(\'/api/organizations/{organization_id}/batch_exports/{id}/\', \'patch\'), (\'/api/projects/{project_id}/batch_exports/{id}/\', \'patch\')]. resolving with numeral suffixes.', 'Warning: operationId "batch_exports_destroy" has collisions [(\'/api/organizations/{organization_id}/batch_exports/{id}/\', \'delete\'), (\'/api/projects/{project_id}/batch_exports/{id}/\', \'delete\')]. resolving with numeral suffixes.', - 'Warning: operationId "batch_exports_backfill_create" has collisions [(\'/api/organizations/{organization_id}/batch_exports/{id}/backfill/\', \'post\'), (\'/api/projects/{project_id}/batch_exports/{id}/backfill/\', \'post\')]. resolving with numeral suffixes.', + 'Warning: operationId "batch_exports_list" has collisions [(\'/api/organizations/{organization_id}/batch_exports/\', \'get\'), (\'/api/projects/{project_id}/batch_exports/\', \'get\')]. resolving with numeral suffixes.', 'Warning: operationId "batch_exports_logs_retrieve" has collisions [(\'/api/organizations/{organization_id}/batch_exports/{id}/logs/\', \'get\'), (\'/api/projects/{project_id}/batch_exports/{id}/logs/\', \'get\')]. resolving with numeral suffixes.', + 'Warning: operationId "batch_exports_partial_update" has collisions [(\'/api/organizations/{organization_id}/batch_exports/{id}/\', \'patch\'), (\'/api/projects/{project_id}/batch_exports/{id}/\', \'patch\')]. resolving with numeral suffixes.', 'Warning: operationId "batch_exports_pause_create" has collisions [(\'/api/organizations/{organization_id}/batch_exports/{id}/pause/\', \'post\'), (\'/api/projects/{project_id}/batch_exports/{id}/pause/\', \'post\')]. resolving with numeral suffixes.', + 'Warning: operationId "batch_exports_retrieve" has collisions [(\'/api/organizations/{organization_id}/batch_exports/{id}/\', \'get\'), (\'/api/projects/{project_id}/batch_exports/{id}/\', \'get\')]. resolving with numeral suffixes.', 'Warning: operationId "batch_exports_unpause_create" has collisions [(\'/api/organizations/{organization_id}/batch_exports/{id}/unpause/\', \'post\'), (\'/api/projects/{project_id}/batch_exports/{id}/unpause/\', \'post\')]. resolving with numeral suffixes.', - 'Warning: operationId "app_metrics_historical_exports_retrieve" has collisions [(\'/api/projects/{project_id}/app_metrics/{plugin_config_id}/historical_exports/\', \'get\'), (\'/api/projects/{project_id}/app_metrics/{plugin_config_id}/historical_exports/{id}/\', \'get\')]. resolving with numeral suffixes.', + 'Warning: operationId "batch_exports_update" has collisions [(\'/api/organizations/{organization_id}/batch_exports/{id}/\', \'put\'), (\'/api/projects/{project_id}/batch_exports/{id}/\', \'put\')]. resolving with numeral suffixes.', 'Warning: operationId "cohorts_activity_retrieve" has collisions [(\'/api/projects/{project_id}/cohorts/{id}/activity/\', \'get\'), (\'/api/projects/{project_id}/cohorts/activity/\', \'get\')]. resolving with numeral suffixes.', + 'Warning: operationId "create" has collisions [(\'/api/organizations/\', \'post\'), (\'/api/organizations/{organization_id}/projects/\', \'post\')]. resolving with numeral suffixes.', + 'Warning: operationId "destroy" has collisions [(\'/api/organizations/{id}/\', \'delete\'), (\'/api/organizations/{organization_id}/projects/{id}/\', \'delete\')]. resolving with numeral suffixes.', + 'Warning: operationId "environments_app_metrics_historical_exports_retrieve" has collisions [(\'/api/environments/{project_id}/app_metrics/{plugin_config_id}/historical_exports/\', \'get\'), (\'/api/environments/{project_id}/app_metrics/{plugin_config_id}/historical_exports/{id}/\', \'get\')]. resolving with numeral suffixes.', + 'Warning: operationId "environments_insights_activity_retrieve" has collisions [(\'/api/environments/{project_id}/insights/{id}/activity/\', \'get\'), (\'/api/environments/{project_id}/insights/activity/\', \'get\')]. resolving with numeral suffixes.', + 'Warning: operationId "environments_persons_activity_retrieve" has collisions [(\'/api/environments/{project_id}/persons/{id}/activity/\', \'get\'), (\'/api/environments/{project_id}/persons/activity/\', \'get\')]. resolving with numeral suffixes.', 'Warning: operationId "event_definitions_retrieve" has collisions [(\'/api/projects/{project_id}/event_definitions/\', \'get\'), (\'/api/projects/{project_id}/event_definitions/{id}/\', \'get\')]. resolving with numeral suffixes.', 'Warning: operationId "feature_flags_activity_retrieve" has collisions [(\'/api/projects/{project_id}/feature_flags/{id}/activity/\', \'get\'), (\'/api/projects/{project_id}/feature_flags/activity/\', \'get\')]. resolving with numeral suffixes.', 'Warning: operationId "insights_activity_retrieve" has collisions [(\'/api/projects/{project_id}/insights/{id}/activity/\', \'get\'), (\'/api/projects/{project_id}/insights/activity/\', \'get\')]. resolving with numeral suffixes.', + 'Warning: operationId "list" has collisions [(\'/api/organizations/\', \'get\'), (\'/api/organizations/{organization_id}/projects/\', \'get\')]. resolving with numeral suffixes.', 'Warning: operationId "notebooks_activity_retrieve" has collisions [(\'/api/projects/{project_id}/notebooks/{short_id}/activity/\', \'get\'), (\'/api/projects/{project_id}/notebooks/activity/\', \'get\')]. resolving with numeral suffixes.', + 'Warning: operationId "partial_update" has collisions [(\'/api/organizations/{id}/\', \'patch\'), (\'/api/organizations/{organization_id}/projects/{id}/\', \'patch\')]. resolving with numeral suffixes.', 'Warning: operationId "persons_activity_retrieve" has collisions [(\'/api/projects/{project_id}/persons/{id}/activity/\', \'get\'), (\'/api/projects/{project_id}/persons/activity/\', \'get\')]. resolving with numeral suffixes.', 'Warning: operationId "property_definitions_retrieve" has collisions [(\'/api/projects/{project_id}/property_definitions/\', \'get\'), (\'/api/projects/{project_id}/property_definitions/{id}/\', \'get\')]. resolving with numeral suffixes.', + 'Warning: operationId "retrieve" has collisions [(\'/api/organizations/{id}/\', \'get\'), (\'/api/organizations/{organization_id}/projects/{id}/\', \'get\')]. resolving with numeral suffixes.', 'Warning: operationId "surveys_activity_retrieve" has collisions [(\'/api/projects/{project_id}/surveys/{id}/activity/\', \'get\'), (\'/api/projects/{project_id}/surveys/activity/\', \'get\')]. resolving with numeral suffixes.', - '', + 'Warning: operationId "update" has collisions [(\'/api/organizations/{id}/\', \'put\'), (\'/api/organizations/{organization_id}/projects/{id}/\', \'put\')]. resolving with numeral suffixes.', ]) # --- diff --git a/posthog/api/test/test_api_docs.py b/posthog/api/test/test_api_docs.py index f7f7ab1d418b0..d5e1fc9b3bbf6 100644 --- a/posthog/api/test/test_api_docs.py +++ b/posthog/api/test/test_api_docs.py @@ -30,4 +30,4 @@ def test_api_docs_generation_warnings_snapshot(self) -> None: # we log lots of warnings when generating the schema warnings = self._capsys.readouterr().err.split("\n") - assert warnings == self._snapshot + assert sorted(warnings) == self._snapshot diff --git a/posthog/clickhouse/migrations/0083_recreate_sessions_v1_after_limiting_teams.py b/posthog/clickhouse/migrations/0083_recreate_sessions_v1_after_limiting_teams.py new file mode 100644 index 0000000000000..d0da2eb158b6d --- /dev/null +++ b/posthog/clickhouse/migrations/0083_recreate_sessions_v1_after_limiting_teams.py @@ -0,0 +1,8 @@ +from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions +from posthog.models.sessions.sql import DROP_SESSION_MATERIALIZED_VIEW_SQL, SESSIONS_TABLE_MV_SQL + +operations = [ + # drop the mv, and recreate it with the new part of the WHERE clause + run_sql_with_exceptions(DROP_SESSION_MATERIALIZED_VIEW_SQL()), + run_sql_with_exceptions(SESSIONS_TABLE_MV_SQL()), +] diff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambr index bc5a623ed0cdc..1fc54f30fdc78 100644 --- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr +++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr @@ -2160,7 +2160,7 @@ sumIf(1, event='$autocapture') as autocapture_count FROM posthog_test.sharded_events - WHERE `$session_id` IS NOT NULL AND `$session_id` != '' + WHERE `$session_id` IS NOT NULL AND `$session_id` != '' AND team_id IN (1, 2, 13610, 19279, 21173, 29929, 32050, 9910, 11775, 21129, 31490) GROUP BY `$session_id`, team_id diff --git a/posthog/clickhouse/test/test_sessions_model.py b/posthog/clickhouse/test/test_sessions_model.py index 0e3631e0ab3cd..0042456a03d95 100644 --- a/posthog/clickhouse/test/test_sessions_model.py +++ b/posthog/clickhouse/test/test_sessions_model.py @@ -1,8 +1,10 @@ from posthog.clickhouse.client import sync_execute, query_with_columns +from posthog.models import Team from posthog.test.base import ( _create_event, ClickhouseTestMixin, BaseTest, + ClickhouseDestroyTablesMixin, ) distinct_id_counter = 0 @@ -21,7 +23,12 @@ def create_session_id(): return f"s{session_id_counter}" -class TestSessionsModel(ClickhouseTestMixin, BaseTest): +# only certain team ids can insert events into this legacy sessions table, see sessions/sql.py for more info +TEAM_ID = 2 +TEAM = Team(id=TEAM_ID) + + +class TestSessionsModel(ClickhouseDestroyTablesMixin, ClickhouseTestMixin, BaseTest): def select_by_session_id(self, session_id): return query_with_columns( """ @@ -34,7 +41,7 @@ def select_by_session_id(self, session_id): """, { "session_id": session_id, - "team_id": self.team.id, + "team_id": TEAM_ID, }, ) @@ -42,7 +49,7 @@ def test_it_creates_session_when_creating_event(self): distinct_id = create_distinct_id() session_id = create_session_id() _create_event( - team=self.team, + team=TEAM, event="$pageview", distinct_id=distinct_id, properties={"$current_url": "/", "$session_id": session_id}, @@ -60,7 +67,7 @@ def test_it_creates_session_when_creating_event(self): """, { "distinct_id": distinct_id, - "team_id": self.team.id, + "team_id": TEAM_ID, }, ) @@ -72,14 +79,14 @@ def test_handles_different_distinct_id_across_same_session(self): session_id = create_session_id() _create_event( - team=self.team, + team=TEAM, event="$pageview", distinct_id=distinct_id1, properties={"$session_id": session_id}, timestamp="2024-03-08", ) _create_event( - team=self.team, + team=TEAM, event="$pageview", distinct_id=distinct_id2, properties={"$session_id": session_id}, @@ -96,28 +103,28 @@ def test_handles_entry_and_exit_urls(self): session_id = create_session_id() _create_event( - team=self.team, + team=TEAM, event="$pageview", distinct_id=distinct_id, properties={"$current_url": "/entry", "$session_id": session_id}, timestamp="2024-03-08:01", ) _create_event( - team=self.team, + team=TEAM, event="$pageview", distinct_id=distinct_id, properties={"$current_url": "/middle", "$session_id": session_id}, timestamp="2024-03-08:02", ) _create_event( - team=self.team, + team=TEAM, event="$pageview", distinct_id=distinct_id, properties={"$current_url": "/middle", "$session_id": session_id}, timestamp="2024-03-08:03", ) _create_event( - team=self.team, + team=TEAM, event="$pageview", distinct_id=distinct_id, properties={"$current_url": "/exit", "$session_id": session_id}, @@ -136,14 +143,14 @@ def test_handles_initial_utm_properties(self): session_id = create_session_id() _create_event( - team=self.team, + team=TEAM, event="$pageview", distinct_id=distinct_id, properties={"$session_id": session_id, "utm_source": "source"}, timestamp="2024-03-08", ) _create_event( - team=self.team, + team=TEAM, event="$pageview", distinct_id=distinct_id, properties={"$session_id": session_id, "utm_source": "other_source"}, @@ -159,35 +166,35 @@ def test_counts_pageviews_autocaptures_and_events(self): session_id = create_session_id() _create_event( - team=self.team, + team=TEAM, event="$pageview", distinct_id=distinct_id, properties={"$session_id": session_id}, timestamp="2024-03-08", ) _create_event( - team=self.team, + team=TEAM, event="$autocapture", distinct_id=distinct_id, properties={"$session_id": session_id}, timestamp="2024-03-08", ) _create_event( - team=self.team, + team=TEAM, event="$autocapture", distinct_id=distinct_id, properties={"$session_id": session_id}, timestamp="2024-03-08", ) _create_event( - team=self.team, + team=TEAM, event="other event", distinct_id=distinct_id, properties={"$session_id": session_id}, timestamp="2024-03-08", ) _create_event( - team=self.team, + team=TEAM, event="$pageleave", distinct_id=distinct_id, properties={"$session_id": session_id}, @@ -209,14 +216,14 @@ def test_separates_sessions_across_same_user(self): session_id3 = create_session_id() _create_event( - team=self.team, + team=TEAM, event="$pageview", distinct_id=distinct_id, properties={"$session_id": session_id1}, timestamp="2024-03-08", ) _create_event( - team=self.team, + team=TEAM, event="$pageview", distinct_id=distinct_id, properties={"$session_id": session_id2}, @@ -235,7 +242,7 @@ def test_select_from_sessions(self): distinct_id = create_distinct_id() session_id = create_session_id() _create_event( - team=self.team, + team=TEAM, event="$pageview", distinct_id=distinct_id, properties={"$session_id": session_id}, @@ -260,7 +267,7 @@ def test_select_from_sessions(self): """, { "session_id": session_id, - "team_id": self.team.id, + "team_id": TEAM_ID, }, ) self.assertEqual(len(responses), 1) @@ -270,7 +277,7 @@ def test_select_from_sessions_mv(self): distinct_id = create_distinct_id() session_id = create_session_id() _create_event( - team=self.team, + team=TEAM, event="$pageview", distinct_id=distinct_id, properties={"$session_id": session_id}, @@ -295,7 +302,7 @@ def test_select_from_sessions_mv(self): """, { "session_id": session_id, - "team_id": self.team.id, + "team_id": TEAM_ID, }, ) self.assertEqual(len(responses), 1) diff --git a/posthog/hogql/database/schema/test/test_sessions_v1.py b/posthog/hogql/database/schema/test/test_sessions_v1.py index eefd04197deab..77f41fd2f6bbc 100644 --- a/posthog/hogql/database/schema/test/test_sessions_v1.py +++ b/posthog/hogql/database/schema/test/test_sessions_v1.py @@ -8,6 +8,7 @@ ) from posthog.hogql.parser import parse_select from posthog.hogql.query import execute_hogql_query +from posthog.models import Team from posthog.models.property_definition import PropertyType from posthog.models.utils import uuid7 from posthog.schema import HogQLQueryModifiers, BounceRatePageViewMode, SessionTableVersion @@ -15,16 +16,20 @@ APIBaseTest, ClickhouseTestMixin, _create_event, - _create_person, + ClickhouseDestroyTablesMixin, ) +# only certain team ids can insert events into this legacy sessions table, see sessions/sql.py for more info +TEAM_ID = 2 +TEAM = Team(id=TEAM_ID, pk=TEAM_ID) -class TestSessionsV1(ClickhouseTestMixin, APIBaseTest): + +class TestSessionsV1(ClickhouseDestroyTablesMixin, ClickhouseTestMixin, APIBaseTest): def __execute(self, query): modifiers = HogQLQueryModifiers(sessionTableVersion=SessionTableVersion.V1) return execute_hogql_query( query=query, - team=self.team, + team=TEAM, modifiers=modifiers, ) @@ -33,7 +38,7 @@ def test_select_star(self): _create_event( event="$pageview", - team=self.team, + team=TEAM, distinct_id="d1", properties={"$current_url": "https://example.com", "$session_id": session_id}, ) @@ -56,7 +61,7 @@ def test_select_event_sessions_star(self): _create_event( event="$pageview", - team=self.team, + team=TEAM, distinct_id="d1", properties={"$current_url": "https://example.com", "$session_id": session_id}, ) @@ -93,7 +98,7 @@ def test_channel_type(self): _create_event( event="$pageview", - team=self.team, + team=TEAM, distinct_id="d1", properties={"gad_source": "1", "$session_id": session_id}, ) @@ -116,7 +121,7 @@ def test_event_dot_session_dot_channel_type(self): _create_event( event="$pageview", - team=self.team, + team=TEAM, distinct_id="d1", properties={"gad_source": "1", "$session_id": session_id}, ) @@ -139,7 +144,7 @@ def test_events_session_dot_channel_type(self): _create_event( event="$pageview", - team=self.team, + team=TEAM, distinct_id="d1", properties={"gad_source": "1", "$session_id": session_id}, ) @@ -157,57 +162,26 @@ def test_events_session_dot_channel_type(self): "Paid Search", ) - def test_persons_and_sessions_on_events(self): - p1 = _create_person(distinct_ids=["d1"], team=self.team) - p2 = _create_person(distinct_ids=["d2"], team=self.team) - - s1 = "session_test_persons_and_sessions_on_events_1" - s2 = "session_test_persons_and_sessions_on_events_2" - - _create_event( - event="$pageview", - team=self.team, - distinct_id="d1", - properties={"$session_id": s1, "utm_source": "source1"}, - ) - _create_event( - event="$pageview", - team=self.team, - distinct_id="d2", - properties={"$session_id": s2, "utm_source": "source2"}, - ) - - response = self.__execute( - parse_select( - "select events.person_id, session.$entry_utm_source from events where $session_id = {session_id} or $session_id = {session_id2} order by 2 asc", - placeholders={"session_id": ast.Constant(value=s1), "session_id2": ast.Constant(value=s2)}, - ), - ) - - [row1, row2] = response.results or [] - self.assertEqual(row1, (p1.uuid, "source1")) - self.assertEqual(row2, (p2.uuid, "source2")) - @parameterized.expand([(BounceRatePageViewMode.UNIQ_URLS,), (BounceRatePageViewMode.COUNT_PAGEVIEWS,)]) def test_bounce_rate(self, bounceRatePageViewMode): # person with 2 different sessions _create_event( event="$pageview", - team=self.team, + team=TEAM, distinct_id="d1", properties={"$session_id": "s1a", "$current_url": "https://example.com/1"}, timestamp="2023-12-02", ) _create_event( event="$pageview", - team=self.team, + team=TEAM, distinct_id="d1", properties={"$session_id": "s1a", "$current_url": "https://example.com/2"}, timestamp="2023-12-03", ) _create_event( event="$pageview", - team=self.team, + team=TEAM, distinct_id="d1", properties={"$session_id": "s1b", "$current_url": "https://example.com/3"}, timestamp="2023-12-12", @@ -215,7 +189,7 @@ def test_bounce_rate(self, bounceRatePageViewMode): # session with 1 pageview _create_event( event="$pageview", - team=self.team, + team=TEAM, distinct_id="d2", properties={"$session_id": "s2", "$current_url": "https://example.com/4"}, timestamp="2023-12-11", @@ -223,14 +197,14 @@ def test_bounce_rate(self, bounceRatePageViewMode): # session with 1 pageview and 1 autocapture _create_event( event="$pageview", - team=self.team, + team=TEAM, distinct_id="d3", properties={"$session_id": "s3", "$current_url": "https://example.com/5"}, timestamp="2023-12-11", ) _create_event( event="$autocapture", - team=self.team, + team=TEAM, distinct_id="d3", properties={"$session_id": "s3", "$current_url": "https://example.com/5"}, timestamp="2023-12-11", @@ -238,14 +212,14 @@ def test_bounce_rate(self, bounceRatePageViewMode): # short session with a pageleave _create_event( event="$pageview", - team=self.team, + team=TEAM, distinct_id="d4", properties={"$session_id": "s4", "$current_url": "https://example.com/6"}, timestamp="2023-12-11T12:00:00", ) _create_event( event="$pageleave", - team=self.team, + team=TEAM, distinct_id="d4", properties={"$session_id": "s4", "$current_url": "https://example.com/6"}, timestamp="2023-12-11T12:00:01", @@ -253,14 +227,14 @@ def test_bounce_rate(self, bounceRatePageViewMode): # long session with a pageleave _create_event( event="$pageview", - team=self.team, + team=TEAM, distinct_id="d5", properties={"$session_id": "s5", "$current_url": "https://example.com/7"}, timestamp="2023-12-11T12:00:00", ) _create_event( event="$pageleave", - team=self.team, + team=TEAM, distinct_id="d5", properties={"$session_id": "s5", "$current_url": "https://example.com/7"}, timestamp="2023-12-11T12:00:11", @@ -269,7 +243,7 @@ def test_bounce_rate(self, bounceRatePageViewMode): parse_select( "select $is_bounce, session_id from sessions ORDER BY session_id", ), - self.team, + TEAM, modifiers=HogQLQueryModifiers( bounceRatePageViewMode=bounceRatePageViewMode, sessionTableVersion=SessionTableVersion.V1 ), @@ -291,7 +265,7 @@ def test_can_use_v1_and_v2_fields(self): _create_event( event="$pageview", - team=self.team, + team=TEAM, distinct_id="d1", properties={ "$current_url": "https://example.com/pathname", @@ -372,4 +346,4 @@ def test_entry_utm(self): def test_can_get_values_for_all(self): results = get_lazy_session_table_properties_v1(None) for prop in results: - get_lazy_session_table_values_v1(key=prop["id"], team=self.team, search_term=None) + get_lazy_session_table_values_v1(key=prop["id"], team=TEAM, search_term=None) diff --git a/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr b/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr index f018e96ef067a..06e46e0ca30b0 100644 --- a/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr +++ b/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr @@ -31,7 +31,7 @@ FROM events LEFT JOIN ( SELECT person_static_cohort.person_id AS cohort_person_id, 1 AS matched, person_static_cohort.cohort_id AS cohort_id FROM person_static_cohort - WHERE and(equals(person_static_cohort.team_id, 420), in(person_static_cohort.cohort_id, [4]))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id) + WHERE and(equals(person_static_cohort.team_id, 420), in(person_static_cohort.cohort_id, [6]))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id) WHERE and(equals(events.team_id, 420), 1, ifNull(equals(__in_cohort.matched, 1), 0)) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0 @@ -42,7 +42,7 @@ FROM events LEFT JOIN ( SELECT person_id AS cohort_person_id, 1 AS matched, cohort_id FROM static_cohort_people - WHERE in(cohort_id, [4])) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id) + WHERE in(cohort_id, [6])) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id) WHERE and(1, equals(__in_cohort.matched, 1)) LIMIT 100 ''' @@ -55,7 +55,7 @@ FROM events LEFT JOIN ( SELECT person_static_cohort.person_id AS cohort_person_id, 1 AS matched, person_static_cohort.cohort_id AS cohort_id FROM person_static_cohort - WHERE and(equals(person_static_cohort.team_id, 420), in(person_static_cohort.cohort_id, [5]))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id) + WHERE and(equals(person_static_cohort.team_id, 420), in(person_static_cohort.cohort_id, [7]))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id) WHERE and(equals(events.team_id, 420), 1, ifNull(equals(__in_cohort.matched, 1), 0)) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0 @@ -66,7 +66,7 @@ FROM events LEFT JOIN ( SELECT person_id AS cohort_person_id, 1 AS matched, cohort_id FROM static_cohort_people - WHERE in(cohort_id, [5])) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id) + WHERE in(cohort_id, [7])) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id) WHERE and(1, equals(__in_cohort.matched, 1)) LIMIT 100 ''' diff --git a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr index 6027f7ca7bb42..4ae57feb8cb96 100644 --- a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr +++ b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr @@ -851,14 +851,49 @@ # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.1 ''' - /* celery:posthog.tasks.tasks.sync_insight_caching_state */ - SELECT team_id, - date_diff('second', max(timestamp), now()) AS age - FROM events - WHERE timestamp > date_sub(DAY, 3, now()) - AND timestamp < now() - GROUP BY team_id - ORDER BY age; + SELECT groupArray(1)(date)[1] AS date, + arrayFold((acc, x) -> arrayMap(i -> plus(acc[i], x[i]), range(1, plus(length(date), 1))), groupArray(ifNull(total, 0)), arrayWithConstant(length(date), reinterpretAsFloat64(0))) AS total, + if(ifNull(ifNull(greaterOrEquals(row_number, 25), 0), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(ifNull(count, 0)), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + breakdown_value AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(DISTINCT if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id)) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1.0 + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC) + WHERE isNotNull(breakdown_value) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 ''' # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.10 @@ -1075,38 +1110,143 @@ # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.2 ''' - /* celery:posthog.tasks.tasks.sync_insight_caching_state */ - SELECT team_id, - date_diff('second', max(timestamp), now()) AS age - FROM events - WHERE timestamp > date_sub(DAY, 3, now()) - AND timestamp < now() - GROUP BY team_id - ORDER BY age; + SELECT groupArray(1)(date)[1] AS date, + arrayFold((acc, x) -> arrayMap(i -> plus(acc[i], x[i]), range(1, plus(length(date), 1))), groupArray(ifNull(total, 0)), arrayWithConstant(length(date), reinterpretAsFloat64(0))) AS total, + if(ifNull(ifNull(greaterOrEquals(row_number, 25), 0), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(ifNull(count, 0)), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + breakdown_value AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(DISTINCT if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id)) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1.0 + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC) + WHERE isNotNull(breakdown_value) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 ''' # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.3 ''' - /* celery:posthog.tasks.tasks.sync_insight_caching_state */ - SELECT team_id, - date_diff('second', max(timestamp), now()) AS age - FROM events - WHERE timestamp > date_sub(DAY, 3, now()) - AND timestamp < now() - GROUP BY team_id - ORDER BY age; + SELECT groupArray(1)(date)[1] AS date, + arrayFold((acc, x) -> arrayMap(i -> plus(acc[i], x[i]), range(1, plus(length(date), 1))), groupArray(ifNull(total, 0)), arrayWithConstant(length(date), reinterpretAsFloat64(0))) AS total, + arrayMap(i -> if(ifNull(ifNull(greaterOrEquals(row_number, 25), 0), 0), '$$_posthog_breakdown_other_$$', i), breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(ifNull(count, 0)), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + breakdown_value AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + [ifNull(toString(breakdown_value_1), '$$_posthog_breakdown_null_$$')] AS breakdown_value + FROM + (SELECT count(DISTINCT if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id)) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value_1 + FROM events AS e SAMPLE 1.0 + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')) + GROUP BY day_start, + breakdown_value_1) + GROUP BY day_start, + breakdown_value_1 + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(has(breakdown_value, '$$_posthog_breakdown_other_$$'), 2, if(has(breakdown_value, '$$_posthog_breakdown_null_$$'), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC) + WHERE arrayExists(x -> isNotNull(x), breakdown_value) + GROUP BY breakdown_value + ORDER BY if(has(breakdown_value, '$$_posthog_breakdown_other_$$'), 2, if(has(breakdown_value, '$$_posthog_breakdown_null_$$'), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 ''' # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.4 ''' - /* celery:posthog.tasks.tasks.sync_insight_caching_state */ - SELECT team_id, - date_diff('second', max(timestamp), now()) AS age - FROM events - WHERE timestamp > date_sub(DAY, 3, now()) - AND timestamp < now() - GROUP BY team_id - ORDER BY age; + SELECT groupArray(1)(date)[1] AS date, + arrayFold((acc, x) -> arrayMap(i -> plus(acc[i], x[i]), range(1, plus(length(date), 1))), groupArray(ifNull(total, 0)), arrayWithConstant(length(date), reinterpretAsFloat64(0))) AS total, + arrayMap(i -> if(ifNull(ifNull(greaterOrEquals(row_number, 25), 0), 0), '$$_posthog_breakdown_other_$$', i), breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(ifNull(count, 0)), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + breakdown_value AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + [ifNull(toString(breakdown_value_1), '$$_posthog_breakdown_null_$$')] AS breakdown_value + FROM + (SELECT count(DISTINCT if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id)) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value_1 + FROM events AS e SAMPLE 1.0 + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')) + GROUP BY day_start, + breakdown_value_1) + GROUP BY day_start, + breakdown_value_1 + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(has(breakdown_value, '$$_posthog_breakdown_other_$$'), 2, if(has(breakdown_value, '$$_posthog_breakdown_null_$$'), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC) + WHERE arrayExists(x -> isNotNull(x), breakdown_value) + GROUP BY breakdown_value + ORDER BY if(has(breakdown_value, '$$_posthog_breakdown_other_$$'), 2, if(has(breakdown_value, '$$_posthog_breakdown_null_$$'), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 ''' # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.5 diff --git a/posthog/hogql_queries/web_analytics/test/test_session_attribution_explorer_query_runner.py b/posthog/hogql_queries/web_analytics/test/test_session_attribution_explorer_query_runner.py index d285ff2aa25f3..06ba1ddfeabbe 100644 --- a/posthog/hogql_queries/web_analytics/test/test_session_attribution_explorer_query_runner.py +++ b/posthog/hogql_queries/web_analytics/test/test_session_attribution_explorer_query_runner.py @@ -1,6 +1,5 @@ from typing import Optional -from parameterized import parameterized from posthog.hogql.constants import LimitContext from posthog.hogql_queries.web_analytics.session_attribution_explorer_query_runner import ( @@ -80,7 +79,7 @@ def _run_session_attribution_query( self, date_from: Optional[str] = None, date_to: Optional[str] = None, - session_table_version: SessionTableVersion = SessionTableVersion.V1, + session_table_version: SessionTableVersion = SessionTableVersion.V2, group_by: Optional[list[SessionAttributionGroupBy]] = None, limit_context: Optional[LimitContext] = None, properties: Optional[list[SessionPropertyFilter]] = None, @@ -94,20 +93,14 @@ def _run_session_attribution_query( runner = SessionAttributionExplorerQueryRunner(team=self.team, query=query, limit_context=limit_context) return runner.calculate() - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_no_crash_when_no_data(self, session_table_version: SessionTableVersion): - results = self._run_session_attribution_query( - session_table_version=session_table_version, - ).results + def test_no_crash_when_no_data(self): + results = self._run_session_attribution_query().results assert results == [(0, [], [], [], [], [], [], [])] - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_group_by_nothing(self, session_table_version: SessionTableVersion): + def test_group_by_nothing(self): self._create_data() - results = self._run_session_attribution_query( - session_table_version=session_table_version, - ).results + results = self._run_session_attribution_query().results assert results == [ ( @@ -122,12 +115,10 @@ def test_group_by_nothing(self, session_table_version: SessionTableVersion): ) ] - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_group_by_initial_url(self, session_table_version: SessionTableVersion): + def test_group_by_initial_url(self): self._create_data() results = self._run_session_attribution_query( - session_table_version=session_table_version, group_by=[SessionAttributionGroupBy.INITIAL_URL], ).results @@ -164,12 +155,10 @@ def test_group_by_initial_url(self, session_table_version: SessionTableVersion): ), ] - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_group_channel_medium_source(self, session_table_version: SessionTableVersion): + def test_group_channel_medium_source(self): self._create_data() results = self._run_session_attribution_query( - session_table_version=session_table_version, group_by=[ SessionAttributionGroupBy.CHANNEL_TYPE, SessionAttributionGroupBy.MEDIUM, @@ -191,12 +180,10 @@ def test_group_channel_medium_source(self, session_table_version: SessionTableVe (1, "Referral", ["referring_domain2"], "source2", "medium2", ["campaign2"], [], ["http://example.com/2"]), ] - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_filters(self, session_table_version: SessionTableVersion): + def test_filters(self): self._create_data() results = self._run_session_attribution_query( - session_table_version=session_table_version, group_by=[ SessionAttributionGroupBy.CHANNEL_TYPE, SessionAttributionGroupBy.MEDIUM, diff --git a/posthog/hogql_queries/web_analytics/test/test_web_overview.py b/posthog/hogql_queries/web_analytics/test/test_web_overview.py index bc41d4d0a6785..3e9b570f57b9e 100644 --- a/posthog/hogql_queries/web_analytics/test/test_web_overview.py +++ b/posthog/hogql_queries/web_analytics/test/test_web_overview.py @@ -2,7 +2,6 @@ from unittest.mock import MagicMock, patch from freezegun import freeze_time -from parameterized import parameterized from posthog.clickhouse.client.execute import sync_execute from posthog.hogql.constants import LimitContext @@ -72,7 +71,7 @@ def _run_web_overview_query( self, date_from: str, date_to: str, - session_table_version: SessionTableVersion = SessionTableVersion.V1, + session_table_version: SessionTableVersion = SessionTableVersion.V2, compare: bool = True, limit_context: Optional[LimitContext] = None, filter_test_accounts: Optional[bool] = False, @@ -97,19 +96,16 @@ def _run_web_overview_query( runner = WebOverviewQueryRunner(team=self.team, query=query, limit_context=limit_context) return runner.calculate() - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_no_crash_when_no_data(self, session_table_version: SessionTableVersion): + def test_no_crash_when_no_data(self): results = self._run_web_overview_query( "2023-12-08", "2023-12-15", - session_table_version=session_table_version, ).results assert [item.key for item in results] == ["visitors", "views", "sessions", "session duration", "bounce rate"] results = self._run_web_overview_query( "2023-12-08", "2023-12-15", - session_table_version=session_table_version, includeLCPScore=True, ).results assert [item.key for item in results] == [ @@ -132,9 +128,7 @@ def test_no_crash_when_no_data(self, session_table_version: SessionTableVersion) } ], ) - results = self._run_web_overview_query( - "2023-12-08", "2023-12-15", session_table_version=session_table_version, action=action - ).results + results = self._run_web_overview_query("2023-12-08", "2023-12-15", action=action).results assert [item.key for item in results] == [ "visitors", @@ -143,8 +137,7 @@ def test_no_crash_when_no_data(self, session_table_version: SessionTableVersion) "conversion rate", ] - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_increase_in_users(self, session_table_version: SessionTableVersion): + def test_increase_in_users(self): s1a = str(uuid7("2023-12-02")) s1b = str(uuid7("2023-12-12")) s2 = str(uuid7("2023-12-11")) @@ -159,7 +152,6 @@ def test_increase_in_users(self, session_table_version: SessionTableVersion): results = self._run_web_overview_query( "2023-12-08", "2023-12-15", - session_table_version=session_table_version, ).results visitors = results[0] @@ -192,8 +184,7 @@ def test_increase_in_users(self, session_table_version: SessionTableVersion): self.assertEqual(0, bounce.previous) self.assertEqual(None, bounce.changeFromPreviousPct) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_all_time(self, session_table_version: SessionTableVersion): + def test_all_time(self): s1a = str(uuid7("2023-12-02")) s1b = str(uuid7("2023-12-12")) s2 = str(uuid7("2023-12-11")) @@ -208,7 +199,6 @@ def test_all_time(self, session_table_version: SessionTableVersion): "all", "2023-12-15", compare=False, - session_table_version=session_table_version, ).results visitors = results[0] @@ -241,15 +231,12 @@ def test_all_time(self, session_table_version: SessionTableVersion): self.assertEqual(None, bounce.previous) self.assertEqual(None, bounce.changeFromPreviousPct) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_filter_test_accounts(self, session_table_version: SessionTableVersion): + def test_filter_test_accounts(self): s1 = str(uuid7("2023-12-02")) # Create 1 test account self._create_events([("test", [("2023-12-02", s1), ("2023-12-03", s1)])]) - results = self._run_web_overview_query( - "2023-12-01", "2023-12-03", session_table_version=session_table_version, filter_test_accounts=True - ).results + results = self._run_web_overview_query("2023-12-01", "2023-12-03", filter_test_accounts=True).results visitors = results[0] self.assertEqual(0, visitors.value) @@ -267,21 +254,17 @@ def test_filter_test_accounts(self, session_table_version: SessionTableVersion): self.assertEqual("bounce rate", bounce.key) self.assertEqual(None, bounce.value) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_dont_filter_test_accounts(self, session_table_version: SessionTableVersion): + def test_dont_filter_test_accounts(self): s1 = str(uuid7("2023-12-02")) # Create 1 test account self._create_events([("test", [("2023-12-02", s1), ("2023-12-03", s1)])]) - results = self._run_web_overview_query( - "2023-12-01", "2023-12-03", session_table_version=session_table_version, filter_test_accounts=False - ).results + results = self._run_web_overview_query("2023-12-01", "2023-12-03", filter_test_accounts=False).results visitors = results[0] self.assertEqual(1, visitors.value) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_correctly_counts_pageviews_in_long_running_session(self, session_table_version: SessionTableVersion): + def test_correctly_counts_pageviews_in_long_running_session(self): # this test is important when using the v1 sessions table as the raw sessions table will have 3 entries, one per day s1 = str(uuid7("2023-12-01")) self._create_events( @@ -293,7 +276,6 @@ def test_correctly_counts_pageviews_in_long_running_session(self, session_table_ results = self._run_web_overview_query( "2023-12-01", "2023-12-03", - session_table_version=session_table_version, ).results visitors = results[0] diff --git a/posthog/hogql_queries/web_analytics/test/test_web_stats_table.py b/posthog/hogql_queries/web_analytics/test/test_web_stats_table.py index 10ce9ab1ebcb4..865424f5bc862 100644 --- a/posthog/hogql_queries/web_analytics/test/test_web_stats_table.py +++ b/posthog/hogql_queries/web_analytics/test/test_web_stats_table.py @@ -1,7 +1,6 @@ from typing import Optional from freezegun import freeze_time -from parameterized import parameterized from posthog.hogql_queries.web_analytics.stats_table import WebStatsTableQueryRunner from posthog.models import Cohort @@ -108,7 +107,7 @@ def _run_web_stats_table_query( include_bounce_rate=False, include_scroll_depth=False, properties=None, - session_table_version: SessionTableVersion = SessionTableVersion.V1, + session_table_version: SessionTableVersion = SessionTableVersion.V2, filter_test_accounts: Optional[bool] = False, ): modifiers = HogQLQueryModifiers(sessionTableVersion=session_table_version) @@ -126,15 +125,14 @@ def _run_web_stats_table_query( runner = WebStatsTableQueryRunner(team=self.team, query=query, modifiers=modifiers) return runner.calculate() - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_no_crash_when_no_data(self, session_table_version: SessionTableVersion): + def test_no_crash_when_no_data(self): results = self._run_web_stats_table_query( - "2023-12-08", "2023-12-15", session_table_version=session_table_version + "2023-12-08", + "2023-12-15", ).results self.assertEqual([], results) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_increase_in_users(self, session_table_version: SessionTableVersion): + def test_increase_in_users(self): s1a = str(uuid7("2023-12-02")) s1b = str(uuid7("2023-12-13")) s2 = str(uuid7("2023-12-10")) @@ -145,9 +143,7 @@ def test_increase_in_users(self, session_table_version: SessionTableVersion): ] ) - results = self._run_web_stats_table_query( - "2023-12-01", "2023-12-11", session_table_version=session_table_version - ).results + results = self._run_web_stats_table_query("2023-12-01", "2023-12-11").results self.assertEqual( [ @@ -157,8 +153,7 @@ def test_increase_in_users(self, session_table_version: SessionTableVersion): results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_all_time(self, session_table_version: SessionTableVersion): + def test_all_time(self): s1a = str(uuid7("2023-12-02")) s1b = str(uuid7("2023-12-13")) s2 = str(uuid7("2023-12-10")) @@ -169,9 +164,7 @@ def test_all_time(self, session_table_version: SessionTableVersion): ] ) - results = self._run_web_stats_table_query( - "all", "2023-12-15", session_table_version=session_table_version - ).results + results = self._run_web_stats_table_query("all", "2023-12-15").results self.assertEqual( [ @@ -182,38 +175,31 @@ def test_all_time(self, session_table_version: SessionTableVersion): results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_filter_test_accounts(self, session_table_version: SessionTableVersion): + def test_filter_test_accounts(self): s1 = str(uuid7("2023-12-02")) # Create 1 test account self._create_events([("test", [("2023-12-02", s1, "/"), ("2023-12-03", s1, "/login")])]) - results = self._run_web_stats_table_query( - "2023-12-01", "2023-12-03", session_table_version=session_table_version, filter_test_accounts=True - ).results + results = self._run_web_stats_table_query("2023-12-01", "2023-12-03", filter_test_accounts=True).results self.assertEqual( [], results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_dont_filter_test_accounts(self, session_table_version: SessionTableVersion): + def test_dont_filter_test_accounts(self): s1 = str(uuid7("2023-12-02")) # Create 1 test account self._create_events([("test", [("2023-12-02", s1, "/"), ("2023-12-03", s1, "/login")])]) - results = self._run_web_stats_table_query( - "2023-12-01", "2023-12-03", session_table_version=session_table_version, filter_test_accounts=False - ).results + results = self._run_web_stats_table_query("2023-12-01", "2023-12-03", filter_test_accounts=False).results self.assertEqual( [["/", 1, 1], ["/login", 1, 1]], results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_breakdown_channel_type_doesnt_throw(self, session_table_version: SessionTableVersion): + def test_breakdown_channel_type_doesnt_throw(self): s1a = str(uuid7("2023-12-02")) s1b = str(uuid7("2023-12-13")) s2 = str(uuid7("2023-12-10")) @@ -229,7 +215,6 @@ def test_breakdown_channel_type_doesnt_throw(self, session_table_version: Sessio "2023-12-01", "2023-12-03", breakdown_by=WebStatsBreakdown.INITIAL_CHANNEL_TYPE, - session_table_version=session_table_version, ).results self.assertEqual( @@ -237,8 +222,7 @@ def test_breakdown_channel_type_doesnt_throw(self, session_table_version: Sessio len(results), ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_limit(self, session_table_version: SessionTableVersion): + def test_limit(self): s1 = str(uuid7("2023-12-02")) s2 = str(uuid7("2023-12-10")) self._create_events( @@ -248,9 +232,7 @@ def test_limit(self, session_table_version: SessionTableVersion): ] ) - response_1 = self._run_web_stats_table_query( - "all", "2023-12-15", limit=1, session_table_version=session_table_version - ) + response_1 = self._run_web_stats_table_query("all", "2023-12-15", limit=1) self.assertEqual( [ ["/", 2, 2], @@ -269,8 +251,7 @@ def test_limit(self, session_table_version: SessionTableVersion): ) self.assertEqual(False, response_2.hasMore) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_path_filters(self, session_table_version: SessionTableVersion): + def test_path_filters(self): s1 = str(uuid7("2023-12-02")) s2 = str(uuid7("2023-12-10")) s3 = str(uuid7("2023-12-10")) @@ -295,7 +276,6 @@ def test_path_filters(self, session_table_version: SessionTableVersion): {"regex": "thing_a", "alias": "thing_b"}, {"regex": "thing_b", "alias": "thing_c"}, ], - session_table_version=session_table_version, ).results self.assertEqual( @@ -308,8 +288,7 @@ def test_path_filters(self, session_table_version: SessionTableVersion): results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_scroll_depth_bounce_rate_one_user(self, session_table_version: SessionTableVersion): + def test_scroll_depth_bounce_rate_one_user(self): self._create_pageviews( "p1", [ @@ -325,7 +304,6 @@ def test_scroll_depth_bounce_rate_one_user(self, session_table_version: SessionT breakdown_by=WebStatsBreakdown.PAGE, include_scroll_depth=True, include_bounce_rate=True, - session_table_version=session_table_version, ).results self.assertEqual( @@ -337,8 +315,7 @@ def test_scroll_depth_bounce_rate_one_user(self, session_table_version: SessionT results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_scroll_depth_bounce_rate(self, session_table_version: SessionTableVersion): + def test_scroll_depth_bounce_rate(self): self._create_pageviews( "p1", [ @@ -369,7 +346,6 @@ def test_scroll_depth_bounce_rate(self, session_table_version: SessionTableVersi breakdown_by=WebStatsBreakdown.PAGE, include_scroll_depth=True, include_bounce_rate=True, - session_table_version=session_table_version, ).results self.assertEqual( @@ -381,8 +357,7 @@ def test_scroll_depth_bounce_rate(self, session_table_version: SessionTableVersi results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_scroll_depth_bounce_rate_with_filter(self, session_table_version: SessionTableVersion): + def test_scroll_depth_bounce_rate_with_filter(self): self._create_pageviews( "p1", [ @@ -414,7 +389,6 @@ def test_scroll_depth_bounce_rate_with_filter(self, session_table_version: Sessi include_scroll_depth=True, include_bounce_rate=True, properties=[EventPropertyFilter(key="$pathname", operator=PropertyOperator.EXACT, value="/a")], - session_table_version=session_table_version, ).results self.assertEqual( @@ -424,8 +398,7 @@ def test_scroll_depth_bounce_rate_with_filter(self, session_table_version: Sessi results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_scroll_depth_bounce_rate_path_cleaning(self, session_table_version: SessionTableVersion): + def test_scroll_depth_bounce_rate_path_cleaning(self): self._create_pageviews( "p1", [ @@ -446,7 +419,6 @@ def test_scroll_depth_bounce_rate_path_cleaning(self, session_table_version: Ses {"regex": "\\/b\\/\\d+", "alias": "/b/:id"}, {"regex": "\\/c\\/\\d+", "alias": "/c/:id"}, ], - session_table_version=session_table_version, ).results self.assertEqual( @@ -458,8 +430,7 @@ def test_scroll_depth_bounce_rate_path_cleaning(self, session_table_version: Ses results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_bounce_rate_one_user(self, session_table_version: SessionTableVersion): + def test_bounce_rate_one_user(self): self._create_pageviews( "p1", [ @@ -474,7 +445,6 @@ def test_bounce_rate_one_user(self, session_table_version: SessionTableVersion): "2023-12-15", breakdown_by=WebStatsBreakdown.PAGE, include_bounce_rate=True, - session_table_version=session_table_version, ).results self.assertEqual( @@ -486,8 +456,7 @@ def test_bounce_rate_one_user(self, session_table_version: SessionTableVersion): results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_bounce_rate(self, session_table_version: SessionTableVersion): + def test_bounce_rate(self): self._create_pageviews( "p1", [ @@ -517,7 +486,6 @@ def test_bounce_rate(self, session_table_version: SessionTableVersion): "2023-12-15", breakdown_by=WebStatsBreakdown.PAGE, include_bounce_rate=True, - session_table_version=session_table_version, ).results self.assertEqual( @@ -529,8 +497,7 @@ def test_bounce_rate(self, session_table_version: SessionTableVersion): results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_bounce_rate_with_property(self, session_table_version: SessionTableVersion): + def test_bounce_rate_with_property(self): self._create_pageviews( "p1", [ @@ -561,7 +528,6 @@ def test_bounce_rate_with_property(self, session_table_version: SessionTableVers breakdown_by=WebStatsBreakdown.PAGE, include_bounce_rate=True, properties=[EventPropertyFilter(key="$pathname", operator=PropertyOperator.EXACT, value="/a")], - session_table_version=session_table_version, ).results self.assertEqual( @@ -571,8 +537,7 @@ def test_bounce_rate_with_property(self, session_table_version: SessionTableVers results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_bounce_rate_path_cleaning(self, session_table_version: SessionTableVersion): + def test_bounce_rate_path_cleaning(self): self._create_pageviews( "p1", [ @@ -592,7 +557,6 @@ def test_bounce_rate_path_cleaning(self, session_table_version: SessionTableVers {"regex": "\\/b\\/\\d+", "alias": "/b/:id"}, {"regex": "\\/c\\/\\d+", "alias": "/c/:id"}, ], - session_table_version=session_table_version, ).results self.assertEqual( @@ -604,8 +568,7 @@ def test_bounce_rate_path_cleaning(self, session_table_version: SessionTableVers results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_entry_bounce_rate_one_user(self, session_table_version: SessionTableVersion): + def test_entry_bounce_rate_one_user(self): self._create_pageviews( "p1", [ @@ -620,7 +583,6 @@ def test_entry_bounce_rate_one_user(self, session_table_version: SessionTableVer "2023-12-15", breakdown_by=WebStatsBreakdown.INITIAL_PAGE, include_bounce_rate=True, - session_table_version=session_table_version, ).results self.assertEqual( @@ -630,8 +592,7 @@ def test_entry_bounce_rate_one_user(self, session_table_version: SessionTableVer results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_entry_bounce_rate(self, session_table_version: SessionTableVersion): + def test_entry_bounce_rate(self): self._create_pageviews( "p1", [ @@ -661,7 +622,6 @@ def test_entry_bounce_rate(self, session_table_version: SessionTableVersion): "2023-12-15", breakdown_by=WebStatsBreakdown.INITIAL_PAGE, include_bounce_rate=True, - session_table_version=session_table_version, ).results self.assertEqual( @@ -671,8 +631,7 @@ def test_entry_bounce_rate(self, session_table_version: SessionTableVersion): results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_entry_bounce_rate_with_property(self, session_table_version: SessionTableVersion): + def test_entry_bounce_rate_with_property(self): self._create_pageviews( "p1", [ @@ -703,7 +662,6 @@ def test_entry_bounce_rate_with_property(self, session_table_version: SessionTab breakdown_by=WebStatsBreakdown.INITIAL_PAGE, include_bounce_rate=True, properties=[EventPropertyFilter(key="$pathname", operator=PropertyOperator.EXACT, value="/a")], - session_table_version=session_table_version, ).results self.assertEqual( @@ -713,8 +671,7 @@ def test_entry_bounce_rate_with_property(self, session_table_version: SessionTab results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_entry_bounce_rate_path_cleaning(self, session_table_version: SessionTableVersion): + def test_entry_bounce_rate_path_cleaning(self): self._create_pageviews( "p1", [ @@ -734,7 +691,6 @@ def test_entry_bounce_rate_path_cleaning(self, session_table_version: SessionTab {"regex": "\\/b\\/\\d+", "alias": "/b/:id"}, {"regex": "\\/c\\/\\d+", "alias": "/c/:id"}, ], - session_table_version=session_table_version, ).results self.assertEqual( @@ -744,8 +700,7 @@ def test_entry_bounce_rate_path_cleaning(self, session_table_version: SessionTab results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_source_medium_campaign(self, session_table_version: SessionTableVersion): + def test_source_medium_campaign(self): d1 = "d1" s1 = str(uuid7("2024-06-26")) @@ -785,7 +740,6 @@ def test_source_medium_campaign(self, session_table_version: SessionTableVersion "all", "2024-06-27", breakdown_by=WebStatsBreakdown.INITIAL_UTM_SOURCE_MEDIUM_CAMPAIGN, - session_table_version=session_table_version, ).results self.assertEqual( @@ -793,8 +747,7 @@ def test_source_medium_campaign(self, session_table_version: SessionTableVersion results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_null_in_utm_tags(self, session_table_version: SessionTableVersion): + def test_null_in_utm_tags(self): d1 = "d1" s1 = str(uuid7("2024-06-26")) @@ -836,7 +789,6 @@ def test_null_in_utm_tags(self, session_table_version: SessionTableVersion): "all", "2024-06-27", breakdown_by=WebStatsBreakdown.INITIAL_UTM_SOURCE, - session_table_version=session_table_version, ).results self.assertEqual( @@ -844,8 +796,7 @@ def test_null_in_utm_tags(self, session_table_version: SessionTableVersion): results, ) - @parameterized.expand([[SessionTableVersion.V1], [SessionTableVersion.V2]]) - def test_is_not_set_filter(self, session_table_version: SessionTableVersion): + def test_is_not_set_filter(self): d1 = "d1" s1 = str(uuid7("2024-06-26")) @@ -888,7 +839,6 @@ def test_is_not_set_filter(self, session_table_version: SessionTableVersion): "2024-06-27", breakdown_by=WebStatsBreakdown.INITIAL_UTM_SOURCE, properties=[EventPropertyFilter(key="utm_source", operator=PropertyOperator.IS_NOT_SET)], - session_table_version=session_table_version, ).results self.assertEqual( diff --git a/posthog/models/sessions/sql.py b/posthog/models/sessions/sql.py index 680e1f7ff6f7b..93d3083a37763 100644 --- a/posthog/models/sessions/sql.py +++ b/posthog/models/sessions/sql.py @@ -7,6 +7,7 @@ AggregatingMergeTree, ) +# V1 Sessions table TABLE_BASE_NAME = "sessions" SESSIONS_DATA_TABLE = lambda: f"sharded_{TABLE_BASE_NAME}" @@ -21,6 +22,32 @@ ) DROP_SESSION_VIEW_SQL = lambda: f"DROP VIEW IF EXISTS {TABLE_BASE_NAME}_v ON CLUSTER '{settings.CLICKHOUSE_CLUSTER}'" +# Only teams that were grandfathered into the V1 sessions table are allowed to use it. Everyone else should use V2, +# i.e. raw_sessions. These teams were those who were seen to have changed their session table version in these metabase +# queries: +# US: https://metabase.prod-us.posthog.dev/question#eyJkYXRhc2V0X3F1ZXJ5Ijp7InR5cGUiOiJuYXRpdmUiLCJuYXRpdmUiOnsicXVlcnkiOiJTRUxFQ1QgdGVhbV9pZCwgc1xuRlJPTSAoXG4gICAgU0VMRUNUIG1vZGlmaWVycy0-PidzZXNzaW9uVGFibGVWZXJzaW9uJyBBUyBzLCBpZCBhcyB0ZWFtX2lkXG4gICAgRlJPTSBwb3N0aG9nX3RlYW1cbikgc3ViXG5XSEVSRSBzICE9ICcnIiwidGVtcGxhdGUtdGFncyI6e319LCJkYXRhYmFzZSI6MzR9LCJkaXNwbGF5IjoidGFibGUiLCJwYXJhbWV0ZXJzIjpbXSwidmlzdWFsaXphdGlvbl9zZXR0aW5ncyI6e319 +# EU: https://metabase.prod-eu.posthog.dev/question#eyJkYXRhc2V0X3F1ZXJ5Ijp7InR5cGUiOiJuYXRpdmUiLCJuYXRpdmUiOnsicXVlcnkiOiJTRUxFQ1QgdGVhbV9pZCwgc1xuRlJPTSAoXG4gICAgU0VMRUNUIG1vZGlmaWVycy0-PidzZXNzaW9uVGFibGVWZXJzaW9uJyBBUyBzLCBpZCBhcyB0ZWFtX2lkXG4gICAgRlJPTSBwb3N0aG9nX3RlYW1cbikgc3ViXG5XSEVSRSBzICE9ICcnIiwidGVtcGxhdGUtdGFncyI6e319LCJkYXRhYmFzZSI6MzR9LCJkaXNwbGF5IjoidGFibGUiLCJwYXJhbWV0ZXJzIjpbXSwidmlzdWFsaXphdGlvbl9zZXR0aW5ncyI6e319 +# or had contacted support about an issue. +# This list exists because we want to reduce the number of writes happening to this table, and so we don't write to it +# for any team not in this list. Adding a team to this is possible if needed, but would require changing this MV in +# production and backfilling this table with the management command backfill_sessions_table. +ALLOWED_TEAM_IDS = [ + # posthog + 1, + 2, + # US query + 13610, # zendesk: https://posthoghelp.zendesk.com/agent/tickets/18001 + 19279, + 21173, + 29929, + 32050, + # EU query + 9910, + 11775, + 21129, + 31490, +] +ALLOWED_TEAM_IDS_SQL = ", ".join(str(team_id) for team_id in ALLOWED_TEAM_IDS) # if updating these column definitions # you'll need to update the explicit column definitions in the materialized view creation statement below @@ -144,7 +171,7 @@ def source_column(column_name: str) -> str: sumIf(1, event='$autocapture') as autocapture_count FROM {database}.sharded_events -WHERE `$session_id` IS NOT NULL AND `$session_id` != '' +WHERE `$session_id` IS NOT NULL AND `$session_id` != '' AND team_id IN ({allowed_team_ids}) GROUP BY `$session_id`, team_id """.format( database=settings.CLICKHOUSE_DATABASE, @@ -168,6 +195,7 @@ def source_column(column_name: str) -> str: mc_cid_property=source_column("mc_cid"), igshid_property=source_column("igshid"), ttclid_property=source_column("ttclid"), + allowed_team_ids=ALLOWED_TEAM_IDS_SQL, ) )