From e3acb16cb264a8b3c8bdae4288bd279c921ebe8a Mon Sep 17 00:00:00 2001 From: Robbie Date: Thu, 25 Apr 2024 16:10:52 +0100 Subject: [PATCH] feat(web-analytics): Session filters 1 (#21512) * Handle hogql session properties * Add test for two lazy joins in one query (session & person) * Support passing session properties in web analytics queries * Fix typings * Working session definition fetching * Working session property definitions * Remove property allow list in web analytics * Put session properties first * Add session values * Rename duration back to $session_duration * Fix "keeps infiniteListCounts in sync" * Fix "setting search query filters events" * Fix taxonomy tests * Change session api to GenericViewSet * Add local properties back * Support datetime and boolean session properties * Update mypy baseline * Hide duration as a property and asterisk field * Add AsyncReturnType * Make taxonoicFilterLogic tests more reliable (maybe) * Fix duplicates from bad rebase * Only show session table session properties if feature flag enabled * Improve error message for non-hogql non-duration session property * Put session table properties behind FF * Revert taxonomicFilterLogic tests * Update query snapshots * Rename TaxonomicFilterGroupType.Sessions to TaxonomicFilterGroupType.SessionProperties * Use ViewSet instead of GenericViewSet * Add tests for session properties and session property values api * Formatting * New typing * Update query snapshots --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- frontend/src/lib/api.ts | 21 +++ .../DefinitionPopover/DefinitionPopover.tsx | 3 +- .../DefinitionPopoverContents.tsx | 45 +++--- .../definitionPopoverLogic.ts | 6 + .../lib/components/DefinitionPopover/utils.ts | 1 + .../components/PropertyFilters/utils.test.ts | 4 +- .../lib/components/PropertyFilters/utils.ts | 16 +- .../TaxonomicFilter/InfiniteList.tsx | 2 + .../taxonomicFilterLogic.test.ts | 22 +-- .../TaxonomicFilter/taxonomicFilterLogic.tsx | 32 ++-- .../lib/components/TaxonomicFilter/types.ts | 2 +- frontend/src/lib/constants.tsx | 1 + frontend/src/lib/taxonomy.test.tsx | 4 +- frontend/src/lib/taxonomy.tsx | 19 +-- frontend/src/lib/utils.tsx | 2 + .../src/models/propertyDefinitionsModel.ts | 83 +++++++---- .../nodes/InsightViz/GlobalAndOrFilters.tsx | 2 +- .../queries/nodes/InsightViz/TrendsSeries.tsx | 2 +- frontend/src/queries/schema.json | 3 + frontend/src/queries/schema.ts | 3 +- .../ActionFilterRow/ActionFilterRow.tsx | 2 +- .../TaxonomicBreakdownPopover.tsx | 2 +- .../web-analytics/WebPropertyFilters.tsx | 48 ++---- frontend/src/test/mocks.ts | 11 +- frontend/src/types.ts | 1 + mypy-baseline.txt | 9 +- posthog/api/__init__.py | 2 + posthog/api/property_definition.py | 12 +- posthog/api/session.py | 56 +++++++ .../api/test/__snapshots__/test_api_docs.ambr | 2 + posthog/api/test/test_session.py | 137 ++++++++++++++++++ posthog/hogql/database/schema/channel_type.py | 20 +++ posthog/hogql/database/schema/sessions.py | 113 ++++++++++++++- .../database/schema/test/test_sessions.py | 33 +++++ .../web_analytics_query_runner.py | 5 +- posthog/models/property/util.py | 2 +- posthog/models/sessions/sql.py | 41 ++++++ posthog/schema.py | 8 +- 38 files changed, 641 insertions(+), 136 deletions(-) create mode 100644 posthog/api/session.py create mode 100644 posthog/api/test/test_session.py diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 5e3a3aab0604f..09e3924e21deb 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -388,6 +388,10 @@ class ApiRequest { .withQueryString(queryParams) } + public sessionPropertyDefinitions(teamId?: TeamType['id']): ApiRequest { + return this.projectsDetail(teamId).addPathComponent('sessions').addPathComponent('property_definitions') + } + public dataManagementActivity(teamId?: TeamType['id']): ApiRequest { return this.projectsDetail(teamId).addPathComponent('data_management').addPathComponent('activity') } @@ -1212,6 +1216,23 @@ const api = { }, }, + sessions: { + async propertyDefinitions({ + teamId = ApiConfig.getCurrentTeamId(), + search, + properties, + }: { + teamId?: TeamType['id'] + search?: string + properties?: string[] + }): Promise> { + return new ApiRequest() + .sessionPropertyDefinitions(teamId) + .withQueryString(toParams({ search, ...(properties ? { properties: properties.join(',') } : {}) })) + .get() + }, + }, + cohorts: { async get(cohortId: CohortType['id']): Promise { return await new ApiRequest().cohortsDetail(cohortId).get() diff --git a/frontend/src/lib/components/DefinitionPopover/DefinitionPopover.tsx b/frontend/src/lib/components/DefinitionPopover/DefinitionPopover.tsx index 4d93fd38f2ae9..807f1097e45c2 100644 --- a/frontend/src/lib/components/DefinitionPopover/DefinitionPopover.tsx +++ b/frontend/src/lib/components/DefinitionPopover/DefinitionPopover.tsx @@ -124,7 +124,8 @@ function Example({ value }: { value?: string }): JSX.Element { type === TaxonomicFilterGroupType.EventFeatureFlags || type === TaxonomicFilterGroupType.PersonProperties || type === TaxonomicFilterGroupType.GroupsPrefix || - type === TaxonomicFilterGroupType.Metadata + type === TaxonomicFilterGroupType.Metadata || + type === TaxonomicFilterGroupType.SessionProperties ) { data = getCoreFilterDefinition(value, type) } else if (type === TaxonomicFilterGroupType.Elements) { diff --git a/frontend/src/lib/components/DefinitionPopover/DefinitionPopoverContents.tsx b/frontend/src/lib/components/DefinitionPopover/DefinitionPopoverContents.tsx index d6419882b3327..f578e4d37b07f 100644 --- a/frontend/src/lib/components/DefinitionPopover/DefinitionPopoverContents.tsx +++ b/frontend/src/lib/components/DefinitionPopover/DefinitionPopoverContents.tsx @@ -78,6 +78,7 @@ function DefinitionView({ group }: { group: TaxonomicFilterGroup }): JSX.Element isCohort, isDataWarehouse, isProperty, + hasSentAs, } = useValues(definitionPopoverLogic) const { setLocalDefinition } = useActions(definitionPopoverLogic) @@ -142,13 +143,17 @@ function DefinitionView({ group }: { group: TaxonomicFilterGroup }): JSX.Element /> - - - {_definition.name}} - /> - + {hasSentAs ? ( + <> + + + {_definition.name}} + /> + + + ) : null} ) } @@ -176,17 +181,21 @@ function DefinitionView({ group }: { group: TaxonomicFilterGroup }): JSX.Element - - - - {_definition.name !== '' ? _definition.name : (empty string)} - - } - /> - + {hasSentAs ? ( + <> + + + + {_definition.name !== '' ? _definition.name : (empty string)} + + } + /> + + + ) : null} ) } diff --git a/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts b/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts index c3baac7ce76ca..38d9af8eb5311 100644 --- a/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts +++ b/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts @@ -176,11 +176,17 @@ export const definitionPopoverLogic = kea([ [ TaxonomicFilterGroupType.PersonProperties, TaxonomicFilterGroupType.EventProperties, + TaxonomicFilterGroupType.SessionProperties, TaxonomicFilterGroupType.EventFeatureFlags, TaxonomicFilterGroupType.NumericalEventProperties, TaxonomicFilterGroupType.Metadata, ].includes(type) || type.startsWith(TaxonomicFilterGroupType.GroupsPrefix), ], + hasSentAs: [ + (s) => [s.type, s.isProperty, s.isEvent], + (type, isProperty, isEvent) => + isEvent || (isProperty && type !== TaxonomicFilterGroupType.SessionProperties), + ], isCohort: [(s) => [s.type], (type) => type === TaxonomicFilterGroupType.Cohorts], isDataWarehouse: [(s) => [s.type], (type) => type === TaxonomicFilterGroupType.DataWarehouse], viewFullDetailUrl: [ diff --git a/frontend/src/lib/components/DefinitionPopover/utils.ts b/frontend/src/lib/components/DefinitionPopover/utils.ts index f828aea981fc2..a12c76237da42 100644 --- a/frontend/src/lib/components/DefinitionPopover/utils.ts +++ b/frontend/src/lib/components/DefinitionPopover/utils.ts @@ -48,6 +48,7 @@ export function getSingularType(type: TaxonomicFilterGroupType): string { case TaxonomicFilterGroupType.EventProperties: case TaxonomicFilterGroupType.PersonProperties: case TaxonomicFilterGroupType.GroupsPrefix: // Group properties + case TaxonomicFilterGroupType.SessionProperties: return 'property' case TaxonomicFilterGroupType.EventFeatureFlags: return 'feature' diff --git a/frontend/src/lib/components/PropertyFilters/utils.test.ts b/frontend/src/lib/components/PropertyFilters/utils.test.ts index 33ad74f8e35d6..da3f32d7e1b3c 100644 --- a/frontend/src/lib/components/PropertyFilters/utils.test.ts +++ b/frontend/src/lib/components/PropertyFilters/utils.test.ts @@ -83,7 +83,7 @@ describe('propertyFilterTypeToTaxonomicFilterType()', () => { ...baseFilter, type: PropertyFilterType.Session, } as SessionPropertyFilter) - ).toEqual(TaxonomicFilterGroupType.Sessions) + ).toEqual(TaxonomicFilterGroupType.SessionProperties) expect(propertyFilterTypeToTaxonomicFilterType({ ...baseFilter, type: PropertyFilterType.HogQL })).toEqual( TaxonomicFilterGroupType.HogQLExpression ) @@ -122,7 +122,7 @@ describe('breakdownFilterToTaxonomicFilterType()', () => { TaxonomicFilterGroupType.EventProperties ) expect(breakdownFilterToTaxonomicFilterType({ ...baseFilter, breakdown_type: 'session' })).toEqual( - TaxonomicFilterGroupType.Sessions + TaxonomicFilterGroupType.SessionProperties ) expect(breakdownFilterToTaxonomicFilterType({ ...baseFilter, breakdown_type: 'hogql' })).toEqual( TaxonomicFilterGroupType.HogQLExpression diff --git a/frontend/src/lib/components/PropertyFilters/utils.ts b/frontend/src/lib/components/PropertyFilters/utils.ts index ad135c0525b0e..63833cbd7acb5 100644 --- a/frontend/src/lib/components/PropertyFilters/utils.ts +++ b/frontend/src/lib/components/PropertyFilters/utils.ts @@ -97,7 +97,7 @@ export const PROPERTY_FILTER_TYPE_TO_TAXONOMIC_FILTER_GROUP_TYPE: Omit< [PropertyFilterType.Feature]: TaxonomicFilterGroupType.EventFeatureFlags, [PropertyFilterType.Cohort]: TaxonomicFilterGroupType.Cohorts, [PropertyFilterType.Element]: TaxonomicFilterGroupType.Elements, - [PropertyFilterType.Session]: TaxonomicFilterGroupType.Sessions, + [PropertyFilterType.Session]: TaxonomicFilterGroupType.SessionProperties, [PropertyFilterType.HogQL]: TaxonomicFilterGroupType.HogQLExpression, [PropertyFilterType.Group]: TaxonomicFilterGroupType.GroupsPrefix, [PropertyFilterType.DataWarehouse]: TaxonomicFilterGroupType.DataWarehouse, @@ -183,10 +183,14 @@ export function isEventPropertyFilter(filter?: AnyFilterLike | null): filter is export function isPersonPropertyFilter(filter?: AnyFilterLike | null): filter is PersonPropertyFilter { return filter?.type === PropertyFilterType.Person } -export function isEventPropertyOrPersonPropertyFilter( +export function isEventPersonOrSessionPropertyFilter( filter?: AnyFilterLike | null -): filter is EventPropertyFilter | PersonPropertyFilter { - return filter?.type === PropertyFilterType.Event || filter?.type === PropertyFilterType.Person +): filter is EventPropertyFilter | PersonPropertyFilter | SessionPropertyFilter { + return ( + filter?.type === PropertyFilterType.Event || + filter?.type === PropertyFilterType.Person || + filter?.type === PropertyFilterType.Session + ) } export function isElementPropertyFilter(filter?: AnyFilterLike | null): filter is ElementPropertyFilter { return filter?.type === PropertyFilterType.Element @@ -264,7 +268,7 @@ const propertyFilterMapping: Partial
@@ -160,6 +161,7 @@ const selectedItemHasPopover = ( TaxonomicFilterGroupType.Cohorts, TaxonomicFilterGroupType.CohortsWithAllUsers, TaxonomicFilterGroupType.Metadata, + TaxonomicFilterGroupType.SessionProperties, ].includes(listGroupType) || listGroupType.startsWith(TaxonomicFilterGroupType.GroupsPrefix)) ) diff --git a/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.test.ts b/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.test.ts index 2c6f0ff84c2db..0dbbe75ddd38d 100644 --- a/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.test.ts +++ b/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.test.ts @@ -45,7 +45,7 @@ describe('taxonomicFilterLogic', () => { TaxonomicFilterGroupType.Events, TaxonomicFilterGroupType.Actions, TaxonomicFilterGroupType.Elements, - TaxonomicFilterGroupType.Sessions, + TaxonomicFilterGroupType.SessionProperties, ], } logic = taxonomicFilterLogic(logicProps) @@ -62,7 +62,7 @@ describe('taxonomicFilterLogic', () => { infiniteListLogic({ ...logic.props, listGroupType: TaxonomicFilterGroupType.Events }), infiniteListLogic({ ...logic.props, listGroupType: TaxonomicFilterGroupType.Actions }), infiniteListLogic({ ...logic.props, listGroupType: TaxonomicFilterGroupType.Elements }), - infiniteListLogic({ ...logic.props, listGroupType: TaxonomicFilterGroupType.Sessions }), + infiniteListLogic({ ...logic.props, listGroupType: TaxonomicFilterGroupType.SessionProperties }), ]) expect( infiniteListLogic({ ...logic.props, listGroupType: TaxonomicFilterGroupType.Cohorts }).isMounted() @@ -76,7 +76,7 @@ describe('taxonomicFilterLogic', () => { [TaxonomicFilterGroupType.Events]: 1, [TaxonomicFilterGroupType.Actions]: 0, [TaxonomicFilterGroupType.Elements]: 4, - [TaxonomicFilterGroupType.Sessions]: 1, + [TaxonomicFilterGroupType.SessionProperties]: 1, }, }) .toDispatchActions(['infiniteListResultsReceived']) @@ -87,7 +87,7 @@ describe('taxonomicFilterLogic', () => { [TaxonomicFilterGroupType.Events]: 157, [TaxonomicFilterGroupType.Actions]: 0, // not mocked [TaxonomicFilterGroupType.Elements]: 4, - [TaxonomicFilterGroupType.Sessions]: 1, + [TaxonomicFilterGroupType.SessionProperties]: 1, }, }) }) @@ -110,7 +110,7 @@ describe('taxonomicFilterLogic', () => { [TaxonomicFilterGroupType.Events]: 4, [TaxonomicFilterGroupType.Actions]: 0, [TaxonomicFilterGroupType.Elements]: 0, - [TaxonomicFilterGroupType.Sessions]: 0, + [TaxonomicFilterGroupType.SessionProperties]: 0, }, }) @@ -127,7 +127,7 @@ describe('taxonomicFilterLogic', () => { [TaxonomicFilterGroupType.Events]: 0, [TaxonomicFilterGroupType.Actions]: 0, [TaxonomicFilterGroupType.Elements]: 1, - [TaxonomicFilterGroupType.Sessions]: 0, + [TaxonomicFilterGroupType.SessionProperties]: 0, }, }) @@ -144,7 +144,7 @@ describe('taxonomicFilterLogic', () => { [TaxonomicFilterGroupType.Events]: 0, [TaxonomicFilterGroupType.Actions]: 0, [TaxonomicFilterGroupType.Elements]: 0, - [TaxonomicFilterGroupType.Sessions]: 0, + [TaxonomicFilterGroupType.SessionProperties]: 0, }, }) @@ -161,13 +161,13 @@ describe('taxonomicFilterLogic', () => { [TaxonomicFilterGroupType.Events]: 157, [TaxonomicFilterGroupType.Actions]: 0, [TaxonomicFilterGroupType.Elements]: 4, - [TaxonomicFilterGroupType.Sessions]: 1, + [TaxonomicFilterGroupType.SessionProperties]: 1, }, }) // move right, skipping Actions await expectLogic(logic, () => logic.actions.tabRight()).toMatchValues({ - activeTab: TaxonomicFilterGroupType.Sessions, + activeTab: TaxonomicFilterGroupType.SessionProperties, }) await expectLogic(logic, () => logic.actions.tabRight()).toMatchValues({ activeTab: TaxonomicFilterGroupType.Events, @@ -181,7 +181,7 @@ describe('taxonomicFilterLogic', () => { activeTab: TaxonomicFilterGroupType.Events, }) await expectLogic(logic, () => logic.actions.tabLeft()).toMatchValues({ - activeTab: TaxonomicFilterGroupType.Sessions, + activeTab: TaxonomicFilterGroupType.SessionProperties, }) await expectLogic(logic, () => logic.actions.tabLeft()).toMatchValues({ activeTab: TaxonomicFilterGroupType.Elements, @@ -201,7 +201,7 @@ describe('taxonomicFilterLogic', () => { [TaxonomicFilterGroupType.Events]: 4, [TaxonomicFilterGroupType.Actions]: 0, [TaxonomicFilterGroupType.Elements]: 0, - [TaxonomicFilterGroupType.Sessions]: 0, + [TaxonomicFilterGroupType.SessionProperties]: 0, }, }) }) diff --git a/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.tsx b/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.tsx index 966b33c5527ed..f35dbda2f7cf5 100644 --- a/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.tsx +++ b/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.tsx @@ -11,7 +11,9 @@ import { TaxonomicFilterLogicProps, TaxonomicFilterValue, } from 'lib/components/TaxonomicFilter/types' +import { FEATURE_FLAGS } from 'lib/constants' import { IconCohort } from 'lib/lemon-ui/icons' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { CORE_FILTER_DEFINITIONS_BY_GROUP } from 'lib/taxonomy' import { capitalizeFirstLetter, pluralize, toParams } from 'lib/utils' import { getEventDefinitionIcon, getPropertyDefinitionIcon } from 'scenes/data-management/events/DefinitionHeader' @@ -168,6 +170,7 @@ export const taxonomicFilterLogic = kea([ s.metadataSource, s.excludedProperties, s.propertyAllowList, + featureFlagLogic.selectors.featureFlags, ], ( teamId, @@ -177,7 +180,8 @@ export const taxonomicFilterLogic = kea([ schemaColumns, metadataSource, excludedProperties, - propertyAllowList + propertyAllowList, + featureFlags ): TaxonomicFilterGroup[] => { const groups: TaxonomicFilterGroup[] = [ { @@ -486,18 +490,26 @@ export const taxonomicFilterLogic = kea([ getPopoverHeader: () => 'Notebooks', }, { - name: 'Sessions', + name: 'Session Properties', searchPlaceholder: 'sessions', - type: TaxonomicFilterGroupType.Sessions, - options: [ - { - name: 'Session duration', - value: '$session_duration', - }, - ], + type: TaxonomicFilterGroupType.SessionProperties, + options: featureFlags[FEATURE_FLAGS.SESSION_TABLE_PROPERTY_FILTERS] + ? undefined + : [ + { + id: '$session_duration', + name: '$session_duration', + property_type: 'Duration', + is_numerical: true, + }, + ], getName: (option: any) => option.name, - getValue: (option: any) => option.value, + getValue: (option) => option.name, getPopoverHeader: () => 'Session', + endpoint: featureFlags[FEATURE_FLAGS.SESSION_TABLE_PROPERTY_FILTERS] + ? `api/projects/${teamId}/sessions/property_definitions` + : undefined, + getIcon: getPropertyDefinitionIcon, }, { name: 'HogQL', diff --git a/frontend/src/lib/components/TaxonomicFilter/types.ts b/frontend/src/lib/components/TaxonomicFilter/types.ts index 8b46784a19b7e..37f1c95b45daa 100644 --- a/frontend/src/lib/components/TaxonomicFilter/types.ts +++ b/frontend/src/lib/components/TaxonomicFilter/types.ts @@ -105,7 +105,7 @@ export enum TaxonomicFilterGroupType { Plugins = 'plugins', Dashboards = 'dashboards', GroupNamesPrefix = 'name_groups', - Sessions = 'sessions', + SessionProperties = 'session_properties', HogQLExpression = 'hogql_expression', Notebooks = 'notebooks', } diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx index 0db68836f76d5..86450c67f19e0 100644 --- a/frontend/src/lib/constants.tsx +++ b/frontend/src/lib/constants.tsx @@ -212,6 +212,7 @@ export const FEATURE_FLAGS = { EMAIL_VERIFICATION_TICKET_SUBMISSION: 'email-verification-ticket-submission', // owner: #team-growth TOOLBAR_HEATMAPS: 'toolbar-heatmaps', // owner: #team-replay THEME: 'theme', // owner: @aprilfools + SESSION_TABLE_PROPERTY_FILTERS: 'session-table-property-filters', // owner: @robbie-c } as const export type FeatureFlagKey = (typeof FEATURE_FLAGS)[keyof typeof FEATURE_FLAGS] diff --git a/frontend/src/lib/taxonomy.test.tsx b/frontend/src/lib/taxonomy.test.tsx index 3805f9d7670ad..e8820fbad0c32 100644 --- a/frontend/src/lib/taxonomy.test.tsx +++ b/frontend/src/lib/taxonomy.test.tsx @@ -26,10 +26,10 @@ describe('taxonomy', () => { }) describe('session properties', () => { - const sessionPropertyNames = Object.keys(CORE_FILTER_DEFINITIONS_BY_GROUP.sessions) + const sessionPropertyNames = Object.keys(CORE_FILTER_DEFINITIONS_BY_GROUP.session_properties) it('should have an $initial_referring_domain property', () => { const property: CoreFilterDefinition = - CORE_FILTER_DEFINITIONS_BY_GROUP.sessions['$initial_referring_domain'] + CORE_FILTER_DEFINITIONS_BY_GROUP.session_properties['$initial_referring_domain'] expect(property.label).toEqual('Initial Referring Domain') }) it(`should have every property in SESSION_PROPERTIES_ADAPTED_FROM_PERSON`, () => { diff --git a/frontend/src/lib/taxonomy.tsx b/frontend/src/lib/taxonomy.tsx index 9b5297cf62955..d52ee74c583c0 100644 --- a/frontend/src/lib/taxonomy.tsx +++ b/frontend/src/lib/taxonomy.tsx @@ -993,7 +993,7 @@ export const CORE_FILTER_DEFINITIONS_BY_GROUP = { }, numerical_event_properties: {}, // Same as event properties, see assignment below person_properties: {}, // Currently person properties are the same as event properties, see assignment below - sessions: { + session_properties: { $session_duration: { label: 'Session duration', description: ( @@ -1006,13 +1006,13 @@ export const CORE_FILTER_DEFINITIONS_BY_GROUP = { ), examples: ['01:04:12'], }, - $min_timestamp: { - label: 'First timestamp', + $start_timestamp: { + label: 'Start timestamp', description: The timestamp of the first event from this session., examples: [new Date().toISOString()], }, - $max_timestamp: { - label: 'Last timestamp', + $end_timestamp: { + label: 'End timestamp', description: The timestamp of the last event from this session, examples: [new Date().toISOString()], }, @@ -1022,7 +1022,7 @@ export const CORE_FILTER_DEFINITIONS_BY_GROUP = { examples: ['https://example.com/interesting-article?parameter=true'], }, $exit_url: { - label: 'Entry URL', + label: 'Exit URL', description: The last URL visited in this session, examples: ['https://example.com/interesting-article?parameter=true'], }, @@ -1036,7 +1036,7 @@ export const CORE_FILTER_DEFINITIONS_BY_GROUP = { description: The number of autocapture events in this session, examples: ['123'], }, - $initial_channel_type: { + $channel_type: { label: 'Channel type', description: What type of acquisition channel this traffic came from., examples: ['Paid Search', 'Organic Video', 'Direct'], @@ -1079,20 +1079,21 @@ for (const [key, value] of Object.entries(CORE_FILTER_DEFINITIONS_BY_GROUP.event CORE_FILTER_DEFINITIONS_BY_GROUP.person_properties[key] = value } if (SESSION_INITIAL_PROPERTIES_ADAPTED_FROM_EVENTS.has(key)) { - CORE_FILTER_DEFINITIONS_BY_GROUP.sessions[`$initial_${key.replace(/^\$/, '')}`] = { + CORE_FILTER_DEFINITIONS_BY_GROUP.session_properties[`$initial_${key.replace(/^\$/, '')}`] = { ...value, label: `Initial ${value.label}`, description: 'description' in value ? `${value.description} Data from the first event in this session.` : 'Data from the first event in this session.', + examples: 'examples' in value ? value.examples : undefined, } } } // We treat `$session_duration` as an event property in the context of series `math`, but it's fake in a sense CORE_FILTER_DEFINITIONS_BY_GROUP.event_properties.$session_duration = - CORE_FILTER_DEFINITIONS_BY_GROUP.sessions.$session_duration + CORE_FILTER_DEFINITIONS_BY_GROUP.session_properties.$session_duration export const PROPERTY_KEYS = Object.keys(CORE_FILTER_DEFINITIONS_BY_GROUP.event_properties) diff --git a/frontend/src/lib/utils.tsx b/frontend/src/lib/utils.tsx index ab32f34b314f3..20b8114375939 100644 --- a/frontend/src/lib/utils.tsx +++ b/frontend/src/lib/utils.tsx @@ -1597,6 +1597,8 @@ export function promiseResolveReject(): { return { resolve: resolve!, reject: reject!, promise } } +export type AsyncReturnType any> = T extends (...args: any) => Promise ? R : any + export function calculateDays(timeValue: number, timeUnit: TimeUnitType): number { if (timeUnit === TimeUnitType.Year) { return timeValue * 365 diff --git a/frontend/src/models/propertyDefinitionsModel.ts b/frontend/src/models/propertyDefinitionsModel.ts index 338e60a5e956f..0bc2dfae5c11b 100644 --- a/frontend/src/models/propertyDefinitionsModel.ts +++ b/frontend/src/models/propertyDefinitionsModel.ts @@ -1,5 +1,5 @@ -import { actions, kea, listeners, path, reducers, selectors } from 'kea' -import api, { ApiMethodOptions } from 'lib/api' +import { actions, connect, kea, listeners, path, reducers, selectors } from 'kea' +import api, { ApiMethodOptions, CountedPaginatedResponse } from 'lib/api' import { TaxonomicFilterValue } from 'lib/components/TaxonomicFilter/types' import { dayjs } from 'lib/dayjs' import { captureTimeToSeeData } from 'lib/internalMetrics' @@ -95,8 +95,42 @@ const checkOrLoadPropertyDefinition = ( return null } +const getEndpoint = ( + teamId: number, + type: PropertyDefinitionType, + propertyKey: string, + eventNames: string[] | undefined, + newInput: string | undefined +): string => { + let eventParams = '' + for (const eventName of eventNames || []) { + eventParams += `&event_name=${eventName}` + } + + if (type === PropertyDefinitionType.Session) { + return ( + `api/projects/${teamId}/${type}s/values/?key=` + + encodeURIComponent(propertyKey) + + (newInput ? '&value=' + encodeURIComponent(newInput) : '') + + eventParams + ) + } + + return ( + 'api/' + + type + + '/values/?key=' + + encodeURIComponent(propertyKey) + + (newInput ? '&value=' + encodeURIComponent(newInput) : '') + + eventParams + ) +} + export const propertyDefinitionsModel = kea([ path(['models', 'propertyDefinitionsModel']), + connect({ + values: [teamLogic, ['currentTeamId']], + }), actions({ // public loadPropertyDefinitions: ( @@ -125,10 +159,12 @@ export const propertyDefinitionsModel = kea([ propertyDefinitionStorage: [ { ...localProperties } as PropertyDefinitionStorage, { - updatePropertyDefinitions: (state, { propertyDefinitions }) => ({ - ...state, - ...propertyDefinitions, - }), + updatePropertyDefinitions: (state, { propertyDefinitions }) => { + return { + ...state, + ...propertyDefinitions, + } + }, }, ], options: [ @@ -179,7 +215,7 @@ export const propertyDefinitionsModel = kea([ // take the first 50 pending properties to avoid the 4k query param length limit const allPending = values.pendingProperties.slice(0, 50) const pendingByType: Record< - 'event' | 'person' | 'group/0' | 'group/1' | 'group/2' | 'group/3' | 'group/4', + 'event' | 'person' | 'group/0' | 'group/1' | 'group/2' | 'group/3' | 'group/4' | 'session', string[] > = { event: [], @@ -189,6 +225,7 @@ export const propertyDefinitionsModel = kea([ 'group/2': [], 'group/3': [], 'group/4': [], + session: [], } for (const key of allPending) { let [type, ...rest] = key.split('/') @@ -226,10 +263,17 @@ export const propertyDefinitionsModel = kea([ } // and then fetch them - const propertyDefinitions = await api.propertyDefinitions.list({ - properties: pending, - ...queryParams, - }) + let propertyDefinitions: CountedPaginatedResponse + if (type === 'session') { + propertyDefinitions = await api.sessions.propertyDefinitions({ + properties: pending, + }) + } else { + propertyDefinitions = await api.propertyDefinitions.list({ + properties: pending, + ...queryParams, + }) + } for (const propertyDefinition of propertyDefinitions.results) { newProperties[`${type}/${propertyDefinition.name}`] = propertyDefinition @@ -268,10 +312,10 @@ export const propertyDefinitionsModel = kea([ }, loadPropertyValues: async ({ endpoint, type, newInput, propertyKey, eventNames }, breakpoint) => { - if (['cohort', 'session'].includes(type)) { + if (['cohort'].includes(type)) { return } - if (!propertyKey) { + if (!propertyKey || values.currentTeamId === null) { return } @@ -286,19 +330,8 @@ export const propertyDefinitionsModel = kea([ signal: cache.abortController.signal, } - let eventParams = '' - for (const eventName of eventNames || []) { - eventParams += `&event_name=${eventName}` - } - const propValues: PropValue[] = await api.get( - endpoint || - 'api/' + - type + - '/values/?key=' + - encodeURIComponent(propertyKey) + - (newInput ? '&value=' + encodeURIComponent(newInput) : '') + - eventParams, + endpoint || getEndpoint(values.currentTeamId, type, propertyKey, eventNames, newInput), methodOptions ) breakpoint() diff --git a/frontend/src/queries/nodes/InsightViz/GlobalAndOrFilters.tsx b/frontend/src/queries/nodes/InsightViz/GlobalAndOrFilters.tsx index 8485b9a71ee84..351e82136133a 100644 --- a/frontend/src/queries/nodes/InsightViz/GlobalAndOrFilters.tsx +++ b/frontend/src/queries/nodes/InsightViz/GlobalAndOrFilters.tsx @@ -27,7 +27,7 @@ export function GlobalAndOrFilters({ insightProps }: EditorFilterProps): JSX.Ele ...groupsTaxonomicTypes, TaxonomicFilterGroupType.Cohorts, TaxonomicFilterGroupType.Elements, - ...(isTrends ? [TaxonomicFilterGroupType.Sessions] : []), + ...(isTrends ? [TaxonomicFilterGroupType.SessionProperties] : []), TaxonomicFilterGroupType.HogQLExpression, ...(featureFlags[FEATURE_FLAGS.DATA_WAREHOUSE] && featureFlags[FEATURE_FLAGS.HOGQL_INSIGHTS] ? [TaxonomicFilterGroupType.DataWarehousePersonProperties] diff --git a/frontend/src/queries/nodes/InsightViz/TrendsSeries.tsx b/frontend/src/queries/nodes/InsightViz/TrendsSeries.tsx index 38b96c2162aca..dd75cd96a6f97 100644 --- a/frontend/src/queries/nodes/InsightViz/TrendsSeries.tsx +++ b/frontend/src/queries/nodes/InsightViz/TrendsSeries.tsx @@ -34,7 +34,7 @@ export function TrendsSeries(): JSX.Element | null { ...groupsTaxonomicTypes, TaxonomicFilterGroupType.Cohorts, TaxonomicFilterGroupType.Elements, - ...(isTrends ? [TaxonomicFilterGroupType.Sessions] : []), + ...(isTrends ? [TaxonomicFilterGroupType.SessionProperties] : []), TaxonomicFilterGroupType.HogQLExpression, TaxonomicFilterGroupType.DataWarehouseProperties, ...(featureFlags[FEATURE_FLAGS.DATA_WAREHOUSE] && featureFlags[FEATURE_FLAGS.HOGQL_INSIGHTS] diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index 868156528ece8..1ef498660d74a 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -5667,6 +5667,9 @@ }, { "$ref": "#/definitions/PersonPropertyFilter" + }, + { + "$ref": "#/definitions/SessionPropertyFilter" } ] }, diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index 01708078b175b..a923dfaec77cf 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -23,6 +23,7 @@ import { PropertyGroupFilter, PropertyMathType, RetentionFilterType, + SessionPropertyFilter, StickinessFilterType, TrendsFilterType, } from '~/types' @@ -985,7 +986,7 @@ export interface SessionsTimelineQuery extends DataNode { before?: string response?: SessionsTimelineQueryResponse } -export type WebAnalyticsPropertyFilter = EventPropertyFilter | PersonPropertyFilter +export type WebAnalyticsPropertyFilter = EventPropertyFilter | PersonPropertyFilter | SessionPropertyFilter export type WebAnalyticsPropertyFilters = WebAnalyticsPropertyFilter[] export interface WebAnalyticsQueryBase { diff --git a/frontend/src/scenes/insights/filters/ActionFilter/ActionFilterRow/ActionFilterRow.tsx b/frontend/src/scenes/insights/filters/ActionFilter/ActionFilterRow/ActionFilterRow.tsx index 0cb3eaeb086b3..f544e601e506d 100644 --- a/frontend/src/scenes/insights/filters/ActionFilter/ActionFilterRow/ActionFilterRow.tsx +++ b/frontend/src/scenes/insights/filters/ActionFilter/ActionFilterRow/ActionFilterRow.tsx @@ -390,7 +390,7 @@ export function ActionFilterRow({ groupTypes={[ TaxonomicFilterGroupType.DataWarehouseProperties, TaxonomicFilterGroupType.NumericalEventProperties, - TaxonomicFilterGroupType.Sessions, + TaxonomicFilterGroupType.SessionProperties, ]} schemaColumns={ filter.type == TaxonomicFilterGroupType.DataWarehouse && filter.name diff --git a/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownPopover.tsx b/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownPopover.tsx index abbcb7020c116..9a8df4a8c392f 100644 --- a/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownPopover.tsx +++ b/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownPopover.tsx @@ -28,7 +28,7 @@ export const TaxonomicBreakdownPopover = ({ open, setOpen, children }: Taxonomic TaxonomicFilterGroupType.EventFeatureFlags, ...groupsTaxonomicTypes, TaxonomicFilterGroupType.CohortsWithAllUsers, - ...(includeSessions ? [TaxonomicFilterGroupType.Sessions] : []), + ...(includeSessions ? [TaxonomicFilterGroupType.SessionProperties] : []), TaxonomicFilterGroupType.HogQLExpression, TaxonomicFilterGroupType.DataWarehouseProperties, ] diff --git a/frontend/src/scenes/web-analytics/WebPropertyFilters.tsx b/frontend/src/scenes/web-analytics/WebPropertyFilters.tsx index 0f65abec6f0c9..5237a78e3cb8b 100644 --- a/frontend/src/scenes/web-analytics/WebPropertyFilters.tsx +++ b/frontend/src/scenes/web-analytics/WebPropertyFilters.tsx @@ -1,6 +1,9 @@ +import { useValues } from 'kea' import { PropertyFilters } from 'lib/components/PropertyFilters/PropertyFilters' -import { isEventPropertyOrPersonPropertyFilter } from 'lib/components/PropertyFilters/utils' +import { isEventPersonOrSessionPropertyFilter } from 'lib/components/PropertyFilters/utils' import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types' +import { FEATURE_FLAGS } from 'lib/constants' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { WebAnalyticsPropertyFilters } from '~/queries/schema' @@ -11,42 +14,23 @@ export const WebPropertyFilters = ({ webAnalyticsFilters: WebAnalyticsPropertyFilters setWebAnalyticsFilters: (filters: WebAnalyticsPropertyFilters) => void }): JSX.Element => { + const { featureFlags } = useValues(featureFlagLogic) + return ( setWebAnalyticsFilters(filters.filter(isEventPropertyOrPersonPropertyFilter))} + taxonomicGroupTypes={ + featureFlags[FEATURE_FLAGS.SESSION_TABLE_PROPERTY_FILTERS] + ? [ + TaxonomicFilterGroupType.SessionProperties, + TaxonomicFilterGroupType.EventProperties, + TaxonomicFilterGroupType.PersonProperties, + ] + : [TaxonomicFilterGroupType.EventProperties, TaxonomicFilterGroupType.PersonProperties] + } + onChange={(filters) => setWebAnalyticsFilters(filters.filter(isEventPersonOrSessionPropertyFilter))} propertyFilters={webAnalyticsFilters} pageKey="web-analytics" eventNames={['$pageview', '$pageleave', '$autocapture']} - propertyAllowList={{ - [TaxonomicFilterGroupType.EventProperties]: [ - '$pathname', - '$host', - '$browser', - '$os', - '$device_type', - '$geoip_country_code', - '$geoip_subdivision_1_code', - '$geoip_city_name', - // re-enable after https://github.com/PostHog/posthog-js/pull/875 is merged - // '$client_session_initial_pathname', - // '$client_session_initial_referring_host', - // '$client_session_initial_utm_source', - // '$client_session_initial_utm_campaign', - // '$client_session_initial_utm_medium', - // '$client_session_initial_utm_content', - // '$client_session_initial_utm_term', - ], - [TaxonomicFilterGroupType.PersonProperties]: [ - '$initial_pathname', - '$initial_referring_domain', - '$initial_utm_source', - '$initial_utm_campaign', - '$initial_utm_medium', - '$initial_utm_content', - '$initial_utm_term', - ], - }} /> ) } diff --git a/frontend/src/test/mocks.ts b/frontend/src/test/mocks.ts index dcd926d4f1e7a..78cba1619bfb8 100644 --- a/frontend/src/test/mocks.ts +++ b/frontend/src/test/mocks.ts @@ -51,7 +51,7 @@ export const mockEventDefinitions: EventDefinition[] = [ 'test event', '$click', '$autocapture', - 'search', + 'search term', 'other event', ...Array(150), ].map((name, index) => ({ @@ -89,6 +89,15 @@ export const mockEventPropertyDefinitions: PropertyDefinition[] = [ is_seen_on_filtered_events: (name || '').includes('$'), })) +export const mockSessionPropertyDefinitions: PropertyDefinition[] = ['$session_duration', '$initial_utm_source'].map( + (name) => ({ + ...mockEventPropertyDefinition, + id: name, + name: name, + description: `${name} is the best!`, + }) +) + export const mockPersonProperty = { name: '$browser_version', count: 1, diff --git a/frontend/src/types.ts b/frontend/src/types.ts index cfdc09496ef93..955735d6b5bd3 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -2815,6 +2815,7 @@ export enum PropertyDefinitionType { Event = 'event', Person = 'person', Group = 'group', + Session = 'session', } export interface PropertyDefinition { diff --git a/mypy-baseline.txt b/mypy-baseline.txt index 9b607b6222cd3..b1e56d812c2f0 100644 --- a/mypy-baseline.txt +++ b/mypy-baseline.txt @@ -66,11 +66,8 @@ posthog/hogql/database/schema/person_distinct_ids.py:0: error: Argument 1 to "se posthog/hogql/database/schema/person_distinct_id_overrides.py:0: error: Argument 1 to "select_from_person_distinct_id_overrides_table" has incompatible type "dict[str, list[str]]"; expected "dict[str, list[str | int]]" [arg-type] posthog/plugins/utils.py:0: error: Subclass of "str" and "bytes" cannot exist: would have incompatible method signatures [unreachable] posthog/plugins/utils.py:0: error: Statement is unreachable [unreachable] +posthog/clickhouse/kafka_engine.py:0: error: Argument 1 to "join" of "str" has incompatible type "list"; expected "Iterable[str]" [arg-type] posthog/models/filters/base_filter.py:0: error: "HogQLContext" has no attribute "person_on_events_mode" [attr-defined] -posthog/hogql/database/database.py:0: error: "FieldOrTable" has no attribute "fields" [attr-defined] -posthog/hogql/database/database.py:0: error: Incompatible types (expression has type "Literal['view', 'lazy_table']", TypedDict item "type" has type "Literal['integer', 'float', 'string', 'datetime', 'date', 'boolean', 'array', 'json', 'lazy_table', 'virtual_table', 'field_traverser', 'expression']") [typeddict-item] -posthog/warehouse/models/datawarehouse_saved_query.py:0: error: Argument 1 to "create_hogql_database" has incompatible type "int | None"; expected "int" [arg-type] -posthog/warehouse/models/datawarehouse_saved_query.py:0: error: Incompatible types in assignment (expression has type "Expr", variable has type "SelectQuery | SelectUnionQuery") [assignment] posthog/models/user.py:0: error: Incompatible types in assignment (expression has type "None", base class "AbstractUser" defined the type as "CharField[str | int | Combinable, str]") [assignment] posthog/models/user.py:0: error: Incompatible types in assignment (expression has type "posthog.models.user.UserManager", base class "AbstractUser" defined the type as "django.contrib.auth.models.UserManager[AbstractUser]") [assignment] posthog/models/user.py:0: error: Cannot override writeable attribute with read-only property [override] @@ -82,6 +79,10 @@ posthog/models/user.py:0: note: bool posthog/models/user.py:0: error: "User" has no attribute "social_auth" [attr-defined] posthog/models/user.py:0: error: "User" has no attribute "social_auth" [attr-defined] posthog/models/person/person.py:0: error: Incompatible types in assignment (expression has type "list[Never]", variable has type "ValuesQuerySet[PersonDistinctId, str]") [assignment] +posthog/hogql/database/database.py:0: error: "FieldOrTable" has no attribute "fields" [attr-defined] +posthog/hogql/database/database.py:0: error: Incompatible types (expression has type "Literal['view', 'lazy_table']", TypedDict item "type" has type "Literal['integer', 'float', 'string', 'datetime', 'date', 'boolean', 'array', 'json', 'lazy_table', 'virtual_table', 'field_traverser', 'expression']") [typeddict-item] +posthog/warehouse/models/datawarehouse_saved_query.py:0: error: Argument 1 to "create_hogql_database" has incompatible type "int | None"; expected "int" [arg-type] +posthog/warehouse/models/datawarehouse_saved_query.py:0: error: Incompatible types in assignment (expression has type "Expr", variable has type "SelectQuery | SelectUnionQuery") [assignment] posthog/models/feature_flag/flag_matching.py:0: error: Statement is unreachable [unreachable] posthog/hogql_queries/utils/query_date_range.py:0: error: Incompatible return value type (got "str", expected "Literal['hour', 'day', 'week', 'month']") [return-value] posthog/hogql_queries/utils/query_date_range.py:0: error: Item "None" of "dict[str, int] | None" has no attribute "get" [union-attr] diff --git a/posthog/api/__init__.py b/posthog/api/__init__.py index e9828f74e7f1d..453a2c4417d82 100644 --- a/posthog/api/__init__.py +++ b/posthog/api/__init__.py @@ -5,6 +5,7 @@ from posthog.settings import EE_AVAILABLE from posthog.warehouse.api import external_data_source, saved_query, table, view_link, external_data_schema from ..heatmaps.heatmaps_api import LegacyHeatmapViewSet, HeatmapViewSet +from .session import SessionViewSet from ..session_recordings.session_recording_api import SessionRecordingViewSet from . import ( activity_log, @@ -333,6 +334,7 @@ def api_not_found(request): ["team_id"], ) projects_router.register(r"heatmaps", HeatmapViewSet, "project_heatmaps", ["team_id"]) +projects_router.register(r"sessions", SessionViewSet, "project_sessions", ["team_id"]) if EE_AVAILABLE: from ee.clickhouse.views.experiments import ClickhouseExperimentsViewSet diff --git a/posthog/api/property_definition.py b/posthog/api/property_definition.py index 6a87fc6f348d7..7db63497d5a9d 100644 --- a/posthog/api/property_definition.py +++ b/posthog/api/property_definition.py @@ -35,7 +35,7 @@ class PropertyDefinitionQuerySerializer(serializers.Serializer): ) type = serializers.ChoiceField( - choices=["event", "person", "group"], + choices=["event", "person", "group", "session"], help_text="What property definitions to return", default="event", ) @@ -192,6 +192,16 @@ def with_type_filter(self, type: str, group_type_index: Optional[int]): "group_type_index": group_type_index, }, ) + elif type == "session": + return dataclasses.replace( + self, + should_join_event_property=False, + params={ + **self.params, + "type": PropertyDefinition.Type.SESSION, + "group_type_index": -1, + }, + ) def with_event_property_filter( self, event_names: Optional[str], filter_by_event_names: Optional[bool] diff --git a/posthog/api/session.py b/posthog/api/session.py new file mode 100644 index 0000000000000..b4c79600d1999 --- /dev/null +++ b/posthog/api/session.py @@ -0,0 +1,56 @@ +import json + +from rest_framework import request, response, viewsets +from rest_framework.decorators import action +from rest_framework.exceptions import ValidationError + +from posthog.api.routing import TeamAndOrgViewSetMixin +from posthog.hogql.database.schema.sessions import get_lazy_session_table_properties, get_lazy_session_table_values +from posthog.rate_limit import ( + ClickHouseBurstRateThrottle, + ClickHouseSustainedRateThrottle, +) +from posthog.utils import convert_property_value, flatten + + +class SessionViewSet( + TeamAndOrgViewSetMixin, + viewsets.ViewSet, +): + scope_object = "query" + throttle_classes = [ClickHouseBurstRateThrottle, ClickHouseSustainedRateThrottle] + + @action(methods=["GET"], detail=False) + def values(self, request: request.Request, **kwargs) -> response.Response: + team = self.team + + key = request.GET.get("key") + search_term = request.GET.get("value") + + if not key: + raise ValidationError(detail=f"Key not provided") + + result = get_lazy_session_table_values(key, search_term=search_term, team=team) + + flattened = [] + for value in result: + try: + # Try loading as json for dicts or arrays + flattened.append(json.loads(value[0])) + except json.decoder.JSONDecodeError: + flattened.append(value[0]) + return response.Response([{"name": convert_property_value(value)} for value in flatten(flattened)]) + + @action(methods=["GET"], detail=False) + def property_definitions(self, request: request.Request, **kwargs) -> response.Response: + search = request.GET.get("search") + + # unlike e.g. event properties, there's a very limited number of session properties, + # so we can just return them all + results = get_lazy_session_table_properties(search) + return response.Response( + { + "count": len(results), + "results": results, + } + ) diff --git a/posthog/api/test/__snapshots__/test_api_docs.ambr b/posthog/api/test/__snapshots__/test_api_docs.ambr index b4a5bb2780673..37bb741043ef9 100644 --- a/posthog/api/test/__snapshots__/test_api_docs.ambr +++ b/posthog/api/test/__snapshots__/test_api_docs.ambr @@ -84,6 +84,8 @@ '/home/runner/work/posthog/posthog/posthog/session_recordings/session_recording_api.py: Warning [SessionRecordingViewSet > SessionRecordingSerializer]: could not resolve field on model with path "viewed". This is likely a custom field that does some unknown magic. Maybe consider annotating the field/property? Defaulting to "string". (Exception: SessionRecording has no field named \'viewed\')', '/home/runner/work/posthog/posthog/posthog/api/person.py: Warning [SessionRecordingViewSet > SessionRecordingSerializer > MinimalPersonSerializer]: unable to resolve type hint for function "get_distinct_ids". Consider using a type hint or @extend_schema_field. Defaulting to string.', '/home/runner/work/posthog/posthog/posthog/session_recordings/session_recording_api.py: Warning [SessionRecordingViewSet > SessionRecordingSerializer]: unable to resolve type hint for function "storage". Consider using a type hint or @extend_schema_field. Defaulting to string.', + '/home/runner/work/posthog/posthog/posthog/api/session.py: Error [SessionViewSet]: unable to guess serializer. This is graceful fallback handling for APIViews. Consider using GenericAPIView as view base class, if view is under your control. Either way you may want to add a serializer_class (or method). Ignoring view for now.', + '/home/runner/work/posthog/posthog/posthog/api/session.py: Warning [SessionViewSet]: could not derive type of path parameter "project_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/ee/api/subscription.py: Warning [SubscriptionViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.subscription.Subscription" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/ee/api/subscription.py: Warning [SubscriptionViewSet > SubscriptionSerializer]: unable to resolve type hint for function "summary". Consider using a type hint or @extend_schema_field. Defaulting to string.', '/home/runner/work/posthog/posthog/posthog/api/survey.py: Warning [SurveyViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.feedback.survey.Survey" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', diff --git a/posthog/api/test/test_session.py b/posthog/api/test/test_session.py new file mode 100644 index 0000000000000..46fcafabd7c13 --- /dev/null +++ b/posthog/api/test/test_session.py @@ -0,0 +1,137 @@ +import uuid + +from rest_framework import status + +from posthog.models.event.util import create_event +from posthog.test.base import APIBaseTest + + +class TestSessionsAPI(APIBaseTest): + def setUp(self) -> None: + super().setUp() + + create_event( + team=self.team, + event="$pageview", + distinct_id="d1", + properties={"$session_id": "s1", "utm_source": "google"}, + event_uuid=(uuid.uuid4()), + ) + create_event( + team=self.team, + event="$pageview", + distinct_id="d1", + properties={"$session_id": "s1", "utm_source": "youtube"}, + event_uuid=(uuid.uuid4()), + ) + + def test_expected_session_properties(self): + response = self.client.get(f"/api/projects/{self.team.pk}/sessions/property_definitions/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + actual_properties = {entry["name"] for entry in response.json()["results"]} + expected_properties = { + "$autocapture_count", + "$channel_type", + "$end_timestamp", + "$entry_url", + "$exit_url", + "$initial_gad_source", + "$initial_gclid", + "$initial_referring_domain", + "$initial_utm_campaign", + "$initial_utm_content", + "$initial_utm_medium", + "$initial_utm_source", + "$initial_utm_term", + "$pageview_count", + "$session_duration", + "$start_timestamp", + } + assert actual_properties == expected_properties + + def test_search_session_properties(self): + response = self.client.get(f"/api/projects/{self.team.pk}/sessions/property_definitions/?search=utm") + self.assertEqual(response.status_code, status.HTTP_200_OK) + actual_properties = {entry["name"] for entry in response.json()["results"]} + expected_properties = { + "$initial_utm_campaign", + "$initial_utm_content", + "$initial_utm_medium", + "$initial_utm_source", + "$initial_utm_term", + } + assert actual_properties == expected_properties + + def test_empty_search_session_properties(self): + response = self.client.get(f"/api/projects/{self.team.pk}/sessions/property_definitions/?search=doesnotexist") + self.assertEqual(response.status_code, status.HTTP_200_OK) + assert len(response.json()["results"]) == 0 + + def test_list_channel_type_values(self): + response = self.client.get(f"/api/projects/{self.team.pk}/sessions/values/?key=$channel_type") + self.assertEqual(response.status_code, status.HTTP_200_OK) + actual_values = {entry["name"] for entry in response.json()} + expected_values = { + "Affiliate", + "Audio", + "Cross Network", + "Direct", + "Email", + "Organic Search", + "Organic Shopping", + "Organic Video", + "Other", + "Paid Other", + "Paid Search", + "Paid Shopping", + "Paid Video", + "Push", + "Referral", + "SMS", + } + assert actual_values == expected_values + + def test_search_channel_type_values(self): + response = self.client.get(f"/api/projects/{self.team.pk}/sessions/values/?key=$channel_type&value=paid") + self.assertEqual(response.status_code, status.HTTP_200_OK) + actual_values = {entry["name"] for entry in response.json()} + expected_values = { + "Paid Other", + "Paid Search", + "Paid Shopping", + "Paid Video", + } + assert actual_values == expected_values + + def test_list_session_property_values(self): + response = self.client.get(f"/api/projects/{self.team.pk}/sessions/values/?key=$initial_utm_source") + self.assertEqual(response.status_code, status.HTTP_200_OK) + actual_values = {entry["name"] for entry in response.json()} + expected_values = { + "google", + "youtube", + } + assert actual_values == expected_values + + def test_search_session_property_values(self): + response = self.client.get(f"/api/projects/{self.team.pk}/sessions/values/?key=$initial_utm_source&value=tub") + self.assertEqual(response.status_code, status.HTTP_200_OK) + actual_values = {entry["name"] for entry in response.json()} + expected_values = { + "youtube", + } + assert actual_values == expected_values + + def test_search_session_property_no_matching_values(self): + response = self.client.get( + f"/api/projects/{self.team.pk}/sessions/values/?key=$initial_utm_source&value=doesnotexist" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + assert len(response.json()) == 0 + + def test_search_missing_session_property_values(self): + response = self.client.get( + f"/api/projects/{self.team.pk}/sessions/values/?key=$initial_utm_source&value=doesnotexist" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + assert len(response.json()) == 0 diff --git a/posthog/hogql/database/schema/channel_type.py b/posthog/hogql/database/schema/channel_type.py index 39c9b31d36918..c45c71458d7d1 100644 --- a/posthog/hogql/database/schema/channel_type.py +++ b/posthog/hogql/database/schema/channel_type.py @@ -139,3 +139,23 @@ def wrap_with_null_if_empty(expr: ast.Expr) -> ast.Expr: "gad_source": wrap_with_null_if_empty(gad_source), }, ) + + +POSSIBLE_CHANNEL_TYPES = [ + "Cross Network", + "Paid Search", + "Paid Video", + "Paid Shopping", + "Paid Other", + "Direct", + "Organic Search", + "Organic Video", + "Organic Shopping", + "Push", + "SMS", + "Audio", + "Email", + "Referral", + "Affiliate", + "Other", +] diff --git a/posthog/hogql/database/schema/sessions.py b/posthog/hogql/database/schema/sessions.py index 0bd6bfef09caf..63f0e4f98e79f 100644 --- a/posthog/hogql/database/schema/sessions.py +++ b/posthog/hogql/database/schema/sessions.py @@ -1,4 +1,4 @@ -from typing import cast, Any, TYPE_CHECKING +from typing import cast, Any, Optional, TYPE_CHECKING from posthog.hogql import ast from posthog.hogql.context import HogQLContext @@ -11,13 +11,21 @@ StringArrayDatabaseField, DatabaseField, LazyTable, + FloatDatabaseField, + BooleanDatabaseField, ) -from posthog.hogql.database.schema.channel_type import create_channel_type_expr +from posthog.hogql.database.schema.channel_type import create_channel_type_expr, POSSIBLE_CHANNEL_TYPES from posthog.hogql.database.schema.util.session_where_clause_extractor import SessionMinTimestampWhereClauseExtractor from posthog.hogql.errors import ResolutionError +from posthog.models.property_definition import PropertyType +from posthog.models.sessions.sql import ( + SELECT_SESSION_PROP_STRING_VALUES_SQL_WITH_FILTER, + SELECT_SESSION_PROP_STRING_VALUES_SQL, +) +from posthog.queries.insight import insight_sync_execute if TYPE_CHECKING: - pass + from posthog.models.team import Team RAW_SESSIONS_FIELDS: dict[str, FieldOrTable] = { "id": StringDatabaseField(name="session_id"), @@ -200,6 +208,11 @@ def to_printed_clickhouse(self, context): def to_printed_hogql(self): return "sessions" + def avoid_asterisk_fields(self) -> list[str]: + return [ + "duration", # alias of $session_duration, deprecated but included for backwards compatibility + ] + def join_events_table_to_sessions_table( from_table: str, to_table: str, requested_fields: dict[str, Any], context: HogQLContext, node: ast.SelectQuery @@ -220,3 +233,97 @@ def join_events_table_to_sessions_table( ) ) return join_expr + + +def get_lazy_session_table_properties(search: Optional[str]): + # some fields shouldn't appear as properties + hidden_fields = {"team_id", "distinct_id", "session_id", "id", "$event_count_map", "$urls", "duration"} + + # some fields should have a specific property type which isn't derivable from the type of database field + property_type_overrides = { + "$session_duration": PropertyType.Duration, + } + + def get_property_type(field_name: str, field_definition: FieldOrTable): + if field_name in property_type_overrides: + return property_type_overrides[field_name] + if isinstance(field_definition, IntegerDatabaseField) or isinstance(field_definition, FloatDatabaseField): + return PropertyType.Numeric + if isinstance(field_definition, DateTimeDatabaseField): + return PropertyType.Datetime + if isinstance(field_definition, BooleanDatabaseField): + return PropertyType.Boolean + return PropertyType.String + + results = [ + { + "id": field_name, + "name": field_name, + "is_numerical": isinstance(field_definition, IntegerDatabaseField) + or isinstance(field_definition, FloatDatabaseField), + "property_type": get_property_type(field_name, field_definition), + "is_seen_on_filtered_events": None, + "tags": [], + } + for field_name, field_definition in LAZY_SESSIONS_FIELDS.items() + if (not search or search.lower() in field_name.lower()) and field_name not in hidden_fields + ] + return results + + +SESSION_PROPERTY_TO_RAW_SESSIONS_EXPR_MAP = { + "$initial_referring_domain": "finalizeAggregation(initial_referring_domain)", + "$initial_utm_source": "finalizeAggregation(initial_utm_source)", + "$initial_utm_campaign": "finalizeAggregation(initial_utm_campaign)", + "$initial_utm_medium": "finalizeAggregation(initial_utm_medium)", + "$initial_utm_term": "finalizeAggregation(initial_utm_term)", + "$initial_utm_content": "finalizeAggregation(initial_utm_content)", + "$initial_gclid": "finalizeAggregation(initial_gclid)", + "$initial_gad_source": "finalizeAggregation(initial_gad_source)", + "$initial_gclsrc": "finalizeAggregation(initial_gclsrc)", + "$initial_dclid": "finalizeAggregation(initial_dclid)", + "$initial_gbraid": "finalizeAggregation(initial_gbraid)", + "$initial_wbraid": "finalizeAggregation(initial_wbraid)", + "$initial_fbclid": "finalizeAggregation(initial_fbclid)", + "$initial_msclkid": "finalizeAggregation(initial_msclkid)", + "$initial_twclid": "finalizeAggregation(initial_twclid)", + "$initial_li_fat_id": "finalizeAggregation(initial_li_fat_id)", + "$initial_mc_cid": "finalizeAggregation(initial_mc_cid)", + "$initial_igshid": "finalizeAggregation(initial_igshid)", + "$initial_ttclid": "finalizeAggregation(initial_ttclid)", + "$entry_url": "finalizeAggregation(entry_url)", + "$exit_url": "finalizeAggregation(exit_url)", +} + + +def get_lazy_session_table_values(key: str, search_term: Optional[str], team: "Team"): + # the sessions table does not have a properties json object like the events and person tables + + if key == "$channel_type": + return [[name] for name in POSSIBLE_CHANNEL_TYPES if not search_term or search_term.lower() in name.lower()] + + expr = SESSION_PROPERTY_TO_RAW_SESSIONS_EXPR_MAP.get(key) + + if not expr: + return [] + + field_definition = LAZY_SESSIONS_FIELDS.get(key) + if not field_definition: + return [] + + if isinstance(field_definition, StringDatabaseField): + if search_term: + return insight_sync_execute( + SELECT_SESSION_PROP_STRING_VALUES_SQL_WITH_FILTER.format(property_expr=expr), + {"team_id": team.pk, "key": key, "value": "%{}%".format(search_term)}, + query_type="get_session_property_values_with_value", + team_id=team.pk, + ) + return insight_sync_execute( + SELECT_SESSION_PROP_STRING_VALUES_SQL.format(property_expr=expr), + {"team_id": team.pk, "key": key}, + query_type="get_session_property_values", + team_id=team.pk, + ) + + return [] diff --git a/posthog/hogql/database/schema/test/test_sessions.py b/posthog/hogql/database/schema/test/test_sessions.py index 2f4728fb9b558..230ffc1bc1897 100644 --- a/posthog/hogql/database/schema/test/test_sessions.py +++ b/posthog/hogql/database/schema/test/test_sessions.py @@ -5,6 +5,7 @@ APIBaseTest, ClickhouseTestMixin, _create_event, + _create_person, ) @@ -103,3 +104,35 @@ def test_events_session_dot_channel_type(self): result[0], "Paid Search", ) + + def test_persons_and_sessions_on_events(self): + p1 = _create_person(distinct_ids=["d1"], team=self.team) + p2 = _create_person(distinct_ids=["d2"], team=self.team) + + s1 = "session_test_persons_and_sessions_on_events_1" + s2 = "session_test_persons_and_sessions_on_events_2" + + _create_event( + event="$pageview", + team=self.team, + distinct_id="d1", + properties={"$session_id": s1, "utm_source": "source1"}, + ) + _create_event( + event="$pageview", + team=self.team, + distinct_id="d2", + properties={"$session_id": s2, "utm_source": "source2"}, + ) + + response = execute_hogql_query( + parse_select( + "select events.person_id, session.$initial_utm_source from events where $session_id = {session_id} or $session_id = {session_id2} order by 2 asc", + placeholders={"session_id": ast.Constant(value=s1), "session_id2": ast.Constant(value=s2)}, + ), + self.team, + ) + + [row1, row2] = response.results or [] + self.assertEqual(row1, (p1.uuid, "source1")) + self.assertEqual(row2, (p2.uuid, "source2")) diff --git a/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py b/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py index fb1288ac1bd26..cd6a218e7ea0f 100644 --- a/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py +++ b/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py @@ -24,6 +24,7 @@ WebStatsTableQuery, PersonPropertyFilter, SamplingRate, + SessionPropertyFilter, ) from posthog.utils import generate_cache_key, get_safe_cache @@ -51,7 +52,9 @@ def pathname_property_filter(self) -> Optional[EventPropertyFilter]: return None @cached_property - def property_filters_without_pathname(self) -> list[Union[EventPropertyFilter, PersonPropertyFilter]]: + def property_filters_without_pathname( + self, + ) -> list[Union[EventPropertyFilter, PersonPropertyFilter, SessionPropertyFilter]]: return [p for p in self.query.properties if p.key != "$pathname"] def session_where(self, include_previous_period: Optional[bool] = None): diff --git a/posthog/models/property/util.py b/posthog/models/property/util.py index b1ce9b6087aaa..de2602539e6ec 100644 --- a/posthog/models/property/util.py +++ b/posthog/models/property/util.py @@ -930,7 +930,7 @@ def get_session_property_filter_statement(prop: Property, idx: int, prepend: str ) else: - raise exceptions.ValidationError(f"Property '{prop.key}' is not allowed in session property filters.") + raise exceptions.ValidationError(f"Session property '{prop.key}' is only valid in HogQL queries.") def clear_excess_levels(prop: Union["PropertyGroup", "Property"], skip=False): diff --git a/posthog/models/sessions/sql.py b/posthog/models/sessions/sql.py index 6bebc73e023f4..22d3431099f94 100644 --- a/posthog/models/sessions/sql.py +++ b/posthog/models/sessions/sql.py @@ -260,3 +260,44 @@ def source_column(column_name: str) -> str: GROUP BY session_id, team_id """ ) + +SELECT_SESSION_PROP_STRING_VALUES_SQL = """ +SELECT + value, + count(value) +FROM ( + SELECT + {property_expr} as value + FROM + sessions + WHERE + team_id = %(team_id)s AND + {property_expr} IS NOT NULL AND + {property_expr} != '' + ORDER BY session_id DESC + LIMIT 100000 +) +GROUP BY value +ORDER BY count(value) DESC +LIMIT 20 +""" + +SELECT_SESSION_PROP_STRING_VALUES_SQL_WITH_FILTER = """ +SELECT + value, + count(value) +FROM ( + SELECT + {property_expr} as value + FROM + sessions + WHERE + team_id = %(team_id)s AND + {property_expr} ILIKE %(value)s + ORDER BY session_id DESC + LIMIT 100000 +) +GROUP BY value +ORDER BY count(value) DESC +LIMIT 20 +""" diff --git a/posthog/schema.py b/posthog/schema.py index 46ad0beb9a11a..281cd8ef3a039 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -1699,7 +1699,7 @@ class WebAnalyticsQueryBase(BaseModel): ) dateRange: Optional[DateRange] = None modifiers: Optional[HogQLQueryModifiers] = None - properties: list[Union[EventPropertyFilter, PersonPropertyFilter]] + properties: list[Union[EventPropertyFilter, PersonPropertyFilter, SessionPropertyFilter]] sampling: Optional[Sampling] = None useSessionsTable: Optional[bool] = None @@ -1712,7 +1712,7 @@ class WebOverviewQuery(BaseModel): dateRange: Optional[DateRange] = None kind: Literal["WebOverviewQuery"] = "WebOverviewQuery" modifiers: Optional[HogQLQueryModifiers] = None - properties: list[Union[EventPropertyFilter, PersonPropertyFilter]] + properties: list[Union[EventPropertyFilter, PersonPropertyFilter, SessionPropertyFilter]] response: Optional[WebOverviewQueryResponse] = None sampling: Optional[Sampling] = None useSessionsTable: Optional[bool] = None @@ -1730,7 +1730,7 @@ class WebStatsTableQuery(BaseModel): kind: Literal["WebStatsTableQuery"] = "WebStatsTableQuery" limit: Optional[int] = None modifiers: Optional[HogQLQueryModifiers] = None - properties: list[Union[EventPropertyFilter, PersonPropertyFilter]] + properties: list[Union[EventPropertyFilter, PersonPropertyFilter, SessionPropertyFilter]] response: Optional[WebStatsTableQueryResponse] = None sampling: Optional[Sampling] = None useSessionsTable: Optional[bool] = None @@ -1743,7 +1743,7 @@ class WebTopClicksQuery(BaseModel): dateRange: Optional[DateRange] = None kind: Literal["WebTopClicksQuery"] = "WebTopClicksQuery" modifiers: Optional[HogQLQueryModifiers] = None - properties: list[Union[EventPropertyFilter, PersonPropertyFilter]] + properties: list[Union[EventPropertyFilter, PersonPropertyFilter, SessionPropertyFilter]] response: Optional[WebTopClicksQueryResponse] = None sampling: Optional[Sampling] = None useSessionsTable: Optional[bool] = None