From bb996d1f175a0b755f50f3f75bc44a9ab5a9c828 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Thu, 5 Oct 2023 14:36:40 +0100 Subject: [PATCH] feat: support filtering recordings by groups (#17797) --- .../AdvancedSessionRecordingsFilters.tsx | 19 +---- ...sion_recording_list_from_replay_summary.py | 16 ++++ ...on_recording_list_from_session_replay.ambr | 59 +++++++++++++- ...sion_recording_list_from_session_replay.py | 81 ++++++++++++++++++- 4 files changed, 156 insertions(+), 19 deletions(-) diff --git a/frontend/src/scenes/session-recordings/filters/AdvancedSessionRecordingsFilters.tsx b/frontend/src/scenes/session-recordings/filters/AdvancedSessionRecordingsFilters.tsx index c6debeab21c8d..a7f25cfc7f30b 100644 --- a/frontend/src/scenes/session-recordings/filters/AdvancedSessionRecordingsFilters.tsx +++ b/frontend/src/scenes/session-recordings/filters/AdvancedSessionRecordingsFilters.tsx @@ -4,23 +4,16 @@ import { PropertyFilters } from 'lib/components/PropertyFilters/PropertyFilters' import { MathAvailability } from 'scenes/insights/filters/ActionFilter/ActionFilterRow/ActionFilterRow' import { LemonLabel } from 'lib/lemon-ui/LemonLabel/LemonLabel' -import { - EntityTypes, - FilterType, - FilterableLogLevel, - RecordingDurationFilter, - RecordingFilters, - PropertyFilterType, -} from '~/types' +import { EntityTypes, FilterableLogLevel, FilterType, RecordingDurationFilter, RecordingFilters } from '~/types' import { DateFilter } from 'lib/components/DateFilter/DateFilter' import { DurationFilter } from './DurationFilter' import { LemonButtonWithDropdown, LemonCheckbox, LemonInput, LemonTag, Tooltip } from '@posthog/lemon-ui' import { TestAccountFilter } from 'scenes/insights/filters/TestAccountFilter' -import { teamLogic } from 'scenes/teamLogic' import { useValues } from 'kea' import { FEATURE_FLAGS } from 'lib/constants' import { FlaggedFeature } from 'lib/components/FlaggedFeature' import { useDebounce } from 'use-debounce' +import { groupsModel } from '~/models/groupsModel' export const AdvancedSessionRecordingsFilters = ({ filters, @@ -35,18 +28,13 @@ export const AdvancedSessionRecordingsFilters = ({ setLocalFilters: (localFilters: FilterType) => void showPropertyFilters?: boolean }): JSX.Element => { - const { currentTeam } = useValues(teamLogic) - - const hasGroupFilters = (currentTeam?.test_account_filters || []) - .map((x) => x.type) - .includes(PropertyFilterType.Group) + const { groupsTaxonomicTypes } = useValues(groupsModel) return (
setFilters({ filter_test_accounts: testFilters.filter_test_accounts })} - disabledReason={hasGroupFilters ? 'Session replay does not support group filters' : false} /> Time and duration @@ -103,6 +91,7 @@ export const AdvancedSessionRecordingsFilters = ({ TaxonomicFilterGroupType.EventFeatureFlags, TaxonomicFilterGroupType.Elements, TaxonomicFilterGroupType.HogQLExpression, + ...groupsTaxonomicTypes, ]} propertyFiltersPopover addFilterDefaultOptions={{ diff --git a/posthog/session_recordings/queries/session_recording_list_from_replay_summary.py b/posthog/session_recordings/queries/session_recording_list_from_replay_summary.py index 2588fbfaae0a8..ba3b590204825 100644 --- a/posthog/session_recordings/queries/session_recording_list_from_replay_summary.py +++ b/posthog/session_recordings/queries/session_recording_list_from_replay_summary.py @@ -279,6 +279,7 @@ def ttl_days(self): {event_filter_having_events_select} `$session_id` FROM events e + {groups_query} -- sometimes we have to join on persons so we can access e.g. person_props in filters {persons_join} PREWHERE @@ -365,6 +366,17 @@ def build_event_filters(self) -> SummaryEventFiltersSQL: params=params, ) + def _get_groups_query(self) -> Tuple[str, Dict]: + try: + from ee.clickhouse.queries.groups_join_query import GroupsJoinQuery + except ImportError: + # if EE not available then we use a no-op version + from posthog.queries.groups_join_query import GroupsJoinQuery + + return GroupsJoinQuery( + self._filter, self._team_id, self._column_optimizer, person_on_events_mode=self._person_on_events_mode + ).get_join_query() + # We want to select events beyond the range of the recording to handle the case where # a recording spans the time boundaries @cached_property @@ -397,6 +409,8 @@ def get_query(self, select_event_ids: bool = False) -> Tuple[str, Dict[str, Any] event_filters_params = event_filters.params events_timestamp_clause, events_timestamp_params = self._get_events_timestamp_clause + groups_query, groups_params = self._get_groups_query() + # these will be applied to the events table, # so we only want property filters that make sense in that context prop_query, prop_params = self._get_prop_groups( @@ -427,6 +441,7 @@ def get_query(self, select_event_ids: bool = False) -> Tuple[str, Dict[str, Any] provided_session_ids_clause=provided_session_ids_clause, persons_join=persons_join, persons_sub_query=persons_sub_query, + groups_query=groups_query, ), { **base_params, @@ -436,6 +451,7 @@ def get_query(self, select_event_ids: bool = False) -> Tuple[str, Dict[str, Any] **event_filters_params, **prop_params, **persons_select_params, + **groups_params, }, ) diff --git a/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr b/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr index 1091b39b7af83..d643d388619f5 100644 --- a/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr +++ b/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr @@ -1260,6 +1260,59 @@ OFFSET 0 ' --- +# name: TestClickhouseSessionRecordingsListFromSessionReplay.test_event_filter_with_group_filter + ' + + SELECT s.session_id, + any(s.team_id), + any(s.distinct_id), + min(s.min_first_timestamp) as start_time, + max(s.max_last_timestamp) as end_time, + dateDiff('SECOND', start_time, end_time) as duration, + argMinMerge(s.first_url) as first_url, + sum(s.click_count), + sum(s.keypress_count), + sum(s.mouse_activity_count), + sum(s.active_milliseconds)/1000 as active_seconds, + duration-active_seconds as inactive_seconds, + sum(s.console_log_count) as console_log_count, + sum(s.console_warn_count) as console_warn_count, + sum(s.console_error_count) as console_error_count + FROM session_replay_events s + WHERE s.team_id = 2 + AND s.min_first_timestamp >= '2020-12-31 20:00:00' + AND s.min_first_timestamp >= '2021-01-14 00:00:00' + AND s.max_last_timestamp <= '2021-01-21 20:00:00' + AND s.session_id in + (select `$session_id` as session_id + from + (SELECT groupUniqArray(event) as event_names, + `$session_id` + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_1 + FROM groups + WHERE team_id = 2 + AND group_type_index = 1 + GROUP BY group_key) groups_1 ON "$group_1" == groups_1.group_key PREWHERE team_id = 2 + AND e.timestamp >= '2020-12-31 20:00:00' + AND e.timestamp <= now() + WHERE notEmpty(`$session_id`) + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' + AND (event = '$pageview' + AND (has(['org one'], replaceRegexpAll(JSONExtractRaw(group_properties_1, 'name'), '^"|"$', '')))) + GROUP BY `$session_id` + HAVING 1=1 + AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) + GROUP BY session_id + HAVING 1=1 + ORDER BY start_time DESC + LIMIT 51 + OFFSET 0 + ' +--- # name: TestClickhouseSessionRecordingsListFromSessionReplay.test_event_filter_with_hogql_event_properties_test_accounts_excluded ' @@ -2682,7 +2735,7 @@ AND s.min_first_timestamp >= '2020-12-31 20:00:00' AND s.min_first_timestamp >= '2021-01-14 00:00:00' AND s.max_last_timestamp <= '2021-01-21 20:00:00' - AND "session_id" in ['with-errors-session-fb398196-a87f-4f98-91eb-e7dccad4efcf', 'with-two-session-a4cc7feb-93e3-48c5-9daa-0e95538c5bb4', 'with-warns-session-b90d7779-7f4d-409c-bf01-e45e2c621b83'] + AND "session_id" in ['with-errors-session-497cfe26-1ce1-4a2d-b81d-1b8e171b6564', 'with-two-session-52d93c81-84c8-490d-92fe-cd80ee9a2483', 'with-warns-session-20fe20bd-4364-4376-b221-9a284b53c7b7'] GROUP BY session_id HAVING 1=1 AND (console_warn_count > 0 @@ -2729,7 +2782,7 @@ AND s.min_first_timestamp >= '2020-12-31 20:00:00' AND s.min_first_timestamp >= '2021-01-14 00:00:00' AND s.max_last_timestamp <= '2021-01-21 20:00:00' - AND "session_id" in ['with-warns-session-b90d7779-7f4d-409c-bf01-e45e2c621b83'] + AND "session_id" in ['with-warns-session-20fe20bd-4364-4376-b221-9a284b53c7b7'] GROUP BY session_id HAVING 1=1 AND (console_warn_count > 0 @@ -2776,7 +2829,7 @@ AND s.min_first_timestamp >= '2020-12-31 20:00:00' AND s.min_first_timestamp >= '2021-01-14 00:00:00' AND s.max_last_timestamp <= '2021-01-21 20:00:00' - AND "session_id" in ['with-warns-session-b90d7779-7f4d-409c-bf01-e45e2c621b83'] + AND "session_id" in ['with-warns-session-20fe20bd-4364-4376-b221-9a284b53c7b7'] GROUP BY session_id HAVING 1=1 AND (console_warn_count > 0 diff --git a/posthog/session_recordings/queries/test/test_session_recording_list_from_session_replay.py b/posthog/session_recordings/queries/test/test_session_recording_list_from_session_replay.py index 6abcd8d9389b8..b1261c1b4b946 100644 --- a/posthog/session_recordings/queries/test/test_session_recording_list_from_session_replay.py +++ b/posthog/session_recordings/queries/test/test_session_recording_list_from_session_replay.py @@ -9,10 +9,11 @@ from posthog.clickhouse.log_entries import TRUNCATE_LOG_ENTRIES_TABLE_SQL from posthog.cloud_utils import TEST_clear_cloud_cache from posthog.constants import AvailableFeature -from posthog.models import Person, Cohort +from posthog.models import Person, Cohort, GroupTypeMapping from posthog.models.action import Action from posthog.models.action_step import ActionStep from posthog.models.filters.session_recordings_filter import SessionRecordingsFilter +from posthog.models.group.util import create_group from posthog.session_recordings.sql.session_replay_event_sql import TRUNCATE_SESSION_REPLAY_EVENTS_TABLE_SQL from posthog.models.team import Team from posthog.session_recordings.queries.session_recording_list_from_replay_summary import ( @@ -2561,3 +2562,81 @@ def _a_session_with_two_events(self, team: Team, session_id: str) -> None: event_name="$pageleave", properties={"$session_id": session_id, "$window_id": "1"}, ) + + @freeze_time("2021-01-21T20:00:00.000Z") + @snapshot_clickhouse_queries + def test_event_filter_with_group_filter(self): + Person.objects.create(team=self.team, distinct_ids=["user"], properties={"email": "bla"}) + session_id = f"test_event_filter_with_group_filter-ONE-{uuid4()}" + different_group_session = f"test_event_filter_with_group_filter-TWO-{uuid4()}" + + produce_replay_summary( + distinct_id="user", + session_id=session_id, + first_timestamp=self.base_time, + team_id=self.team.pk, + ) + produce_replay_summary( + distinct_id="user", + session_id=different_group_session, + first_timestamp=self.base_time, + team_id=self.team.pk, + ) + + GroupTypeMapping.objects.create(team=self.team, group_type="project", group_type_index=0) + create_group( + team_id=self.team.pk, group_type_index=0, group_key="project:1", properties={"name": "project one"} + ) + + GroupTypeMapping.objects.create(team=self.team, group_type="organization", group_type_index=1) + create_group(team_id=self.team.pk, group_type_index=1, group_key="org:1", properties={"name": "org one"}) + + self.create_event( + "user", + self.base_time, + team=self.team, + event_name="$pageview", + properties={ + "$session_id": session_id, + "$window_id": "1", + "$group_1": "org:1", + }, + ) + self.create_event( + "user", + self.base_time, + team=self.team, + event_name="$pageview", + properties={ + "$session_id": different_group_session, + "$window_id": "1", + "$group_0": "project:1", + }, + ) + + filter = SessionRecordingsFilter( + team=self.team, + data={ + "events": [ + { + "id": "$pageview", + "type": "events", + "order": 0, + "name": "$pageview", + "properties": [ + { + "key": "name", + "value": ["org one"], + "operator": "exact", + "type": "group", + "group_type_index": 1, + } + ], + } + ], + }, + ) + session_recording_list_instance = SessionRecordingListFromReplaySummary(filter=filter, team=self.team) + (session_recordings, _) = session_recording_list_instance.run() + + self.assertEqual([sr["session_id"] for sr in session_recordings], [session_id])