From 6a93be3166b395abcf8378709db4937dbcfadcaf Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 24 Jun 2024 16:38:42 +0200 Subject: [PATCH] chore(insights): Yeet the unbaked groups-on-events (#22764) * chore(insights): Yeet the unbaked groups-on-events * Update TestAutoProjectMiddleware * Update verifiedDomainsLogic.test.ts.snap * Update funnel_correlation.py * Update test_property.py * Update query snapshots * Update UI snapshots for `chromium` (2) * Update query snapshots * Update UI snapshots for `chromium` (2) * Update query snapshots * Update query snapshots * Update query snapshots --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- .github/actions/run-backend-tests/action.yml | 1 - ee/clickhouse/models/test/test_property.py | 93 +- .../queries/funnels/funnel_correlation.py | 32 +- ee/clickhouse/queries/groups_join_query.py | 4 - frontend/src/lib/api.mock.ts | 1 - .../SystemStatus/systemStatusLogic.ts | 1 - .../verifiedDomainsLogic.test.ts.snap | 1 - frontend/src/scenes/teamActivityDescriber.tsx | 1 - frontend/src/types.ts | 1 - posthog/api/team.py | 11 +- .../test_properties_timeline.ambr | 137 +- posthog/api/test/test_properties_timeline.py | 1464 ++++++++--------- posthog/api/test/test_team.py | 4 - posthog/models/property/util.py | 24 +- posthog/models/team/team.py | 9 - posthog/queries/breakdown_props.py | 4 +- posthog/queries/funnels/base.py | 29 +- posthog/queries/trends/breakdown.py | 16 - posthog/settings/dynamic_settings.py | 6 - posthog/test/test_middleware.py | 2 +- 20 files changed, 818 insertions(+), 1023 deletions(-) diff --git a/.github/actions/run-backend-tests/action.yml b/.github/actions/run-backend-tests/action.yml index 39cba842f237b..d7c9689f55901 100644 --- a/.github/actions/run-backend-tests/action.yml +++ b/.github/actions/run-backend-tests/action.yml @@ -148,7 +148,6 @@ runs: if: ${{ inputs.segment == 'Core' }} env: PERSON_ON_EVENTS_V2_ENABLED: ${{ inputs.person-on-events }} - GROUPS_ON_EVENTS_ENABLED: ${{ inputs.person-on-events }} shell: bash run: | # async_migrations covered in ci-async-migrations.yml pytest ${{ diff --git a/ee/clickhouse/models/test/test_property.py b/ee/clickhouse/models/test/test_property.py index 6348697d84435..fd1791438ceeb 100644 --- a/ee/clickhouse/models/test/test_property.py +++ b/ee/clickhouse/models/test/test_property.py @@ -14,7 +14,6 @@ from posthog.models.filters import Filter from posthog.models.instance_setting import ( get_instance_setting, - override_instance_config, ) from posthog.models.organization import Organization from posthog.models.property import Property, TableWithProperties @@ -1037,38 +1036,6 @@ def test_get_property_string_expr(self): ) self.assertEqual(string_expr, ('"mat_pp_some_mat_prop2"', True)) - def test_get_property_string_expr_groups(self): - if not get_instance_setting("GROUPS_ON_EVENTS_ENABLED"): - return - - materialize("events", "some_mat_prop3", table_column="group2_properties") - - string_expr = get_property_string_expr( - "events", - "some_mat_prop3", - "x", - "properties", - table_alias="e", - materialised_table_column="group2_properties", - ) - self.assertEqual(string_expr, ('e."mat_gp2_some_mat_prop3"', True)) - - string_expr = get_property_string_expr( - "events", - "some_mat_prop3", - "'x'", - "gp_props_alias", - table_alias="e", - materialised_table_column="group1_properties", - ) - self.assertEqual( - string_expr, - ( - "replaceRegexpAll(JSONExtractRaw(e.gp_props_alias, 'x'), '^\"|\"$', '')", - False, - ), - ) - @pytest.mark.django_db def test_parse_prop_clauses_defaults(snapshot): @@ -1255,22 +1222,7 @@ def test_breakdown_query_expression( {"breakdown_param_1": "$browser"}, ), ('array("mat_pp_$browser") AS value', {"breakdown_param_1": "$browser"}), - ), - ( - ["$browser", "$browser_version"], - "events", - "prop", - "properties", - "group2_properties", - ( - "array(replaceRegexpAll(JSONExtractRaw(properties, %(breakdown_param_1)s), '^\"|\"$', ''),replaceRegexpAll(JSONExtractRaw(properties, %(breakdown_param_2)s), '^\"|\"$', '')) AS prop", - {"breakdown_param_1": "$browser", "breakdown_param_2": "$browser_version"}, - ), - ( - """array("mat_gp2_$browser",replaceRegexpAll(JSONExtractRaw(properties, %(breakdown_param_2)s), '^\"|\"$', '')) AS prop""", - {"breakdown_param_1": "$browser", "breakdown_param_2": "$browser_version"}, - ), - ), + ) ] @@ -1289,31 +1241,30 @@ def test_breakdown_query_expression_materialised( expected_with: str, expected_without: str, ): - with override_instance_config("GROUPS_ON_EVENTS_ENABLED", True): - from posthog.models.team import util + from posthog.models.team import util - util.can_enable_actor_on_events = True + util.can_enable_actor_on_events = True - materialize(table, breakdown[0], table_column="properties") - actual = get_single_or_multi_property_string_expr( - breakdown, - table, - query_alias, - column, - materialised_table_column=materialise_column, - ) - assert actual == expected_with - - materialize(table, breakdown[0], table_column=materialise_column) # type: ignore - actual = get_single_or_multi_property_string_expr( - breakdown, - table, - query_alias, - column, - materialised_table_column=materialise_column, - ) + materialize(table, breakdown[0], table_column="properties") + actual = get_single_or_multi_property_string_expr( + breakdown, + table, + query_alias, + column, + materialised_table_column=materialise_column, + ) + assert actual == expected_with + + materialize(table, breakdown[0], table_column=materialise_column) # type: ignore + actual = get_single_or_multi_property_string_expr( + breakdown, + table, + query_alias, + column, + materialised_table_column=materialise_column, + ) - assert actual == expected_without + assert actual == expected_without @pytest.fixture diff --git a/ee/clickhouse/queries/funnels/funnel_correlation.py b/ee/clickhouse/queries/funnels/funnel_correlation.py index 9bf35aaa67b8c..d23c459c7ef8d 100644 --- a/ee/clickhouse/queries/funnels/funnel_correlation.py +++ b/ee/clickhouse/queries/funnels/funnel_correlation.py @@ -24,7 +24,6 @@ from posthog.models.filters import Filter from posthog.models.property.util import get_property_string_expr from posthog.models.team import Team -from posthog.models.team.team import groups_on_events_querying_enabled from posthog.queries.funnels.utils import get_funnel_order_actor_class from posthog.queries.insight import insight_sync_execute from posthog.queries.person_distinct_id_query import get_team_distinct_ids_query @@ -161,23 +160,7 @@ def properties_to_include(self) -> list[str]: for property_name in cast(list, self._filter.correlation_property_names): if self._filter.aggregation_group_type_index is not None: - if not groups_on_events_querying_enabled(): - continue - - if "$all" == property_name: - return [f"group{self._filter.aggregation_group_type_index}_properties"] - - possible_mat_col = mat_event_cols.get( - ( - property_name, - f"group{self._filter.aggregation_group_type_index}_properties", - ) - ) - if possible_mat_col is not None: - props_to_include.append(possible_mat_col) - else: - props_to_include.append(f"group{self._filter.aggregation_group_type_index}_properties") - + continue # We don't support group properties on events at this time else: if "$all" == property_name: return [f"person_properties"] @@ -499,12 +482,6 @@ def _get_events_join_query(self) -> str: def _get_aggregation_join_query(self): if self._filter.aggregation_group_type_index is None: - if ( - alias_poe_mode_for_legacy(self._team.person_on_events_mode) != PersonsOnEventsMode.DISABLED - and groups_on_events_querying_enabled() - ): - return "", {} - person_query, person_query_params = PersonQuery( self._filter, self._team.pk, @@ -524,12 +501,9 @@ def _get_aggregation_join_query(self): def _get_properties_prop_clause(self): if ( alias_poe_mode_for_legacy(self._team.person_on_events_mode) != PersonsOnEventsMode.DISABLED - and groups_on_events_querying_enabled() + and self._filter.aggregation_group_type_index is None ): - group_properties_field = f"group{self._filter.aggregation_group_type_index}_properties" - aggregation_properties_alias = ( - "person_properties" if self._filter.aggregation_group_type_index is None else group_properties_field - ) + aggregation_properties_alias = "person_properties" else: group_properties_field = f"groups_{self._filter.aggregation_group_type_index}.group_properties_{self._filter.aggregation_group_type_index}" aggregation_properties_alias = ( diff --git a/ee/clickhouse/queries/groups_join_query.py b/ee/clickhouse/queries/groups_join_query.py index 70334d0f2df1d..5a48bc0d0efa2 100644 --- a/ee/clickhouse/queries/groups_join_query.py +++ b/ee/clickhouse/queries/groups_join_query.py @@ -7,7 +7,6 @@ from posthog.models.filters.stickiness_filter import StickinessFilter from posthog.models.filters.utils import GroupTypeIndex from posthog.models.property.util import parse_prop_grouped_clauses -from posthog.models.team.team import groups_on_events_querying_enabled from posthog.queries.util import PersonPropertiesMode, alias_poe_mode_for_legacy from posthog.schema import PersonsOnEventsMode @@ -38,9 +37,6 @@ def __init__( def get_join_query(self) -> tuple[str, dict]: join_queries, params = [], {} - if self._person_on_events_mode != PersonsOnEventsMode.DISABLED and groups_on_events_querying_enabled(): - return "", {} - for group_type_index in self._column_optimizer.group_types_to_query: var = f"group_index_{group_type_index}" group_join_key = self._join_key or f'"$group_{group_type_index}"' diff --git a/frontend/src/lib/api.mock.ts b/frontend/src/lib/api.mock.ts index 676f55a6fb866..4021fb67b004c 100644 --- a/frontend/src/lib/api.mock.ts +++ b/frontend/src/lib/api.mock.ts @@ -86,7 +86,6 @@ export const MOCK_DEFAULT_TEAM: TeamType = { primary_dashboard: 1, live_events_columns: null, person_on_events_querying_enabled: true, - groups_on_events_querying_enabled: true, live_events_token: '123', } diff --git a/frontend/src/scenes/instance/SystemStatus/systemStatusLogic.ts b/frontend/src/scenes/instance/SystemStatus/systemStatusLogic.ts index 48384258b378c..5ee7e0b6e3c97 100644 --- a/frontend/src/scenes/instance/SystemStatus/systemStatusLogic.ts +++ b/frontend/src/scenes/instance/SystemStatus/systemStatusLogic.ts @@ -39,7 +39,6 @@ const EDITABLE_INSTANCE_SETTINGS = [ 'EMAIL_REPLY_TO', 'AGGREGATE_BY_DISTINCT_IDS_TEAMS', 'PERSON_ON_EVENTS_ENABLED', - 'GROUPS_ON_EVENTS_ENABLED', 'STRICT_CACHING_TEAMS', 'SLACK_APP_CLIENT_ID', 'SLACK_APP_CLIENT_SECRET', diff --git a/frontend/src/scenes/settings/organization/VerifiedDomains/__snapshots__/verifiedDomainsLogic.test.ts.snap b/frontend/src/scenes/settings/organization/VerifiedDomains/__snapshots__/verifiedDomainsLogic.test.ts.snap index 9ae32414a859c..8f407e7d5330e 100644 --- a/frontend/src/scenes/settings/organization/VerifiedDomains/__snapshots__/verifiedDomainsLogic.test.ts.snap +++ b/frontend/src/scenes/settings/organization/VerifiedDomains/__snapshots__/verifiedDomainsLogic.test.ts.snap @@ -61,7 +61,6 @@ exports[`verifiedDomainsLogic values has proper defaults 1`] = ` "data-attr", ], "effective_membership_level": 8, - "groups_on_events_querying_enabled": true, "has_group_types": true, "heatmaps_opt_in": true, "id": 997, diff --git a/frontend/src/scenes/teamActivityDescriber.tsx b/frontend/src/scenes/teamActivityDescriber.tsx index 992c3d3708dea..708651fb9a6ab 100644 --- a/frontend/src/scenes/teamActivityDescriber.tsx +++ b/frontend/src/scenes/teamActivityDescriber.tsx @@ -222,7 +222,6 @@ const teamActionsMapping: Record< correlation_config: () => null, data_attributes: () => null, effective_membership_level: () => null, - groups_on_events_querying_enabled: () => null, has_group_types: () => null, ingested_event: () => null, is_demo: () => null, diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 4ce45fa13fc91..9b14ea8d07f3a 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -507,7 +507,6 @@ export interface TeamType extends TeamBasicType { */ correlation_config: CorrelationConfigType | null person_on_events_querying_enabled: boolean - groups_on_events_querying_enabled: boolean extra_settings?: Record modifiers?: HogQLQueryModifiers default_modifiers?: HogQLQueryModifiers diff --git a/posthog/api/team.py b/posthog/api/team.py index 0b2d7b850013e..667dbfa74243d 100644 --- a/posthog/api/team.py +++ b/posthog/api/team.py @@ -30,10 +30,7 @@ from posthog.models.organization import OrganizationMembership from posthog.models.personal_api_key import APIScopeObjectOrNotSupported from posthog.models.signals import mute_selected_signals -from posthog.models.team.team import ( - groups_on_events_querying_enabled, - set_team_in_cache, -) +from posthog.models.team.team import set_team_in_cache from posthog.models.team.util import delete_batch_exports, delete_bulky_postgres_data from posthog.models.utils import UUIDT, generate_random_token_project from posthog.permissions import ( @@ -117,7 +114,6 @@ class Meta: class TeamSerializer(serializers.ModelSerializer, UserPermissionsSerializerMixin): effective_membership_level = serializers.SerializerMethodField() has_group_types = serializers.SerializerMethodField() - groups_on_events_querying_enabled = serializers.SerializerMethodField() live_events_token = serializers.SerializerMethodField() class Meta: @@ -162,7 +158,6 @@ class Meta: "live_events_columns", "recording_domains", "person_on_events_querying_enabled", - "groups_on_events_querying_enabled", "inject_web_apps", "extra_settings", "modifiers", @@ -184,7 +179,6 @@ class Meta: "has_group_types", "default_modifiers", "person_on_events_querying_enabled", - "groups_on_events_querying_enabled", "live_events_token", ) @@ -194,9 +188,6 @@ def get_effective_membership_level(self, team: Team) -> Optional[OrganizationMem def get_has_group_types(self, team: Team) -> bool: return GroupTypeMapping.objects.filter(team=team).exists() - def get_groups_on_events_querying_enabled(self, team: Team) -> bool: - return groups_on_events_querying_enabled() - def get_live_events_token(self, team: Team) -> Optional[str]: return encode_jwt( {"team_id": team.id, "api_token": team.api_token}, diff --git a/posthog/api/test/__snapshots__/test_properties_timeline.ambr b/posthog/api/test/__snapshots__/test_properties_timeline.ambr index 496592e781895..ac8cf04120ea6 100644 --- a/posthog/api/test/__snapshots__/test_properties_timeline.ambr +++ b/posthog/api/test/__snapshots__/test_properties_timeline.ambr @@ -1,5 +1,6 @@ +# serializer version: 1 # name: TestPersonPropertiesTimeline.test_timeline_for_existing_actor_with_six_events_but_only_two_relevant_changes - ' + ''' /* user_id:0 request:_snapshot_ */ SELECT timestamp, properties, end_event_number - start_event_number AS relevant_event_count @@ -20,7 +21,7 @@ e."person_properties" AS "person_properties" FROM events e WHERE team_id = 2 - AND person_id = '12345678-0000-0000-0000-000000000001' + AND person_id = '00000000-0000-0000-0000-000000000000' AND event = '$pageview' AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-05 23:59:59', 'UTC') @@ -32,10 +33,10 @@ OR relevant_property_values != previous_relevant_property_values OR timestamp IS NULL ) WHERE timestamp IS NOT NULL - ' ---- + ''' +# --- # name: TestPersonPropertiesTimeline.test_timeline_for_existing_actor_with_six_events_but_only_two_relevant_changes_without_events - ' + ''' /* user_id:0 request:_snapshot_ */ SELECT timestamp, properties, end_event_number - start_event_number AS relevant_event_count @@ -56,7 +57,7 @@ e."person_properties" AS "person_properties" FROM events e WHERE team_id = 2 - AND person_id = '12345678-0000-0000-0000-000000000001' + AND person_id = '00000000-0000-0000-0000-000000000000' AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-05 23:59:59', 'UTC') ORDER BY timestamp ASC) @@ -67,10 +68,10 @@ OR relevant_property_values != previous_relevant_property_values OR timestamp IS NULL ) WHERE timestamp IS NOT NULL - ' ---- + ''' +# --- # name: TestPersonPropertiesTimeline.test_timeline_for_existing_actor_with_six_events_but_only_two_relevant_changes_without_filters - ' + ''' /* user_id:0 request:_snapshot_ */ SELECT timestamp, properties, end_event_number - start_event_number AS relevant_event_count @@ -91,7 +92,7 @@ e."person_properties" AS "person_properties" FROM events e WHERE team_id = 2 - AND person_id = '12345678-0000-0000-0000-000000000001' + AND person_id = '00000000-0000-0000-0000-000000000000' AND event = '$pageview' AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-05 23:59:59', 'UTC') @@ -103,10 +104,10 @@ OR relevant_property_values != previous_relevant_property_values OR timestamp IS NULL ) WHERE timestamp IS NOT NULL - ' ---- + ''' +# --- # name: TestPersonPropertiesTimeline.test_timeline_for_existing_actor_with_three_events_and_return_to_previous_value - ' + ''' /* user_id:0 request:_snapshot_ */ SELECT timestamp, properties, end_event_number - start_event_number AS relevant_event_count @@ -127,7 +128,7 @@ e."person_properties" AS "person_properties" FROM events e WHERE team_id = 2 - AND person_id = '12345678-0000-0000-0000-000000000001' + AND person_id = '00000000-0000-0000-0000-000000000000' AND event = '$pageview' AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-05 23:59:59', 'UTC') @@ -139,10 +140,10 @@ OR relevant_property_values != previous_relevant_property_values OR timestamp IS NULL ) WHERE timestamp IS NOT NULL - ' ---- + ''' +# --- # name: TestPersonPropertiesTimeline.test_timeline_for_existing_person_with_three_events_and_return_to_previous_value_at_single_day_point - ' + ''' /* user_id:0 request:_snapshot_ */ SELECT timestamp, properties, end_event_number - start_event_number AS relevant_event_count @@ -163,7 +164,7 @@ e."person_properties" AS "person_properties" FROM events e WHERE team_id = 2 - AND person_id = '12345678-0000-0000-0000-000000000001' + AND person_id = '00000000-0000-0000-0000-000000000000' AND event = '$pageview' AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-02 23:59:59', 'UTC') @@ -175,10 +176,10 @@ OR relevant_property_values != previous_relevant_property_values OR timestamp IS NULL ) WHERE timestamp IS NOT NULL - ' ---- + ''' +# --- # name: TestPersonPropertiesTimeline.test_timeline_for_existing_person_with_three_events_and_return_to_previous_value_at_single_hour_point - ' + ''' /* user_id:0 request:_snapshot_ */ SELECT timestamp, properties, end_event_number - start_event_number AS relevant_event_count @@ -199,7 +200,7 @@ e."person_properties" AS "person_properties" FROM events e WHERE team_id = 2 - AND person_id = '12345678-0000-0000-0000-000000000001' + AND person_id = '00000000-0000-0000-0000-000000000000' AND event = '$pageview' AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-02 00:00:00', 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-02 01:00:00', 'UTC') @@ -211,10 +212,10 @@ OR relevant_property_values != previous_relevant_property_values OR timestamp IS NULL ) WHERE timestamp IS NOT NULL - ' ---- + ''' +# --- # name: TestPersonPropertiesTimeline.test_timeline_for_existing_person_with_three_events_and_return_to_previous_value_at_single_month_point - ' + ''' /* user_id:0 request:_snapshot_ */ SELECT timestamp, properties, end_event_number - start_event_number AS relevant_event_count @@ -235,7 +236,7 @@ e."person_properties" AS "person_properties" FROM events e WHERE team_id = 2 - AND person_id = '12345678-0000-0000-0000-000000000001' + AND person_id = '00000000-0000-0000-0000-000000000000' AND event = '$pageview' AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfMonth(toDateTime('2020-01-01 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-31 23:59:59', 'UTC') @@ -247,10 +248,10 @@ OR relevant_property_values != previous_relevant_property_values OR timestamp IS NULL ) WHERE timestamp IS NOT NULL - ' ---- + ''' +# --- # name: TestPersonPropertiesTimeline.test_timeline_for_existing_person_with_three_events_and_return_to_previous_value_using_relative_date_from - ' + ''' /* user_id:0 request:_snapshot_ */ SELECT timestamp, properties, end_event_number - start_event_number AS relevant_event_count @@ -271,7 +272,7 @@ e."person_properties" AS "person_properties" FROM events e WHERE team_id = 2 - AND person_id = '12345678-0000-0000-0000-000000000001' + AND person_id = '00000000-0000-0000-0000-000000000000' AND event = '$pageview' AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-02 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-09 23:59:59', 'UTC') @@ -283,10 +284,10 @@ OR relevant_property_values != previous_relevant_property_values OR timestamp IS NULL ) WHERE timestamp IS NOT NULL - ' ---- + ''' +# --- # name: TestPersonPropertiesTimeline.test_timeline_for_new_actor_with_one_event_before_range - ' + ''' /* user_id:0 request:_snapshot_ */ SELECT timestamp, properties, end_event_number - start_event_number AS relevant_event_count @@ -307,7 +308,7 @@ e."person_properties" AS "person_properties" FROM events e WHERE team_id = 2 - AND person_id = '12345678-0000-0000-0000-000000000001' + AND person_id = '00000000-0000-0000-0000-000000000000' AND event = '$pageview' AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-05 23:59:59', 'UTC') @@ -319,10 +320,10 @@ OR relevant_property_values != previous_relevant_property_values OR timestamp IS NULL ) WHERE timestamp IS NOT NULL - ' ---- + ''' +# --- # name: TestPersonPropertiesTimeline.test_timeline_for_new_actor_with_one_event_before_range_materialized - ' + ''' /* user_id:0 request:_snapshot_ */ SELECT timestamp, properties, end_event_number - start_event_number AS relevant_event_count @@ -344,7 +345,7 @@ e."person_properties" AS "person_properties" FROM events e WHERE team_id = 2 - AND person_id = '12345678-0000-0000-0000-000000000001' + AND person_id = '00000000-0000-0000-0000-000000000000' AND event = '$pageview' AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-05 23:59:59', 'UTC') @@ -357,10 +358,10 @@ OR relevant_property_values != previous_relevant_property_values OR timestamp IS NULL ) WHERE timestamp IS NOT NULL - ' ---- + ''' +# --- # name: TestPersonPropertiesTimeline.test_timeline_for_new_actor_with_one_event_in_range - ' + ''' /* user_id:0 request:_snapshot_ */ SELECT timestamp, properties, end_event_number - start_event_number AS relevant_event_count @@ -381,7 +382,7 @@ e."person_properties" AS "person_properties" FROM events e WHERE team_id = 2 - AND person_id = '12345678-0000-0000-0000-000000000001' + AND person_id = '00000000-0000-0000-0000-000000000000' AND event = '$pageview' AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-05 23:59:59', 'UTC') @@ -393,10 +394,10 @@ OR relevant_property_values != previous_relevant_property_values OR timestamp IS NULL ) WHERE timestamp IS NOT NULL - ' ---- + ''' +# --- # name: TestPersonPropertiesTimeline.test_timeline_for_new_actor_with_one_event_in_range_materialized - ' + ''' /* user_id:0 request:_snapshot_ */ SELECT timestamp, properties, end_event_number - start_event_number AS relevant_event_count @@ -418,7 +419,7 @@ e."person_properties" AS "person_properties" FROM events e WHERE team_id = 2 - AND person_id = '12345678-0000-0000-0000-000000000001' + AND person_id = '00000000-0000-0000-0000-000000000000' AND event = '$pageview' AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-05 23:59:59', 'UTC') @@ -431,10 +432,10 @@ OR relevant_property_values != previous_relevant_property_values OR timestamp IS NULL ) WHERE timestamp IS NOT NULL - ' ---- + ''' +# --- # name: TestPersonPropertiesTimeline.test_timeline_with_two_events_in_range_using_breakdown - ' + ''' /* user_id:0 request:_snapshot_ */ SELECT timestamp, properties, end_event_number - start_event_number AS relevant_event_count @@ -445,7 +446,7 @@ ORDER BY timestamp ASC ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) AS end_event_number FROM (SELECT timestamp, person_properties AS properties, - array(replaceRegexpAll(JSONExtractRaw(person_properties, 'foo'), '^"|"$', ''), replaceRegexpAll(JSONExtractRaw(person_properties, 'bar'), '^"|"$', '')) AS relevant_property_values, + array(replaceRegexpAll(JSONExtractRaw(person_properties, 'bar'), '^"|"$', ''), replaceRegexpAll(JSONExtractRaw(person_properties, 'foo'), '^"|"$', '')) AS relevant_property_values, lagInFrame(relevant_property_values) OVER ( ORDER BY timestamp ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS previous_relevant_property_values, row_number() OVER ( @@ -455,7 +456,7 @@ e."person_properties" AS "person_properties" FROM events e WHERE team_id = 2 - AND person_id = '12345678-0000-0000-0000-000000000001' + AND person_id = '00000000-0000-0000-0000-000000000000' AND event = '$pageview' AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-05 23:59:59', 'UTC') @@ -467,10 +468,10 @@ OR relevant_property_values != previous_relevant_property_values OR timestamp IS NULL ) WHERE timestamp IS NOT NULL - ' ---- + ''' +# --- # name: TestPersonPropertiesTimeline.test_timeline_with_two_events_in_range_using_breakdown_materialized - ' + ''' /* user_id:0 request:_snapshot_ */ SELECT timestamp, properties, end_event_number - start_event_number AS relevant_event_count @@ -481,7 +482,7 @@ ORDER BY timestamp ASC ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) AS end_event_number FROM (SELECT timestamp, person_properties AS properties, - array("mat_pp_foo", "mat_pp_bar") AS relevant_property_values, + array("mat_pp_bar", "mat_pp_foo") AS relevant_property_values, lagInFrame(relevant_property_values) OVER ( ORDER BY timestamp ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS previous_relevant_property_values, row_number() OVER ( @@ -493,7 +494,7 @@ e."person_properties" AS "person_properties" FROM events e WHERE team_id = 2 - AND person_id = '12345678-0000-0000-0000-000000000001' + AND person_id = '00000000-0000-0000-0000-000000000000' AND event = '$pageview' AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-05 23:59:59', 'UTC') @@ -507,10 +508,10 @@ OR relevant_property_values != previous_relevant_property_values OR timestamp IS NULL ) WHERE timestamp IS NOT NULL - ' ---- + ''' +# --- # name: TestPersonPropertiesTimeline.test_timeline_with_two_events_in_range_using_filter_on_series - ' + ''' /* user_id:0 request:_snapshot_ */ SELECT timestamp, properties, end_event_number - start_event_number AS relevant_event_count @@ -521,7 +522,7 @@ ORDER BY timestamp ASC ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) AS end_event_number FROM (SELECT timestamp, person_properties AS properties, - array(replaceRegexpAll(JSONExtractRaw(person_properties, 'foo'), '^"|"$', ''), replaceRegexpAll(JSONExtractRaw(person_properties, 'bar'), '^"|"$', '')) AS relevant_property_values, + array(replaceRegexpAll(JSONExtractRaw(person_properties, 'bar'), '^"|"$', ''), replaceRegexpAll(JSONExtractRaw(person_properties, 'foo'), '^"|"$', '')) AS relevant_property_values, lagInFrame(relevant_property_values) OVER ( ORDER BY timestamp ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS previous_relevant_property_values, row_number() OVER ( @@ -531,7 +532,7 @@ e."person_properties" AS "person_properties" FROM events e WHERE team_id = 2 - AND person_id = '12345678-0000-0000-0000-000000000001' + AND person_id = '00000000-0000-0000-0000-000000000000' AND event = '$pageview' AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-05 23:59:59', 'UTC') @@ -543,10 +544,10 @@ OR relevant_property_values != previous_relevant_property_values OR timestamp IS NULL ) WHERE timestamp IS NOT NULL - ' ---- + ''' +# --- # name: TestPersonPropertiesTimeline.test_timeline_with_two_events_in_range_using_filter_on_series_materialized - ' + ''' /* user_id:0 request:_snapshot_ */ SELECT timestamp, properties, end_event_number - start_event_number AS relevant_event_count @@ -557,7 +558,7 @@ ORDER BY timestamp ASC ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) AS end_event_number FROM (SELECT timestamp, person_properties AS properties, - array("mat_pp_foo", "mat_pp_bar") AS relevant_property_values, + array("mat_pp_bar", "mat_pp_foo") AS relevant_property_values, lagInFrame(relevant_property_values) OVER ( ORDER BY timestamp ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS previous_relevant_property_values, row_number() OVER ( @@ -569,7 +570,7 @@ e."person_properties" AS "person_properties" FROM events e WHERE team_id = 2 - AND person_id = '12345678-0000-0000-0000-000000000001' + AND person_id = '00000000-0000-0000-0000-000000000000' AND event = '$pageview' AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-05 23:59:59', 'UTC') @@ -583,5 +584,5 @@ OR relevant_property_values != previous_relevant_property_values OR timestamp IS NULL ) WHERE timestamp IS NOT NULL - ' ---- + ''' +# --- diff --git a/posthog/api/test/test_properties_timeline.py b/posthog/api/test/test_properties_timeline.py index d8b8a11e9099a..b4eb9a080f17d 100644 --- a/posthog/api/test/test_properties_timeline.py +++ b/posthog/api/test/test_properties_timeline.py @@ -1,17 +1,15 @@ import json import random import uuid -from typing import Any, Literal, Optional +from typing import Optional from freezegun.api import freeze_time from rest_framework import status from posthog.models.filters.mixins.base import BreakdownType -from posthog.models.group.util import create_group from posthog.queries.properties_timeline.properties_timeline import ( PropertiesTimelineResult, ) -from posthog.settings.dynamic_settings import CONSTANCE_CONFIG from posthog.test.base import ( APIBaseTest, ClickhouseTestMixin, @@ -22,612 +20,520 @@ snapshot_clickhouse_queries, ) - -def properties_timeline_test_factory(actor_type: Literal["person", "group"]): - materialized_column_kwargs = ( - {"person_properties": ["foo", "bar"]} if actor_type == "person" else {"group_properties": ["foo", "bar"]} - ) - main_actor_id = uuid.UUID("12345678-0000-0000-0000-000000000001") if actor_type == "person" else "test" - - class TestPropertiesTimeline(ClickhouseTestMixin, APIBaseTest): - maxDiff = None - - def _create_actor(self, properties: dict) -> str: - """Create actor of relevant type and return its UUID (for persons) or key (for groups).""" - if actor_type == "person": - person = _create_person( - team=self.team, - distinct_ids=["abcd"], - uuid=main_actor_id, - properties=properties, - ) - return str(person.uuid) - else: - group = create_group( - team_id=self.team.pk, - group_type_index=0, - group_key=str(main_actor_id), - properties=properties, - ) - return group.group_key - - def _create_event(self, event: str, timestamp: str, actor_properties: dict): - create_event_kwargs: dict[str, Any] = {} - if actor_type == "person": - create_event_kwargs["person_id"] = main_actor_id - create_event_kwargs["person_properties"] = actor_properties - else: - create_event_kwargs["properties"] = {"$group_0": main_actor_id} - create_event_kwargs["group_0_properties"] = actor_properties - - _create_event( - team=self.team, - event=event, - timestamp=timestamp, - distinct_id=str(random.randint(1, 1000)), - **create_event_kwargs, - ) - - def _get_timeline_result( - self, - *, - events: Optional[list] = None, - actions: Optional[list] = None, - properties: Optional[list] = None, - breakdown: Optional[str] = None, - breakdown_type: Optional[BreakdownType] = None, - date_from: Optional[str], - date_to: Optional[str], - display: str = "ActionsTable", - interval: Optional[str] = None, - expected_status: int = status.HTTP_200_OK, - ) -> PropertiesTimelineResult: - url = ( - f"/api/person/{main_actor_id}/properties_timeline" - f"?events={json.dumps(events or [])}&actions={json.dumps(actions or [])}" - f"&properties={json.dumps(properties or [])}&display={display}" - f"&date_from={date_from or ''}&date_to={date_to or ''}&interval={interval or ''}" - f"&breakdown={breakdown or ''}&breakdown_type={(breakdown_type or actor_type) if breakdown else ''}" - ) - properties_timeline = self.client.get(url) - self.assertEqual(properties_timeline.status_code, expected_status) - return properties_timeline.json() - - @also_test_with_materialized_columns(**materialized_column_kwargs) - @snapshot_clickhouse_queries - def test_timeline_for_new_actor_with_one_event_in_range(self): - self._create_actor({"foo": "abc", "bar": 123}) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 123}, - timestamp="2020-01-01T00:00:00Z", # Exactly the same as date_from - ) - flush_persons_and_events() - - timeline = self._get_timeline_result( - events=[ - { - "id": "$pageview", - } - ], - properties=[{"key": "bar", "value": "xyz", "type": "person"}], - date_from="2020-01-01", - date_to="2020-01-05", - ) - - self.assertEqual( - timeline, +MATERIALIZED_COLUMN_KWARGS = {"person_properties": ["foo", "bar"]} +TEST_PERSON_ID = uuid.UUID("12345678-0000-0000-0000-000000000001") + + +class TestPersonPropertiesTimeline(ClickhouseTestMixin, APIBaseTest): + maxDiff = None + + def _create_person(self, properties: dict) -> str: + """Create person and return its UUID.""" + person = _create_person( + team=self.team, + distinct_ids=["abcd"], + uuid=TEST_PERSON_ID, + properties=properties, + ) + return str(person.uuid) + + def _create_event(self, event: str, timestamp: str, actor_properties: dict): + _create_event( + team=self.team, + event=event, + timestamp=timestamp, + distinct_id=str(random.randint(1, 1000)), + person_id=TEST_PERSON_ID, + person_properties=actor_properties, + ) + + def _get_timeline_result( + self, + *, + events: Optional[list] = None, + actions: Optional[list] = None, + properties: Optional[list] = None, + breakdown: Optional[str] = None, + breakdown_type: Optional[BreakdownType] = None, + date_from: Optional[str], + date_to: Optional[str], + display: str = "ActionsTable", + interval: Optional[str] = None, + expected_status: int = status.HTTP_200_OK, + ) -> PropertiesTimelineResult: + url = ( + f"/api/person/{TEST_PERSON_ID}/properties_timeline" + f"?events={json.dumps(events or [])}&actions={json.dumps(actions or [])}" + f"&properties={json.dumps(properties or [])}&display={display}" + f"&date_from={date_from or ''}&date_to={date_to or ''}&interval={interval or ''}" + f"&breakdown={breakdown or ''}&breakdown_type={(breakdown_type or 'person') if breakdown else ''}" + ) + properties_timeline = self.client.get(url) + self.assertEqual(properties_timeline.status_code, expected_status) + return properties_timeline.json() + + @also_test_with_materialized_columns(**MATERIALIZED_COLUMN_KWARGS) + @snapshot_clickhouse_queries + def test_timeline_for_new_actor_with_one_event_in_range(self): + self._create_person({"foo": "abc", "bar": 123}) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 123}, + timestamp="2020-01-01T00:00:00Z", # Exactly the same as date_from + ) + flush_persons_and_events() + + timeline = self._get_timeline_result( + events=[ { - "points": [ - { - "properties": {"foo": "abc", "bar": 123}, - "relevant_event_count": 1, - "timestamp": "2020-01-01T00:00:00Z", - } - ], - "crucial_property_keys": ["bar"], - "effective_date_from": "2020-01-01T00:00:00+00:00", - "effective_date_to": "2020-01-05T23:59:59.999999+00:00", - }, - ) - - @also_test_with_materialized_columns(**materialized_column_kwargs) - @snapshot_clickhouse_queries - def test_timeline_for_new_actor_with_one_event_before_range(self): - self._create_actor({"foo": "abc", "bar": 123}) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 123}, - timestamp="2019-12-27T00:00:00Z", # Before date_from - ) - flush_persons_and_events() - - timeline = self._get_timeline_result( - events=[ + "id": "$pageview", + } + ], + properties=[{"key": "bar", "value": "xyz", "type": "person"}], + date_from="2020-01-01", + date_to="2020-01-05", + ) + + self.assertEqual( + timeline, + { + "points": [ { - "id": "$pageview", + "properties": {"foo": "abc", "bar": 123}, + "relevant_event_count": 1, + "timestamp": "2020-01-01T00:00:00Z", } ], - properties=[{"key": "bar", "value": "xyz", "type": "person"}], - date_from="2020-01-01", - date_to="2020-01-05", - ) - - self.assertEqual( - timeline, + "crucial_property_keys": ["bar"], + "effective_date_from": "2020-01-01T00:00:00+00:00", + "effective_date_to": "2020-01-05T23:59:59.999999+00:00", + }, + ) + + @also_test_with_materialized_columns(**MATERIALIZED_COLUMN_KWARGS) + @snapshot_clickhouse_queries + def test_timeline_for_new_actor_with_one_event_before_range(self): + self._create_person({"foo": "abc", "bar": 123}) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 123}, + timestamp="2019-12-27T00:00:00Z", # Before date_from + ) + flush_persons_and_events() + + timeline = self._get_timeline_result( + events=[ { - "points": [], # No relevant events in range - "crucial_property_keys": ["bar"], - "effective_date_from": "2020-01-01T00:00:00+00:00", - "effective_date_to": "2020-01-05T23:59:59.999999+00:00", - }, - ) - - @also_test_with_materialized_columns(**materialized_column_kwargs) - @snapshot_clickhouse_queries - def test_timeline_with_two_events_in_range_using_filter_on_series(self): - self._create_actor({"foo": "abc", "bar": 123}) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 123}, - timestamp="2020-01-01T00:00:00Z", # Exactly the same as date_from - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "klm", "bar": 123}, - timestamp="2020-01-01T21:37:00Z", - ) - flush_persons_and_events() - - timeline = self._get_timeline_result( - events=[ - { - "id": "$pageview", - "properties": [ - { - "key": "foo", - "type": "person", - "value": ["whatever"], - "operator": "exact", - }, - { - "key": "fin", - "type": "event", - "value": ["anything"], - "operator": "exact", - }, - ], - } - ], - properties=[{"key": "bar", "value": "xyz", "type": "person"}], - date_from="2020-01-01", - date_to="2020-01-05", - ) - - self.assertEqual( - timeline, + "id": "$pageview", + } + ], + properties=[{"key": "bar", "value": "xyz", "type": "person"}], + date_from="2020-01-01", + date_to="2020-01-05", + ) + + self.assertEqual( + timeline, + { + "points": [], # No relevant events in range + "crucial_property_keys": ["bar"], + "effective_date_from": "2020-01-01T00:00:00+00:00", + "effective_date_to": "2020-01-05T23:59:59.999999+00:00", + }, + ) + + @also_test_with_materialized_columns(**MATERIALIZED_COLUMN_KWARGS) + @snapshot_clickhouse_queries + def test_timeline_with_two_events_in_range_using_filter_on_series(self): + self._create_person({"foo": "abc", "bar": 123}) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 123}, + timestamp="2020-01-01T00:00:00Z", # Exactly the same as date_from + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "klm", "bar": 123}, + timestamp="2020-01-01T21:37:00Z", + ) + flush_persons_and_events() + + timeline = self._get_timeline_result( + events=[ { - "points": [ + "id": "$pageview", + "properties": [ { - "properties": {"foo": "abc", "bar": 123}, - "relevant_event_count": 1, - "timestamp": "2020-01-01T00:00:00Z", + "key": "foo", + "type": "person", + "value": ["whatever"], + "operator": "exact", }, { - "properties": {"foo": "klm", "bar": 123}, - "relevant_event_count": 1, - "timestamp": "2020-01-01T21:37:00Z", + "key": "fin", + "type": "event", + "value": ["anything"], + "operator": "exact", }, ], - "crucial_property_keys": ["bar", "foo"], - "effective_date_from": "2020-01-01T00:00:00+00:00", - "effective_date_to": "2020-01-05T23:59:59.999999+00:00", - }, - ) - - @also_test_with_materialized_columns(**materialized_column_kwargs) - @snapshot_clickhouse_queries - def test_timeline_with_two_events_in_range_using_breakdown(self): - self._create_actor({"foo": "abc", "bar": 123}) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 123}, - timestamp="2020-01-01T00:00:00Z", # Exactly the same as date_from - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "klm", "bar": 123}, - timestamp="2020-01-01T21:37:00Z", - ) - flush_persons_and_events() - - timeline = self._get_timeline_result( - events=[ + } + ], + properties=[{"key": "bar", "value": "xyz", "type": "person"}], + date_from="2020-01-01", + date_to="2020-01-05", + ) + + self.assertEqual( + timeline, + { + "points": [ { - "id": "$pageview", - } + "properties": {"foo": "abc", "bar": 123}, + "relevant_event_count": 1, + "timestamp": "2020-01-01T00:00:00Z", + }, + { + "properties": {"foo": "klm", "bar": 123}, + "relevant_event_count": 1, + "timestamp": "2020-01-01T21:37:00Z", + }, ], - breakdown="foo", - properties=[{"key": "bar", "value": "xyz", "type": "person"}], - date_from="2020-01-01", - date_to="2020-01-05", - ) - - self.assertEqual( - timeline, + "crucial_property_keys": ["bar", "foo"], + "effective_date_from": "2020-01-01T00:00:00+00:00", + "effective_date_to": "2020-01-05T23:59:59.999999+00:00", + }, + ) + + @also_test_with_materialized_columns(**MATERIALIZED_COLUMN_KWARGS) + @snapshot_clickhouse_queries + def test_timeline_with_two_events_in_range_using_breakdown(self): + self._create_person({"foo": "abc", "bar": 123}) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 123}, + timestamp="2020-01-01T00:00:00Z", # Exactly the same as date_from + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "klm", "bar": 123}, + timestamp="2020-01-01T21:37:00Z", + ) + flush_persons_and_events() + + timeline = self._get_timeline_result( + events=[ { - "points": [ - { - "properties": {"foo": "abc", "bar": 123}, - "relevant_event_count": 1, - "timestamp": "2020-01-01T00:00:00Z", - }, - { - "properties": {"foo": "klm", "bar": 123}, - "relevant_event_count": 1, - "timestamp": "2020-01-01T21:37:00Z", - }, - ], - "crucial_property_keys": ["bar", "foo"], - "effective_date_from": "2020-01-01T00:00:00+00:00", - "effective_date_to": "2020-01-05T23:59:59.999999+00:00", - }, - ) - - @snapshot_clickhouse_queries - @also_test_with_materialized_columns(**materialized_column_kwargs) - def test_timeline_for_existing_actor_with_three_events_and_return_to_previous_value(self): - self._create_actor({"foo": "abc", "bar": 123}) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 456}, # Initial bar - timestamp="2020-01-02T00:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 123}, # Changed bar - timestamp="2020-01-03T00:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={ - "foo": "abc", - "bar": 456, - }, # Changed bar back to initial value - timestamp="2020-01-04T00:00:00Z", - ) - flush_persons_and_events() - - timeline = self._get_timeline_result( - events=[ + "id": "$pageview", + } + ], + breakdown="foo", + properties=[{"key": "bar", "value": "xyz", "type": "person"}], + date_from="2020-01-01", + date_to="2020-01-05", + ) + + self.assertEqual( + timeline, + { + "points": [ { - "id": "$pageview", - } + "properties": {"foo": "abc", "bar": 123}, + "relevant_event_count": 1, + "timestamp": "2020-01-01T00:00:00Z", + }, + { + "properties": {"foo": "klm", "bar": 123}, + "relevant_event_count": 1, + "timestamp": "2020-01-01T21:37:00Z", + }, ], - properties=[{"key": "bar", "value": "xyz", "type": "person"}], - date_from="2020-01-01", - date_to="2020-01-05", - ) - - self.assertEqual( - timeline, + "crucial_property_keys": ["bar", "foo"], + "effective_date_from": "2020-01-01T00:00:00+00:00", + "effective_date_to": "2020-01-05T23:59:59.999999+00:00", + }, + ) + + @snapshot_clickhouse_queries + @also_test_with_materialized_columns(**MATERIALIZED_COLUMN_KWARGS) + def test_timeline_for_existing_actor_with_three_events_and_return_to_previous_value(self): + self._create_person({"foo": "abc", "bar": 123}) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 456}, # Initial bar + timestamp="2020-01-02T00:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 123}, # Changed bar + timestamp="2020-01-03T00:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={ + "foo": "abc", + "bar": 456, + }, # Changed bar back to initial value + timestamp="2020-01-04T00:00:00Z", + ) + flush_persons_and_events() + + timeline = self._get_timeline_result( + events=[ { - "points": [ - { - "properties": {"foo": "abc", "bar": 456}, - "relevant_event_count": 1, - "timestamp": "2020-01-02T00:00:00Z", - }, - { - "properties": {"foo": "abc", "bar": 123}, - "relevant_event_count": 1, - "timestamp": "2020-01-03T00:00:00Z", - }, - { - "properties": {"foo": "abc", "bar": 456}, - "relevant_event_count": 1, - "timestamp": "2020-01-04T00:00:00Z", - }, - ], - "crucial_property_keys": ["bar"], - "effective_date_from": "2020-01-01T00:00:00+00:00", - "effective_date_to": "2020-01-05T23:59:59.999999+00:00", - }, - ) - - @snapshot_clickhouse_queries - @also_test_with_materialized_columns(person_properties=["bar"], materialize_only_with_person_on_events=True) - def test_timeline_for_existing_person_with_three_events_and_return_to_previous_value_at_single_day_point(self): - self._create_actor(properties={"foo": "abc", "bar": 123}) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 456}, # Initial bar - timestamp="2020-01-02T00:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 123}, # Changed bar - timestamp="2020-01-02T07:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={ - "foo": "abc", - "bar": 456, - }, # Changed bar back to initial value - timestamp="2020-01-02T14:00:00Z", - ) - flush_persons_and_events() - - timeline = self._get_timeline_result( - events=[ + "id": "$pageview", + } + ], + properties=[{"key": "bar", "value": "xyz", "type": "person"}], + date_from="2020-01-01", + date_to="2020-01-05", + ) + + self.assertEqual( + timeline, + { + "points": [ { - "id": "$pageview", - } + "properties": {"foo": "abc", "bar": 456}, + "relevant_event_count": 1, + "timestamp": "2020-01-02T00:00:00Z", + }, + { + "properties": {"foo": "abc", "bar": 123}, + "relevant_event_count": 1, + "timestamp": "2020-01-03T00:00:00Z", + }, + { + "properties": {"foo": "abc", "bar": 456}, + "relevant_event_count": 1, + "timestamp": "2020-01-04T00:00:00Z", + }, ], - properties=[{"key": "bar", "value": "xyz", "type": "person"}], - display="ActionsLineGraph", - date_from="2020-01-02T00:00:00Z", - date_to="2020-01-02T00:00:00Z", - # For some legacy reason data point-specific date_from and date_to are the same in the persons modal - # The backend needs interval to offset date_from properly - interval="day", - ) - - self.assertEqual( - timeline, + "crucial_property_keys": ["bar"], + "effective_date_from": "2020-01-01T00:00:00+00:00", + "effective_date_to": "2020-01-05T23:59:59.999999+00:00", + }, + ) + + @snapshot_clickhouse_queries + @also_test_with_materialized_columns(person_properties=["bar"], materialize_only_with_person_on_events=True) + def test_timeline_for_existing_person_with_three_events_and_return_to_previous_value_at_single_day_point(self): + self._create_person(properties={"foo": "abc", "bar": 123}) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 456}, # Initial bar + timestamp="2020-01-02T00:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 123}, # Changed bar + timestamp="2020-01-02T07:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={ + "foo": "abc", + "bar": 456, + }, # Changed bar back to initial value + timestamp="2020-01-02T14:00:00Z", + ) + flush_persons_and_events() + + timeline = self._get_timeline_result( + events=[ { - "points": [ - { - "properties": {"foo": "abc", "bar": 456}, - "relevant_event_count": 1, - "timestamp": "2020-01-02T00:00:00Z", - }, - { - "properties": {"foo": "abc", "bar": 123}, - "relevant_event_count": 1, - "timestamp": "2020-01-02T07:00:00Z", - }, - { - "properties": {"foo": "abc", "bar": 456}, - "relevant_event_count": 1, - "timestamp": "2020-01-02T14:00:00Z", - }, - ], - "crucial_property_keys": ["bar"], - "effective_date_from": "2020-01-02T00:00:00+00:00", - "effective_date_to": "2020-01-02T23:59:59.999999+00:00", - }, - ) - - @snapshot_clickhouse_queries - @also_test_with_materialized_columns(person_properties=["bar"], materialize_only_with_person_on_events=True) - def test_timeline_for_existing_person_with_three_events_and_return_to_previous_value_at_single_hour_point(self): - self._create_actor(properties={"foo": "abc", "bar": 123}) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 456}, # Initial bar - timestamp="2020-01-02T00:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 123}, # Changed bar - timestamp="2020-01-02T00:20:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={ - "foo": "abc", - "bar": 456, - }, # Changed bar back to initial value - timestamp="2020-01-02T00:40:00Z", - ) - flush_persons_and_events() - - timeline = self._get_timeline_result( - events=[ + "id": "$pageview", + } + ], + properties=[{"key": "bar", "value": "xyz", "type": "person"}], + display="ActionsLineGraph", + date_from="2020-01-02T00:00:00Z", + date_to="2020-01-02T00:00:00Z", + # For some legacy reason data point-specific date_from and date_to are the same in the persons modal + # The backend needs interval to offset date_from properly + interval="day", + ) + + self.assertEqual( + timeline, + { + "points": [ { - "id": "$pageview", - } + "properties": {"foo": "abc", "bar": 456}, + "relevant_event_count": 1, + "timestamp": "2020-01-02T00:00:00Z", + }, + { + "properties": {"foo": "abc", "bar": 123}, + "relevant_event_count": 1, + "timestamp": "2020-01-02T07:00:00Z", + }, + { + "properties": {"foo": "abc", "bar": 456}, + "relevant_event_count": 1, + "timestamp": "2020-01-02T14:00:00Z", + }, ], - properties=[{"key": "bar", "value": "xyz", "type": "person"}], - display="ActionsLineGraph", - date_from="2020-01-02T00:00:00Z", - date_to="2020-01-02T00:00:00Z", - interval="hour", - ) - - self.assertEqual( - timeline, + "crucial_property_keys": ["bar"], + "effective_date_from": "2020-01-02T00:00:00+00:00", + "effective_date_to": "2020-01-02T23:59:59.999999+00:00", + }, + ) + + @snapshot_clickhouse_queries + @also_test_with_materialized_columns(person_properties=["bar"], materialize_only_with_person_on_events=True) + def test_timeline_for_existing_person_with_three_events_and_return_to_previous_value_at_single_hour_point(self): + self._create_person(properties={"foo": "abc", "bar": 123}) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 456}, # Initial bar + timestamp="2020-01-02T00:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 123}, # Changed bar + timestamp="2020-01-02T00:20:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={ + "foo": "abc", + "bar": 456, + }, # Changed bar back to initial value + timestamp="2020-01-02T00:40:00Z", + ) + flush_persons_and_events() + + timeline = self._get_timeline_result( + events=[ { - "points": [ - { - "properties": {"foo": "abc", "bar": 456}, - "relevant_event_count": 1, - "timestamp": "2020-01-02T00:00:00Z", - }, - { - "properties": {"foo": "abc", "bar": 123}, - "relevant_event_count": 1, - "timestamp": "2020-01-02T00:20:00Z", - }, - { - "properties": {"foo": "abc", "bar": 456}, - "relevant_event_count": 1, - "timestamp": "2020-01-02T00:40:00Z", - }, - ], - "crucial_property_keys": ["bar"], - "effective_date_from": "2020-01-02T00:00:00+00:00", - "effective_date_to": "2020-01-02T01:00:00+00:00", - }, - ) - - @snapshot_clickhouse_queries - @also_test_with_materialized_columns(person_properties=["bar"], materialize_only_with_person_on_events=True) - def test_timeline_for_existing_person_with_three_events_and_return_to_previous_value_at_single_month_point( - self, - ): - self._create_actor(properties={"foo": "abc", "bar": 123}) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 456}, # Initial bar - timestamp="2020-01-01T00:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 123}, # Changed bar - timestamp="2020-01-02T00:20:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={ - "foo": "abc", - "bar": 456, - }, # Changed bar back to initial value - timestamp="2020-01-31T00:40:00Z", - ) - flush_persons_and_events() - - timeline = self._get_timeline_result( - events=[ + "id": "$pageview", + } + ], + properties=[{"key": "bar", "value": "xyz", "type": "person"}], + display="ActionsLineGraph", + date_from="2020-01-02T00:00:00Z", + date_to="2020-01-02T00:00:00Z", + interval="hour", + ) + + self.assertEqual( + timeline, + { + "points": [ { - "id": "$pageview", - } + "properties": {"foo": "abc", "bar": 456}, + "relevant_event_count": 1, + "timestamp": "2020-01-02T00:00:00Z", + }, + { + "properties": {"foo": "abc", "bar": 123}, + "relevant_event_count": 1, + "timestamp": "2020-01-02T00:20:00Z", + }, + { + "properties": {"foo": "abc", "bar": 456}, + "relevant_event_count": 1, + "timestamp": "2020-01-02T00:40:00Z", + }, ], - properties=[{"key": "bar", "value": "xyz", "type": "person"}], - display="ActionsLineGraph", - date_from="2020-01-01", - date_to="2020-01-01", - interval="month", - ) - - self.assertEqual( - timeline, - { - "points": [ - { - "properties": {"foo": "abc", "bar": 456}, - "relevant_event_count": 1, - "timestamp": "2020-01-01T00:00:00Z", - }, - { - "properties": {"foo": "abc", "bar": 123}, - "relevant_event_count": 1, - "timestamp": "2020-01-02T00:20:00Z", - }, - { - "properties": {"foo": "abc", "bar": 456}, - "relevant_event_count": 1, - "timestamp": "2020-01-31T00:40:00Z", - }, - ], - "crucial_property_keys": ["bar"], - "effective_date_from": "2020-01-01T00:00:00+00:00", - "effective_date_to": "2020-01-31T23:59:59.999999+00:00", - }, - ) - - @snapshot_clickhouse_queries - @also_test_with_materialized_columns(person_properties=["bar"], materialize_only_with_person_on_events=True) - def test_timeline_for_existing_person_with_three_events_and_return_to_previous_value_using_relative_date_from( - self, - ): - self._create_actor(properties={"foo": "abc", "bar": 123}) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 456}, # Initial bar - timestamp="2020-01-02T00:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 123}, # Changed bar - timestamp="2020-01-02T00:20:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={ - "foo": "abc", - "bar": 456, - }, # Changed bar back to initial value - timestamp="2020-01-06T00:40:00Z", - ) - flush_persons_and_events() - - with freeze_time("2020-01-09T21:37:00Z"): - timeline = self._get_timeline_result( - events=[ - { - "id": "$pageview", - } - ], - properties=[{"key": "bar", "value": "xyz", "type": "person"}], - date_from="-7d", - date_to=None, - ) - - self.assertEqual( - timeline, + "crucial_property_keys": ["bar"], + "effective_date_from": "2020-01-02T00:00:00+00:00", + "effective_date_to": "2020-01-02T01:00:00+00:00", + }, + ) + + @snapshot_clickhouse_queries + @also_test_with_materialized_columns(person_properties=["bar"], materialize_only_with_person_on_events=True) + def test_timeline_for_existing_person_with_three_events_and_return_to_previous_value_at_single_month_point( + self, + ): + self._create_person(properties={"foo": "abc", "bar": 123}) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 456}, # Initial bar + timestamp="2020-01-01T00:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 123}, # Changed bar + timestamp="2020-01-02T00:20:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={ + "foo": "abc", + "bar": 456, + }, # Changed bar back to initial value + timestamp="2020-01-31T00:40:00Z", + ) + flush_persons_and_events() + + timeline = self._get_timeline_result( + events=[ { - "points": [ - { - "properties": {"foo": "abc", "bar": 456}, - "relevant_event_count": 1, - "timestamp": "2020-01-02T00:00:00Z", - }, - { - "properties": {"foo": "abc", "bar": 123}, - "relevant_event_count": 1, - "timestamp": "2020-01-02T00:20:00Z", - }, - { - "properties": {"foo": "abc", "bar": 456}, - "relevant_event_count": 1, - "timestamp": "2020-01-06T00:40:00Z", - }, - ], - "crucial_property_keys": ["bar"], - "effective_date_from": "2020-01-02T00:00:00+00:00", - "effective_date_to": "2020-01-09T23:59:59.999999+00:00", - }, - ) - - @snapshot_clickhouse_queries - @also_test_with_materialized_columns(**materialized_column_kwargs) - def test_timeline_for_existing_actor_with_six_events_but_only_two_relevant_changes(self): - self._create_actor({"foo": "abc", "bar": 123}) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 456}, - timestamp="2020-01-01T00:00:00Z", # Exactly the same as date_from - ) - self._create_event( - event="whatever", # This event is not a $pageview, so it must be ignored here - actor_properties={"foo": "abc", "bar": 123}, # Changed bar - timestamp="2020-01-02T00:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 123}, # Changed bar - timestamp="2020-01-02T01:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 123}, - timestamp="2020-01-03T01:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 123}, - timestamp="2020-01-04T19:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 789}, # Changed bar - timestamp="2020-01-04T19:00:01Z", - ) - flush_persons_and_events() - + "id": "$pageview", + } + ], + properties=[{"key": "bar", "value": "xyz", "type": "person"}], + display="ActionsLineGraph", + date_from="2020-01-01", + date_to="2020-01-01", + interval="month", + ) + + self.assertEqual( + timeline, + { + "points": [ + { + "properties": {"foo": "abc", "bar": 456}, + "relevant_event_count": 1, + "timestamp": "2020-01-01T00:00:00Z", + }, + { + "properties": {"foo": "abc", "bar": 123}, + "relevant_event_count": 1, + "timestamp": "2020-01-02T00:20:00Z", + }, + { + "properties": {"foo": "abc", "bar": 456}, + "relevant_event_count": 1, + "timestamp": "2020-01-31T00:40:00Z", + }, + ], + "crucial_property_keys": ["bar"], + "effective_date_from": "2020-01-01T00:00:00+00:00", + "effective_date_to": "2020-01-31T23:59:59.999999+00:00", + }, + ) + + @snapshot_clickhouse_queries + @also_test_with_materialized_columns(person_properties=["bar"], materialize_only_with_person_on_events=True) + def test_timeline_for_existing_person_with_three_events_and_return_to_previous_value_using_relative_date_from( + self, + ): + self._create_person(properties={"foo": "abc", "bar": 123}) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 456}, # Initial bar + timestamp="2020-01-02T00:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 123}, # Changed bar + timestamp="2020-01-02T00:20:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={ + "foo": "abc", + "bar": 456, + }, # Changed bar back to initial value + timestamp="2020-01-06T00:40:00Z", + ) + flush_persons_and_events() + + with freeze_time("2020-01-09T21:37:00Z"): timeline = self._get_timeline_result( events=[ { @@ -635,177 +541,233 @@ def test_timeline_for_existing_actor_with_six_events_but_only_two_relevant_chang } ], properties=[{"key": "bar", "value": "xyz", "type": "person"}], - date_from="2020-01-01", - date_to="2020-01-05", - ) - - self.assertEqual( - timeline, - { - "points": [ - { - "properties": {"foo": "abc", "bar": 456}, - "relevant_event_count": 1, - "timestamp": "2020-01-01T00:00:00Z", - }, - { - "properties": {"foo": "abc", "bar": 123}, - "relevant_event_count": 3, - "timestamp": "2020-01-02T01:00:00Z", - }, - { - "properties": {"foo": "abc", "bar": 789}, - "relevant_event_count": 1, - "timestamp": "2020-01-04T19:00:01Z", - }, - ], - "crucial_property_keys": ["bar"], - "effective_date_from": "2020-01-01T00:00:00+00:00", - "effective_date_to": "2020-01-05T23:59:59.999999+00:00", - }, + date_from="-7d", + date_to=None, ) - @snapshot_clickhouse_queries - def test_timeline_for_existing_actor_with_six_events_but_only_two_relevant_changes_without_filters(self): - self._create_actor({"foo": "abc", "bar": 123}) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 456}, - timestamp="2020-01-01T00:00:00Z", # Exactly the same as date_from - ) - self._create_event( - event="whatever", # This event is not a $pageview, but this doesn't matter here anyway - actor_properties={"foo": "abc", "bar": 123}, # Changed bar - timestamp="2020-01-01T01:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 123}, # Changed bar - timestamp="2020-01-02T01:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 123}, - timestamp="2020-01-03T01:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 123}, - timestamp="2020-01-04T19:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 789}, # Changed bar - timestamp="2020-01-04T19:00:01Z", - ) - flush_persons_and_events() - - timeline = self._get_timeline_result( - events=[ + self.assertEqual( + timeline, + { + "points": [ { - "id": "$pageview", - } + "properties": {"foo": "abc", "bar": 456}, + "relevant_event_count": 1, + "timestamp": "2020-01-02T00:00:00Z", + }, + { + "properties": {"foo": "abc", "bar": 123}, + "relevant_event_count": 1, + "timestamp": "2020-01-02T00:20:00Z", + }, + { + "properties": {"foo": "abc", "bar": 456}, + "relevant_event_count": 1, + "timestamp": "2020-01-06T00:40:00Z", + }, ], - date_from="2020-01-01", - date_to="2020-01-05", - ) - - self.assertEqual( - timeline, # Without filters, NO changes are relevant + "crucial_property_keys": ["bar"], + "effective_date_from": "2020-01-02T00:00:00+00:00", + "effective_date_to": "2020-01-09T23:59:59.999999+00:00", + }, + ) + + @snapshot_clickhouse_queries + @also_test_with_materialized_columns(**MATERIALIZED_COLUMN_KWARGS) + def test_timeline_for_existing_actor_with_six_events_but_only_two_relevant_changes(self): + self._create_person({"foo": "abc", "bar": 123}) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 456}, + timestamp="2020-01-01T00:00:00Z", # Exactly the same as date_from + ) + self._create_event( + event="whatever", # This event is not a $pageview, so it must be ignored here + actor_properties={"foo": "abc", "bar": 123}, # Changed bar + timestamp="2020-01-02T00:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 123}, # Changed bar + timestamp="2020-01-02T01:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 123}, + timestamp="2020-01-03T01:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 123}, + timestamp="2020-01-04T19:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 789}, # Changed bar + timestamp="2020-01-04T19:00:01Z", + ) + flush_persons_and_events() + + timeline = self._get_timeline_result( + events=[ { - "points": [ - { - "properties": {"foo": "abc", "bar": 456}, - "relevant_event_count": 5, - "timestamp": "2020-01-01T00:00:00Z", - }, - ], - "crucial_property_keys": [], - "effective_date_from": "2020-01-01T00:00:00+00:00", - "effective_date_to": "2020-01-05T23:59:59.999999+00:00", - }, - ) - - @snapshot_clickhouse_queries - def test_timeline_for_existing_actor_with_six_events_but_only_two_relevant_changes_without_events(self): - self._create_actor({"foo": "abc", "bar": 123}) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 456}, - timestamp="2020-01-01T00:00:00Z", # Exactly the same as date_from - ) - self._create_event( - event="whatever", # This event is not a $pageview, but with no events/actions specified, it will be counted - actor_properties={"foo": "abc", "bar": 123}, # Changed bar - timestamp="2020-01-01T01:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 123}, # Changed bar - timestamp="2020-01-02T01:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 123}, - timestamp="2020-01-03T01:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 123}, - timestamp="2020-01-04T19:00:00Z", - ) - self._create_event( - event="$pageview", - actor_properties={"foo": "abc", "bar": 789}, # Changed bar - timestamp="2020-01-04T19:00:01Z", - ) - flush_persons_and_events() - - timeline = self._get_timeline_result( - properties=[{"key": "bar", "value": "xyz", "type": "person"}], - date_from="2020-01-01", - date_to="2020-01-05", - ) - - self.assertEqual( - timeline, + "id": "$pageview", + } + ], + properties=[{"key": "bar", "value": "xyz", "type": "person"}], + date_from="2020-01-01", + date_to="2020-01-05", + ) + + self.assertEqual( + timeline, + { + "points": [ + { + "properties": {"foo": "abc", "bar": 456}, + "relevant_event_count": 1, + "timestamp": "2020-01-01T00:00:00Z", + }, + { + "properties": {"foo": "abc", "bar": 123}, + "relevant_event_count": 3, + "timestamp": "2020-01-02T01:00:00Z", + }, + { + "properties": {"foo": "abc", "bar": 789}, + "relevant_event_count": 1, + "timestamp": "2020-01-04T19:00:01Z", + }, + ], + "crucial_property_keys": ["bar"], + "effective_date_from": "2020-01-01T00:00:00+00:00", + "effective_date_to": "2020-01-05T23:59:59.999999+00:00", + }, + ) + + @snapshot_clickhouse_queries + def test_timeline_for_existing_actor_with_six_events_but_only_two_relevant_changes_without_filters(self): + self._create_person({"foo": "abc", "bar": 123}) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 456}, + timestamp="2020-01-01T00:00:00Z", # Exactly the same as date_from + ) + self._create_event( + event="whatever", # This event is not a $pageview, but this doesn't matter here anyway + actor_properties={"foo": "abc", "bar": 123}, # Changed bar + timestamp="2020-01-01T01:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 123}, # Changed bar + timestamp="2020-01-02T01:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 123}, + timestamp="2020-01-03T01:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 123}, + timestamp="2020-01-04T19:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 789}, # Changed bar + timestamp="2020-01-04T19:00:01Z", + ) + flush_persons_and_events() + + timeline = self._get_timeline_result( + events=[ { - "points": [ - { - "properties": {"foo": "abc", "bar": 456}, - "relevant_event_count": 1, - "timestamp": "2020-01-01T00:00:00Z", - }, - { - "properties": {"foo": "abc", "bar": 123}, - "relevant_event_count": 4, - "timestamp": "2020-01-01T01:00:00Z", # whatever event - }, - { - "properties": {"foo": "abc", "bar": 789}, - "relevant_event_count": 1, - "timestamp": "2020-01-04T19:00:01Z", - }, - ], - "crucial_property_keys": ["bar"], - "effective_date_from": "2020-01-01T00:00:00+00:00", - "effective_date_to": "2020-01-05T23:59:59.999999+00:00", - }, - ) - - return TestPropertiesTimeline - - -# Only test person properties timeline if persons-on-events is enabled -# Using CONSTANCE_CONFIG instead of get_instance_setting, becasue DB access is only allowed _inside_ the Test* class -if CONSTANCE_CONFIG["PERSON_ON_EVENTS_ENABLED"][0]: - - class TestPersonPropertiesTimeline(properties_timeline_test_factory(actor_type="person")): # type: ignore - pass - - -# TODO: Uncomment below when groups-on-events is released, and make sure everything works -# if CONSTANCE_CONFIG["GROUPS_ON_EVENTS_ENABLED"][0]: -# -# class TestGroupPropertiesTimeline(properties_timeline_test_factory(actor_type="group")): # type: ignore -# pass + "id": "$pageview", + } + ], + date_from="2020-01-01", + date_to="2020-01-05", + ) + + self.assertEqual( + timeline, # Without filters, NO changes are relevant + { + "points": [ + { + "properties": {"foo": "abc", "bar": 456}, + "relevant_event_count": 5, + "timestamp": "2020-01-01T00:00:00Z", + }, + ], + "crucial_property_keys": [], + "effective_date_from": "2020-01-01T00:00:00+00:00", + "effective_date_to": "2020-01-05T23:59:59.999999+00:00", + }, + ) + + @snapshot_clickhouse_queries + def test_timeline_for_existing_actor_with_six_events_but_only_two_relevant_changes_without_events(self): + self._create_person({"foo": "abc", "bar": 123}) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 456}, + timestamp="2020-01-01T00:00:00Z", # Exactly the same as date_from + ) + self._create_event( + event="whatever", # This event is not a $pageview, but with no events/actions specified, it will be counted + actor_properties={"foo": "abc", "bar": 123}, # Changed bar + timestamp="2020-01-01T01:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 123}, # Changed bar + timestamp="2020-01-02T01:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 123}, + timestamp="2020-01-03T01:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 123}, + timestamp="2020-01-04T19:00:00Z", + ) + self._create_event( + event="$pageview", + actor_properties={"foo": "abc", "bar": 789}, # Changed bar + timestamp="2020-01-04T19:00:01Z", + ) + flush_persons_and_events() + + timeline = self._get_timeline_result( + properties=[{"key": "bar", "value": "xyz", "type": "person"}], + date_from="2020-01-01", + date_to="2020-01-05", + ) + + self.assertEqual( + timeline, + { + "points": [ + { + "properties": {"foo": "abc", "bar": 456}, + "relevant_event_count": 1, + "timestamp": "2020-01-01T00:00:00Z", + }, + { + "properties": {"foo": "abc", "bar": 123}, + "relevant_event_count": 4, + "timestamp": "2020-01-01T01:00:00Z", # whatever event + }, + { + "properties": {"foo": "abc", "bar": 789}, + "relevant_event_count": 1, + "timestamp": "2020-01-04T19:00:01Z", + }, + ], + "crucial_property_keys": ["bar"], + "effective_date_from": "2020-01-01T00:00:00+00:00", + "effective_date_to": "2020-01-05T23:59:59.999999+00:00", + }, + ) diff --git a/posthog/api/test/test_team.py b/posthog/api/test/test_team.py index 6a6825f598f42..1eb35ff162669 100644 --- a/posthog/api/test/test_team.py +++ b/posthog/api/test/test_team.py @@ -73,10 +73,6 @@ def test_retrieve_project(self): response_data["person_on_events_querying_enabled"], get_instance_setting("PERSON_ON_EVENTS_ENABLED") or get_instance_setting("PERSON_ON_EVENTS_V2_ENABLED"), ) - self.assertEqual( - response_data["groups_on_events_querying_enabled"], - get_instance_setting("GROUPS_ON_EVENTS_ENABLED"), - ) # TODO: These assertions will no longer make sense when we fully remove these attributes from the model self.assertNotIn("event_names", response_data) diff --git a/posthog/models/property/util.py b/posthog/models/property/util.py index de2602539e6ec..0a28a3f3f0683 100644 --- a/posthog/models/property/util.py +++ b/posthog/models/property/util.py @@ -49,7 +49,6 @@ PropertyName, ) from posthog.models.property.property import ValueT -from posthog.models.team.team import groups_on_events_querying_enabled from posthog.queries.person_distinct_id_query import get_team_distinct_ids_query from posthog.session_recordings.queries.session_query import SessionQuery from posthog.queries.util import PersonPropertiesMode @@ -288,27 +287,6 @@ def parse_prop_clauses( if query: final.append(f"{property_operator} {query}") params.update(filter_params) - elif ( - prop.type == "group" - and person_properties_mode - in [ - PersonPropertiesMode.DIRECT_ON_EVENTS, - PersonPropertiesMode.DIRECT_ON_EVENTS_WITH_POE_V2, - ] - and groups_on_events_querying_enabled() - ): - group_column = f"group{prop.group_type_index}_properties" - filter_query, filter_params = prop_filter_json_extract( - prop, - idx, - prepend, - prop_var=group_column, - allow_denormalized_props=True, - property_operator=property_operator, - use_event_column=group_column, - ) - final.append(filter_query) - params.update(filter_params) elif prop.type == "group": if group_properties_joined: filter_query, filter_params = prop_filter_json_extract( @@ -740,7 +718,7 @@ def get_property_string_expr( if ( allow_denormalized_props and (property_name, materialised_table_column) in materialized_columns - and ("group" not in materialised_table_column or groups_on_events_querying_enabled()) + and "group" not in materialised_table_column ): return ( f'{table_string}"{materialized_columns[(property_name, materialised_table_column)]}"', diff --git a/posthog/models/team/team.py b/posthog/models/team/team.py index e06104ce1c2a6..e798a906386f1 100644 --- a/posthog/models/team/team.py +++ b/posthog/models/team/team.py @@ -482,15 +482,6 @@ def delete_team_in_cache_on_delete(sender, instance: Team, **kwargs): set_team_in_cache(instance.api_token, None) -def groups_on_events_querying_enabled(): - """ - Returns whether to allow querying groups columns on events. - - Remove all usages of this when the feature is released to everyone. - """ - return get_instance_setting("GROUPS_ON_EVENTS_ENABLED") - - def check_is_feature_available_for_team(team_id: int, feature_key: str, current_usage: Optional[int] = None): available_product_features: Optional[list[dict[str, str]]] = ( Team.objects.select_related("organization") diff --git a/posthog/queries/breakdown_props.py b/posthog/queries/breakdown_props.py index 010d61aa8d16c..8475866f4dddc 100644 --- a/posthog/queries/breakdown_props.py +++ b/posthog/queries/breakdown_props.py @@ -22,7 +22,6 @@ parse_prop_grouped_clauses, ) from posthog.models.team import Team -from posthog.models.team.team import groups_on_events_querying_enabled from posthog.queries.column_optimizer.column_optimizer import ColumnOptimizer from posthog.queries.groups_join_query import GroupsJoinQuery from posthog.queries.insight import insight_sync_execute @@ -95,8 +94,7 @@ def get_breakdown_prop_values( outer_properties: Optional[PropertyGroup] = props_to_filter person_id_joined_alias = "e.person_id" - if not groups_on_events_querying_enabled(): - groups_join_clause, groups_join_params = GroupsJoinQuery(filter, team.pk, column_optimizer).get_join_query() + groups_join_clause, groups_join_params = GroupsJoinQuery(filter, team.pk, column_optimizer).get_join_query() else: outer_properties = ( column_optimizer.property_optimizer.parse_property_groups(props_to_filter).outer diff --git a/posthog/queries/funnels/base.py b/posthog/queries/funnels/base.py index ea337267a2c57..b72e223ab24aa 100644 --- a/posthog/queries/funnels/base.py +++ b/posthog/queries/funnels/base.py @@ -24,7 +24,6 @@ get_single_or_multi_property_string_expr, parse_prop_grouped_clauses, ) -from posthog.models.team.team import groups_on_events_querying_enabled from posthog.queries.breakdown_props import ( format_breakdown_cohort_join_query, get_breakdown_cohort_name, @@ -760,27 +759,13 @@ def _get_breakdown_select_prop(self) -> tuple[str, dict[str, Any]]: # :TRICKY: We only support string breakdown for group properties assert isinstance(self._filter.breakdown, str) - if ( - alias_poe_mode_for_legacy(self._team.person_on_events_mode) != PersonsOnEventsMode.DISABLED - and groups_on_events_querying_enabled() - ): - properties_field = f"group{self._filter.breakdown_group_type_index}_properties" - expression, _ = get_property_string_expr( - table="events", - property_name=self._filter.breakdown, - var="%(breakdown)s", - column=properties_field, - allow_denormalized_props=True, - materialised_table_column=properties_field, - ) - else: - properties_field = f"group_properties_{self._filter.breakdown_group_type_index}" - expression, _ = get_property_string_expr( - table="groups", - property_name=self._filter.breakdown, - var="%(breakdown)s", - column=properties_field, - ) + properties_field = f"group_properties_{self._filter.breakdown_group_type_index}" + expression, _ = get_property_string_expr( + table="groups", + property_name=self._filter.breakdown, + var="%(breakdown)s", + column=properties_field, + ) basic_prop_selector = f"{expression} AS prop_basic" elif self._filter.breakdown_type == "hogql": from posthog.hogql.hogql import translate_hogql diff --git a/posthog/queries/trends/breakdown.py b/posthog/queries/trends/breakdown.py index 663ccd42ccff6..28c9e3d4d2ab5 100644 --- a/posthog/queries/trends/breakdown.py +++ b/posthog/queries/trends/breakdown.py @@ -30,7 +30,6 @@ parse_prop_grouped_clauses, ) from posthog.models.team import Team -from posthog.models.team.team import groups_on_events_querying_enabled from posthog.queries.breakdown_props import ( ALL_USERS_COHORT_ID, format_breakdown_cohort_join_query, @@ -520,7 +519,6 @@ def _get_breakdown_value(self, breakdown: str) -> str: from posthog.hogql.hogql import translate_hogql breakdown_value = translate_hogql(breakdown, self.filter.hogql_context) - elif self.filter.breakdown_type == "session": if breakdown == "$session_duration": # Return the session duration expression right away because it's already an number, @@ -528,20 +526,6 @@ def _get_breakdown_value(self, breakdown: str) -> str: breakdown_value = f"{SessionQuery.SESSION_TABLE_ALIAS}.session_duration" else: raise ValidationError(f'Invalid breakdown "{breakdown}" for breakdown type "session"') - - elif ( - self.person_on_events_mode != PersonsOnEventsMode.DISABLED - and self.filter.breakdown_type == "group" - and groups_on_events_querying_enabled() - ): - properties_field = f"group{self.filter.breakdown_group_type_index}_properties" - breakdown_value, _ = get_property_string_expr( - "events", - breakdown, - "%(key)s", - properties_field, - materialised_table_column=properties_field, - ) elif self.person_on_events_mode != PersonsOnEventsMode.DISABLED and self.filter.breakdown_type != "group": if self.filter.breakdown_type == "person": breakdown_value, _ = get_property_string_expr( diff --git a/posthog/settings/dynamic_settings.py b/posthog/settings/dynamic_settings.py index 7f16237912f6d..0f883965d2f45 100644 --- a/posthog/settings/dynamic_settings.py +++ b/posthog/settings/dynamic_settings.py @@ -41,11 +41,6 @@ "Whether to use query path using person_id and person_properties on events or the old query", bool, ), - "GROUPS_ON_EVENTS_ENABLED": ( - get_from_env("GROUPS_ON_EVENTS_ENABLED", False, type_cast=str_to_bool), - "Whether to use query path using group_properties on events or the old query", - bool, - ), "AUTO_START_ASYNC_MIGRATIONS": ( get_from_env("AUTO_START_ASYNC_MIGRATIONS", False, type_cast=str_to_bool), "Whether the earliest unapplied async migration should be triggered automatically on server startup.", @@ -212,7 +207,6 @@ "ASYNC_MIGRATIONS_OPT_OUT_EMAILS", "PERSON_ON_EVENTS_ENABLED", "PERSON_ON_EVENTS_V2_ENABLED", - "GROUPS_ON_EVENTS_ENABLED", "STRICT_CACHING_TEAMS", "SLACK_APP_CLIENT_ID", "SLACK_APP_CLIENT_SECRET", diff --git a/posthog/test/test_middleware.py b/posthog/test/test_middleware.py index dfbed651e918a..ce8bfeb71b7bb 100644 --- a/posthog/test/test_middleware.py +++ b/posthog/test/test_middleware.py @@ -124,7 +124,7 @@ class TestAutoProjectMiddleware(APIBaseTest): @classmethod def setUpTestData(cls): super().setUpTestData() - cls.base_app_num_queries = 41 + cls.base_app_num_queries = 40 # Create another team that the user does have access to cls.second_team = create_team(organization=cls.organization, name="Second Life")